]> git.saurik.com Git - apple/xnu.git/blame - bsd/kern/pthread_support.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-1699.32.7.tar.gz
[apple/xnu.git] / bsd / kern / pthread_support.c
CommitLineData
2d21ac55 1/*
b0d623f7 2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
2d21ac55
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* Copyright (c) 1995-2005 Apple Computer, Inc. All Rights Reserved */
29/*
30 * pthread_support.c
31 */
32
b0d623f7 33#if PSYNCH
2d21ac55
A		 34
35#include <sys/param.h>
b0d623f7 36#include <sys/queue.h>
2d21ac55
A		 37#include <sys/resourcevar.h>
38#include <sys/proc_internal.h>
39#include <sys/kauth.h>
40#include <sys/systm.h>
41#include <sys/timeb.h>
42#include <sys/times.h>
b0d623f7 43#include <sys/time.h>
2d21ac55 44#include <sys/acct.h>
2d21ac55
A		 45#include <sys/kernel.h>
46#include <sys/wait.h>
47#include <sys/signalvar.h>
48#include <sys/syslog.h>
49#include <sys/stat.h>
50#include <sys/lock.h>
51#include <sys/kdebug.h>
2d21ac55 52#include <sys/sysproto.h>
b0d623f7 53#include <sys/pthread_internal.h>
2d21ac55 54#include <sys/vm.h>
b0d623f7
A		 55#include <sys/user.h>
56
57#include <mach/mach_types.h>
58#include <mach/vm_prot.h>
59#include <mach/semaphore.h>
60#include <mach/sync_policy.h>
61#include <mach/task.h>
62#include <kern/kern_types.h>
63#include <kern/task.h>
64#include <kern/clock.h>
65#include <mach/kern_return.h>
2d21ac55 66#include <kern/thread.h>
b0d623f7
A		 67#include <kern/sched_prim.h>
68#include <kern/thread_call.h>
69#include <kern/kalloc.h>
6d2010ae 70#include <kern/zalloc.h>
b0d623f7
A		 71#include <kern/sched_prim.h>
72#include <kern/processor.h>
73#include <kern/affinity.h>
74#include <kern/wait_queue.h>
6d2010ae 75#include <kern/mach_param.h>
b0d623f7
A		 76#include <mach/mach_vm.h>
77#include <mach/mach_param.h>
78#include <mach/thread_policy.h>
79#include <mach/message.h>
80#include <mach/port.h>
81#include <vm/vm_protos.h>
82#include <vm/vm_map.h>
83#include <mach/vm_region.h>
2d21ac55 84
b0d623f7 85#include <libkern/OSAtomic.h>
2d21ac55 86
6d2010ae
A		 87#include <pexpert/pexpert.h>
88
89#define __PSYNCH_DEBUG__ 0 /* debug panic actions */
90#define _PSYNCH_TRACE_ 1 /* kdebug trace */
91
92#define __TESTMODE__ 2 /* 0 - return error on user error conditions */
93 /* 1 - log error on user error conditions */
94 /* 2 - abort caller on user error conditions */
95 /* 3 - panic on user error conditions */
96static int __test_panics__;
97static int __test_aborts__;
98static int __test_prints__;
99
100static inline void __FAILEDUSERTEST__(const char *str)
101{
102 proc_t p;
103
104 if (__test_panics__ != 0)
105 panic(str);
106
107 if (__test_aborts__ != 0 || __test_prints__ != 0)
108 p = current_proc();
109
110 if (__test_prints__ != 0)
111 printf("PSYNCH: pid[%d]: %s\n", p->p_pid, str);
112
113 if (__test_aborts__ != 0)
114 psignal(p, SIGABRT);
115}
2d21ac55 116
b0d623f7
A		 117#if _PSYNCH_TRACE_
118#define _PSYNCH_TRACE_MLWAIT 0x9000000
119#define _PSYNCH_TRACE_MLDROP 0x9000004
120#define _PSYNCH_TRACE_CVWAIT 0x9000008
121#define _PSYNCH_TRACE_CVSIGNAL 0x900000c
122#define _PSYNCH_TRACE_CVBROAD 0x9000010
123#define _PSYNCH_TRACE_KMDROP 0x9000014
124#define _PSYNCH_TRACE_RWRDLOCK 0x9000018
125#define _PSYNCH_TRACE_RWLRDLOCK 0x900001c
126#define _PSYNCH_TRACE_RWWRLOCK 0x9000020
127#define _PSYNCH_TRACE_RWYWRLOCK 0x9000024
128#define _PSYNCH_TRACE_RWUPGRADE 0x9000028
129#define _PSYNCH_TRACE_RWDOWNGRADE 0x900002c
130#define _PSYNCH_TRACE_RWUNLOCK 0x9000030
131#define _PSYNCH_TRACE_RWUNLOCK2 0x9000034
132#define _PSYNCH_TRACE_RWHANDLEU 0x9000038
133#define _PSYNCH_TRACE_FSEQTILL 0x9000040
6d2010ae
A		 134#define _PSYNCH_TRACE_CLRPRE 0x9000044
135#define _PSYNCH_TRACE_CVHBROAD 0x9000048
136#define _PSYNCH_TRACE_CVSEQ 0x900004c
137#define _PSYNCH_TRACE_THWAKEUP 0x9000050
b0d623f7
A		 138/* user side */
139#define _PSYNCH_TRACE_UM_LOCK 0x9000060
140#define _PSYNCH_TRACE_UM_UNLOCK 0x9000064
141#define _PSYNCH_TRACE_UM_MHOLD 0x9000068
142#define _PSYNCH_TRACE_UM_MDROP 0x900006c
143#define _PSYNCH_TRACE_UM_CVWAIT 0x9000070
144#define _PSYNCH_TRACE_UM_CVSIG 0x9000074
145#define _PSYNCH_TRACE_UM_CVBRD 0x9000078
2d21ac55 146
6d2010ae
A		 147proc_t pthread_debug_proc = PROC_NULL;
148static inline void __PTHREAD_TRACE_DEBUG(uint32_t debugid, uintptr_t arg1,
149 uintptr_t arg2,
150 uintptr_t arg3,
151 uintptr_t arg4,
152 uintptr_t arg5)
153{
154 proc_t p = current_proc();
155
156 if ((pthread_debug_proc != NULL) && (p == pthread_debug_proc))
157 KERNEL_DEBUG_CONSTANT(debugid, arg1, arg2, arg3, arg4, arg5);
158}
159
b0d623f7 160#endif /* _PSYNCH_TRACE_ */
2d21ac55 161
6d2010ae
A		 162#define ECVCERORR 256
163#define ECVPERORR 512
164
b0d623f7
A		 165lck_mtx_t * pthread_list_mlock;
166
167#define PTHHASH(addr) (&pthashtbl[(addr) & pthhash])
168extern LIST_HEAD(pthhashhead, ksyn_wait_queue) *pth_glob_hashtbl;
169struct pthhashhead * pth_glob_hashtbl;
170u_long pthhash;
171
172LIST_HEAD(, ksyn_wait_queue) pth_free_list;
6d2010ae
A		 173int num_total_kwq = 0; /* number of kwq in use currently */
174int num_infreekwq = 0; /* number of kwq in free list */
175int num_freekwq = 0; /* number of kwq actually freed from the free the list */
176int num_reusekwq = 0; /* number of kwq pulled back for reuse from free list */
177int num_addedfreekwq = 0; /* number of added free kwq from the last instance */
178int num_lastfreekwqcount = 0; /* the free count from the last time */
b0d623f7
A		 179
180static int PTH_HASHSIZE = 100;
181
6d2010ae
A		 182static zone_t kwq_zone; /* zone for allocation of ksyn_queue */
183static zone_t kwe_zone; /* zone for allocation of ksyn_waitq_element */
b0d623f7
A		 184
185#define SEQFIT 0
186#define FIRSTFIT 1
187
188struct ksyn_queue {
6d2010ae 189 TAILQ_HEAD(ksynq_kwelist_head, ksyn_waitq_element) ksynq_kwelist;
b0d623f7
A		 190 uint32_t ksynq_count; /* number of entries in queue */
191 uint32_t ksynq_firstnum; /* lowest seq in queue */
192 uint32_t ksynq_lastnum; /* highest seq in queue */
193};
6d2010ae 194typedef struct ksyn_queue * ksyn_queue_t;
b0d623f7
A		 195
196#define KSYN_QUEUE_READ 0
197#define KSYN_QUEUE_LREAD 1
198#define KSYN_QUEUE_WRITER 2
199#define KSYN_QUEUE_YWRITER 3
200#define KSYN_QUEUE_UPGRADE 4
201#define KSYN_QUEUE_MAX 5
202
203struct ksyn_wait_queue {
204 LIST_ENTRY(ksyn_wait_queue) kw_hash;
205 LIST_ENTRY(ksyn_wait_queue) kw_list;
b0d623f7
A		 206 user_addr_t kw_addr;
207 uint64_t kw_owner;
208 uint64_t kw_object; /* object backing in shared mode */
209 uint64_t kw_offset; /* offset inside the object in shared mode */
210 int kw_flags; /* mutex, cvar options/flags */
211 int kw_pflags; /* flags under listlock protection */
212 struct timeval kw_ts; /* timeval need for upkeep before free */
213 int kw_iocount; /* inuse reference */
6d2010ae 214 int kw_dropcount; /* current users unlocking... */
b0d623f7
A		 215
216 int kw_type; /* queue type like mutex, cvar, etc */
217 uint32_t kw_inqueue; /* num of waiters held */
6d2010ae 218 uint32_t kw_fakecount; /* number of error/prepost fakes */
b0d623f7
A		 219 uint32_t kw_highseq; /* highest seq in the queue */
220 uint32_t kw_lowseq; /* lowest seq in the queue */
6d2010ae
A		 221 uint32_t kw_lword; /* L value from userland */
222 uint32_t kw_uword; /* U world value from userland */
223 uint32_t kw_sword; /* S word value from userland */
b0d623f7 224 uint32_t kw_lastunlockseq; /* the last seq that unlocked */
6d2010ae
A		 225/* for CV to be used as the seq kernel has seen so far */
226#define kw_cvkernelseq kw_lastunlockseq
227 uint32_t kw_lastseqword; /* the last seq that unlocked */
228/* for mutex and cvar we need to track I bit values */
229 uint32_t kw_nextseqword; /* the last seq that unlocked; with num of waiters */
230#define kw_initrecv kw_nextseqword /* number of incoming waiters with Ibit seen sofar */
231 uint32_t kw_overlapwatch; /* chance for overlaps */
232#define kw_initcount kw_overlapwatch /* number of incoming waiters with Ibit expected */
233 uint32_t kw_initcountseq; /* highest seq with Ibit on for mutex and cvar*/
b0d623f7
A		 234 uint32_t kw_pre_rwwc; /* prepost count */
235 uint32_t kw_pre_lockseq; /* prepost target seq */
6d2010ae 236 uint32_t kw_pre_sseq; /* prepost target sword, in cvar used for mutexowned */
b0d623f7
A		 237 uint32_t kw_pre_intrcount; /* prepost of missed wakeup due to intrs */
238 uint32_t kw_pre_intrseq; /* prepost of missed wakeup limit seq */
239 uint32_t kw_pre_intrretbits; /* return bits value for missed wakeup threads */
240 uint32_t kw_pre_intrtype; /* type of failed wakueps*/
241
242 int kw_kflags;
b0d623f7
A		 243 struct ksyn_queue kw_ksynqueues[KSYN_QUEUE_MAX]; /* queues to hold threads */
244 lck_mtx_t kw_lock; /* mutex lock protecting this structure */
b0d623f7 245};
b0d623f7
A		 246typedef struct ksyn_wait_queue * ksyn_wait_queue_t;
247
b0d623f7 248#define PTHRW_INC 0x100
6d2010ae 249#define PTHRW_BIT_MASK 0x000000ff
b0d623f7
A		 250
251#define PTHRW_COUNT_SHIFT 8
252#define PTHRW_COUNT_MASK 0xffffff00
253#define PTHRW_MAX_READERS 0xffffff00
254
6d2010ae
A		 255/* New model bits on Lword */
256#define PTH_RWL_KBIT 0x01 /* users cannot acquire in user mode */
257#define PTH_RWL_EBIT 0x02 /* exclusive lock in progress */
258#define PTH_RWL_WBIT 0x04 /* write waiters pending in kernel */
259#define PTH_RWL_PBIT 0x04 /* prepost (cv) pending in kernel */
260#define PTH_RWL_YBIT 0x08 /* yielding write waiters pending in kernel */
261#define PTH_RWL_RETRYBIT 0x08 /* mutex retry wait */
262#define PTH_RWL_LBIT 0x10 /* long read in progress */
263#define PTH_RWL_MTXNONE 0x10 /* indicates the cvwait does not have mutex held */
264#define PTH_RWL_UBIT 0x20 /* upgrade request pending */
265#define PTH_RWL_MTX_WAIT 0x20 /* in cvar in mutex wait */
266#define PTH_RWL_RBIT 0x40 /* reader pending in kernel(not used) */
267#define PTH_RWL_MBIT 0x40 /* overlapping grants from kernel */
268#define PTH_RWL_TRYLKBIT 0x40 /* trylock attempt (mutex only) */
269#define PTH_RWL_IBIT 0x80 /* lcok reset, held untill first succeesful unlock */
270
271
272/* UBIT values for mutex, cvar */
273#define PTH_RWU_SBIT 0x01
274#define PTH_RWU_BBIT 0x02
275
276#define PTHRW_RWL_INIT PTH_RWL_IBIT /* reset state on the lock bits (U)*/
277
278/* New model bits on Sword */
279#define PTH_RWS_SBIT 0x01 /* kernel transition seq not set yet*/
280#define PTH_RWS_IBIT 0x02 /* Sequence is not set on return from kernel */
281#define PTH_RWS_CV_CBIT PTH_RWS_SBIT /* kernel has cleared all info w.r.s.t CV */
282#define PTH_RWS_CV_PBIT PTH_RWS_IBIT /* kernel has prepost/fake structs only,no waiters */
283#define PTH_RWS_CV_MBIT PTH_RWL_MBIT /* to indicate prepost return */
284#define PTH_RWS_WSVBIT 0x04 /* save W bit */
285#define PTH_RWS_USVBIT 0x08 /* save U bit */
286#define PTH_RWS_YSVBIT 0x10 /* save Y bit */
287#define PTHRW_RWS_INIT PTH_RWS_SBIT /* reset on the lock bits (U)*/
288#define PTHRW_RWS_SAVEMASK (PTH_RWS_WSVBIT|PTH_RWS_USVBIT|PTH_RWS_YSVBIT) /*save bits mask*/
289#define PTHRW_SW_Reset_BIT_MASK 0x000000fe /* remove S bit and get rest of the bits */
290
291#define PTHRW_RWS_INIT PTH_RWS_SBIT /* reset on the lock bits (U)*/
292
293
294#define PTHRW_UN_BIT_MASK 0x000000bf /* remove overlap bit */
295
296
297#define PTHREAD_MTX_TID_SWITCHING (uint64_t)-1
298
299/* new L word defns */
300#define is_rwl_readinuser(x) ((((x) & (PTH_RWL_UBIT | PTH_RWL_KBIT)) == 0)||(((x) & PTH_RWL_LBIT) != 0))
301#define is_rwl_ebit_set(x) (((x) & PTH_RWL_EBIT) != 0)
302#define is_rwl_lbit_set(x) (((x) & PTH_RWL_LBIT) != 0)
303#define is_rwl_readoverlap(x) (((x) & PTH_RWL_MBIT) != 0)
304#define is_rw_ubit_set(x) (((x) & PTH_RWL_UBIT) != 0)
305
306/* S word checks */
307#define is_rws_setseq(x) (((x) & PTH_RWS_SBIT))
308#define is_rws_setunlockinit(x) (((x) & PTH_RWS_IBIT))
309
b0d623f7
A		 310/* first contended seq that kernel sees */
311#define KW_MTXFIRST_KSEQ 0x200
312#define KW_CVFIRST_KSEQ 1
313#define KW_RWFIRST_KSEQ 0x200
314
6d2010ae
A		 315int is_seqlower(uint32_t x, uint32_t y);
316int is_seqlower_eq(uint32_t x, uint32_t y);
317int is_seqhigher(uint32_t x, uint32_t y);
318int is_seqhigher_eq(uint32_t x, uint32_t y);
319int find_diff(uint32_t upto, uint32_t lowest);
b0d623f7 320
b0d623f7
A		 321
322static inline int diff_genseq(uint32_t x, uint32_t y) {
323 if (x > y) {
324 return(x-y);
325 } else {
326 return((PTHRW_MAX_READERS - y) + x + PTHRW_INC);
2d21ac55 327 }
2d21ac55
A		 328}
329
b0d623f7
A		 330#define TID_ZERO (uint64_t)0
331
332/* bits needed in handling the rwlock unlock */
333#define PTH_RW_TYPE_READ 0x01
334#define PTH_RW_TYPE_LREAD 0x02
335#define PTH_RW_TYPE_WRITE 0x04
336#define PTH_RW_TYPE_YWRITE 0x08
337#define PTH_RW_TYPE_UPGRADE 0x10
338#define PTH_RW_TYPE_MASK 0xff
339#define PTH_RW_TYPE_SHIFT 8
340
341#define PTH_RWSHFT_TYPE_READ 0x0100
342#define PTH_RWSHFT_TYPE_LREAD 0x0200
343#define PTH_RWSHFT_TYPE_WRITE 0x0400
344#define PTH_RWSHFT_TYPE_YWRITE 0x0800
345#define PTH_RWSHFT_TYPE_MASK 0xff00
346
347/*
348 * Mutex protocol attributes
349 */
350#define PTHREAD_PRIO_NONE 0
351#define PTHREAD_PRIO_INHERIT 1
352#define PTHREAD_PRIO_PROTECT 2
353#define PTHREAD_PROTOCOL_FLAGS_MASK 0x3
354
355/*
356 * Mutex type attributes
357 */
358#define PTHREAD_MUTEX_NORMAL 0
359#define PTHREAD_MUTEX_ERRORCHECK 4
360#define PTHREAD_MUTEX_RECURSIVE 8
361#define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_NORMAL
362#define PTHREAD_TYPE_FLAGS_MASK 0xc
363
364/*
365 * Mutex pshared attributes
366 */
367#define PTHREAD_PROCESS_SHARED 0x10
368#define PTHREAD_PROCESS_PRIVATE 0x20
369#define PTHREAD_PSHARED_FLAGS_MASK 0x30
370
371/*
372 * Mutex policy attributes
373 */
374#define _PTHREAD_MUTEX_POLICY_NONE 0
375#define _PTHREAD_MUTEX_POLICY_FAIRSHARE 0x040 /* 1 */
376#define _PTHREAD_MUTEX_POLICY_FIRSTFIT 0x080 /* 2 */
377#define _PTHREAD_MUTEX_POLICY_REALTIME 0x0c0 /* 3 */
378#define _PTHREAD_MUTEX_POLICY_ADAPTIVE 0x100 /* 4 */
379#define _PTHREAD_MUTEX_POLICY_PRIPROTECT 0x140 /* 5 */
380#define _PTHREAD_MUTEX_POLICY_PRIINHERIT 0x180 /* 6 */
381#define PTHREAD_POLICY_FLAGS_MASK 0x1c0
382
383#define _PTHREAD_MTX_OPT_HOLDLOCK 0x200
6d2010ae
A		 384#define _PTHREAD_MTX_OPT_NOMTX 0x400
385
386#define _PTHREAD_MTX_OPT_NOTIFY 0x1000
387#define _PTHREAD_MTX_OPT_MUTEX 0x2000 /* this is a mutex type */
b0d623f7 388
6d2010ae
A		 389#define _PTHREAD_RWLOCK_UPGRADE_TRY 0x10000
390
391/* pflags */
b0d623f7
A		 392#define KSYN_WQ_INLIST 1
393#define KSYN_WQ_INHASH 2
394#define KSYN_WQ_SHARED 4
6d2010ae 395#define KSYN_WQ_WAITING 8 /* threads waiting for this wq to be available */
b0d623f7
A		 396#define KSYN_WQ_FLIST 0X10 /* in free list to be freed after a short delay */
397
6d2010ae
A		 398/* kflags */
399#define KSYN_KWF_INITCLEARED 1 /* the init status found and preposts cleared */
400#define KSYN_KWF_ZEROEDOUT 2 /* the lword, etc are inited to 0 */
401
b0d623f7
A		 402#define KSYN_CLEANUP_DEADLINE 10
403int psynch_cleanupset;
404thread_call_t psynch_thcall;
2d21ac55 405
b0d623f7 406#define KSYN_WQTYPE_INWAIT 0x1000
6d2010ae 407#define KSYN_WQTYPE_INDROP 0x2000
b0d623f7
A		 408#define KSYN_WQTYPE_MTX 0x1
409#define KSYN_WQTYPE_CVAR 0x2
410#define KSYN_WQTYPE_RWLOCK 0x4
411#define KSYN_WQTYPE_SEMA 0x8
412#define KSYN_WQTYPE_BARR 0x10
6d2010ae 413#define KSYN_WQTYPE_MASK 0x00ff
b0d623f7
A		 414
415#define KSYN_MTX_MAX 0x0fffffff
6d2010ae 416#define KSYN_WQTYPE_MUTEXDROP (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX)
b0d623f7
A		 417
418#define KW_UNLOCK_PREPOST 0x01
419#define KW_UNLOCK_PREPOST_UPGRADE 0x02
420#define KW_UNLOCK_PREPOST_DOWNGRADE 0x04
421#define KW_UNLOCK_PREPOST_READLOCK 0x08
422#define KW_UNLOCK_PREPOST_LREADLOCK 0x10
423#define KW_UNLOCK_PREPOST_WRLOCK 0x20
424#define KW_UNLOCK_PREPOST_YWRLOCK 0x40
425
426#define CLEAR_PREPOST_BITS(kwq) {\
427 kwq->kw_pre_lockseq = 0; \
6d2010ae 428 kwq->kw_pre_sseq = PTHRW_RWS_INIT; \
b0d623f7 429 kwq->kw_pre_rwwc = 0; \
b0d623f7
A		 430 }
431
6d2010ae
A		 432#define CLEAR_INITCOUNT_BITS(kwq) {\
433 kwq->kw_initcount = 0; \
434 kwq->kw_initrecv = 0; \
435 kwq->kw_initcountseq = 0; \
b0d623f7
A		 436 }
437
438#define CLEAR_INTR_PREPOST_BITS(kwq) {\
439 kwq->kw_pre_intrcount = 0; \
440 kwq->kw_pre_intrseq = 0; \
441 kwq->kw_pre_intrretbits = 0; \
442 kwq->kw_pre_intrtype = 0; \
443 }
6d2010ae
A		 444
445#define CLEAR_REINIT_BITS(kwq) {\
446 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) { \
447 if((kwq->kw_inqueue != 0) && (kwq->kw_inqueue != kwq->kw_fakecount)) \
448 panic("CV:entries in queue durinmg reinit %d:%d\n",kwq->kw_inqueue, kwq->kw_fakecount); \
449 };\
450 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_RWLOCK) { \
451 kwq->kw_nextseqword = PTHRW_RWS_INIT; \
452 kwq->kw_overlapwatch = 0; \
453 }; \
454 kwq->kw_pre_lockseq = 0; \
455 kwq->kw_pre_rwwc = 0; \
456 kwq->kw_pre_sseq = PTHRW_RWS_INIT; \
457 kwq->kw_lastunlockseq = PTHRW_RWL_INIT; \
458 kwq->kw_lastseqword = PTHRW_RWS_INIT; \
459 kwq->kw_pre_intrcount = 0; \
460 kwq->kw_pre_intrseq = 0; \
461 kwq->kw_pre_intrretbits = 0; \
462 kwq->kw_pre_intrtype = 0; \
463 kwq->kw_lword = 0; \
464 kwq->kw_uword = 0; \
465 kwq->kw_sword = PTHRW_RWS_INIT; \
466 }
467
b0d623f7
A		 468void pthread_list_lock(void);
469void pthread_list_unlock(void);
470void pthread_list_lock_spin(void);
471void pthread_list_lock_convert_spin(void);
472void ksyn_wqlock(ksyn_wait_queue_t kwq);
473void ksyn_wqunlock(ksyn_wait_queue_t kwq);
474ksyn_wait_queue_t ksyn_wq_hash_lookup(user_addr_t mutex, proc_t p, int flags, uint64_t object, uint64_t offset);
475int ksyn_wqfind(user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int flags, int wqtype , ksyn_wait_queue_t * wq);
6d2010ae 476void ksyn_wqrelease(ksyn_wait_queue_t mkwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype);
b0d623f7 477extern int ksyn_findobj(uint64_t mutex, uint64_t * object, uint64_t * offset);
6d2010ae
A		 478static void UPDATE_CVKWQ(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int wqtype);
479extern thread_t port_name_to_thread(mach_port_name_t port_name);
480
7ddcb079 481kern_return_t ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int log, thread_continue_t, void * parameter);
6d2010ae
A		 482kern_return_t ksyn_wakeup_thread(ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe);
483void ksyn_freeallkwe(ksyn_queue_t kq);
484
485uint32_t psynch_mutexdrop_internal(ksyn_wait_queue_t kwq, uint32_t lkseq, uint32_t ugen, int flags);
486int kwq_handle_unlock(ksyn_wait_queue_t, uint32_t mgen, uint32_t rw_wc, uint32_t * updatep, int flags, int *blockp, uint32_t premgen);
487
b0d623f7 488void ksyn_queue_init(ksyn_queue_t kq);
6d2010ae
A		 489int ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, struct uthread * uth, ksyn_waitq_element_t kwe, int firstfit);
490ksyn_waitq_element_t ksyn_queue_removefirst(ksyn_queue_t kq, ksyn_wait_queue_t kwq);
491void ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe);
492int ksyn_queue_move_tofree(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t upto, ksyn_queue_t freeq, int all, int reease);
b0d623f7
A		 493void update_low_high(ksyn_wait_queue_t kwq, uint32_t lockseq);
494uint32_t find_nextlowseq(ksyn_wait_queue_t kwq);
495uint32_t find_nexthighseq(ksyn_wait_queue_t kwq);
6d2010ae 496
b0d623f7 497int find_seq_till(ksyn_wait_queue_t kwq, uint32_t upto, uint32_t nwaiters, uint32_t *countp);
b0d623f7 498uint32_t ksyn_queue_count_tolowest(ksyn_queue_t kq, uint32_t upto);
6d2010ae
A		 499
500ksyn_waitq_element_t ksyn_queue_find_cvpreposeq(ksyn_queue_t kq, uint32_t cgen);
501uint32_t ksyn_queue_cvcount_entries(ksyn_queue_t kq, uint32_t upto, uint32_t from, int * numwaitersp, int * numintrp, int * numprepop);
502void ksyn_handle_cvbroad(ksyn_wait_queue_t ckwq, uint32_t upto, uint32_t *updatep);
503void ksyn_cvupdate_fixup(ksyn_wait_queue_t ckwq, uint32_t *updatep, ksyn_queue_t kfreeq, int release);
504ksyn_waitq_element_t ksyn_queue_find_signalseq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t toseq, uint32_t lockseq);
505ksyn_waitq_element_t ksyn_queue_find_threadseq(ksyn_wait_queue_t ckwq, ksyn_queue_t kq, thread_t th, uint32_t toseq);
7ddcb079
A		 506void psynch_cvcontinue(void *, wait_result_t);
507void psynch_mtxcontinue(void *, wait_result_t);
6d2010ae 508
b0d623f7
A		 509int ksyn_wakeupreaders(ksyn_wait_queue_t kwq, uint32_t limitread, int longreadset, int allreaders, uint32_t updatebits, int * wokenp);
510int kwq_find_rw_lowest(ksyn_wait_queue_t kwq, int flags, uint32_t premgen, int * type, uint32_t lowest[]);
6d2010ae
A		 511ksyn_waitq_element_t ksyn_queue_find_seq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t seq, int remove);
512int kwq_handle_overlap(ksyn_wait_queue_t kwq, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, uint32_t *updatebitsp, int flags , int * blockp);
b0d623f7
A		 513int kwq_handle_downgrade(ksyn_wait_queue_t kwq, uint32_t mgen, int flags, uint32_t premgen, int * blockp);
514
b0d623f7 515static void
6d2010ae 516UPDATE_CVKWQ(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, __unused uint64_t tid, __unused int wqtype)
2d21ac55 517{
6d2010ae
A		 518 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) {
519 if ((kwq->kw_kflags & KSYN_KWF_ZEROEDOUT) != 0) {
520 /* the values of L,U and S are cleared out due to L==S in previous transition */
521 kwq->kw_lword = mgen;
522 kwq->kw_uword = ugen;
523 kwq->kw_sword = rw_wc;
524 kwq->kw_kflags &= ~KSYN_KWF_ZEROEDOUT;
525 }
526 if (is_seqhigher((mgen & PTHRW_COUNT_MASK), (kwq->kw_lword & PTHRW_COUNT_MASK)) != 0)
527 kwq->kw_lword = mgen;
528 if (is_seqhigher((ugen & PTHRW_COUNT_MASK), (kwq->kw_uword & PTHRW_COUNT_MASK)) != 0)
529 kwq->kw_uword = ugen;
530 if ((rw_wc & PTH_RWS_CV_CBIT) != 0) {
531 if(is_seqlower(kwq->kw_cvkernelseq, (rw_wc & PTHRW_COUNT_MASK)) != 0) {
532 kwq->kw_cvkernelseq = (rw_wc & PTHRW_COUNT_MASK);
533 }
534 if (is_seqhigher((rw_wc & PTHRW_COUNT_MASK), (kwq->kw_sword & PTHRW_COUNT_MASK)) != 0)
535 kwq->kw_sword = rw_wc;
536 }
537 }
b0d623f7 538}
2d21ac55 539
6d2010ae 540
b0d623f7
A		 541/* to protect the hashes, iocounts, freelist */
542void
543pthread_list_lock(void)
544{
545 lck_mtx_lock(pthread_list_mlock);
2d21ac55
A		 546}
547
b0d623f7
A		 548void
549pthread_list_lock_spin(void)
550{
551 lck_mtx_lock_spin(pthread_list_mlock);
552}
2d21ac55
A		 553
554void
b0d623f7 555pthread_list_lock_convert_spin(void)
2d21ac55 556{
b0d623f7
A		 557 lck_mtx_convert_spin(pthread_list_mlock);
558}
559
560
561void
562pthread_list_unlock(void)
563{
564 lck_mtx_unlock(pthread_list_mlock);
565}
566
567/* to protect the indiv queue */
568void
569ksyn_wqlock(ksyn_wait_queue_t kwq)
570{
571
572 lck_mtx_lock(&kwq->kw_lock);
573}
574
575void
576ksyn_wqunlock(ksyn_wait_queue_t kwq)
577{
578 lck_mtx_unlock(&kwq->kw_lock);
579}
580
581
582/* routine to drop the mutex unlocks , used both for mutexunlock system call and drop during cond wait */
6d2010ae 583uint32_t
b0d623f7
A		 584psynch_mutexdrop_internal(ksyn_wait_queue_t kwq, uint32_t lkseq, uint32_t ugen, int flags)
585{
6d2010ae 586 uint32_t nextgen, low_writer, updatebits, returnbits = 0;
b0d623f7 587 int firstfit = flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT;
6d2010ae 588 ksyn_waitq_element_t kwe = NULL;
b0d623f7 589 kern_return_t kret = KERN_SUCCESS;
b0d623f7
A		 590
591 nextgen = (ugen + PTHRW_INC);
592
593#if _PSYNCH_TRACE_
6d2010ae 594 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_START, (uint32_t)kwq->kw_addr, lkseq, ugen, flags, 0);
b0d623f7
A		 595#endif /* _PSYNCH_TRACE_ */
596
597 ksyn_wqlock(kwq);
598
599redrive:
6d2010ae 600
b0d623f7 601 if (kwq->kw_inqueue != 0) {
6d2010ae
A		 602 updatebits = (kwq->kw_highseq & PTHRW_COUNT_MASK) | (PTH_RWL_EBIT | PTH_RWL_KBIT);
603 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
b0d623f7
A		 604 if (firstfit != 0)
605 {
b0d623f7 606 /* first fit , pick any one */
6d2010ae
A		 607 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
608 kwe->kwe_psynchretval = updatebits;
609 kwe->kwe_kwqqueue = NULL;
b0d623f7 610
b0d623f7 611#if _PSYNCH_TRACE_
6d2010ae 612 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf1, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
b0d623f7 613#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 614
615 kret = ksyn_wakeup_thread(kwq, kwe);
616#if __TESTPANICS__
b0d623f7
A		 617 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
618 panic("psynch_mutexdrop_internal: panic unable to wakeup firstfit mutex thread\n");
6d2010ae 619#endif /* __TESTPANICS__ */
b0d623f7
A		 620 if (kret == KERN_NOT_WAITING)
621 goto redrive;
622 } else {
623 /* handle fairshare */
624 low_writer = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
625 low_writer &= PTHRW_COUNT_MASK;
626
627 if (low_writer == nextgen) {
b0d623f7 628 /* next seq to be granted found */
6d2010ae 629 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
b0d623f7 630
6d2010ae
A		 631 /* since the grant could be cv, make sure mutex wait is set incase the thread interrupted out */
632 kwe->kwe_psynchretval = updatebits | PTH_RWL_MTX_WAIT;
633 kwe->kwe_kwqqueue = NULL;
b0d623f7 634
6d2010ae
A		 635#if _PSYNCH_TRACE_
636 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
637#endif /* _PSYNCH_TRACE_ */
638
639 kret = ksyn_wakeup_thread(kwq, kwe);
640#if __TESTPANICS__
b0d623f7
A		 641 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
642 panic("psynch_mutexdrop_internal: panic unable to wakeup fairshare mutex thread\n");
6d2010ae
A		 643#endif /* __TESTPANICS__ */
644 if (kret == KERN_NOT_WAITING) {
645 /* interrupt post */
646 kwq->kw_pre_intrcount = 1;
647 kwq->kw_pre_intrseq = nextgen;
648 kwq->kw_pre_intrretbits = updatebits;
649 kwq->kw_pre_intrtype = PTH_RW_TYPE_WRITE;
650#if _PSYNCH_TRACE_
651 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfafafaf1, nextgen, kwq->kw_pre_intrretbits, 0);
652#endif /* _PSYNCH_TRACE_ */
653 }
b0d623f7
A		 654
655 } else if (is_seqhigher(low_writer, nextgen) != 0) {
b0d623f7 656 kwq->kw_pre_rwwc++;
6d2010ae
A		 657
658 if (kwq->kw_pre_rwwc > 1) {
659 __FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (1)\n");
660 goto out;
661 }
662
b0d623f7 663 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
b0d623f7 664#if _PSYNCH_TRACE_
6d2010ae 665 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7 666#endif /* _PSYNCH_TRACE_ */
6d2010ae 667 } else {
b0d623f7 668
6d2010ae
A		 669 //__FAILEDUSERTEST__("psynch_mutexdrop_internal: FS mutex unlock sequence higher than the lowest one is queue\n");
670
671 kwe = ksyn_queue_find_seq(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (nextgen & PTHRW_COUNT_MASK), 1);
672 if (kwe != NULL) {
673 /* next seq to be granted found */
674 /* since the grant could be cv, make sure mutex wait is set incase the thread interrupted out */
675 kwe->kwe_psynchretval = updatebits | PTH_RWL_MTX_WAIT;
676 kwe->kwe_kwqqueue = NULL;
b0d623f7 677#if _PSYNCH_TRACE_
6d2010ae 678 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
b0d623f7 679#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 680 kret = ksyn_wakeup_thread(kwq, kwe);
681#if __TESTPANICS__
b0d623f7
A		 682 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
683 panic("psynch_mutexdrop_internal: panic unable to wakeup fairshare mutex thread\n");
6d2010ae 684#endif /* __TESTPANICS__ */
b0d623f7
A		 685 if (kret == KERN_NOT_WAITING)
686 goto redrive;
687 } else {
688 /* next seq to be granted not found, prepost */
b0d623f7 689 kwq->kw_pre_rwwc++;
6d2010ae
A		 690
691 if (kwq->kw_pre_rwwc > 1) {
692 __FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (2)\n");
693 goto out;
694 }
695
b0d623f7 696 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
6d2010ae
A		 697#if _PSYNCH_TRACE_
698 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
699#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 700 }
701 }
702 }
703 } else {
6d2010ae
A		 704
705 /* if firstfit the last one could be spurious */
706 if (firstfit == 0) {
707 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
708 kwq->kw_pre_rwwc++;
709
710 if (kwq->kw_pre_rwwc > 1) {
711 __FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (3)\n");
712 goto out;
713 }
714
715 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
b0d623f7 716#if _PSYNCH_TRACE_
6d2010ae 717 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7 718#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 719 } else {
720 /* first fit case */
b0d623f7 721#if _PSYNCH_TRACE_
6d2010ae 722 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_lastunlockseq, kwq->kw_pre_lockseq, 0);
b0d623f7 723#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 724 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
725 /* not set or the new lkseq is higher */
726 if ((kwq->kw_pre_rwwc == 0) || (is_seqlower(kwq->kw_pre_lockseq, lkseq) == 0))
727 kwq->kw_pre_lockseq = (lkseq & PTHRW_COUNT_MASK);
728 kwq->kw_pre_rwwc = 1;
729#if _PSYNCH_TRACE_
730 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
731#endif /* _PSYNCH_TRACE_ */
732
733 /* indicate prepost content in kernel */
734 returnbits = lkseq | PTH_RWL_PBIT;
b0d623f7 735 }
2d21ac55 736 }
b0d623f7 737
6d2010ae 738out:
b0d623f7
A		 739 ksyn_wqunlock(kwq);
740
741#if _PSYNCH_TRACE_
6d2010ae 742 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0xeeeeeeed, 0, 0, 0);
b0d623f7 743#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 744 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX));
745 return(returnbits);
2d21ac55
A		 746}
747
b0d623f7
A		 748/*
749 * psynch_mutexwait: This system call is used for contended psynch mutexes to block.
750 */
2d21ac55 751
b0d623f7
A		 752int
753psynch_mutexwait(__unused proc_t p, struct psynch_mutexwait_args * uap, uint32_t * retval)
2d21ac55 754{
b0d623f7
A		 755 user_addr_t mutex = uap->mutex;
756 uint32_t mgen = uap->mgen;
757 uint32_t ugen = uap->ugen;
758 uint64_t tid = uap->tid;
759 int flags = uap->flags;
760 ksyn_wait_queue_t kwq;
761 int error=0;
6d2010ae 762 int ins_flags, retry;
b0d623f7
A		 763 uthread_t uth;
764 int firstfit = flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT;
6d2010ae
A		 765 uint32_t lockseq, updatebits=0;
766 ksyn_waitq_element_t kwe;
7ddcb079 767 kern_return_t kret;
b0d623f7
A		 768
769#if _PSYNCH_TRACE_
6d2010ae
A		 770 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_START, (uint32_t)mutex, mgen, ugen, flags, 0);
771 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)mutex, mgen, ugen, (uint32_t)tid, 0);
b0d623f7
A		 772#endif /* _PSYNCH_TRACE_ */
773
774 uth = current_uthread();
775
6d2010ae
A		 776 kwe = &uth->uu_kwe;
777 kwe->kwe_lockseq = uap->mgen;
778 kwe->kwe_uth = uth;
779 kwe->kwe_psynchretval = 0;
780 kwe->kwe_kwqqueue = NULL;
b0d623f7
A		 781 lockseq = (uap->mgen & PTHRW_COUNT_MASK);
782
783 if (firstfit == 0) {
784 ins_flags = SEQFIT;
785 } else {
786 /* first fit */
787 ins_flags = FIRSTFIT;
788 }
789
790 error = ksyn_wqfind(mutex, mgen, ugen, 0, tid, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX), &kwq);
791 if (error != 0) {
792#if _PSYNCH_TRACE_
6d2010ae 793 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 1, 0xdeadbeef, error, 0);
b0d623f7
A		 794#endif /* _PSYNCH_TRACE_ */
795 return(error);
796 }
797
798 ksyn_wqlock(kwq);
799
6d2010ae
A		 800
801 if ((mgen & PTH_RWL_RETRYBIT) != 0) {
802 retry = 1;
803 mgen &= ~PTH_RWL_RETRYBIT;
804 }
805
806 /* handle first the missed wakeups */
807 if ((kwq->kw_pre_intrcount != 0) &&
808 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_WRITE)) &&
809 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
810 kwq->kw_pre_intrcount--;
811 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
812 if (kwq->kw_pre_intrcount==0)
813 CLEAR_INTR_PREPOST_BITS(kwq);
814 ksyn_wqunlock(kwq);
815 *retval = kwe->kwe_psynchretval;
816#if _PSYNCH_TRACE_
817 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)mutex, 0xfafafaf1, kwe->kwe_psynchretval, kwq->kw_pre_intrcount, 0);
818#endif /* _PSYNCH_TRACE_ */
819 goto out;
820 }
821
822 if ((kwq->kw_pre_rwwc != 0) && ((ins_flags == FIRSTFIT) || ((lockseq & PTHRW_COUNT_MASK) == (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK) ))) {
b0d623f7
A		 823 /* got preposted lock */
824 kwq->kw_pre_rwwc--;
825 if (kwq->kw_pre_rwwc == 0) {
826 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 827 kwq->kw_lastunlockseq = PTHRW_RWL_INIT;
828 if (kwq->kw_inqueue == 0) {
829 updatebits = lockseq | (PTH_RWL_KBIT | PTH_RWL_EBIT);
830 } else {
831 updatebits = (kwq->kw_highseq & PTHRW_COUNT_MASK) | (PTH_RWL_KBIT | PTH_RWL_EBIT);
832 }
833 updatebits &= ~PTH_RWL_MTX_WAIT;
834
835 kwe->kwe_psynchretval = updatebits;
836
837 if (updatebits == 0) {
838 __FAILEDUSERTEST__("psynch_mutexwait(prepost): returning 0 lseq in mutexwait with no EBIT \n");
839 }
840 ksyn_wqunlock(kwq);
841 *retval = updatebits;
842#if _PSYNCH_TRACE_
843 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
844#endif /* _PSYNCH_TRACE_ */
845 goto out;
b0d623f7 846 } else {
6d2010ae 847 __FAILEDUSERTEST__("psynch_mutexwait: more than one prepost\n");
b0d623f7 848 kwq->kw_pre_lockseq += PTHRW_INC; /* look for next one */
6d2010ae
A		 849 ksyn_wqunlock(kwq);
850 error = EINVAL;
851 goto out;
b0d623f7 852 }
b0d623f7
A		 853 }
854
6d2010ae
A		 855#if _PSYNCH_TRACE_
856 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfeedfeed, mgen, ins_flags, 0);
857#endif /* _PSYNCH_TRACE_ */
b0d623f7 858
6d2010ae
A		 859 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], mgen, uth, kwe, ins_flags);
860 if (error != 0) {
861 ksyn_wqunlock(kwq);
862#if _PSYNCH_TRACE_
863 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 2, 0xdeadbeef, error, 0);
864#endif /* _PSYNCH_TRACE_ */
865 goto out;
866 }
867
7ddcb079
A		 868 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, psynch_mtxcontinue, (void *)kwq);
869
870 psynch_mtxcontinue((void *)kwq, kret);
871
872 /* not expected to return from unix_syscall_return */
873 panic("psynch_mtxcontinue returned from unix_syscall_return");
874
875out:
876 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX));
877#if _PSYNCH_TRACE_
878 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 0xeeeeeeed, updatebits, error, 0);
879#endif /* _PSYNCH_TRACE_ */
880
881 return(error);
882}
883
884void
885psynch_mtxcontinue(void * parameter, wait_result_t result)
886{
887 int error = 0;
888 uint32_t updatebits = 0;
889 uthread_t uth = current_uthread();
890 ksyn_wait_queue_t kwq = (ksyn_wait_queue_t)parameter;
891 ksyn_waitq_element_t kwe;
892
893 kwe = &uth->uu_kwe;
894
895 switch (result) {
896 case THREAD_TIMED_OUT:
897 error = ETIMEDOUT;
898 break;
899 case THREAD_INTERRUPTED:
900 error = EINTR;
901 break;
902 default:
903 error = 0;
904 break;
905 }
b0d623f7
A		 906
907 if (error != 0) {
908 ksyn_wqlock(kwq);
6d2010ae 909
b0d623f7 910#if _PSYNCH_TRACE_
7ddcb079 911 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, 0xdeadbeef, error, 0);
b0d623f7 912#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 913 if (kwe->kwe_kwqqueue != NULL)
914 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
b0d623f7
A		 915 ksyn_wqunlock(kwq);
916 } else {
6d2010ae
A		 917 updatebits = kwe->kwe_psynchretval;
918 updatebits &= ~PTH_RWL_MTX_WAIT;
7ddcb079 919 uth->uu_rval[0] = updatebits;
6d2010ae
A		 920
921 if (updatebits == 0)
922 __FAILEDUSERTEST__("psynch_mutexwait: returning 0 lseq in mutexwait with no EBIT \n");
b0d623f7 923 }
6d2010ae 924 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX));
b0d623f7 925#if _PSYNCH_TRACE_
7ddcb079 926 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0xeeeeeeed, updatebits, error, 0);
b0d623f7
A		 927#endif /* _PSYNCH_TRACE_ */
928
7ddcb079 929 unix_syscall_return(error);
2d21ac55
A		 930}
931
b0d623f7
A		 932/*
933 * psynch_mutexdrop: This system call is used for unlock postings on contended psynch mutexes.
934 */
935int
6d2010ae 936psynch_mutexdrop(__unused proc_t p, struct psynch_mutexdrop_args * uap, uint32_t * retval)
b0d623f7
A		 937{
938 user_addr_t mutex = uap->mutex;
939 uint32_t mgen = uap->mgen;
b0d623f7
A		 940 uint32_t ugen = uap->ugen;
941 uint64_t tid = uap->tid;
942 int flags = uap->flags;
943 ksyn_wait_queue_t kwq;
6d2010ae 944 uint32_t updateval;
b0d623f7
A		 945 int error=0;
946
6d2010ae 947 error = ksyn_wqfind(mutex, mgen, ugen, 0, tid, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX), &kwq);
b0d623f7 948 if (error != 0) {
b0d623f7
A		 949 return(error);
950 }
6d2010ae
A		 951
952 updateval = psynch_mutexdrop_internal(kwq, mgen, ugen, flags);
b0d623f7 953 /* drops the kwq reference */
6d2010ae
A		 954
955 *retval = updateval;
b0d623f7
A		 956 return(0);
957
958}
2d21ac55 959
b0d623f7
A		 960/*
961 * psynch_cvbroad: This system call is used for broadcast posting on blocked waiters of psynch cvars.
962 */
963int
6d2010ae 964psynch_cvbroad(__unused proc_t p, struct psynch_cvbroad_args * uap, uint32_t * retval)
2d21ac55 965{
b0d623f7 966 user_addr_t cond = uap->cv;
6d2010ae
A		 967 uint64_t cvlsgen = uap->cvlsgen;
968 uint64_t cvudgen = uap->cvudgen;
969 uint32_t cgen, cugen, csgen, diffgen;
970 uint32_t uptoseq, fromseq;
b0d623f7 971 int flags = uap->flags;
6d2010ae 972 ksyn_wait_queue_t ckwq;
b0d623f7 973 int error=0;
6d2010ae
A		 974 uint32_t updatebits = 0;
975 uint32_t count;
976 struct ksyn_queue kfreeq;
977
978 csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
979 cgen = ((uint32_t)(cvlsgen & 0xffffffff));
980 cugen = (uint32_t)((cvudgen >> 32) & 0xffffffff);
981 diffgen = ((uint32_t)(cvudgen & 0xffffffff));
982 count = (diffgen >> PTHRW_COUNT_SHIFT);
2d21ac55 983
b0d623f7 984#if _PSYNCH_TRACE_
6d2010ae
A		 985 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
986 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_NONE, (uint32_t)cond, 0xcbcbcbc1, diffgen,flags, 0);
b0d623f7 987#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 988
989 uptoseq = cgen & PTHRW_COUNT_MASK;
990 fromseq = (cugen & PTHRW_COUNT_MASK) + PTHRW_INC;
991
992 if (is_seqhigher(fromseq, uptoseq) || is_seqhigher((csgen & PTHRW_COUNT_MASK), uptoseq)) {
993 __FAILEDUSERTEST__("cvbroad: invalid L, U and S values\n");
994 return EINVAL;
995 }
996 if (count > (uint32_t)task_threadmax) {
997 __FAILEDUSERTEST__("cvbroad: difference greater than maximum possible thread count\n");
998 return EBUSY;
999 }
1000
1001 ckwq = NULL;
1002
1003 error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
b0d623f7
A		 1004 if (error != 0) {
1005#if _PSYNCH_TRACE_
6d2010ae 1006 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
b0d623f7
A		 1007#endif /* _PSYNCH_TRACE_ */
1008 return(error);
1009 }
2d21ac55 1010
6d2010ae 1011 *retval = 0;
b0d623f7
A		 1012
1013 ksyn_wqlock(ckwq);
b0d623f7 1014
6d2010ae
A		 1015 /* update L, U and S... */
1016 UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
b0d623f7 1017
6d2010ae
A		 1018 /* broadcast wakeups/prepost handling */
1019 ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
b0d623f7 1020
6d2010ae
A		 1021 /* set C or P bits and free if needed */
1022 ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1023 ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
1024 ksyn_wqunlock(ckwq);
1025
1026 *retval = updatebits;
1027
1028 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_CVAR));
b0d623f7 1029#if _PSYNCH_TRACE_
6d2010ae 1030 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, (uint32_t)*retval, error, 0);
b0d623f7
A		 1031#endif /* _PSYNCH_TRACE_ */
1032
1033 return(error);
2d21ac55
A		 1034}
1035
6d2010ae
A		 1036ksyn_waitq_element_t
1037ksyn_queue_find_threadseq(ksyn_wait_queue_t ckwq, __unused ksyn_queue_t kq, thread_t th, uint32_t upto)
1038{
1039 uthread_t uth = get_bsdthread_info(th);
1040 ksyn_waitq_element_t kwe = &uth->uu_kwe;
1041
1042 if (kwe->kwe_kwqqueue != ckwq ||
1043 is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), upto)) {
1044 /* the thread is not waiting in the cv (or wasn't when the wakeup happened) */
1045 return NULL;
1046 }
1047 return kwe;
1048}
1049
b0d623f7
A		 1050/*
1051 * psynch_cvsignal: This system call is used for signalling the blocked waiters of psynch cvars.
1052 */
1053int
6d2010ae 1054psynch_cvsignal(__unused proc_t p, struct psynch_cvsignal_args * uap, uint32_t * retval)
b0d623f7
A		 1055{
1056 user_addr_t cond = uap->cv;
6d2010ae
A		 1057 uint64_t cvlsgen = uap->cvlsgen;
1058 uint32_t cgen, csgen, signalseq, uptoseq;
b0d623f7 1059 uint32_t cugen = uap->cvugen;
b0d623f7
A		 1060 int threadport = uap->thread_port;
1061 int flags = uap->flags;
6d2010ae
A		 1062 ksyn_wait_queue_t ckwq = NULL;
1063 ksyn_waitq_element_t kwe, nkwe = NULL;
1064 ksyn_queue_t kq;
1065 int error=0;
b0d623f7 1066 thread_t th = THREAD_NULL;
6d2010ae
A		 1067 uint32_t updatebits = 0;
1068 kern_return_t kret;
1069 struct ksyn_queue kfreeq;
b0d623f7
A		 1070
1071
6d2010ae
A		 1072 csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
1073 cgen = ((uint32_t)(cvlsgen & 0xffffffff));
b0d623f7 1074
b0d623f7 1075#if _PSYNCH_TRACE_
6d2010ae 1076 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, threadport, 0);
b0d623f7 1077#endif /* _PSYNCH_TRACE_ */
b0d623f7 1078
6d2010ae
A		 1079 uptoseq = cgen & PTHRW_COUNT_MASK;
1080 signalseq = (cugen & PTHRW_COUNT_MASK) + PTHRW_INC;
b0d623f7 1081
6d2010ae
A		 1082 /* validate sane L, U, and S values */
1083 if (((threadport == 0) && (is_seqhigher(signalseq, uptoseq))) || is_seqhigher((csgen & PTHRW_COUNT_MASK), uptoseq)) {
1084 __FAILEDUSERTEST__("psync_cvsignal; invalid sequence numbers\n");
1085 error = EINVAL;
1086 goto out;
b0d623f7 1087 }
b0d623f7 1088
6d2010ae 1089 /* If we are looking for a specific thread, grab a reference for it */
b0d623f7
A		 1090 if (threadport != 0) {
1091 th = (thread_t)port_name_to_thread((mach_port_name_t)threadport);
1092 if (th == THREAD_NULL) {
b0d623f7
A		 1093 error = ESRCH;
1094 goto out;
1095 }
1096 }
1097
6d2010ae
A		 1098 error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
1099 if (error != 0) {
b0d623f7 1100#if _PSYNCH_TRACE_
6d2010ae
A		 1101 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
1102#endif /* _PSYNCH_TRACE_ */
1103 goto out;
1104 }
1105
1106 ksyn_wqlock(ckwq);
1107
1108 /* update L, U and S... */
1109 UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
1110
1111 kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
1112
1113retry:
1114 /* Only bother if we aren't already balanced */
1115 if ((ckwq->kw_lword & PTHRW_COUNT_MASK) != (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
1116
1117 kwe = (th != NULL) ? ksyn_queue_find_threadseq(ckwq, kq, th, uptoseq) :
1118 ksyn_queue_find_signalseq(ckwq, kq, uptoseq, signalseq);
1119 if (kwe != NULL) {
1120 switch (kwe->kwe_flags) {
1121
1122 case KWE_THREAD_BROADCAST:
1123 /* broadcasts swallow our signal */
1124 break;
1125
1126 case KWE_THREAD_PREPOST:
1127 /* merge in with existing prepost at our same uptoseq */
1128 kwe->kwe_count += 1;
1129 break;
1130
1131 case KWE_THREAD_INWAIT:
1132 if (is_seqlower((kwe->kwe_lockseq & PTHRW_COUNT_MASK), signalseq)) {
1133 /*
1134 * A valid thread in our range, but lower than our signal.
1135 * Matching it may leave our match with nobody to wake it if/when
1136 * it arrives (the signal originally meant for this thread might
1137 * not successfully wake it).
1138 *
1139 * Convert to broadcast - may cause some spurious wakeups
1140 * (allowed by spec), but avoids starvation (better choice).
1141 */
1142#if _PSYNCH_TRACE_
1143 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc1c1c1c1, uptoseq, 0, 0);
b0d623f7 1144#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1145 ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
1146 } else {
1147 ksyn_queue_removeitem(ckwq, kq, kwe);
1148 kwe->kwe_psynchretval = PTH_RWL_MTX_WAIT;
1149 kwe->kwe_kwqqueue = NULL;
b0d623f7 1150#if _PSYNCH_TRACE_
6d2010ae 1151 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
b0d623f7 1152#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1153 kret = ksyn_wakeup_thread(ckwq, kwe);
1154#if __TESTPANICS__
1155 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
1156 panic("ksyn_wakeup_thread: panic waking up condition waiter\n");
1157#endif /* __TESTPANICS__ */
1158 updatebits += PTHRW_INC;
b0d623f7 1159 }
6d2010ae
A		 1160
1161 ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1162 break;
1163
1164 default:
1165 panic("unknown kweflags\n");
1166 break;
b0d623f7 1167 }
6d2010ae
A		 1168
1169 } else if (th != NULL) {
1170 /*
1171 * Could not find the thread, post a broadcast,
1172 * otherwise the waiter will be stuck. Use to send
1173 * ESRCH here, did lead to rare hangs.
1174 */
1175 ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
1176 ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1177 } else if (nkwe == NULL) {
b0d623f7 1178 ksyn_wqunlock(ckwq);
6d2010ae
A		 1179 nkwe = (ksyn_waitq_element_t)zalloc(kwe_zone);
1180 ksyn_wqlock(ckwq);
1181 goto retry;
1182
1183 } else {
1184 /* no eligible entries - add prepost */
1185 bzero(nkwe, sizeof(struct ksyn_waitq_element));
1186 nkwe->kwe_kwqqueue = ckwq;
1187 nkwe->kwe_flags = KWE_THREAD_PREPOST;
1188 nkwe->kwe_lockseq = uptoseq;
1189 nkwe->kwe_count = 1;
1190 nkwe->kwe_uth = NULL;
1191 nkwe->kwe_psynchretval = 0;
1192
b0d623f7 1193#if _PSYNCH_TRACE_
6d2010ae 1194 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfefe, uptoseq, 0, 0);
b0d623f7 1195#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1196
1197 (void)ksyn_queue_insert(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], uptoseq, NULL, nkwe, SEQFIT);
1198 ckwq->kw_fakecount++;
1199 nkwe = NULL;
b0d623f7 1200 }
6d2010ae
A		 1201
1202 /* set C or P bits and free if needed */
1203 ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
b0d623f7 1204 }
6d2010ae
A		 1205
1206 ksyn_wqunlock(ckwq);
1207 if (nkwe != NULL)
1208 zfree(kwe_zone, nkwe);
1209
1210 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_CVAR));
1211
b0d623f7 1212out:
6d2010ae 1213 if (th != NULL)
b0d623f7 1214 thread_deallocate(th);
6d2010ae
A		 1215 if (error == 0)
1216 *retval = updatebits;
b0d623f7 1217#if _PSYNCH_TRACE_
6d2010ae 1218 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, updatebits, error, 0);
b0d623f7
A		 1219#endif /* _PSYNCH_TRACE_ */
1220
1221 return(error);
1222}
2d21ac55 1223
b0d623f7
A		 1224/*
1225 * psynch_cvwait: This system call is used for psynch cvar waiters to block in kernel.
1226 */
1227int
1228psynch_cvwait(__unused proc_t p, struct psynch_cvwait_args * uap, uint32_t * retval)
2d21ac55 1229{
b0d623f7 1230 user_addr_t cond = uap->cv;
6d2010ae
A		 1231 uint64_t cvlsgen = uap->cvlsgen;
1232 uint32_t cgen, csgen;
b0d623f7
A		 1233 uint32_t cugen = uap->cvugen;
1234 user_addr_t mutex = uap->mutex;
6d2010ae
A		 1235 uint64_t mugen = uap->mugen;
1236 uint32_t mgen, ugen;
1237 int flags = uap->flags;
b0d623f7 1238 ksyn_wait_queue_t kwq, ckwq;
6d2010ae 1239 int error=0, local_error = 0;
b0d623f7 1240 uint64_t abstime = 0;
6d2010ae 1241 uint32_t lockseq, updatebits=0;
b0d623f7
A		 1242 struct timespec ts;
1243 uthread_t uth;
6d2010ae
A		 1244 ksyn_waitq_element_t kwe, nkwe = NULL;
1245 struct ksyn_queue *kq, kfreeq;
7ddcb079 1246 kern_return_t kret;
6d2010ae 1247
b0d623f7
A		 1248 /* for conformance reasons */
1249 __pthread_testcancel(0);
1250
6d2010ae
A		 1251 csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
1252 cgen = ((uint32_t)(cvlsgen & 0xffffffff));
1253 ugen = (uint32_t)((mugen >> 32) & 0xffffffff);
1254 mgen = ((uint32_t)(mugen & 0xffffffff));
1255
b0d623f7 1256#if _PSYNCH_TRACE_
6d2010ae
A		 1257 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
1258 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)mutex, mgen, ugen, flags, 0);
b0d623f7 1259#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1260
1261 lockseq = (cgen & PTHRW_COUNT_MASK);
1262 /*
1263 * In cvwait U word can be out of range as cond could be used only for
1264 * timeouts. However S word needs to be within bounds and validated at
1265 * user level as well.
1266 */
1267 if (is_seqhigher_eq((csgen & PTHRW_COUNT_MASK), lockseq) != 0) {
1268 __FAILEDUSERTEST__("psync_cvwait; invalid sequence numbers\n");
1269 return EINVAL;
b0d623f7 1270 }
6d2010ae
A		 1271
1272 ckwq = kwq = NULL;
1273 error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INWAIT, &ckwq);
b0d623f7
A		 1274 if (error != 0) {
1275#if _PSYNCH_TRACE_
6d2010ae 1276 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)cond, 1, 0xdeadbeef, error, 0);
b0d623f7
A		 1277#endif /* _PSYNCH_TRACE_ */
1278 return(error);
1279 }
1280
b0d623f7 1281
6d2010ae
A		 1282 if (mutex != (user_addr_t)0) {
1283 error = ksyn_wqfind(mutex, mgen, ugen, 0, 0, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX), &kwq);
1284 if (error != 0) {
1285 local_error = error;
1286#if _PSYNCH_TRACE_
1287 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)mutex, 2, 0xdeadbeef, error, 0);
1288#endif /* _PSYNCH_TRACE_ */
b0d623f7 1289 goto out;
2d21ac55 1290 }
b0d623f7 1291
6d2010ae 1292 (void)psynch_mutexdrop_internal(kwq, mgen, ugen, flags);
b0d623f7 1293 /* drops kwq reference */
6d2010ae 1294 kwq = NULL;
2d21ac55 1295 }
b0d623f7 1296
6d2010ae 1297 if (uap->sec != 0 || (uap->nsec & 0x3fffffff) != 0) {
b0d623f7 1298 ts.tv_sec = uap->sec;
6d2010ae
A		 1299 ts.tv_nsec = (uap->nsec & 0x3fffffff);
1300 nanoseconds_to_absolutetime((uint64_t)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec, &abstime );
1301 clock_absolutetime_interval_to_deadline( abstime, &abstime );
b0d623f7 1302 }
6d2010ae 1303
b0d623f7 1304 ksyn_wqlock(ckwq);
6d2010ae
A		 1305
1306 /* update L, U and S... */
1307 UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
1308
1309 /* Look for the sequence for prepost (or conflicting thread */
1310 kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
1311 kwe = ksyn_queue_find_cvpreposeq(kq, lockseq);
1312
1313 if (kwe != NULL) {
1314 switch (kwe->kwe_flags) {
1315
1316 case KWE_THREAD_INWAIT:
1317 ksyn_wqunlock(ckwq);
1318 __FAILEDUSERTEST__("cvwait: thread entry with same sequence already present\n");
1319 local_error = EBUSY;
1320 goto out;
1321
1322 case KWE_THREAD_BROADCAST:
1323 break;
1324
1325 case KWE_THREAD_PREPOST:
1326 if ((kwe->kwe_lockseq & PTHRW_COUNT_MASK) == lockseq) {
1327 /* we can safely consume a reference, so do so */
1328 if (--kwe->kwe_count == 0) {
1329 ksyn_queue_removeitem(ckwq, kq, kwe);
1330 ckwq->kw_fakecount--;
1331 nkwe = kwe;
1332 }
1333 } else {
1334 /*
1335 * consuming a prepost higher than our lock sequence is valid, but
1336 * can leave the higher thread without a match. Convert the entry
1337 * to a broadcast to compensate for this.
1338 */
b0d623f7 1339#if _PSYNCH_TRACE_
6d2010ae 1340 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc2c2c2c2, kwe->kwe_lockseq, 0, 0);
b0d623f7 1341#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1342
1343 ksyn_handle_cvbroad(ckwq, kwe->kwe_lockseq, &updatebits);
1344#if __TESTPANICS__
1345 if (updatebits != 0)
1346 panic("psync_cvwait: convert pre-post to broadcast: woke up %d threads that shouldn't be there\n",
1347 updatebits);
1348#endif /* __TESTPANICS__ */
1349 }
1350
1351 break;
b0d623f7 1352
6d2010ae
A		 1353 default:
1354 panic("psync_cvwait: unexpected wait queue element type\n");
1355 }
1356
1357#if _PSYNCH_TRACE_
1358 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfefefefe, kwe->kwe_lockseq, 0, 0);
1359#endif /* _PSYNCH_TRACE_ */
1360
1361
1362 updatebits = PTHRW_INC;
1363 ckwq->kw_sword += PTHRW_INC;
1364
1365 /* set C or P bits and free if needed */
1366 ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
1367
b0d623f7 1368 error = 0;
6d2010ae
A		 1369 local_error = 0;
1370
1371 *retval = updatebits;
1372
b0d623f7 1373 ksyn_wqunlock(ckwq);
6d2010ae
A		 1374
1375 if (nkwe != NULL)
1376 zfree(kwe_zone, nkwe);
1377
b0d623f7 1378 goto out;
6d2010ae
A		 1379
1380 }
b0d623f7 1381
6d2010ae
A		 1382 uth = current_uthread();
1383 kwe = &uth->uu_kwe;
1384 kwe->kwe_kwqqueue = ckwq;
1385 kwe->kwe_flags = KWE_THREAD_INWAIT;
1386 kwe->kwe_lockseq = lockseq;
1387 kwe->kwe_count = 1;
1388 kwe->kwe_uth = uth;
1389 kwe->kwe_psynchretval = 0;
1390
b0d623f7 1391#if _PSYNCH_TRACE_
6d2010ae 1392 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfeed, cgen, 0, 0);
b0d623f7 1393#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1394
1395 error = ksyn_queue_insert(ckwq, kq, cgen, uth, kwe, SEQFIT);
1396 if (error != 0) {
1397 ksyn_wqunlock(ckwq);
1398 local_error = error;
1399 goto out;
b0d623f7 1400 }
6d2010ae 1401
7ddcb079 1402 kret = ksyn_block_thread_locked(ckwq, abstime, kwe, 1, psynch_cvcontinue, (void *)ckwq);
6d2010ae
A		 1403 /* lock dropped */
1404
7ddcb079
A		 1405 psynch_cvcontinue(ckwq, kret);
1406 /* not expected to return from unix_syscall_return */
1407 panic("psynch_cvcontinue returned from unix_syscall_return");
1408
1409out:
1410#if _PSYNCH_TRACE_
1411 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, (uint32_t)*retval, local_error, 0);
1412#endif /* _PSYNCH_TRACE_ */
1413 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_CVAR));
1414 return(local_error);
1415}
1416
1417
1418void
1419psynch_cvcontinue(void * parameter, wait_result_t result)
1420{
1421 int error = 0, local_error = 0;
1422 uthread_t uth = current_uthread();
1423 ksyn_wait_queue_t ckwq = (ksyn_wait_queue_t)parameter;
1424 ksyn_waitq_element_t kwe;
1425 struct ksyn_queue kfreeq;
1426
1427 switch (result) {
1428 case THREAD_TIMED_OUT:
1429 error = ETIMEDOUT;
1430 break;
1431 case THREAD_INTERRUPTED:
1432 error = EINTR;
1433 break;
1434 default:
1435 error = 0;
1436 break;
1437 }
1438#if _PSYNCH_TRACE_
1439 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf4f3f2f1, (uintptr_t)uth, result, 0, 0);
1440#endif /* _PSYNCH_TRACE_ */
1441
6d2010ae 1442 local_error = error;
7ddcb079
A		 1443 kwe = &uth->uu_kwe;
1444
b0d623f7
A		 1445 if (error != 0) {
1446 ksyn_wqlock(ckwq);
6d2010ae 1447 /* just in case it got woken up as we were granting */
7ddcb079 1448 uth->uu_rval[0] = kwe->kwe_psynchretval;
6d2010ae
A		 1449
1450#if __TESTPANICS__
1451 if ((kwe->kwe_kwqqueue != NULL) && (kwe->kwe_kwqqueue != ckwq))
1452 panic("cvwait waiting on some other kwq\n");
1453
1454#endif /* __TESTPANICS__ */
1455
1456
1457 if (kwe->kwe_kwqqueue != NULL) {
1458 ksyn_queue_removeitem(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
1459 kwe->kwe_kwqqueue = NULL;
1460 }
1461 if ((kwe->kwe_psynchretval & PTH_RWL_MTX_WAIT) != 0) {
7ddcb079 1462 /* the condition var granted.
6d2010ae
A		 1463 * reset the error so that the thread returns back.
1464 */
1465 local_error = 0;
1466 /* no need to set any bits just return as cvsig/broad covers this */
1467 ksyn_wqunlock(ckwq);
6d2010ae
A		 1468 goto out;
1469 }
1470
1471 ckwq->kw_sword += PTHRW_INC;
1472
7ddcb079 1473 /* set C and P bits, in the local error */
6d2010ae 1474 if ((ckwq->kw_lword & PTHRW_COUNT_MASK) == (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
6d2010ae
A		 1475 local_error |= ECVCERORR;
1476 if (ckwq->kw_inqueue != 0) {
7ddcb079 1477 (void)ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (ckwq->kw_lword & PTHRW_COUNT_MASK), &kfreeq, 1, 1);
6d2010ae
A		 1478 }
1479 ckwq->kw_lword = ckwq->kw_uword = ckwq->kw_sword = 0;
1480 ckwq->kw_kflags |= KSYN_KWF_ZEROEDOUT;
1481 } else {
1482 /* everythig in the queue is a fake entry ? */
1483 if ((ckwq->kw_inqueue != 0) && (ckwq->kw_fakecount == ckwq->kw_inqueue)) {
6d2010ae
A		 1484 local_error |= ECVPERORR;
1485 }
1486 }
1487 ksyn_wqunlock(ckwq);
1488
1489 } else {
1490 /* PTH_RWL_MTX_WAIT is removed */
1491 if ((kwe->kwe_psynchretval & PTH_RWS_CV_MBIT) != 0)
7ddcb079 1492 uth->uu_rval[0] = PTHRW_INC | PTH_RWS_CV_CBIT;
6d2010ae 1493 else
7ddcb079 1494 uth->uu_rval[0] = 0;
6d2010ae
A		 1495 local_error = 0;
1496 }
1497out:
1498#if _PSYNCH_TRACE_
7ddcb079 1499 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)ckwq->kw_addr, 0xeeeeeeed, uth->uu_rval[0], local_error, 0);
6d2010ae
A		 1500#endif /* _PSYNCH_TRACE_ */
1501 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_CVAR));
7ddcb079
A		 1502
1503 unix_syscall_return(local_error);
1504
6d2010ae
A		 1505}
1506
1507/*
1508 * psynch_cvclrprepost: This system call clears pending prepost if present.
1509 */
1510int
1511psynch_cvclrprepost(__unused proc_t p, struct psynch_cvclrprepost_args * uap, __unused int * retval)
1512{
1513 user_addr_t cond = uap->cv;
1514 uint32_t cgen = uap->cvgen;
1515 uint32_t cugen = uap->cvugen;
1516 uint32_t csgen = uap->cvsgen;
1517 uint32_t pseq = uap->preposeq;
1518 uint32_t flags = uap->flags;
1519 int error;
1520 ksyn_wait_queue_t ckwq = NULL;
1521 struct ksyn_queue kfreeq;
1522
b0d623f7 1523#if _PSYNCH_TRACE_
6d2010ae
A		 1524 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
1525 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_NONE, (uint32_t)cond, 0xcececece, pseq, flags, 0);
b0d623f7 1526#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1527
1528 if ((flags & _PTHREAD_MTX_OPT_MUTEX) == 0) {
1529 error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
1530 if (error != 0) {
1531 *retval = 0;
1532#if _PSYNCH_TRACE_
1533 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
1534#endif /* _PSYNCH_TRACE_ */
1535 return(error);
b0d623f7 1536 }
6d2010ae
A		 1537
1538 ksyn_wqlock(ckwq);
1539 (void)ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (pseq & PTHRW_COUNT_MASK), &kfreeq, 0, 1);
b0d623f7 1540 ksyn_wqunlock(ckwq);
6d2010ae
A		 1541 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP));
1542 } else {
1543 /* mutex type */
1544 error = ksyn_wqfind(cond, cgen, cugen, 0, 0, flags, (KSYN_WQTYPE_MTX | KSYN_WQTYPE_INDROP), &ckwq);
1545 if (error != 0) {
1546 *retval = 0;
1547#if _PSYNCH_TRACE_
1548 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
1549#endif /* _PSYNCH_TRACE_ */
1550 return(error);
1551 }
b0d623f7 1552
6d2010ae
A		 1553 ksyn_wqlock(ckwq);
1554 if (((flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT) != 0) && (ckwq->kw_pre_rwwc != 0)) {
1555 if (is_seqlower_eq(ckwq->kw_pre_lockseq, cgen) != 0) {
1556 /* clear prepost */
1557 ckwq->kw_pre_rwwc = 0;
1558 ckwq->kw_pre_lockseq = 0;
1559 }
1560 }
1561 ksyn_wqunlock(ckwq);
1562 ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_MTX | KSYN_WQTYPE_INDROP));
b0d623f7 1563 }
6d2010ae 1564
b0d623f7 1565#if _PSYNCH_TRACE_
6d2010ae 1566 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, 0, 0, 0);
b0d623f7 1567#endif /* _PSYNCH_TRACE_ */
6d2010ae 1568 return(0);
2d21ac55
A		 1569}
1570
b0d623f7
A		 1571/* ***************** pthread_rwlock ************************ */
1572/*
1573 * psynch_rw_rdlock: This system call is used for psync rwlock readers to block.
1574 */
1575int
1576psynch_rw_rdlock(__unused proc_t p, struct psynch_rw_rdlock_args * uap, uint32_t * retval)
1577{
1578 user_addr_t rwlock = uap->rwlock;
1579 uint32_t lgen = uap->lgenval;
1580 uint32_t ugen = uap->ugenval;
1581 uint32_t rw_wc = uap->rw_wc;
1582 //uint64_t tid = uap->tid;
1583 int flags = uap->flags;
1584 int error = 0, block;
6d2010ae 1585 uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
b0d623f7
A		 1586 ksyn_wait_queue_t kwq;
1587 uthread_t uth;
6d2010ae
A		 1588 int isinit = lgen & PTHRW_RWL_INIT;
1589 uint32_t returnbits = 0;
1590 ksyn_waitq_element_t kwe;
7ddcb079 1591 kern_return_t kret;
2d21ac55 1592
b0d623f7 1593#if _PSYNCH_TRACE_
6d2010ae 1594 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 1595#endif /* _PSYNCH_TRACE_ */
1596 uth = current_uthread();
1597
1598 /* preserve the seq number */
6d2010ae
A		 1599 kwe = &uth->uu_kwe;
1600 kwe->kwe_lockseq = lgen;
1601 kwe->kwe_uth = uth;
1602 kwe->kwe_psynchretval = 0;
1603 kwe->kwe_kwqqueue = NULL;
1604
b0d623f7
A		 1605 lockseq = lgen & PTHRW_COUNT_MASK;
1606
6d2010ae 1607
b0d623f7
A		 1608 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1609 if (error != 0) {
1610#if _PSYNCH_TRACE_
6d2010ae 1611 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 1612#endif /* _PSYNCH_TRACE_ */
1613 return(error);
1614 }
1615
1616 ksyn_wqlock(kwq);
1617
6d2010ae
A		 1618 if (isinit != 0) {
1619 lgen &= ~PTHRW_RWL_INIT;
1620 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1621 /* first to notice the reset of the lock, clear preposts */
1622 CLEAR_REINIT_BITS(kwq);
1623 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1624#if _PSYNCH_TRACE_
1625 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
1626#endif /* _PSYNCH_TRACE_ */
1627 }
1628 }
1629
b0d623f7
A		 1630 /* handle first the missed wakeups */
1631 if ((kwq->kw_pre_intrcount != 0) &&
1632 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_READ) || (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD)) &&
1633 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1634
1635 kwq->kw_pre_intrcount--;
6d2010ae 1636 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 1637 if (kwq->kw_pre_intrcount==0)
1638 CLEAR_INTR_PREPOST_BITS(kwq);
1639 ksyn_wqunlock(kwq);
1640 goto out;
1641 }
1642
6d2010ae
A		 1643 /* handle overlap first as they are not counted against pre_rwwc */
1644
1645 /* check for overlap and if no pending W bit (indicates writers) */
1646 if ((kwq->kw_overlapwatch != 0) && ((rw_wc & PTHRW_RWS_SAVEMASK) == 0) && ((lgen & PTH_RWL_WBIT) == 0)) {
b0d623f7 1647#if _PSYNCH_TRACE_
6d2010ae 1648 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 10, kwq->kw_nextseqword, kwq->kw_lastseqword, 0);
b0d623f7 1649#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1650 error = kwq_handle_overlap(kwq, lgen, ugen, rw_wc, &updatebits, (KW_UNLOCK_PREPOST_READLOCK|KW_UNLOCK_PREPOST), &block);
1651#if __TESTPANICS__
1652 if (error != 0)
1653 panic("rw_rdlock: kwq_handle_overlap failed %d\n",error);
1654#endif /* __TESTPANICS__ */
1655 if (block == 0) {
1656 error = 0;
1657 kwe->kwe_psynchretval = updatebits;
1658#if _PSYNCH_TRACE_
1659 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xff, updatebits, 0xee, 0);
1660#endif /* _PSYNCH_TRACE_ */
1661 ksyn_wqunlock(kwq);
1662 goto out;
1663 }
b0d623f7
A		 1664 }
1665
1666 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1667#if _PSYNCH_TRACE_
6d2010ae 1668 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 1669#endif /* _PSYNCH_TRACE_ */
1670 kwq->kw_pre_rwwc--;
1671 if (kwq->kw_pre_rwwc == 0) {
1672 preseq = kwq->kw_pre_lockseq;
6d2010ae 1673 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 1674 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 1675 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1676 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1677#if _PSYNCH_TRACE_
1678 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
1679#endif /* _PSYNCH_TRACE_ */
1680 }
1681 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_READLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1682#if __TESTPANICS__
b0d623f7 1683 if (error != 0)
6d2010ae
A		 1684 panic("rw_rdlock: kwq_handle_unlock failed %d\n",error);
1685#endif /* __TESTPANICS__ */
b0d623f7
A		 1686 if (block == 0) {
1687 ksyn_wqunlock(kwq);
1688 goto out;
1689 }
1690 /* insert to q and proceed as ususal */
1691 }
1692 }
1693
6d2010ae 1694
b0d623f7 1695#if _PSYNCH_TRACE_
6d2010ae 1696 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 1697#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1698 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], lgen, uth, kwe, SEQFIT);
1699#if __TESTPANICS__
b0d623f7
A		 1700 if (error != 0)
1701 panic("psynch_rw_rdlock: failed to enqueue\n");
6d2010ae 1702#endif /* __TESTPANICS__ */
7ddcb079 1703 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
b0d623f7 1704 /* drops the kwq lock */
7ddcb079
A		 1705 switch (kret) {
1706 case THREAD_TIMED_OUT:
1707 error = ETIMEDOUT;
1708 break;
1709 case THREAD_INTERRUPTED:
1710 error = EINTR;
1711 break;
1712 default:
1713 error = 0;
1714 break;
1715 }
b0d623f7
A		 1716
1717out:
1718 if (error != 0) {
1719#if _PSYNCH_TRACE_
6d2010ae 1720 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
b0d623f7
A		 1721#endif /* _PSYNCH_TRACE_ */
1722 ksyn_wqlock(kwq);
6d2010ae
A		 1723 if (kwe->kwe_kwqqueue != NULL)
1724 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwe);
b0d623f7
A		 1725 ksyn_wqunlock(kwq);
1726 } else {
1727 /* update bits */
6d2010ae
A		 1728 *retval = kwe->kwe_psynchretval;
1729 returnbits = kwe->kwe_psynchretval;
b0d623f7 1730 }
6d2010ae 1731 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
b0d623f7 1732#if _PSYNCH_TRACE_
6d2010ae 1733 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
b0d623f7
A		 1734#endif /* _PSYNCH_TRACE_ */
1735 return(error);
1736}
1737
1738/*
1739 * psynch_rw_longrdlock: This system call is used for psync rwlock long readers to block.
1740 */
1741int
6d2010ae 1742psynch_rw_longrdlock(__unused proc_t p, __unused struct psynch_rw_longrdlock_args * uap, __unused uint32_t * retval)
2d21ac55 1743{
b0d623f7
A		 1744 user_addr_t rwlock = uap->rwlock;
1745 uint32_t lgen = uap->lgenval;
1746 uint32_t ugen = uap->ugenval;
1747 uint32_t rw_wc = uap->rw_wc;
1748 //uint64_t tid = uap->tid;
1749 int flags = uap->flags;
6d2010ae
A		 1750 int isinit = lgen & PTHRW_RWL_INIT;
1751 uint32_t returnbits=0;
1752 ksyn_waitq_element_t kwe;
7ddcb079 1753 kern_return_t kret;
b0d623f7
A		 1754
1755 ksyn_wait_queue_t kwq;
1756 int error=0, block = 0 ;
1757 uthread_t uth;
6d2010ae 1758 uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
b0d623f7
A		 1759
1760#if _PSYNCH_TRACE_
6d2010ae 1761 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 1762#endif /* _PSYNCH_TRACE_ */
1763 uth = current_uthread();
6d2010ae
A		 1764 kwe = &uth->uu_kwe;
1765 kwe->kwe_lockseq = lgen;
1766 kwe->kwe_uth = uth;
1767 kwe->kwe_psynchretval = 0;
1768 kwe->kwe_kwqqueue = NULL;
b0d623f7 1769 lockseq = (lgen & PTHRW_COUNT_MASK);
6d2010ae 1770
b0d623f7
A		 1771 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1772 if (error != 0) {
1773#if _PSYNCH_TRACE_
6d2010ae 1774 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 1775#endif /* _PSYNCH_TRACE_ */
1776 return(error);
2d21ac55 1777 }
b0d623f7
A		 1778
1779 ksyn_wqlock(kwq);
1780
6d2010ae
A		 1781 if (isinit != 0) {
1782 lgen &= ~PTHRW_RWL_INIT;
1783 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1784 /* first to notice the reset of the lock, clear preposts */
1785 CLEAR_REINIT_BITS(kwq);
1786 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1787#if _PSYNCH_TRACE_
1788 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
1789#endif /* _PSYNCH_TRACE_ */
1790 }
1791 }
1792
b0d623f7
A		 1793 /* handle first the missed wakeups */
1794 if ((kwq->kw_pre_intrcount != 0) &&
1795 (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD) &&
1796 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1797
1798 kwq->kw_pre_intrcount--;
6d2010ae 1799 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 1800 if (kwq->kw_pre_intrcount==0)
1801 CLEAR_INTR_PREPOST_BITS(kwq);
1802 ksyn_wqunlock(kwq);
1803 goto out;
1804 }
1805
b0d623f7
A		 1806
1807 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1808#if _PSYNCH_TRACE_
6d2010ae 1809 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 1810#endif /* _PSYNCH_TRACE_ */
1811 kwq->kw_pre_rwwc--;
1812 if (kwq->kw_pre_rwwc == 0) {
1813 preseq = kwq->kw_pre_lockseq;
6d2010ae 1814 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 1815 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 1816 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1817 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1818#if _PSYNCH_TRACE_
1819 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
1820#endif /* _PSYNCH_TRACE_ */
1821 }
1822 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_LREADLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1823#if __TESTPANICS__
b0d623f7
A		 1824 if (error != 0)
1825 panic("kwq_handle_unlock failed %d\n",error);
6d2010ae 1826#endif /* __TESTPANICS__ */
b0d623f7
A		 1827 if (block == 0) {
1828 ksyn_wqunlock(kwq);
1829 goto out;
1830 }
1831 /* insert to q and proceed as ususal */
1832 }
1833 }
1834
1835#if _PSYNCH_TRACE_
6d2010ae 1836 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 1837#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1838 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], lgen, uth, kwe, SEQFIT);
1839#if __TESTPANICS__
b0d623f7
A		 1840 if (error != 0)
1841 panic("psynch_rw_longrdlock: failed to enqueue\n");
6d2010ae 1842#endif /* __TESTPANICS__ */
b0d623f7 1843
7ddcb079 1844 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
b0d623f7 1845 /* drops the kwq lock */
7ddcb079
A		 1846 switch (kret) {
1847 case THREAD_TIMED_OUT:
1848 error = ETIMEDOUT;
1849 break;
1850 case THREAD_INTERRUPTED:
1851 error = EINTR;
1852 break;
1853 default:
1854 error = 0;
1855 break;
1856 }
b0d623f7
A		 1857out:
1858 if (error != 0) {
1859#if _PSYNCH_TRACE_
6d2010ae 1860 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 1861#endif /* _PSYNCH_TRACE_ */
1862 ksyn_wqlock(kwq);
6d2010ae
A		 1863 if (kwe->kwe_kwqqueue != NULL)
1864 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwe);
b0d623f7
A		 1865 ksyn_wqunlock(kwq);
1866 } else {
1867 /* update bits */
6d2010ae
A		 1868 *retval = kwe->kwe_psynchretval;
1869 returnbits = kwe->kwe_psynchretval;
b0d623f7
A		 1870 }
1871
6d2010ae 1872 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
b0d623f7
A		 1873
1874#if _PSYNCH_TRACE_
6d2010ae 1875 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0, returnbits, error, 0);
b0d623f7
A		 1876#endif /* _PSYNCH_TRACE_ */
1877 return(error);
2d21ac55
A		 1878}
1879
b0d623f7
A		 1880/*
1881 * psynch_rw_wrlock: This system call is used for psync rwlock writers to block.
1882 */
1883int
1884psynch_rw_wrlock(__unused proc_t p, struct psynch_rw_wrlock_args * uap, uint32_t * retval)
1885{
1886 user_addr_t rwlock = uap->rwlock;
1887 uint32_t lgen = uap->lgenval;
1888 uint32_t ugen = uap->ugenval;
1889 uint32_t rw_wc = uap->rw_wc;
1890 //uint64_t tid = uap->tid;
1891 int flags = uap->flags;
1892 int block;
1893 ksyn_wait_queue_t kwq;
1894 int error=0;
1895 uthread_t uth;
6d2010ae
A		 1896 uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
1897 int isinit = lgen & PTHRW_RWL_INIT;
1898 uint32_t returnbits = 0;
1899 ksyn_waitq_element_t kwe;
7ddcb079 1900 kern_return_t kret;
2d21ac55 1901
b0d623f7 1902#if _PSYNCH_TRACE_
6d2010ae 1903 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 1904#endif /* _PSYNCH_TRACE_ */
1905 uth = current_uthread();
6d2010ae
A		 1906 kwe = &uth->uu_kwe;
1907 kwe->kwe_lockseq = lgen;
1908 kwe->kwe_uth = uth;
1909 kwe->kwe_psynchretval = 0;
1910 kwe->kwe_kwqqueue = NULL;
b0d623f7
A		 1911 lockseq = (lgen & PTHRW_COUNT_MASK);
1912
1913 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1914 if (error != 0) {
1915#if _PSYNCH_TRACE_
6d2010ae 1916 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 1917#endif /* _PSYNCH_TRACE_ */
1918 return(error);
1919 }
1920
1921 ksyn_wqlock(kwq);
1922
6d2010ae
A		 1923
1924 if (isinit != 0) {
1925 lgen &= ~PTHRW_RWL_INIT;
1926 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1927 /* first to notice the reset of the lock, clear preposts */
1928 CLEAR_REINIT_BITS(kwq);
1929 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1930#if _PSYNCH_TRACE_
1931 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
1932#endif /* _PSYNCH_TRACE_ */
1933 }
1934 }
1935
1936
b0d623f7
A		 1937 /* handle first the missed wakeups */
1938 if ((kwq->kw_pre_intrcount != 0) &&
1939 (kwq->kw_pre_intrtype == PTH_RW_TYPE_WRITE) &&
1940 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1941
1942 kwq->kw_pre_intrcount--;
6d2010ae 1943 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 1944 if (kwq->kw_pre_intrcount==0)
1945 CLEAR_INTR_PREPOST_BITS(kwq);
1946 ksyn_wqunlock(kwq);
1947 goto out;
1948 }
1949
b0d623f7
A		 1950
1951 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1952#if _PSYNCH_TRACE_
6d2010ae 1953 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 1954#endif /* _PSYNCH_TRACE_ */
1955 kwq->kw_pre_rwwc--;
1956 if (kwq->kw_pre_rwwc == 0) {
1957 preseq = kwq->kw_pre_lockseq;
6d2010ae 1958 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 1959 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 1960 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1961 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1962#if _PSYNCH_TRACE_
1963 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
1964#endif /* _PSYNCH_TRACE_ */
1965 }
1966 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_WRLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1967#if __TESTPANICS__
b0d623f7 1968 if (error != 0)
6d2010ae
A		 1969 panic("rw_wrlock: kwq_handle_unlock failed %d\n",error);
1970#endif /* __TESTPANICS__ */
b0d623f7
A		 1971 if (block == 0) {
1972 ksyn_wqunlock(kwq);
6d2010ae
A		 1973 *retval = updatebits;
1974 goto out1;
b0d623f7
A		 1975 }
1976 /* insert to q and proceed as ususal */
1977 }
1978 }
1979
6d2010ae
A		 1980 /* No overlap watch needed go ahead and block */
1981
b0d623f7 1982#if _PSYNCH_TRACE_
6d2010ae 1983 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 1984#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 1985 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], lgen, uth, kwe, SEQFIT);
1986#if __TESTPANICS__
b0d623f7
A		 1987 if (error != 0)
1988 panic("psynch_rw_wrlock: failed to enqueue\n");
6d2010ae 1989#endif /* __TESTPANICS__ */
b0d623f7 1990
7ddcb079 1991 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
b0d623f7 1992 /* drops the wq lock */
7ddcb079
A		 1993 switch (kret) {
1994 case THREAD_TIMED_OUT:
1995 error = ETIMEDOUT;
1996 break;
1997 case THREAD_INTERRUPTED:
1998 error = EINTR;
1999 break;
2000 default:
2001 error = 0;
2002 break;
2003 }
b0d623f7
A		 2004
2005out:
2006 if (error != 0) {
2007#if _PSYNCH_TRACE_
6d2010ae 2008 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
b0d623f7
A		 2009#endif /* _PSYNCH_TRACE_ */
2010 ksyn_wqlock(kwq);
6d2010ae
A		 2011 if (kwe->kwe_kwqqueue != NULL)
2012 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
b0d623f7
A		 2013 ksyn_wqunlock(kwq);
2014 } else {
2015 /* update bits */
6d2010ae
A		 2016 *retval = kwe->kwe_psynchretval;
2017 returnbits = kwe->kwe_psynchretval;
b0d623f7 2018 }
6d2010ae
A		 2019out1:
2020 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
b0d623f7
A		 2021
2022#if _PSYNCH_TRACE_
6d2010ae 2023 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
b0d623f7
A		 2024#endif /* _PSYNCH_TRACE_ */
2025 return(error);
2026}
2027
2028/*
2029 * psynch_rw_yieldwrlock: This system call is used for psync rwlock yielding writers to block.
2030 */
2031int
6d2010ae 2032psynch_rw_yieldwrlock(__unused proc_t p, __unused struct psynch_rw_yieldwrlock_args * uap, __unused uint32_t * retval)
2d21ac55 2033{
b0d623f7
A		 2034 user_addr_t rwlock = uap->rwlock;
2035 uint32_t lgen = uap->lgenval;
2036 uint32_t ugen = uap->ugenval;
2037 uint32_t rw_wc = uap->rw_wc;
2038 //uint64_t tid = uap->tid;
2039 int flags = uap->flags;
2040 int block;
2041 ksyn_wait_queue_t kwq;
2042 int error=0;
6d2010ae 2043 int isinit = lgen & PTHRW_RWL_INIT;
b0d623f7 2044 uthread_t uth;
6d2010ae
A		 2045 uint32_t returnbits=0;
2046 ksyn_waitq_element_t kwe;
7ddcb079 2047 kern_return_t kret;
b0d623f7
A		 2048
2049#if _PSYNCH_TRACE_
6d2010ae 2050 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7 2051#endif /* _PSYNCH_TRACE_ */
6d2010ae 2052 uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
b0d623f7
A		 2053
2054 uth = current_uthread();
6d2010ae
A		 2055 kwe = &uth->uu_kwe;
2056 kwe->kwe_lockseq = lgen;
2057 kwe->kwe_uth = uth;
2058 kwe->kwe_psynchretval = 0;
2059 kwe->kwe_kwqqueue = NULL;
b0d623f7
A		 2060 lockseq = (lgen & PTHRW_COUNT_MASK);
2061
2062 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
2063 if (error != 0) {
2064#if _PSYNCH_TRACE_
6d2010ae 2065 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 2066#endif /* _PSYNCH_TRACE_ */
2067 return(error);
2068 }
2069
2070 ksyn_wqlock(kwq);
2071
6d2010ae
A		 2072 if (isinit != 0) {
2073 lgen &= ~PTHRW_RWL_INIT;
2074 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
2075 /* first to notice the reset of the lock, clear preposts */
2076 CLEAR_REINIT_BITS(kwq);
2077 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2078#if _PSYNCH_TRACE_
2079 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
2080#endif /* _PSYNCH_TRACE_ */
2081 }
2082 }
2083
b0d623f7
A		 2084 /* handle first the missed wakeups */
2085 if ((kwq->kw_pre_intrcount != 0) &&
2086 (kwq->kw_pre_intrtype == PTH_RW_TYPE_YWRITE) &&
2087 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
2088
2089 kwq->kw_pre_intrcount--;
6d2010ae 2090 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 2091 if (kwq->kw_pre_intrcount==0)
2092 CLEAR_INTR_PREPOST_BITS(kwq);
2093 ksyn_wqunlock(kwq);
2094 goto out;
2095 }
2096
b0d623f7
A		 2097 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
2098#if _PSYNCH_TRACE_
6d2010ae 2099 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 2100#endif /* _PSYNCH_TRACE_ */
2101 kwq->kw_pre_rwwc--;
2102 if (kwq->kw_pre_rwwc == 0) {
2103 preseq = kwq->kw_pre_lockseq;
6d2010ae 2104 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 2105 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 2106 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
2107 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2108#if _PSYNCH_TRACE_
2109 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
2110#endif /* _PSYNCH_TRACE_ */
2111 }
2112 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_YWRLOCK|KW_UNLOCK_PREPOST), &block, lgen);
2113#if __TESTPANICS__
b0d623f7
A		 2114 if (error != 0)
2115 panic("kwq_handle_unlock failed %d\n",error);
6d2010ae 2116#endif /* __TESTPANICS__ */
b0d623f7
A		 2117 if (block == 0) {
2118 ksyn_wqunlock(kwq);
6d2010ae 2119 *retval = updatebits;
b0d623f7
A		 2120 goto out;
2121 }
2122 /* insert to q and proceed as ususal */
2123 }
2124 }
2125
2126#if _PSYNCH_TRACE_
6d2010ae 2127 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 2128#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2129 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], lgen, uth, kwe, SEQFIT);
2130#if __TESTPANICS__
b0d623f7
A		 2131 if (error != 0)
2132 panic("psynch_rw_yieldwrlock: failed to enqueue\n");
6d2010ae 2133#endif /* __TESTPANICS__ */
b0d623f7 2134
7ddcb079
A		 2135 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
2136 switch (kret) {
2137 case THREAD_TIMED_OUT:
2138 error = ETIMEDOUT;
2139 break;
2140 case THREAD_INTERRUPTED:
2141 error = EINTR;
2142 break;
2143 default:
2144 error = 0;
2145 break;
2146 }
b0d623f7
A		 2147
2148out:
2149 if (error != 0) {
2150#if _PSYNCH_TRACE_
6d2010ae 2151 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
b0d623f7
A		 2152#endif /* _PSYNCH_TRACE_ */
2153 ksyn_wqlock(kwq);
6d2010ae
A		 2154 if (kwe->kwe_kwqqueue != NULL)
2155 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwe);
b0d623f7
A		 2156 ksyn_wqunlock(kwq);
2157 } else {
2158 /* update bits */
6d2010ae
A		 2159 *retval = kwe->kwe_psynchretval;
2160 returnbits = kwe->kwe_psynchretval;
b0d623f7
A		 2161 }
2162
6d2010ae 2163 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK));
b0d623f7
A		 2164
2165#if _PSYNCH_TRACE_
6d2010ae 2166 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
b0d623f7
A		 2167#endif /* _PSYNCH_TRACE_ */
2168 return(error);
2169}
2170
6d2010ae 2171#if NOTYET
b0d623f7
A		 2172/*
2173 * psynch_rw_downgrade: This system call is used for wakeup blocked readers who are eligible to run due to downgrade.
2174 */
2175int
2176psynch_rw_downgrade(__unused proc_t p, struct psynch_rw_downgrade_args * uap, __unused int * retval)
2177{
2178 user_addr_t rwlock = uap->rwlock;
2179 uint32_t lgen = uap->lgenval;
2180 uint32_t ugen = uap->ugenval;
2181 uint32_t rw_wc = uap->rw_wc;
2182 //uint64_t tid = uap->tid;
2183 int flags = uap->flags;
2184 uint32_t count = 0;
6d2010ae 2185 int isinit = lgen & PTHRW_RWL_INIT;
b0d623f7
A		 2186 ksyn_wait_queue_t kwq;
2187 int error=0;
2188 uthread_t uth;
2189 uint32_t curgen = 0;
2190
2191#if _PSYNCH_TRACE_
6d2010ae 2192 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 2193#endif /* _PSYNCH_TRACE_ */
2194 uth = current_uthread();
2195
2196 curgen = (lgen & PTHRW_COUNT_MASK);
2197
6d2010ae 2198 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK), &kwq);
b0d623f7
A		 2199 if (error != 0) {
2200#if _PSYNCH_TRACE_
6d2010ae 2201 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 2202#endif /* _PSYNCH_TRACE_ */
2203 return(error);
2204 }
2205
2206 ksyn_wqlock(kwq);
2207
6d2010ae
A		 2208 if ((lgen & PTHRW_RWL_INIT) != 0) {
2209 lgen &= ~PTHRW_RWL_INIT;
2210 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0){
2211 CLEAR_REINIT_BITS(kwq);
2212 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2213#if _PSYNCH_TRACE_
2214 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
2215#endif /* _PSYNCH_TRACE_ */
2216 }
2217 isinit = 1;
2218 }
2219
2220 /* if lastunlock seq is set, ensure the current one is not lower than that, as it would be spurious */
2221 if ((kwq->kw_lastunlockseq != PTHRW_RWL_INIT) && (is_seqlower(ugen, kwq->kw_lastunlockseq)!= 0)) {
b0d623f7 2222 /* spurious updatebits?? */
6d2010ae 2223 error = 0;
b0d623f7
A		 2224 goto out;
2225 }
b0d623f7 2226
6d2010ae
A		 2227
2228
2229 /* If L-U != num of waiters, then it needs to be preposted or spr */
2230 diff = find_diff(lgen, ugen);
2231 /* take count of the downgrade thread itself */
2232 diff--;
2233
2234
2235#if _PSYNCH_TRACE_
2236 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_inqueue, curgen, 0);
2237#endif /* _PSYNCH_TRACE_ */
2238 if (find_seq_till(kwq, curgen, diff, &count) == 0) {
2239 if (count < (uint32_t)diff)
2240 goto prepost;
b0d623f7 2241 }
6d2010ae
A		 2242
2243 /* no prepost and all threads are in place, reset the bit */
2244 if ((isinit != 0) && ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0)){
2245 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2246#if _PSYNCH_TRACE_
2247 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
2248#endif /* _PSYNCH_TRACE_ */
b0d623f7 2249 }
6d2010ae
A		 2250
2251 /* can handle unlock now */
b0d623f7 2252
6d2010ae
A		 2253 CLEAR_PREPOST_BITS(kwq);
2254
b0d623f7
A		 2255dounlock:
2256#if _PSYNCH_TRACE_
6d2010ae 2257 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7
A		 2258#endif /* _PSYNCH_TRACE_ */
2259 error = kwq_handle_downgrade(kwq, lgen, 0, 0, NULL);
2260
6d2010ae 2261#if __TESTPANICS__
b0d623f7
A		 2262 if (error != 0)
2263 panic("psynch_rw_downgrade: failed to wakeup\n");
6d2010ae 2264#endif /* __TESTPANICS__ */
b0d623f7
A		 2265
2266out:
2267 ksyn_wqunlock(kwq);
2268#if _PSYNCH_TRACE_
6d2010ae 2269 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_END, (uint32_t)rwlock, 0, 0, error, 0);
b0d623f7 2270#endif /* _PSYNCH_TRACE_ */
6d2010ae 2271 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK));
b0d623f7
A		 2272
2273 return(error);
2274
2275prepost:
2276 kwq->kw_pre_rwwc = (rw_wc - count);
2277 kwq->kw_pre_lockseq = lgen;
2278#if _PSYNCH_TRACE_
6d2010ae 2279 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 2280#endif /* _PSYNCH_TRACE_ */
2281 error = 0;
2282 goto out;
2283}
2284
2285
2286/*
2287 * psynch_rw_upgrade: This system call is used by an reader to block waiting for upgrade to be granted.
2288 */
2289int
2290psynch_rw_upgrade(__unused proc_t p, struct psynch_rw_upgrade_args * uap, uint32_t * retval)
2291{
2292 user_addr_t rwlock = uap->rwlock;
2293 uint32_t lgen = uap->lgenval;
2294 uint32_t ugen = uap->ugenval;
2295 uint32_t rw_wc = uap->rw_wc;
2296 //uint64_t tid = uap->tid;
2297 int flags = uap->flags;
2298 int block;
2299 ksyn_wait_queue_t kwq;
2300 int error=0;
2301 uthread_t uth;
2302 uint32_t lockseq = 0, updatebits = 0, preseq = 0;
6d2010ae
A		 2303 int isinit = lgen & PTHRW_RWL_INIT;
2304 ksyn_waitq_element_t kwe;
7ddcb079 2305 kern_return_t kret;
b0d623f7
A		 2306
2307#if _PSYNCH_TRACE_
6d2010ae 2308 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 2309#endif /* _PSYNCH_TRACE_ */
2310 uth = current_uthread();
6d2010ae
A		 2311 kwe = &uth->uu_kwe;
2312 kwe->kwe_lockseq = lgen;
2313 kwe->kwe_uth = uth;
2314 kwe->kwe_psynchretval = 0;
2315 kwe->kwe_kwqqueue = NULL;
b0d623f7 2316 lockseq = (lgen & PTHRW_COUNT_MASK);
6d2010ae
A		 2317
2318 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK), &kwq);
b0d623f7
A		 2319 if (error != 0) {
2320#if _PSYNCH_TRACE_
6d2010ae 2321 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 2322#endif /* _PSYNCH_TRACE_ */
2323 return(error);
2324 }
2325
2326 ksyn_wqlock(kwq);
6d2010ae
A		 2327
2328 if (isinit != 0) {
2329 lgen &= ~PTHRW_RWL_INIT;
2330 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
2331 /* first to notice the reset of the lock, clear preposts */
2332 CLEAR_REINIT_BITS(kwq);
2333 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2334#if _PSYNCH_TRACE_
2335 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
2336#endif /* _PSYNCH_TRACE_ */
2337 }
2338 }
2339
b0d623f7
A		 2340 /* handle first the missed wakeups */
2341 if ((kwq->kw_pre_intrcount != 0) &&
6d2010ae 2342 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_READ) || (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD)) &&
b0d623f7
A		 2343 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
2344
2345 kwq->kw_pre_intrcount--;
6d2010ae 2346 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
b0d623f7
A		 2347 if (kwq->kw_pre_intrcount==0)
2348 CLEAR_INTR_PREPOST_BITS(kwq);
2349 ksyn_wqunlock(kwq);
2350 goto out;
2351 }
2352
2353 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
2354#if _PSYNCH_TRACE_
6d2010ae 2355 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7
A		 2356#endif /* _PSYNCH_TRACE_ */
2357 kwq->kw_pre_rwwc--;
2358 if (kwq->kw_pre_rwwc == 0) {
2359 preseq = kwq->kw_pre_lockseq;
6d2010ae 2360 prerw_wc = kwq->kw_pre_sseq;
b0d623f7 2361 CLEAR_PREPOST_BITS(kwq);
6d2010ae
A		 2362 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
2363 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2364#if _PSYNCH_TRACE_
2365 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
2366#endif /* _PSYNCH_TRACE_ */
2367 }
2368 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_UPGRADE|KW_UNLOCK_PREPOST), &block, lgen);
2369#if __TESTPANICS__
b0d623f7 2370 if (error != 0)
6d2010ae
A		 2371 panic("rw_rdlock: kwq_handle_unlock failed %d\n",error);
2372#endif /* __TESTPANICS__ */
b0d623f7
A		 2373 if (block == 0) {
2374 ksyn_wqunlock(kwq);
2375 goto out;
2376 }
2377 /* insert to q and proceed as ususal */
2378 }
2379 }
2380
2381
2382#if _PSYNCH_TRACE_
6d2010ae 2383 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
b0d623f7 2384#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2385 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], lgen, uth, kwe, SEQFIT);
2386#if __TESTPANICS__
b0d623f7
A		 2387 if (error != 0)
2388 panic("psynch_rw_upgrade: failed to enqueue\n");
6d2010ae 2389#endif /* __TESTPANICS__ */
b0d623f7
A		 2390
2391
7ddcb079 2392 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
b0d623f7 2393 /* drops the lock */
7ddcb079
A		 2394 switch (kret) {
2395 case THREAD_TIMED_OUT:
2396 error = ETIMEDOUT;
2397 break;
2398 case THREAD_INTERRUPTED:
2399 error = EINTR;
2400 break;
2401 default:
2402 error = 0;
2403 break;
2404 }
b0d623f7
A		 2405
2406out:
2407 if (error != 0) {
2408#if _PSYNCH_TRACE_
6d2010ae 2409 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
b0d623f7
A		 2410#endif /* _PSYNCH_TRACE_ */
2411 ksyn_wqlock(kwq);
6d2010ae
A		 2412 if (kwe->kwe_kwqqueue != NULL)
2413 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwe);
b0d623f7
A		 2414 ksyn_wqunlock(kwq);
2415 } else {
2416 /* update bits */
6d2010ae 2417 *retval = kwe->kwe_psynchretval;
b0d623f7
A		 2418 }
2419
6d2010ae 2420 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK));
b0d623f7 2421#if _PSYNCH_TRACE_
6d2010ae 2422 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUPGRADE | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7 2423#endif /* _PSYNCH_TRACE_ */
6d2010ae 2424
b0d623f7
A		 2425 return(error);
2426}
2427
6d2010ae
A		 2428#else /* NOTYET */
2429int
2430psynch_rw_upgrade(__unused proc_t p, __unused struct psynch_rw_upgrade_args * uap, __unused uint32_t * retval)
2431{
2432 return(0);
2433}
2434int
2435psynch_rw_downgrade(__unused proc_t p, __unused struct psynch_rw_downgrade_args * uap, __unused int * retval)
2436{
2437 return(0);
2438}
2439#endif /* NOTYET */
b0d623f7
A		 2440/*
2441 * psynch_rw_unlock: This system call is used for unlock state postings. This will grant appropriate
2442 * reader/writer variety lock.
2443 */
2444
2445int
2446psynch_rw_unlock(__unused proc_t p, struct psynch_rw_unlock_args * uap, uint32_t * retval)
2447{
2448 user_addr_t rwlock = uap->rwlock;
2449 uint32_t lgen = uap->lgenval;
2450 uint32_t ugen = uap->ugenval;
2451 uint32_t rw_wc = uap->rw_wc;
2452 uint32_t curgen;
2453 //uint64_t tid = uap->tid;
2454 int flags = uap->flags;
2455 uthread_t uth;
2456 ksyn_wait_queue_t kwq;
2457 uint32_t updatebits = 0;
6d2010ae 2458 int error=0, diff;
b0d623f7 2459 uint32_t count = 0;
6d2010ae 2460 int isinit = 0;
b0d623f7
A		 2461
2462
2463#if _PSYNCH_TRACE_
6d2010ae 2464 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
b0d623f7
A		 2465#endif /* _PSYNCH_TRACE_ */
2466 uth = current_uthread();
2467
6d2010ae 2468 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK), &kwq);
b0d623f7
A		 2469 if (error != 0) {
2470#if _PSYNCH_TRACE_
6d2010ae 2471 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
b0d623f7
A		 2472#endif /* _PSYNCH_TRACE_ */
2473 return(error);
2474 }
2475
2476 curgen = lgen & PTHRW_COUNT_MASK;
2477
2478 ksyn_wqlock(kwq);
2479
6d2010ae
A		 2480 if ((lgen & PTHRW_RWL_INIT) != 0) {
2481 lgen &= ~PTHRW_RWL_INIT;
2482 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0){
2483 CLEAR_REINIT_BITS(kwq);
2484 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2485#if _PSYNCH_TRACE_
2486 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
2487#endif /* _PSYNCH_TRACE_ */
2488 }
2489 isinit = 1;
2490 }
2491
2492 /* if lastunlock seq is set, ensure the current one is not lower than that, as it would be spurious */
2493 if ((kwq->kw_lastunlockseq != PTHRW_RWL_INIT) && (is_seqlower(ugen, kwq->kw_lastunlockseq)!= 0)) {
2494#if _PSYNCH_TRACE_
2495 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, (uint32_t)0xeeeeeeee, rw_wc, kwq->kw_lastunlockseq, 0);
2496#endif /* _PSYNCH_TRACE_ */
2497 error = 0;
b0d623f7
A		 2498 goto out;
2499 }
2500
6d2010ae
A		 2501 /* If L-U != num of waiters, then it needs to be preposted or spr */
2502 diff = find_diff(lgen, ugen);
b0d623f7
A		 2503
2504#if _PSYNCH_TRACE_
6d2010ae 2505 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_inqueue, curgen, 0);
b0d623f7 2506#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2507 if (find_seq_till(kwq, curgen, diff, &count) == 0) {
2508 if ((count == 0) || (count < (uint32_t)diff))
b0d623f7
A		 2509 goto prepost;
2510 }
2511
6d2010ae
A		 2512 /* no prepost and all threads are in place, reset the bit */
2513 if ((isinit != 0) && ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0)){
2514 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2515#if _PSYNCH_TRACE_
2516 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
2517#endif /* _PSYNCH_TRACE_ */
2518 }
b0d623f7
A		 2519
2520 /* can handle unlock now */
2521
2522 CLEAR_PREPOST_BITS(kwq);
b0d623f7
A		 2523
2524#if _PSYNCH_TRACE_
6d2010ae 2525 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, 0, 0, 0);
b0d623f7 2526#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2527 error = kwq_handle_unlock(kwq, lgen, rw_wc, &updatebits, 0, NULL, 0);
2528#if __TESTPANICS__
b0d623f7
A		 2529 if (error != 0)
2530 panic("psynch_rw_unlock: kwq_handle_unlock failed %d\n",error);
6d2010ae 2531#endif /* __TESTPANICS__ */
b0d623f7
A		 2532out:
2533 if (error == 0) {
2534 /* update bits?? */
2535 *retval = updatebits;
2536 }
6d2010ae
A		 2537
2538
b0d623f7
A		 2539 ksyn_wqunlock(kwq);
2540
6d2010ae 2541 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK));
b0d623f7 2542#if _PSYNCH_TRACE_
6d2010ae 2543 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0, updatebits, error, 0);
b0d623f7
A		 2544#endif /* _PSYNCH_TRACE_ */
2545
2546 return(error);
2547
2548prepost:
6d2010ae
A		 2549 /* update if the new seq is higher than prev prepost, or first set */
2550 if ((is_rws_setseq(kwq->kw_pre_sseq) != 0) ||
2551 (is_seqhigher_eq((rw_wc & PTHRW_COUNT_MASK), (kwq->kw_pre_sseq & PTHRW_COUNT_MASK)) != 0)) {
2552 kwq->kw_pre_rwwc = (diff - count);
2553 kwq->kw_pre_lockseq = curgen;
2554 kwq->kw_pre_sseq = rw_wc;
b0d623f7 2555#if _PSYNCH_TRACE_
6d2010ae
A		 2556 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, rw_wc, count, 0);
2557 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
b0d623f7 2558#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 2559 updatebits = lgen; /* let this not do unlock handling */
2560 }
b0d623f7
A		 2561 error = 0;
2562 goto out;
2563}
2564
2565
2566/*
2567 * psynch_rw_unlock2: This system call is used to wakeup pending readers when unlock grant frm kernel
2568 * to new reader arrival races
2569 */
2570int
6d2010ae 2571psynch_rw_unlock2(__unused proc_t p, __unused struct psynch_rw_unlock2_args * uap, __unused uint32_t * retval)
b0d623f7 2572{
6d2010ae 2573 return(ENOTSUP);
b0d623f7
A		 2574}
2575
2576
2577/* ************************************************************************** */
2578void
2579pth_global_hashinit()
2580{
6d2010ae
A		 2581 int arg;
2582
b0d623f7 2583 pth_glob_hashtbl = hashinit(PTH_HASHSIZE * 4, M_PROC, &pthhash);
6d2010ae
A		 2584
2585 /*
2586 * pthtest={0,1,2,3} (override default aborting behavior on pthread sync failures)
2587 * 0 - just return errors
2588 * 1 - print and return errors
2589 * 2 - abort user, print and return errors
2590 * 3 - panic
2591 */
2592 if (!PE_parse_boot_argn("pthtest", &arg, sizeof(arg)))
2593 arg = __TESTMODE__;
2594
2595 if (arg == 3) {
2596 __test_panics__ = 1;
2597 printf("Pthread support PANICS when sync kernel primitives misused\n");
2598 } else if (arg == 2) {
2599 __test_aborts__ = 1;
2600 __test_prints__ = 1;
2601 printf("Pthread support ABORTS when sync kernel primitives misused\n");
2602 } else if (arg == 1) {
2603 __test_prints__ = 1;
2604 printf("Pthread support LOGS when sync kernel primitives misused\n");
2605 }
b0d623f7
A		 2606}
2607
2608void
2609pth_proc_hashinit(proc_t p)
2610{
2611 p->p_pthhash = hashinit(PTH_HASHSIZE, M_PROC, &pthhash);
2612 if (p->p_pthhash == NULL)
2613 panic("pth_proc_hashinit: hash init returned 0\n");
2614}
2615
2616
2617ksyn_wait_queue_t
2618ksyn_wq_hash_lookup(user_addr_t mutex, proc_t p, int flags, uint64_t object, uint64_t objoffset)
2619{
2620 ksyn_wait_queue_t kwq;
2621 struct pthhashhead * hashptr;
2622
2623 if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
2624 {
2625 hashptr = pth_glob_hashtbl;
2626 kwq = (&hashptr[object & pthhash])->lh_first;
2627 if (kwq != 0) {
2628 for (; kwq != NULL; kwq = kwq->kw_hash.le_next) {
2629 if ((kwq->kw_object == object) &&(kwq->kw_offset == objoffset)) {
2630 return (kwq);
2631 }
2632 }
2633 }
2634 } else {
2635 hashptr = p->p_pthhash;
2636 kwq = (&hashptr[mutex & pthhash])->lh_first;
2637 if (kwq != 0)
2638 for (; kwq != NULL; kwq = kwq->kw_hash.le_next) {
2639 if (kwq->kw_addr == mutex) {
2640 return (kwq);
2641 }
2642 }
2643 }
2644 return(NULL);
2645}
2646
2647void
2648pth_proc_hashdelete(proc_t p)
2649{
2650 struct pthhashhead * hashptr;
2651 ksyn_wait_queue_t kwq;
2652 int hashsize = pthhash + 1;
2653 int i;
2654
6d2010ae
A		 2655#if _PSYNCH_TRACE_
2656 if ((pthread_debug_proc != NULL) && (p == pthread_debug_proc))
2657 pthread_debug_proc = PROC_NULL;
2658#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 2659 hashptr = p->p_pthhash;
2660 if (hashptr == NULL)
2661 return;
2662
2663 for(i= 0; i < hashsize; i++) {
2664 while ((kwq = LIST_FIRST(&hashptr[i])) != NULL) {
2665 pthread_list_lock();
2666 if ((kwq->kw_pflags & KSYN_WQ_INHASH) != 0) {
2667 kwq->kw_pflags &= ~KSYN_WQ_INHASH;
2668 LIST_REMOVE(kwq, kw_hash);
2669 }
2670 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2671 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
2672 LIST_REMOVE(kwq, kw_list);
6d2010ae 2673 num_infreekwq--;
b0d623f7 2674 }
6d2010ae 2675 num_freekwq++;
b0d623f7 2676 pthread_list_unlock();
6d2010ae
A		 2677 /* release fake entries if present for cvars */
2678 if (((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) && (kwq->kw_inqueue != 0))
2679 ksyn_freeallkwe(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER]);
b0d623f7 2680 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
6d2010ae 2681 zfree(kwq_zone, kwq);
b0d623f7
A		 2682 }
2683 }
2684 FREE(p->p_pthhash, M_PROC);
2685 p->p_pthhash = NULL;
2686}
2687
6d2010ae
A		 2688/* no lock held for this as the waitqueue is getting freed */
2689void
2690ksyn_freeallkwe(ksyn_queue_t kq)
2691{
2692 ksyn_waitq_element_t kwe;
2693
2694 /* free all the fake entries, dequeue rest */
2695 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
2696 while (kwe != NULL) {
2697 if (kwe->kwe_flags != KWE_THREAD_INWAIT) {
2698 TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
2699 zfree(kwe_zone, kwe);
2700 } else {
2701 TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
2702 }
2703 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
2704 }
2705}
b0d623f7
A		 2706
2707/* find kernel waitqueue, if not present create one. Grants a reference */
2708int
2709ksyn_wqfind(user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int flags, int wqtype, ksyn_wait_queue_t * kwqp)
2710{
2711 ksyn_wait_queue_t kwq;
2712 ksyn_wait_queue_t nkwq;
2713 struct pthhashhead * hashptr;
2714 uint64_t object = 0, offset = 0;
2715 uint64_t hashhint;
2716 proc_t p = current_proc();
6d2010ae
A		 2717 int retry = mgen & PTH_RWL_RETRYBIT;
2718 struct ksyn_queue kfreeq;
b0d623f7
A		 2719 int i;
2720
2721 if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
2722 {
2723 (void)ksyn_findobj(mutex, &object, &offset);
2724 hashhint = object;
2725 hashptr = pth_glob_hashtbl;
2726 } else {
2727 hashptr = p->p_pthhash;
2728 }
2729
6d2010ae
A		 2730 ksyn_queue_init(&kfreeq);
2731
2732 if (((wqtype & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_MTX) && (retry != 0))
2733 mgen &= ~PTH_RWL_RETRYBIT;
2734
2735loop:
b0d623f7
A		 2736 //pthread_list_lock_spin();
2737 pthread_list_lock();
2738
2739 kwq = ksyn_wq_hash_lookup(mutex, p, flags, object, offset);
2740
2741 if (kwq != NULL) {
b0d623f7
A		 2742 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2743 LIST_REMOVE(kwq, kw_list);
2744 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
6d2010ae
A		 2745 num_infreekwq--;
2746 num_reusekwq++;
2747 }
2748 if ((kwq->kw_type & KSYN_WQTYPE_MASK) != (wqtype &KSYN_WQTYPE_MASK)) {
2749 if ((kwq->kw_inqueue == 0) && (kwq->kw_pre_rwwc ==0) && (kwq->kw_pre_intrcount == 0)) {
2750 if (kwq->kw_iocount == 0) {
2751 kwq->kw_addr = mutex;
2752 kwq->kw_flags = flags;
2753 kwq->kw_object = object;
2754 kwq->kw_offset = offset;
2755 kwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
2756 CLEAR_REINIT_BITS(kwq);
2757 CLEAR_INTR_PREPOST_BITS(kwq);
2758 CLEAR_PREPOST_BITS(kwq);
2759 kwq->kw_lword = mgen;
2760 kwq->kw_uword = ugen;
2761 kwq->kw_sword = rw_wc;
2762 kwq->kw_owner = tid;
2763 } else if ((kwq->kw_iocount == 1) && (kwq->kw_dropcount == kwq->kw_iocount)) {
2764 /* if all users are unlockers then wait for it to finish */
2765 kwq->kw_pflags |= KSYN_WQ_WAITING;
2766 /* wait for the wq to be free */
2767 (void)msleep(&kwq->kw_pflags, pthread_list_mlock, PDROP, "ksyn_wqfind", 0);
2768 /* does not have list lock */
2769 goto loop;
2770 } else {
2771 __FAILEDUSERTEST__("address already known to kernel for another (busy) synchronizer type\n");
2772 pthread_list_unlock();
2773 return EBUSY;
2774 }
2775 } else {
2776 __FAILEDUSERTEST__("address already known to kernel for another (busy) synchronizer type(1)\n");
2777 pthread_list_unlock();
2778 return EBUSY;
2779 }
b0d623f7 2780 }
6d2010ae
A		 2781 kwq->kw_iocount++;
2782 if (wqtype == KSYN_WQTYPE_MUTEXDROP)
2783 kwq->kw_dropcount++;
b0d623f7
A		 2784 if (kwqp != NULL)
2785 *kwqp = kwq;
2786 pthread_list_unlock();
2787 return (0);
2788 }
2789
2790 pthread_list_unlock();
2791
6d2010ae 2792 nkwq = (ksyn_wait_queue_t)zalloc(kwq_zone);
b0d623f7
A		 2793 bzero(nkwq, sizeof(struct ksyn_wait_queue));
2794 nkwq->kw_addr = mutex;
2795 nkwq->kw_flags = flags;
2796 nkwq->kw_iocount = 1;
6d2010ae
A		 2797 if (wqtype == KSYN_WQTYPE_MUTEXDROP)
2798 nkwq->kw_dropcount++;
b0d623f7
A		 2799 nkwq->kw_object = object;
2800 nkwq->kw_offset = offset;
2801 nkwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
6d2010ae
A		 2802 nkwq->kw_lastseqword = PTHRW_RWS_INIT;
2803 if (nkwq->kw_type == KSYN_WQTYPE_RWLOCK)
2804 nkwq->kw_nextseqword = PTHRW_RWS_INIT;
2805
2806 nkwq->kw_pre_sseq = PTHRW_RWS_INIT;
2807
2808 CLEAR_PREPOST_BITS(nkwq);
2809 CLEAR_INTR_PREPOST_BITS(nkwq);
2810 CLEAR_REINIT_BITS(nkwq);
2811 nkwq->kw_lword = mgen;
2812 nkwq->kw_uword = ugen;
2813 nkwq->kw_sword = rw_wc;
2814 nkwq->kw_owner = tid;
2815
b0d623f7
A		 2816
2817 for (i=0; i< KSYN_QUEUE_MAX; i++)
2818 ksyn_queue_init(&nkwq->kw_ksynqueues[i]);
2819
b0d623f7
A		 2820 lck_mtx_init(&nkwq->kw_lock, pthread_lck_grp, pthread_lck_attr);
2821
2822 //pthread_list_lock_spin();
2823 pthread_list_lock();
2824 /* see whether it is alread allocated */
2825 kwq = ksyn_wq_hash_lookup(mutex, p, flags, object, offset);
2826
2827 if (kwq != NULL) {
b0d623f7
A		 2828 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2829 LIST_REMOVE(kwq, kw_list);
2830 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
6d2010ae
A		 2831 num_infreekwq--;
2832 num_reusekwq++;
2833 }
2834 if ((kwq->kw_type & KSYN_WQTYPE_MASK) != (wqtype &KSYN_WQTYPE_MASK)) {
2835 if ((kwq->kw_inqueue == 0) && (kwq->kw_pre_rwwc ==0) && (kwq->kw_pre_intrcount == 0)) {
2836 if (kwq->kw_iocount == 0) {
2837 kwq->kw_addr = mutex;
2838 kwq->kw_flags = flags;
2839 kwq->kw_object = object;
2840 kwq->kw_offset = offset;
2841 kwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
2842 CLEAR_REINIT_BITS(kwq);
2843 CLEAR_INTR_PREPOST_BITS(kwq);
2844 CLEAR_PREPOST_BITS(kwq);
2845 kwq->kw_lword = mgen;
2846 kwq->kw_uword = ugen;
2847 kwq->kw_sword = rw_wc;
2848 kwq->kw_owner = tid;
2849 } else if ((kwq->kw_iocount == 1) && (kwq->kw_dropcount == kwq->kw_iocount)) {
2850 kwq->kw_pflags |= KSYN_WQ_WAITING;
2851 /* wait for the wq to be free */
2852 (void)msleep(&kwq->kw_pflags, pthread_list_mlock, PDROP, "ksyn_wqfind", 0);
2853
2854 lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
2855 zfree(kwq_zone, nkwq);
2856 /* will acquire lock again */
2857
2858 goto loop;
2859 } else {
2860 __FAILEDUSERTEST__("address already known to kernel for another [busy] synchronizer type(2)\n");
2861 pthread_list_unlock();
2862 lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
2863 zfree(kwq_zone, nkwq);
2864 return EBUSY;
2865 }
2866 } else {
2867 __FAILEDUSERTEST__("address already known to kernel for another [busy] synchronizer type(3)\n");
2868 pthread_list_unlock();
2869 lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
2870 zfree(kwq_zone, nkwq);
2871 return EBUSY;
2872 }
b0d623f7 2873 }
6d2010ae
A		 2874 kwq->kw_iocount++;
2875 if (wqtype == KSYN_WQTYPE_MUTEXDROP)
2876 kwq->kw_dropcount++;
b0d623f7
A		 2877 if (kwqp != NULL)
2878 *kwqp = kwq;
2879 pthread_list_unlock();
2880 lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
6d2010ae 2881 zfree(kwq_zone, nkwq);
b0d623f7
A		 2882 return (0);
2883 }
2884 kwq = nkwq;
2885
6d2010ae
A		 2886#if _PSYNCH_TRACE_
2887 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, kwq->kw_lword, kwq->kw_uword, kwq->kw_sword, 0xffff, 0);
2888#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 2889 if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
2890 {
2891 kwq->kw_pflags |= KSYN_WQ_SHARED;
2892 LIST_INSERT_HEAD(&hashptr[kwq->kw_object & pthhash], kwq, kw_hash);
2893 } else
2894 LIST_INSERT_HEAD(&hashptr[mutex & pthhash], kwq, kw_hash);
2895
2896 kwq->kw_pflags |= KSYN_WQ_INHASH;
6d2010ae 2897 num_total_kwq++;
b0d623f7
A		 2898
2899 pthread_list_unlock();
2900
2901 if (kwqp != NULL)
2902 *kwqp = kwq;
2903 return (0);
2904}
2905
2906/* Reference from find is dropped here. Starts the free process if needed */
2907void
6d2010ae 2908ksyn_wqrelease(ksyn_wait_queue_t kwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype)
b0d623f7
A		 2909{
2910 uint64_t deadline;
2911 struct timeval t;
2912 int sched = 0;
6d2010ae
A		 2913 ksyn_wait_queue_t free_elem = NULL;
2914 ksyn_wait_queue_t free_elem1 = NULL;
b0d623f7
A		 2915
2916 //pthread_list_lock_spin();
2917 pthread_list_lock();
2918 kwq->kw_iocount--;
6d2010ae
A		 2919 if (wqtype == KSYN_WQTYPE_MUTEXDROP) {
2920 kwq->kw_dropcount--;
2921 }
b0d623f7 2922 if (kwq->kw_iocount == 0) {
6d2010ae
A		 2923 if ((kwq->kw_pflags & KSYN_WQ_WAITING) != 0) {
2924 /* some one is waiting for the waitqueue, wake them up */
2925 kwq->kw_pflags &= ~KSYN_WQ_WAITING;
2926 wakeup(&kwq->kw_pflags);
b0d623f7 2927 }
6d2010ae
A		 2928
2929 if ((kwq->kw_pre_rwwc == 0) && (kwq->kw_inqueue == 0) && (kwq->kw_pre_intrcount == 0)) {
2930 if (qfreenow == 0) {
2931 microuptime(&kwq->kw_ts);
2932 LIST_INSERT_HEAD(&pth_free_list, kwq, kw_list);
2933 kwq->kw_pflags |= KSYN_WQ_FLIST;
2934 num_infreekwq++;
2935 free_elem = NULL;
2936 } else {
2937 /* remove from the only list it is in ie hash */
2938 kwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
2939 LIST_REMOVE(kwq, kw_hash);
2940 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
2941 num_total_kwq--;
2942 num_freekwq++;
2943 free_elem = kwq;
2944 }
2945 } else
2946 free_elem = NULL;
2947 if (qfreenow == 0)
2948 sched = 1;
b0d623f7 2949 }
6d2010ae
A		 2950
2951 if (ckwq != NULL) {
b0d623f7 2952 ckwq->kw_iocount--;
6d2010ae
A		 2953 if (wqtype == KSYN_WQTYPE_MUTEXDROP) {
2954 kwq->kw_dropcount--;
2955 }
b0d623f7 2956 if ( ckwq->kw_iocount == 0) {
6d2010ae
A		 2957 if ((kwq->kw_pflags & KSYN_WQ_WAITING) != 0) {
2958 /* some one is waiting for the waitqueue, wake them up */
2959 kwq->kw_pflags &= ~KSYN_WQ_WAITING;
2960 wakeup(&kwq->kw_pflags);
b0d623f7 2961 }
6d2010ae
A		 2962 if ((ckwq->kw_pre_rwwc == 0) && (ckwq->kw_inqueue == 0) && (ckwq->kw_pre_intrcount == 0)) {
2963 if (qfreenow == 0) {
2964 /* mark for free if we can */
2965 microuptime(&ckwq->kw_ts);
2966 LIST_INSERT_HEAD(&pth_free_list, ckwq, kw_list);
2967 ckwq->kw_pflags |= KSYN_WQ_FLIST;
2968 num_infreekwq++;
2969 free_elem1 = NULL;
2970 } else {
2971 /* remove from the only list it is in ie hash */
2972 ckwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
2973 LIST_REMOVE(ckwq, kw_hash);
2974 lck_mtx_destroy(&ckwq->kw_lock, pthread_lck_grp);
2975 num_total_kwq--;
2976 num_freekwq++;
2977 free_elem1 = ckwq;
2978 }
2979 } else
2980 free_elem1 = NULL;
2981 if (qfreenow == 0)
2982 sched = 1;
b0d623f7
A		 2983 }
2984 }
2985
2986 if (sched == 1 && psynch_cleanupset == 0) {
2987 psynch_cleanupset = 1;
2988 microuptime(&t);
2989 t.tv_sec += KSYN_CLEANUP_DEADLINE;
2990
2991 deadline = tvtoabstime(&t);
2992 thread_call_enter_delayed(psynch_thcall, deadline);
2993 }
2994 pthread_list_unlock();
6d2010ae
A		 2995 if (free_elem != NULL)
2996 zfree(kwq_zone, free_elem);
2997 if (free_elem1 != NULL)
2998 zfree(kwq_zone, free_elem1);
b0d623f7
A		 2999}
3000
3001/* responsible to free the waitqueues */
3002void
3003psynch_wq_cleanup(__unused void * param, __unused void * param1)
3004{
3005 ksyn_wait_queue_t kwq;
3006 struct timeval t;
3007 LIST_HEAD(, ksyn_wait_queue) freelist = {NULL};
3008 int count = 0, delayed = 0, diff;
3009 uint64_t deadline = 0;
3010
3011 //pthread_list_lock_spin();
3012 pthread_list_lock();
3013
6d2010ae
A		 3014 num_addedfreekwq = num_infreekwq - num_lastfreekwqcount;
3015 num_lastfreekwqcount = num_infreekwq;
b0d623f7
A		 3016 microuptime(&t);
3017
3018 LIST_FOREACH(kwq, &pth_free_list, kw_list) {
6d2010ae
A		 3019 if ((kwq->kw_iocount != 0) || (kwq->kw_pre_rwwc != 0) || (kwq->kw_inqueue != 0) || (kwq->kw_pre_intrcount != 0)) {
3020 /* still in use */
b0d623f7
A		 3021 continue;
3022 }
3023 diff = t.tv_sec - kwq->kw_ts.tv_sec;
3024 if (diff < 0)
3025 diff *= -1;
3026 if (diff >= KSYN_CLEANUP_DEADLINE) {
3027 /* out of hash */
3028 kwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
6d2010ae
A		 3029 num_infreekwq--;
3030 num_freekwq++;
b0d623f7
A		 3031 LIST_REMOVE(kwq, kw_hash);
3032 LIST_REMOVE(kwq, kw_list);
3033 LIST_INSERT_HEAD(&freelist, kwq, kw_list);
3034 count ++;
6d2010ae 3035 num_total_kwq--;
b0d623f7
A		 3036 } else {
3037 delayed = 1;
3038 }
3039
3040 }
3041 if (delayed != 0) {
3042 t.tv_sec += KSYN_CLEANUP_DEADLINE;
3043
3044 deadline = tvtoabstime(&t);
3045 thread_call_enter_delayed(psynch_thcall, deadline);
3046 psynch_cleanupset = 1;
3047 } else
3048 psynch_cleanupset = 0;
3049
3050 pthread_list_unlock();
3051
3052
3053 while ((kwq = LIST_FIRST(&freelist)) != NULL) {
3054 LIST_REMOVE(kwq, kw_list);
3055 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
6d2010ae 3056 zfree(kwq_zone, kwq);
b0d623f7
A		 3057 }
3058}
3059
3060
7ddcb079
A		 3061kern_return_t
3062#if _PSYNCH_TRACE_
3063ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int mylog, thread_continue_t continuation, void * parameter)
3064#else
3065ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, __unused int mylog, thread_continue_t continuation, void * parameter)
3066#endif
b0d623f7
A		 3067{
3068 kern_return_t kret;
3069 int error = 0;
6d2010ae
A		 3070#if _PSYNCH_TRACE_
3071 uthread_t uth = NULL;
3072#endif /* _PSYNCH_TRACE_ */
b0d623f7 3073
6d2010ae
A		 3074 kwe->kwe_kwqqueue = (void *)kwq;
3075 assert_wait_deadline(&kwe->kwe_psynchretval, THREAD_ABORTSAFE, abstime);
b0d623f7
A		 3076 ksyn_wqunlock(kwq);
3077
7ddcb079
A		 3078 if (continuation == THREAD_CONTINUE_NULL)
3079 kret = thread_block(NULL);
3080 else
3081 kret = thread_block_parameter(continuation, parameter);
3082
3083#if _PSYNCH_TRACE_
b0d623f7
A		 3084 switch (kret) {
3085 case THREAD_TIMED_OUT:
3086 error = ETIMEDOUT;
3087 break;
3088 case THREAD_INTERRUPTED:
3089 error = EINTR;
3090 break;
3091 }
6d2010ae
A		 3092 uth = current_uthread();
3093#if defined(__i386__)
3094 if (mylog != 0)
3095 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf4f3f2f1, (uint32_t)uth, kret, 0, 0);
3096#else
3097 if (mylog != 0)
3098 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xeeeeeeee, kret, error, 0xeeeeeeee, 0);
3099#endif
3100#endif /* _PSYNCH_TRACE_ */
3101
7ddcb079 3102 return(kret);
b0d623f7
A		 3103}
3104
3105kern_return_t
6d2010ae 3106ksyn_wakeup_thread(__unused ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe)
b0d623f7 3107{
b0d623f7 3108 kern_return_t kret;
6d2010ae
A		 3109#if _PSYNCH_TRACE_
3110 uthread_t uth = NULL;
3111#endif /* _PSYNCH_TRACE_ */
b0d623f7 3112
6d2010ae 3113 kret = thread_wakeup_one((caddr_t)&kwe->kwe_psynchretval);
b0d623f7
A		 3114
3115 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3116 panic("ksyn_wakeup_thread: panic waking up thread %x\n", kret);
6d2010ae
A		 3117#if _PSYNCH_TRACE_
3118 uth = kwe->kwe_uth;
3119#if defined(__i386__)
3120 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf1f2f3f4, (uint32_t)uth, kret, 0, 0);
3121#endif
3122#endif /* _PSYNCH_TRACE_ */
b0d623f7 3123
b0d623f7
A		 3124 return(kret);
3125}
3126
b0d623f7
A		 3127/* find the true shared obect/offset for shared mutexes */
3128int
3129ksyn_findobj(uint64_t mutex, uint64_t * objectp, uint64_t * offsetp)
3130{
3131 vm_page_info_basic_data_t info;
3132 kern_return_t kret;
3133 mach_msg_type_number_t count = VM_PAGE_INFO_BASIC_COUNT;
3134
3135 kret = vm_map_page_info(current_map(), mutex, VM_PAGE_INFO_BASIC,
3136 (vm_page_info_t)&info, &count);
3137
3138 if (kret != KERN_SUCCESS)
3139 return(EINVAL);
3140
3141 if (objectp != NULL)
3142 *objectp = (uint64_t)info.object_id;
3143 if (offsetp != NULL)
3144 *offsetp = (uint64_t)info.offset;
3145
3146 return(0);
3147}
3148
3149
3150/* lowest of kw_fr, kw_flr, kw_fwr, kw_fywr */
3151int
3152kwq_find_rw_lowest(ksyn_wait_queue_t kwq, int flags, uint32_t premgen, int * typep, uint32_t lowest[])
3153{
3154
3155 uint32_t kw_fr, kw_flr, kw_fwr, kw_fywr, low;
3156 int type = 0, lowtype, typenum[4];
3157 uint32_t numbers[4];
3158 int count = 0, i;
3159
3160
3161 if ((kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0)) {
3162 type |= PTH_RWSHFT_TYPE_READ;
3163 /* read entries are present */
3164 if (kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count != 0) {
3165 kw_fr = kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_firstnum;
3166 if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, kw_fr) != 0))
3167 kw_fr = premgen;
3168 } else
3169 kw_fr = premgen;
3170
3171 lowest[KSYN_QUEUE_READ] = kw_fr;
3172 numbers[count]= kw_fr;
3173 typenum[count] = PTH_RW_TYPE_READ;
3174 count++;
3175 } else
3176 lowest[KSYN_QUEUE_READ] = 0;
3177
3178 if ((kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0)) {
3179 type |= PTH_RWSHFT_TYPE_LREAD;
3180 /* read entries are present */
3181 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count != 0) {
3182 kw_flr = kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum;
3183 if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) && (is_seqlower(premgen, kw_flr) != 0))
3184 kw_flr = premgen;
3185 } else
3186 kw_flr = premgen;
3187
3188 lowest[KSYN_QUEUE_LREAD] = kw_flr;
3189 numbers[count]= kw_flr;
3190 typenum[count] = PTH_RW_TYPE_LREAD;
3191 count++;
3192 } else
3193 lowest[KSYN_QUEUE_LREAD] = 0;
3194
3195
3196 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0)) {
3197 type |= PTH_RWSHFT_TYPE_WRITE;
3198 /* read entries are present */
3199 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) {
3200 kw_fwr = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
3201 if (((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) && (is_seqlower(premgen, kw_fwr) != 0))
3202 kw_fwr = premgen;
3203 } else
3204 kw_fwr = premgen;
3205
3206 lowest[KSYN_QUEUE_WRITER] = kw_fwr;
3207 numbers[count]= kw_fwr;
3208 typenum[count] = PTH_RW_TYPE_WRITE;
3209 count++;
3210 } else
3211 lowest[KSYN_QUEUE_WRITER] = 0;
3212
3213 if ((kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0)) {
3214 type |= PTH_RWSHFT_TYPE_YWRITE;
3215 /* read entries are present */
3216 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) {
3217 kw_fywr = kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_firstnum;
3218 if (((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) && (is_seqlower(premgen, kw_fywr) != 0))
3219 kw_fywr = premgen;
3220 } else
3221 kw_fywr = premgen;
3222
3223 lowest[KSYN_QUEUE_YWRITER] = kw_fywr;
3224 numbers[count]= kw_fywr;
3225 typenum[count] = PTH_RW_TYPE_YWRITE;
3226 count++;
3227 } else
3228 lowest[KSYN_QUEUE_YWRITER] = 0;
3229
b0d623f7 3230
6d2010ae 3231#if __TESTPANICS__
b0d623f7
A		 3232 if (count == 0)
3233 panic("nothing in the queue???\n");
6d2010ae 3234#endif /* __TESTPANICS__ */
b0d623f7 3235
6d2010ae 3236 low = numbers[0];
b0d623f7
A		 3237 lowtype = typenum[0];
3238 if (count > 1) {
3239 for (i = 1; i< count; i++) {
3240 if(is_seqlower(numbers[i] , low) != 0) {
3241 low = numbers[i];
3242 lowtype = typenum[i];
3243 }
3244 }
3245 }
3246 type |= lowtype;
3247
3248 if (typep != 0)
3249 *typep = type;
3250 return(0);
3251}
3252
3253/* wakeup readers and longreaders to upto the writer limits */
3254int
3255ksyn_wakeupreaders(ksyn_wait_queue_t kwq, uint32_t limitread, int longreadset, int allreaders, uint32_t updatebits, int * wokenp)
3256{
6d2010ae 3257 ksyn_waitq_element_t kwe = NULL;
b0d623f7
A		 3258 ksyn_queue_t kq;
3259 int failedwakeup = 0;
3260 int numwoken = 0;
3261 kern_return_t kret = KERN_SUCCESS;
b0d623f7
A		 3262 uint32_t lbits = 0;
3263
3264 lbits = updatebits;
3265 if (longreadset != 0) {
3266 /* clear all read and longreads */
6d2010ae
A		 3267 while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwq)) != NULL) {
3268 kwe->kwe_psynchretval = lbits;
3269 kwe->kwe_kwqqueue = NULL;
3270
b0d623f7 3271 numwoken++;
6d2010ae
A		 3272 kret = ksyn_wakeup_thread(kwq, kwe);
3273#if __TESTPANICS__
b0d623f7
A		 3274 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3275 panic("ksyn_wakeupreaders: panic waking up readers\n");
6d2010ae 3276#endif /* __TESTPANICS__ */
b0d623f7
A		 3277 if (kret == KERN_NOT_WAITING) {
3278 failedwakeup++;
3279 }
3280 }
6d2010ae
A		 3281 while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwq)) != NULL) {
3282 kwe->kwe_psynchretval = lbits;
3283 kwe->kwe_kwqqueue = NULL;
b0d623f7 3284 numwoken++;
6d2010ae
A		 3285 kret = ksyn_wakeup_thread(kwq, kwe);
3286#if __TESTPANICS__
b0d623f7
A		 3287 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3288 panic("ksyn_wakeupreaders: panic waking up lreaders\n");
6d2010ae 3289#endif /* __TESTPANICS__ */
b0d623f7
A		 3290 if (kret == KERN_NOT_WAITING) {
3291 failedwakeup++;
3292 }
3293 }
3294 } else {
3295 kq = &kwq->kw_ksynqueues[KSYN_QUEUE_READ];
3296 while ((kq->ksynq_count != 0) && (allreaders || (is_seqlower(kq->ksynq_firstnum, limitread) != 0))) {
6d2010ae
A		 3297 kwe = ksyn_queue_removefirst(kq, kwq);
3298 kwe->kwe_psynchretval = lbits;
3299 kwe->kwe_kwqqueue = NULL;
b0d623f7 3300 numwoken++;
6d2010ae
A		 3301 kret = ksyn_wakeup_thread(kwq, kwe);
3302#if __TESTPANICS__
b0d623f7
A		 3303 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3304 panic("ksyn_wakeupreaders: panic waking up readers\n");
6d2010ae 3305#endif /* __TESTPANICS__ */
b0d623f7
A		 3306 if (kret == KERN_NOT_WAITING) {
3307 failedwakeup++;
3308 }
3309 }
3310 }
3311
3312 if (wokenp != NULL)
3313 *wokenp = numwoken;
3314 return(failedwakeup);
3315}
3316
3317
3318/* This handles the unlock grants for next set on rw_unlock() or on arrival of all preposted waiters */
3319int
6d2010ae 3320kwq_handle_unlock(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t rw_wc, uint32_t * updatep, int flags, int * blockp, uint32_t premgen)
b0d623f7
A		 3321{
3322 uint32_t low_reader, low_writer, low_ywriter, low_lreader,limitrdnum;
3323 int rwtype, error=0;
3324 int longreadset = 0, allreaders, failed;
6d2010ae 3325 uint32_t updatebits=0, numneeded = 0;;
b0d623f7
A		 3326 int prepost = flags & KW_UNLOCK_PREPOST;
3327 thread_t preth = THREAD_NULL;
6d2010ae 3328 ksyn_waitq_element_t kwe;
b0d623f7
A		 3329 uthread_t uth;
3330 thread_t th;
3331 int woken = 0;
3332 int block = 1;
6d2010ae 3333 uint32_t lowest[KSYN_QUEUE_MAX]; /* np need for upgrade as it is handled separately */
b0d623f7 3334 kern_return_t kret = KERN_SUCCESS;
6d2010ae
A		 3335 ksyn_queue_t kq;
3336 int curthreturns = 0;
b0d623f7
A		 3337
3338#if _PSYNCH_TRACE_
6d2010ae 3339 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_START, (uint32_t)kwq->kw_addr, mgen, premgen, rw_wc, 0);
b0d623f7
A		 3340#endif /* _PSYNCH_TRACE_ */
3341 if (prepost != 0) {
3342 preth = current_thread();
3343 }
3344
6d2010ae
A		 3345 kq = &kwq->kw_ksynqueues[KSYN_QUEUE_READ];
3346 kwq->kw_lastseqword = rw_wc;
3347 kwq->kw_lastunlockseq = (rw_wc & PTHRW_COUNT_MASK);
3348 kwq->kw_overlapwatch = 0;
3349
b0d623f7
A		 3350 /* upgrade pending */
3351 if (is_rw_ubit_set(mgen)) {
6d2010ae
A		 3352#if __TESTPANICS__
3353 panic("NO UBIT SHOULD BE SET\n");
3354 updatebits = PTH_RWL_EBIT | PTH_RWL_KBIT;
3355 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0)
3356 updatebits |= PTH_RWL_WBIT;
3357 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0)
3358 updatebits |= PTH_RWL_YBIT;
b0d623f7
A		 3359 if (prepost != 0) {
3360 if((flags & KW_UNLOCK_PREPOST_UPGRADE) != 0) {
3361 /* upgrade thread calling the prepost */
3362 /* upgrade granted */
3363 block = 0;
3364 goto out;
3365 }
3366
3367 }
3368 if (kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE].ksynq_count > 0) {
6d2010ae
A		 3369 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwq);
3370
3371 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3372 kwe->kwe_psynchretval = updatebits;
3373 kwe->kwe_kwqqueue = NULL;
3374 kret = ksyn_wakeup_thread(kwq, kwe);
b0d623f7
A		 3375 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3376 panic("kwq_handle_unlock: panic waking up the upgrade thread \n");
3377 if (kret == KERN_NOT_WAITING) {
3378 kwq->kw_pre_intrcount = 1; /* actually a count */
3379 kwq->kw_pre_intrseq = mgen;
6d2010ae 3380 kwq->kw_pre_intrretbits = kwe->kwe_psynchretval;
b0d623f7
A		 3381 kwq->kw_pre_intrtype = PTH_RW_TYPE_UPGRADE;
3382 }
3383 error = 0;
3384 } else {
3385 panic("panic unable to find the upgrade thread\n");
3386 }
6d2010ae 3387#endif /* __TESTPANICS__ */
b0d623f7
A		 3388 ksyn_wqunlock(kwq);
3389 goto out;
3390 }
3391
3392 error = kwq_find_rw_lowest(kwq, flags, premgen, &rwtype, lowest);
6d2010ae 3393#if __TESTPANICS__
b0d623f7
A		 3394 if (error != 0)
3395 panic("rwunlock: cannot fails to slot next round of threads");
6d2010ae 3396#endif /* __TESTPANICS__ */
b0d623f7
A		 3397
3398#if _PSYNCH_TRACE_
6d2010ae 3399 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 1, rwtype, 0, 0);
b0d623f7
A		 3400#endif /* _PSYNCH_TRACE_ */
3401 low_reader = lowest[KSYN_QUEUE_READ];
3402 low_lreader = lowest[KSYN_QUEUE_LREAD];
3403 low_writer = lowest[KSYN_QUEUE_WRITER];
3404 low_ywriter = lowest[KSYN_QUEUE_YWRITER];
3405
3406
b0d623f7
A		 3407 longreadset = 0;
3408 allreaders = 0;
6d2010ae
A		 3409 updatebits = 0;
3410
3411
b0d623f7
A		 3412 switch (rwtype & PTH_RW_TYPE_MASK) {
3413 case PTH_RW_TYPE_LREAD:
3414 longreadset = 1;
6d2010ae 3415
b0d623f7 3416 case PTH_RW_TYPE_READ: {
6d2010ae
A		 3417 /* what about the preflight which is LREAD or READ ?? */
3418 if ((rwtype & PTH_RWSHFT_TYPE_MASK) != 0) {
3419 if (rwtype & PTH_RWSHFT_TYPE_WRITE)
3420 updatebits |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
3421 if (rwtype & PTH_RWSHFT_TYPE_YWRITE)
3422 updatebits |= PTH_RWL_YBIT;
3423 }
b0d623f7
A		 3424 limitrdnum = 0;
3425 if (longreadset == 0) {
3426 switch (rwtype & (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE)) {
3427 case PTH_RWSHFT_TYPE_WRITE:
3428 limitrdnum = low_writer;
3429 if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
6d2010ae
A		 3430 (is_seqlower(low_lreader, limitrdnum) != 0)) {
3431 longreadset = 1;
3432 }
3433 if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
3434 (is_seqlower(premgen, limitrdnum) != 0)) {
b0d623f7
A		 3435 longreadset = 1;
3436 }
b0d623f7
A		 3437 break;
3438 case PTH_RWSHFT_TYPE_YWRITE:
3439 /* all read ? */
3440 if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
3441 (is_seqlower(low_lreader, low_ywriter) != 0)) {
3442 longreadset = 1;
3443 } else
3444 allreaders = 1;
6d2010ae
A		 3445 if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
3446 (is_seqlower(premgen, low_ywriter) != 0)) {
3447 longreadset = 1;
3448 allreaders = 0;
3449 }
3450
3451
b0d623f7
A		 3452 break;
3453 case (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE):
6d2010ae
A		 3454 if (is_seqlower(low_ywriter, low_writer) != 0) {
3455 limitrdnum = low_ywriter;
3456 } else
3457 limitrdnum = low_writer;
b0d623f7 3458 if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
6d2010ae
A		 3459 (is_seqlower(low_lreader, limitrdnum) != 0)) {
3460 longreadset = 1;
3461 }
3462 if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
3463 (is_seqlower(premgen, limitrdnum) != 0)) {
b0d623f7
A		 3464 longreadset = 1;
3465 }
3466 break;
3467 default: /* no writers at all */
3468 if ((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0)
3469 longreadset = 1;
3470 else
3471 allreaders = 1;
3472 };
3473
3474 }
6d2010ae
A		 3475 numneeded = 0;
3476 if (longreadset != 0) {
3477 updatebits |= PTH_RWL_LBIT;
3478 updatebits &= ~PTH_RWL_KBIT;
3479 if ((flags & (KW_UNLOCK_PREPOST_READLOCK | KW_UNLOCK_PREPOST_LREADLOCK)) != 0)
3480 numneeded += 1;
3481 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3482 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count;
3483 updatebits += (numneeded << PTHRW_COUNT_SHIFT);
3484 kwq->kw_overlapwatch = 1;
3485 } else {
3486 /* no longread, evaluate number of readers */
b0d623f7 3487
6d2010ae
A		 3488 switch (rwtype & (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE)) {
3489 case PTH_RWSHFT_TYPE_WRITE:
3490 limitrdnum = low_writer;
3491 numneeded = ksyn_queue_count_tolowest(kq, limitrdnum);
3492 if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, limitrdnum) != 0)) {
3493 curthreturns = 1;
3494 numneeded += 1;
3495 }
3496 break;
3497 case PTH_RWSHFT_TYPE_YWRITE:
3498 /* all read ? */
3499 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3500 if ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) {
3501 curthreturns = 1;
3502 numneeded += 1;
3503 }
3504 break;
3505 case (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE):
3506 limitrdnum = low_writer;
3507 numneeded = ksyn_queue_count_tolowest(kq, limitrdnum);
3508 if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, limitrdnum) != 0)) {
3509 curthreturns = 1;
3510 numneeded += 1;
3511 }
3512 break;
3513 default: /* no writers at all */
3514 /* no other waiters only readers */
3515 kwq->kw_overlapwatch = 1;
3516 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3517 if ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) {
3518 curthreturns = 1;
3519 numneeded += 1;
3520 }
3521 };
3522
3523 updatebits += (numneeded << PTHRW_COUNT_SHIFT);
3524 }
3525 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
b0d623f7 3526
6d2010ae
A		 3527 if (curthreturns != 0) {
3528 block = 0;
3529 uth = current_uthread();
3530 kwe = &uth->uu_kwe;
3531 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3532 }
3533
b0d623f7
A		 3534
3535 failed = ksyn_wakeupreaders(kwq, limitrdnum, longreadset, allreaders, updatebits, &woken);
6d2010ae
A		 3536#if _PSYNCH_TRACE_
3537 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3538#endif /* _PSYNCH_TRACE_ */
3539
b0d623f7
A		 3540 if (failed != 0) {
3541 kwq->kw_pre_intrcount = failed; /* actually a count */
3542 kwq->kw_pre_intrseq = limitrdnum;
3543 kwq->kw_pre_intrretbits = updatebits;
3544 if (longreadset)
3545 kwq->kw_pre_intrtype = PTH_RW_TYPE_LREAD;
3546 else
3547 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3548 }
3549
b0d623f7
A		 3550 error = 0;
3551
6d2010ae
A		 3552 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) && ((updatebits & PTH_RWL_WBIT) == 0))
3553 panic("kwq_handle_unlock: writer pending but no writebit set %x\n", updatebits);
b0d623f7
A		 3554 }
3555 break;
3556
3557 case PTH_RW_TYPE_WRITE: {
6d2010ae
A		 3558
3559 /* only one thread is goin to be granted */
3560 updatebits |= (PTHRW_INC);
3561 updatebits |= PTH_RWL_KBIT| PTH_RWL_EBIT;
3562
b0d623f7
A		 3563 if (((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) && (low_writer == premgen)) {
3564 block = 0;
3565 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0)
6d2010ae
A		 3566 updatebits |= PTH_RWL_WBIT;
3567 if ((rwtype & PTH_RWSHFT_TYPE_YWRITE) != 0)
3568 updatebits |= PTH_RWL_YBIT;
b0d623f7
A		 3569 th = preth;
3570 uth = get_bsdthread_info(th);
6d2010ae
A		 3571 kwe = &uth->uu_kwe;
3572 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3573 } else {
3574 /* we are not granting writelock to the preposting thread */
6d2010ae 3575 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
b0d623f7
A		 3576
3577 /* if there are writers present or the preposting write thread then W bit is to be set */
3578 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) )
6d2010ae
A		 3579 updatebits |= PTH_RWL_WBIT;
3580 if ((rwtype & PTH_RWSHFT_TYPE_YWRITE) != 0)
3581 updatebits |= PTH_RWL_YBIT;
3582 kwe->kwe_psynchretval = updatebits;
3583 kwe->kwe_kwqqueue = NULL;
b0d623f7 3584 /* setup next in the queue */
6d2010ae
A		 3585 kret = ksyn_wakeup_thread(kwq, kwe);
3586#if _PSYNCH_TRACE_
3587 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, kret, 0, 0);
3588#endif /* _PSYNCH_TRACE_ */
3589#if __TESTPANICS__
b0d623f7
A		 3590 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3591 panic("kwq_handle_unlock: panic waking up writer\n");
6d2010ae 3592#endif /* __TESTPANICS__ */
b0d623f7
A		 3593 if (kret == KERN_NOT_WAITING) {
3594 kwq->kw_pre_intrcount = 1; /* actually a count */
3595 kwq->kw_pre_intrseq = low_writer;
3596 kwq->kw_pre_intrretbits = updatebits;
3597 kwq->kw_pre_intrtype = PTH_RW_TYPE_WRITE;
3598 }
3599 error = 0;
3600 }
6d2010ae
A		 3601 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3602 if ((updatebits & (PTH_RWL_KBIT | PTH_RWL_EBIT)) != (PTH_RWL_KBIT | PTH_RWL_EBIT))
3603 panic("kwq_handle_unlock: writer lock granted but no ke set %x\n", updatebits);
b0d623f7
A		 3604
3605 }
3606 break;
3607
3608 case PTH_RW_TYPE_YWRITE: {
3609 /* can reader locks be granted ahead of this write? */
3610 if ((rwtype & PTH_RWSHFT_TYPE_READ) != 0) {
6d2010ae
A		 3611 if ((rwtype & PTH_RWSHFT_TYPE_MASK) != 0) {
3612 if (rwtype & PTH_RWSHFT_TYPE_WRITE)
3613 updatebits |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
3614 if (rwtype & PTH_RWSHFT_TYPE_YWRITE)
3615 updatebits |= PTH_RWL_YBIT;
3616 }
b0d623f7
A		 3617
3618 if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0) {
3619 /* is lowest reader less than the low writer? */
3620 if (is_seqlower(low_reader,low_writer) == 0)
3621 goto yielditis;
6d2010ae
A		 3622
3623 numneeded = ksyn_queue_count_tolowest(kq, low_writer);
3624 updatebits += (numneeded << PTHRW_COUNT_SHIFT);
b0d623f7
A		 3625 if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, low_writer) != 0)) {
3626 uth = current_uthread();
6d2010ae
A		 3627 kwe = &uth->uu_kwe;
3628 /* add one more */
3629 updatebits += PTHRW_INC;
3630 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3631 block = 0;
3632 }
b0d623f7 3633
6d2010ae
A		 3634 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3635
b0d623f7
A		 3636 /* there will be readers to wakeup , no need to check for woken */
3637 failed = ksyn_wakeupreaders(kwq, low_writer, 0, 0, updatebits, NULL);
6d2010ae
A		 3638#if _PSYNCH_TRACE_
3639 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3640#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 3641 if (failed != 0) {
3642 kwq->kw_pre_intrcount = failed; /* actually a count */
3643 kwq->kw_pre_intrseq = low_writer;
3644 kwq->kw_pre_intrretbits = updatebits;
3645 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3646 }
3647 error = 0;
3648 } else {
3649 /* wakeup all readers */
6d2010ae
A		 3650 numneeded = kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3651 updatebits += (numneeded << PTHRW_COUNT_SHIFT);
b0d623f7
A		 3652 if ((prepost != 0) && ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0)) {
3653 uth = current_uthread();
6d2010ae
A		 3654 kwe = &uth->uu_kwe;
3655 updatebits += PTHRW_INC;
3656 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3657 block = 0;
3658 }
6d2010ae 3659 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
b0d623f7 3660 failed = ksyn_wakeupreaders(kwq, low_writer, 0, 1, updatebits, &woken);
6d2010ae
A		 3661#if _PSYNCH_TRACE_
3662 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3663#endif /* _PSYNCH_TRACE_ */
b0d623f7
A		 3664 if (failed != 0) {
3665 kwq->kw_pre_intrcount = failed; /* actually a count */
3666 kwq->kw_pre_intrseq = kwq->kw_highseq;
3667 kwq->kw_pre_intrretbits = updatebits;
3668 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3669 }
b0d623f7
A		 3670 error = 0;
3671 }
3672 } else {
3673yielditis:
3674 /* no reads, so granting yeilding writes */
6d2010ae
A		 3675 updatebits |= PTHRW_INC;
3676 updatebits |= PTH_RWL_KBIT| PTH_RWL_EBIT;
b0d623f7
A		 3677
3678 if (((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) && (low_writer == premgen)) {
3679 /* preposting yielding write thread is being granted exclusive lock */
3680
3681 block = 0;
3682
3683 if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0)
6d2010ae 3684 updatebits |= PTH_RWL_WBIT;
b0d623f7 3685 else if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0)
6d2010ae 3686 updatebits |= PTH_RWL_YBIT;
b0d623f7
A		 3687
3688 th = preth;
3689 uth = get_bsdthread_info(th);
6d2010ae
A		 3690 kwe = &uth->uu_kwe;
3691 kwe->kwe_psynchretval = updatebits;
b0d623f7
A		 3692 } else {
3693 /* we are granting yield writelock to some other thread */
6d2010ae 3694 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwq);
b0d623f7
A		 3695
3696 if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0)
6d2010ae 3697 updatebits |= PTH_RWL_WBIT;
b0d623f7
A		 3698 /* if there are ywriters present or the preposting ywrite thread then W bit is to be set */
3699 else if ((kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) )
6d2010ae 3700 updatebits |= PTH_RWL_YBIT;
b0d623f7 3701
6d2010ae
A		 3702 kwe->kwe_psynchretval = updatebits;
3703 kwe->kwe_kwqqueue = NULL;
b0d623f7 3704
6d2010ae
A		 3705 kret = ksyn_wakeup_thread(kwq, kwe);
3706#if _PSYNCH_TRACE_
3707 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, kret, 0, 0);
3708#endif /* _PSYNCH_TRACE_ */
3709#if __TESTPANICS__
b0d623f7
A		 3710 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
3711 panic("kwq_handle_unlock : panic waking up readers\n");
6d2010ae 3712#endif /* __TESTPANICS__ */
b0d623f7
A		 3713 if (kret == KERN_NOT_WAITING) {
3714 kwq->kw_pre_intrcount = 1; /* actually a count */
3715 kwq->kw_pre_intrseq = low_ywriter;
3716 kwq->kw_pre_intrretbits = updatebits;
3717 kwq->kw_pre_intrtype = PTH_RW_TYPE_YWRITE;
3718 }
3719 error = 0;
3720 }
6d2010ae 3721 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
b0d623f7
A		 3722 }
3723 }
3724 break;
3725
3726 default:
3727 panic("rwunlock: invalid type for lock grants");
3728
3729 };
3730
b0d623f7
A		 3731
3732out:
6d2010ae
A		 3733 if (updatep != NULL)
3734 *updatep = updatebits;
b0d623f7
A		 3735 if (blockp != NULL)
3736 *blockp = block;
3737#if _PSYNCH_TRACE_
6d2010ae 3738 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0, updatebits, block, 0);
b0d623f7
A		 3739#endif /* _PSYNCH_TRACE_ */
3740 return(error);
3741}
3742
6d2010ae
A		 3743int
3744kwq_handle_overlap(ksyn_wait_queue_t kwq, uint32_t lgenval, __unused uint32_t ugenval, uint32_t rw_wc, uint32_t *updatebitsp, __unused int flags , int * blockp)
3745{
3746 uint32_t highword = kwq->kw_nextseqword & PTHRW_COUNT_MASK;
3747 uint32_t lowword = kwq->kw_lastseqword & PTHRW_COUNT_MASK;
3748 uint32_t val=0;
3749 int withinseq;
3750
3751
3752 /* overlap is set, so no need to check for valid state for overlap */
3753
3754 withinseq = ((is_seqlower_eq(rw_wc, highword) != 0) || (is_seqhigher_eq(lowword, rw_wc) != 0));
3755
3756 if (withinseq != 0) {
3757 if ((kwq->kw_nextseqword & PTH_RWL_LBIT) == 0) {
3758 /* if no writers ahead, overlap granted */
3759 if ((lgenval & PTH_RWL_WBIT) == 0) {
3760 goto grantoverlap;
3761 }
3762 } else {
3763 /* Lbit is set, and writers ahead does not count */
3764 goto grantoverlap;
3765 }
3766 }
3767
3768 *blockp = 1;
3769 return(0);
3770
3771grantoverlap:
3772 /* increase the next expected seq by one */
3773 kwq->kw_nextseqword += PTHRW_INC;
3774 /* set count by one & bits from the nextseq and add M bit */
3775 val = PTHRW_INC;
3776 val |= ((kwq->kw_nextseqword & PTHRW_BIT_MASK) | PTH_RWL_MBIT);
3777 *updatebitsp = val;
3778 *blockp = 0;
3779 return(0);
3780}
b0d623f7 3781
6d2010ae 3782#if NOTYET
b0d623f7
A		 3783/* handle downgrade actions */
3784int
3785kwq_handle_downgrade(ksyn_wait_queue_t kwq, uint32_t mgen, __unused int flags, __unused uint32_t premgen, __unused int * blockp)
3786{
3787 uint32_t updatebits, lowriter = 0;
3788 int longreadset, allreaders, count;
3789
3790 /* can handle downgrade now */
3791 updatebits = mgen;
3792
3793 longreadset = 0;
3794 allreaders = 0;
3795 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count > 0) {
3796 lowriter = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
3797 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count > 0) {
3798 if (is_seqlower(kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum, lowriter) != 0)
3799 longreadset = 1;
3800 }
3801 } else {
3802 allreaders = 1;
3803 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count > 0) {
3804 lowriter = kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_firstnum;
3805 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count > 0) {
3806 if (is_seqlower(kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum, lowriter) != 0)
3807 longreadset = 1;
3808 }
3809 }
3810 }
3811
3812 count = ksyn_wakeupreaders(kwq, lowriter, longreadset, allreaders, updatebits, NULL);
3813 if (count != 0) {
3814 kwq->kw_pre_limrd = count;
3815 kwq->kw_pre_limrdseq = lowriter;
3816 kwq->kw_pre_limrdbits = lowriter;
3817 /* need to handle prepost */
3818 }
3819 return(0);
3820}
6d2010ae
A		 3821
3822#endif /* NOTYET */
3823
b0d623f7
A		 3824/************* Indiv queue support routines ************************/
3825void
3826ksyn_queue_init(ksyn_queue_t kq)
3827{
6d2010ae 3828 TAILQ_INIT(&kq->ksynq_kwelist);
b0d623f7
A		 3829 kq->ksynq_count = 0;
3830 kq->ksynq_firstnum = 0;
3831 kq->ksynq_lastnum = 0;
3832}
3833
b0d623f7 3834int
6d2010ae 3835ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, struct uthread * uth, ksyn_waitq_element_t kwe, int fit)
b0d623f7
A		 3836{
3837 uint32_t lockseq = mgen & PTHRW_COUNT_MASK;
6d2010ae
A		 3838 ksyn_waitq_element_t q_kwe, r_kwe;
3839 int res = 0;
3840 uthread_t nuth = NULL;
b0d623f7
A		 3841
3842 if (kq->ksynq_count == 0) {
6d2010ae 3843 TAILQ_INSERT_HEAD(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3844 kq->ksynq_firstnum = lockseq;
3845 kq->ksynq_lastnum = lockseq;
3846 goto out;
3847 }
3848
3849 if (fit == FIRSTFIT) {
6d2010ae 3850 /* TBD: if retry bit is set for mutex, add it to the head */
b0d623f7 3851 /* firstfit, arriving order */
6d2010ae 3852 TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3853 if (is_seqlower (lockseq, kq->ksynq_firstnum) != 0)
3854 kq->ksynq_firstnum = lockseq;
3855 if (is_seqhigher (lockseq, kq->ksynq_lastnum) != 0)
3856 kq->ksynq_lastnum = lockseq;
3857 goto out;
3858 }
3859
6d2010ae
A		 3860 if ((lockseq == kq->ksynq_firstnum) || (lockseq == kq->ksynq_lastnum)) {
3861 /* During prepost when a thread is getting cancelled, we could have two with same seq */
3862 if (kwe->kwe_flags == KWE_THREAD_PREPOST) {
3863 q_kwe = ksyn_queue_find_seq(kwq, kq, lockseq, 0);
3864 if ((q_kwe != NULL) && ((nuth = (uthread_t)q_kwe->kwe_uth) != NULL) &&
3865 ((nuth->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL)) {
3866 TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
3867 goto out;
3868
3869 } else {
3870 __FAILEDUSERTEST__("ksyn_queue_insert: two threads with same lockseq ");
3871 res = EBUSY;
3872 goto out1;
3873 }
3874 } else {
3875 __FAILEDUSERTEST__("ksyn_queue_insert: two threads with same lockseq ");
3876 res = EBUSY;
3877 goto out1;
3878 }
3879 }
b0d623f7
A		 3880
3881 /* check for next seq one */
3882 if (is_seqlower(kq->ksynq_lastnum, lockseq) != 0) {
6d2010ae 3883 TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3884 kq->ksynq_lastnum = lockseq;
3885 goto out;
3886 }
3887
3888 if (is_seqlower(lockseq, kq->ksynq_firstnum) != 0) {
6d2010ae 3889 TAILQ_INSERT_HEAD(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3890 kq->ksynq_firstnum = lockseq;
3891 goto out;
3892 }
3893
3894 /* goto slow insert mode */
6d2010ae
A		 3895 TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
3896 if (is_seqhigher(q_kwe->kwe_lockseq, lockseq) != 0) {
3897 TAILQ_INSERT_BEFORE(q_kwe, kwe, kwe_list);
b0d623f7
A		 3898 goto out;
3899 }
3900 }
3901
6d2010ae 3902#if __TESTPANICS__
b0d623f7 3903 panic("failed to insert \n");
6d2010ae
A		 3904#endif /* __TESTPANICS__ */
3905
b0d623f7 3906out:
6d2010ae
A		 3907 if (uth != NULL)
3908 kwe->kwe_uth = uth;
b0d623f7
A		 3909 kq->ksynq_count++;
3910 kwq->kw_inqueue++;
3911 update_low_high(kwq, lockseq);
6d2010ae
A		 3912out1:
3913 return(res);
b0d623f7
A		 3914}
3915
6d2010ae 3916ksyn_waitq_element_t
b0d623f7
A		 3917ksyn_queue_removefirst(ksyn_queue_t kq, ksyn_wait_queue_t kwq)
3918{
6d2010ae
A		 3919 ksyn_waitq_element_t kwe = NULL;
3920 ksyn_waitq_element_t q_kwe;
b0d623f7
A		 3921 uint32_t curseq;
3922
3923 if (kq->ksynq_count != 0) {
6d2010ae
A		 3924 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
3925 TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
3926 curseq = kwe->kwe_lockseq & PTHRW_COUNT_MASK;
b0d623f7
A		 3927 kq->ksynq_count--;
3928 kwq->kw_inqueue--;
3929
3930 if(kq->ksynq_count != 0) {
6d2010ae
A		 3931 q_kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
3932 kq->ksynq_firstnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
b0d623f7
A		 3933 } else {
3934 kq->ksynq_firstnum = 0;
3935 kq->ksynq_lastnum = 0;
3936
3937 }
3938 if (kwq->kw_inqueue == 0) {
3939 kwq->kw_lowseq = 0;
3940 kwq->kw_highseq = 0;
3941 } else {
3942 if (kwq->kw_lowseq == curseq)
3943 kwq->kw_lowseq = find_nextlowseq(kwq);
3944 if (kwq->kw_highseq == curseq)
3945 kwq->kw_highseq = find_nexthighseq(kwq);
3946 }
3947 }
6d2010ae 3948 return(kwe);
b0d623f7
A		 3949}
3950
3951void
6d2010ae 3952ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe)
b0d623f7 3953{
6d2010ae 3954 ksyn_waitq_element_t q_kwe;
b0d623f7
A		 3955 uint32_t curseq;
3956
3957 if (kq->ksynq_count > 0) {
6d2010ae 3958 TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
b0d623f7
A		 3959 kq->ksynq_count--;
3960 if(kq->ksynq_count != 0) {
6d2010ae
A		 3961 q_kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
3962 kq->ksynq_firstnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
3963 q_kwe = TAILQ_LAST(&kq->ksynq_kwelist, ksynq_kwelist_head);
3964 kq->ksynq_lastnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
b0d623f7
A		 3965 } else {
3966 kq->ksynq_firstnum = 0;
3967 kq->ksynq_lastnum = 0;
3968
3969 }
3970 kwq->kw_inqueue--;
6d2010ae 3971 curseq = kwe->kwe_lockseq & PTHRW_COUNT_MASK;
b0d623f7
A		 3972 if (kwq->kw_inqueue == 0) {
3973 kwq->kw_lowseq = 0;
3974 kwq->kw_highseq = 0;
3975 } else {
3976 if (kwq->kw_lowseq == curseq)
3977 kwq->kw_lowseq = find_nextlowseq(kwq);
3978 if (kwq->kw_highseq == curseq)
3979 kwq->kw_highseq = find_nexthighseq(kwq);
3980 }
3981 }
3982}
3983
6d2010ae
A		 3984/* find the thread and removes from the queue */
3985ksyn_waitq_element_t
3986ksyn_queue_find_seq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t seq, int remove)
3987{
3988 ksyn_waitq_element_t q_kwe, r_kwe;
3989
3990 /* TBD: bail out if higher seq is seen */
3991 /* case where wrap in the tail of the queue exists */
3992 TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
3993 if ((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK) == seq) {
3994 if (remove != 0)
3995 ksyn_queue_removeitem(kwq, kq, q_kwe);
3996 return(q_kwe);
3997 }
3998 }
3999 return(NULL);
4000}
4001
4002
4003/* find the thread at the target sequence (or a broadcast/prepost at or above) */
4004ksyn_waitq_element_t
4005ksyn_queue_find_cvpreposeq(ksyn_queue_t kq, uint32_t cgen)
4006{
4007 ksyn_waitq_element_t q_kwe, r_kwe;
4008 uint32_t lgen = (cgen & PTHRW_COUNT_MASK);
4009
4010 /* case where wrap in the tail of the queue exists */
4011 TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
4012
4013 /* skip the lower entries */
4014 if (is_seqlower((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), cgen) != 0)
4015 continue;
4016
4017 switch (q_kwe->kwe_flags) {
4018
4019 case KWE_THREAD_INWAIT:
4020 if ((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK) != lgen)
4021 break;
4022 /* fall thru */
4023
4024 case KWE_THREAD_BROADCAST:
4025 case KWE_THREAD_PREPOST:
4026 return (q_kwe);
4027 }
4028 }
4029 return(NULL);
4030}
4031
4032/* look for a thread at lockseq, a */
4033ksyn_waitq_element_t
4034ksyn_queue_find_signalseq(__unused ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t uptoseq, uint32_t signalseq)
4035{
4036 ksyn_waitq_element_t q_kwe, r_kwe, t_kwe = NULL;
4037
4038 /* case where wrap in the tail of the queue exists */
4039 TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
4040
4041 switch (q_kwe->kwe_flags) {
4042
4043 case KWE_THREAD_PREPOST:
4044 if (is_seqhigher((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
4045 return t_kwe;
4046 /* fall thru */
4047
4048 case KWE_THREAD_BROADCAST:
4049 /* match any prepost at our same uptoseq or any broadcast above */
4050 if (is_seqlower((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
4051 continue;
4052 return q_kwe;
4053
4054 case KWE_THREAD_INWAIT:
4055 /*
4056 * Match any (non-cancelled) thread at or below our upto sequence -
4057 * but prefer an exact match to our signal sequence (if present) to
4058 * keep exact matches happening.
4059 */
4060 if (is_seqhigher((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
4061 return t_kwe;
4062
4063 if (q_kwe->kwe_kwqqueue == kwq) {
4064 uthread_t ut = q_kwe->kwe_uth;
4065 if ((ut->uu_flag & ( UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) != UT_CANCEL) {
4066 /* if equal or higher than our signal sequence, return this one */
4067 if (is_seqhigher_eq((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), signalseq))
4068 return q_kwe;
4069
4070 /* otherwise, just remember this eligible thread and move on */
4071 if (t_kwe == NULL)
4072 t_kwe = q_kwe;
4073 }
4074 }
4075 break;
4076
4077 default:
4078 panic("ksyn_queue_find_signalseq(): unknow wait queue element type (%d)\n", q_kwe->kwe_flags);
4079 break;
4080 }
4081 }
4082 return t_kwe;
4083}
4084
4085
4086int
4087ksyn_queue_move_tofree(ksyn_wait_queue_t ckwq, ksyn_queue_t kq, uint32_t upto, ksyn_queue_t kfreeq, int all, int release)
4088{
4089 ksyn_waitq_element_t kwe;
4090 int count = 0;
4091 uint32_t tseq = upto & PTHRW_COUNT_MASK;
4092#if _PSYNCH_TRACE_
4093 uthread_t ut;
4094#endif /* _PSYNCH_TRACE_ */
4095
4096 ksyn_queue_init(kfreeq);
4097
4098 /* free all the entries, must be only fakes.. */
4099 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
4100 while (kwe != NULL) {
4101 if ((all == 0) && (is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), tseq) != 0))
4102 break;
4103 if (kwe->kwe_flags == KWE_THREAD_INWAIT) {
4104 /*
4105 * This scenario is typically noticed when the cvar is
4106 * reinited and the new waiters are waiting. We can
4107 * return them as spurious wait so the cvar state gets
4108 * reset correctly.
4109 */
4110#if _PSYNCH_TRACE_
4111 ut = (uthread_t)kwe->kwe_uth;
4112#endif /* _PSYNCH_TRACE_ */
4113
4114 /* skip canceled ones */
4115 /* wake the rest */
4116 ksyn_queue_removeitem(ckwq, kq, kwe);
4117 /* set M bit to indicate to waking CV to retun Inc val */
4118 kwe->kwe_psynchretval = PTHRW_INC | (PTH_RWS_CV_MBIT | PTH_RWL_MTX_WAIT);
4119 kwe->kwe_kwqqueue = NULL;
4120#if _PSYNCH_TRACE_
4121 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
4122#endif /* _PSYNCH_TRACE_ */
4123 (void)ksyn_wakeup_thread(ckwq, kwe);
4124 } else {
4125 ksyn_queue_removeitem(ckwq, kq, kwe);
4126 TAILQ_INSERT_TAIL(&kfreeq->ksynq_kwelist, kwe, kwe_list);
4127 ckwq->kw_fakecount--;
4128 count++;
4129 }
4130 kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
4131 }
4132
4133 if ((release != 0) && (count != 0)) {
4134 kwe = TAILQ_FIRST(&kfreeq->ksynq_kwelist);
4135 while (kwe != NULL) {
4136 TAILQ_REMOVE(&kfreeq->ksynq_kwelist, kwe, kwe_list);
4137 zfree(kwe_zone, kwe);
4138 kwe = TAILQ_FIRST(&kfreeq->ksynq_kwelist);
4139 }
4140 }
4141
4142 return(count);
4143}
4144
4145/*************************************************************************/
b0d623f7
A		 4146
4147void
4148update_low_high(ksyn_wait_queue_t kwq, uint32_t lockseq)
4149{
4150 if (kwq->kw_inqueue == 1) {
4151 kwq->kw_lowseq = lockseq;
4152 kwq->kw_highseq = lockseq;
4153 } else {
4154 if (is_seqlower(lockseq, kwq->kw_lowseq) != 0)
4155 kwq->kw_lowseq = lockseq;
4156 if (is_seqhigher(lockseq, kwq->kw_highseq) != 0)
4157 kwq->kw_highseq = lockseq;
4158 }
4159}
4160
4161uint32_t
4162find_nextlowseq(ksyn_wait_queue_t kwq)
4163{
4164 uint32_t numbers[4];
4165 int count = 0, i;
4166 uint32_t lowest;
4167
4168 for(i = 0; i< KSYN_QUEUE_MAX; i++) {
4169 if (kwq->kw_ksynqueues[i].ksynq_count != 0) {
4170 numbers[count]= kwq->kw_ksynqueues[i].ksynq_firstnum;
4171 count++;
4172 }
4173 }
4174
4175 if (count == 0)
4176 return(0);
4177 lowest = numbers[0];
4178 if (count > 1) {
4179 for (i = 1; i< count; i++) {
4180 if(is_seqlower(numbers[i] , lowest) != 0)
4181 lowest = numbers[count];
4182
4183 }
4184 }
4185 return(lowest);
4186}
4187
4188uint32_t
4189find_nexthighseq(ksyn_wait_queue_t kwq)
4190{
4191 uint32_t numbers[4];
4192 int count = 0, i;
4193 uint32_t highest;
4194
4195 for(i = 0; i< KSYN_QUEUE_MAX; i++) {
4196 if (kwq->kw_ksynqueues[i].ksynq_count != 0) {
4197 numbers[count]= kwq->kw_ksynqueues[i].ksynq_lastnum;
4198 count++;
4199 }
4200 }
4201
4202
4203
4204 if (count == 0)
4205 return(0);
4206 highest = numbers[0];
4207 if (count > 1) {
4208 for (i = 1; i< count; i++) {
4209 if(is_seqhigher(numbers[i], highest) != 0)
4210 highest = numbers[i];
4211
4212 }
4213 }
4214 return(highest);
4215}
4216
6d2010ae
A		 4217int
4218is_seqlower(uint32_t x, uint32_t y)
4219{
4220 if (x < y) {
4221 if ((y-x) < (PTHRW_MAX_READERS/2))
4222 return(1);
4223 } else {
4224 if ((x-y) > (PTHRW_MAX_READERS/2))
4225 return(1);
4226 }
4227 return(0);
4228}
4229
4230int
4231is_seqlower_eq(uint32_t x, uint32_t y)
4232{
4233 if (x==y)
4234 return(1);
4235 else
4236 return(is_seqlower(x,y));
4237}
4238
4239int
4240is_seqhigher(uint32_t x, uint32_t y)
4241{
4242 if (x > y) {
4243 if ((x-y) < (PTHRW_MAX_READERS/2))
4244 return(1);
4245 } else {
4246 if ((y-x) > (PTHRW_MAX_READERS/2))
4247 return(1);
4248 }
4249 return(0);
4250}
4251
4252int
4253is_seqhigher_eq(uint32_t x, uint32_t y)
4254{
4255 if (x==y)
4256 return(1);
4257 else
4258 return(is_seqhigher(x,y));
4259}
4260
4261
b0d623f7
A		 4262int
4263find_diff(uint32_t upto, uint32_t lowest)
4264{
4265 uint32_t diff;
4266
4267 if (upto == lowest)
4268 return(0);
6d2010ae 4269#if 0
b0d623f7 4270 diff = diff_genseq(upto, lowest);
6d2010ae
A		 4271#else
4272 if (is_seqlower(upto, lowest) != 0)
4273 diff = diff_genseq(lowest, upto);
4274 else
4275 diff = diff_genseq(upto, lowest);
4276#endif
b0d623f7
A		 4277 diff = (diff >> PTHRW_COUNT_SHIFT);
4278 return(diff);
4279}
4280
4281
4282int
4283find_seq_till(ksyn_wait_queue_t kwq, uint32_t upto, uint32_t nwaiters, uint32_t *countp)
4284{
4285 int i;
4286 uint32_t count = 0;
4287
4288
4289#if _PSYNCH_TRACE_
6d2010ae 4290 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_START, 0, 0, upto, nwaiters, 0);
b0d623f7
A		 4291#endif /* _PSYNCH_TRACE_ */
4292
4293 for (i= 0; i< KSYN_QUEUE_MAX; i++) {
4294 count += ksyn_queue_count_tolowest(&kwq->kw_ksynqueues[i], upto);
4295#if _PSYNCH_TRACE_
6d2010ae 4296 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_NONE, 0, 1, i, count, 0);
b0d623f7
A		 4297#endif /* _PSYNCH_TRACE_ */
4298 if (count >= nwaiters) {
4299 break;
4300 }
4301 }
4302
4303 if (countp != NULL) {
4304 *countp = count;
4305 }
4306#if _PSYNCH_TRACE_
6d2010ae 4307 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_END, 0, 0, count, nwaiters, 0);
b0d623f7 4308#endif /* _PSYNCH_TRACE_ */
6d2010ae
A		 4309 if (count == 0)
4310 return(0);
4311 else if (count >= nwaiters)
b0d623f7
A		 4312 return(1);
4313 else
4314 return(0);
4315}
4316
4317
4318uint32_t
4319ksyn_queue_count_tolowest(ksyn_queue_t kq, uint32_t upto)
4320{
4321 uint32_t i = 0;
6d2010ae 4322 ksyn_waitq_element_t kwe, newkwe;
b0d623f7
A		 4323 uint32_t curval;
4324
4325 /* if nothing or the first num is greater than upto, return none */
4326 if ((kq->ksynq_count == 0) || (is_seqhigher(kq->ksynq_firstnum, upto) != 0))
4327 return(0);
4328 if (upto == kq->ksynq_firstnum)
4329 return(1);
4330
6d2010ae
A		 4331 TAILQ_FOREACH_SAFE(kwe, &kq->ksynq_kwelist, kwe_list, newkwe) {
4332 curval = (kwe->kwe_lockseq & PTHRW_COUNT_MASK);
b0d623f7
A		 4333 if (upto == curval) {
4334 i++;
4335 break;
4336 } else if (is_seqhigher(curval, upto) != 0) {
4337 break;
4338 } else {
4339 /* seq is lower */
4340 i++;
4341 }
4342 }
4343 return(i);
4344}
4345
6d2010ae
A		 4346
4347/* handles the cond broadcast of cvar and returns number of woken threads and bits for syscall return */
4348void
4349ksyn_handle_cvbroad(ksyn_wait_queue_t ckwq, uint32_t upto, uint32_t * updatep)
4350{
4351 kern_return_t kret;
4352 ksyn_queue_t kq;
4353 ksyn_waitq_element_t kwe, newkwe;
4354 uint32_t updatebits = 0;
4355 struct ksyn_queue kfreeq;
4356 uthread_t ut;
4357
4358#if _PSYNCH_TRACE_
4359 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_START, 0xcbcbcbc2, upto, 0, 0, 0);
4360#endif /* _PSYNCH_TRACE_ */
4361
4362 ksyn_queue_init(&kfreeq);
4363 kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
4364
4365 retry:
4366 TAILQ_FOREACH_SAFE(kwe, &kq->ksynq_kwelist, kwe_list, newkwe) {
4367
4368 if (is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), upto)) /* outside our range */
4369 break;
4370
4371 /* now handle the one we found (inside the range) */
4372 switch (kwe->kwe_flags) {
4373
4374 case KWE_THREAD_INWAIT:
4375 ut = (uthread_t)kwe->kwe_uth;
4376
4377 /* skip canceled ones */
4378 if (kwe->kwe_kwqqueue != ckwq ||
4379 (ut->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL)
4380 break;
4381
4382 /* wake the rest */
4383 ksyn_queue_removeitem(ckwq, kq, kwe);
4384 kwe->kwe_psynchretval = PTH_RWL_MTX_WAIT;
4385 kwe->kwe_kwqqueue = NULL;
4386#if _PSYNCH_TRACE_
4387 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
4388#endif /* _PSYNCH_TRACE_ */
4389 kret = ksyn_wakeup_thread(ckwq, kwe);
4390#if __TESTPANICS__
4391 if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
4392 panic("ksyn_wakeupreaders: panic waking up readers\n");
4393#endif /* __TESTPANICS__ */
4394 updatebits += PTHRW_INC;
4395 break;
4396
4397 case KWE_THREAD_BROADCAST:
4398 case KWE_THREAD_PREPOST:
4399 ksyn_queue_removeitem(ckwq, kq, kwe);
4400 TAILQ_INSERT_TAIL(&kfreeq.ksynq_kwelist, kwe, kwe_list);
4401 ckwq->kw_fakecount--;
4402 break;
4403
4404 default:
4405 panic("unknown kweflags\n");
4406 break;
b0d623f7 4407 }
6d2010ae
A		 4408 }
4409
4410 /* Need to enter a broadcast in the queue (if not already at L == S) */
4411
4412 if ((ckwq->kw_lword & PTHRW_COUNT_MASK) != (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
4413
4414 newkwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
4415 if (newkwe == NULL) {
4416 ksyn_wqunlock(ckwq);
4417 newkwe = (ksyn_waitq_element_t)zalloc(kwe_zone);
4418 TAILQ_INSERT_TAIL(&kfreeq.ksynq_kwelist, newkwe, kwe_list);
4419 ksyn_wqlock(ckwq);
4420 goto retry;
4421 }
4422
4423 TAILQ_REMOVE(&kfreeq.ksynq_kwelist, newkwe, kwe_list);
4424 bzero(newkwe, sizeof(struct ksyn_waitq_element));
4425 newkwe->kwe_kwqqueue = ckwq;
4426 newkwe->kwe_flags = KWE_THREAD_BROADCAST;
4427 newkwe->kwe_lockseq = upto;
4428 newkwe->kwe_count = 0;
4429 newkwe->kwe_uth = NULL;
4430 newkwe->kwe_psynchretval = 0;
4431
4432#if _PSYNCH_TRACE_
4433 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfeed, upto, 0, 0);
4434#endif /* _PSYNCH_TRACE_ */
4435
4436 (void)ksyn_queue_insert(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], upto, NULL, newkwe, SEQFIT);
4437 ckwq->kw_fakecount++;
4438 }
4439
4440 /* free up any remaining things stumbled across above */
4441 kwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
4442 while (kwe != NULL) {
4443 TAILQ_REMOVE(&kfreeq.ksynq_kwelist, kwe, kwe_list);
4444 zfree(kwe_zone, kwe);
4445 kwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
4446 }
4447
4448 if (updatep != NULL)
4449 *updatep = updatebits;
4450
4451#if _PSYNCH_TRACE_
4452 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_END, 0xeeeeeeed, updatebits, 0, 0, 0);
4453#endif /* _PSYNCH_TRACE_ */
2d21ac55
A		 4454}
4455
6d2010ae
A		 4456void
4457ksyn_cvupdate_fixup(ksyn_wait_queue_t ckwq, uint32_t *updatep, ksyn_queue_t kfreeq, int release)
4458{
4459 uint32_t updatebits = 0;
4460
4461 if (updatep != NULL)
4462 updatebits = *updatep;
4463 if ((ckwq->kw_lword & PTHRW_COUNT_MASK) == (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
4464 updatebits |= PTH_RWS_CV_CBIT;
4465 if (ckwq->kw_inqueue != 0) {
4466 /* FREE THE QUEUE */
4467 ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], ckwq->kw_lword, kfreeq, 0, release);
4468#if __TESTPANICS__
4469 if (ckwq->kw_inqueue != 0)
4470 panic("ksyn_cvupdate_fixup: L == S, but entries in queue beyond S");
4471#endif /* __TESTPANICS__ */
4472 }
4473 ckwq->kw_lword = ckwq->kw_uword = ckwq->kw_sword = 0;
4474 ckwq->kw_kflags |= KSYN_KWF_ZEROEDOUT;
4475 } else if ((ckwq->kw_inqueue != 0) && (ckwq->kw_fakecount == ckwq->kw_inqueue)) {
4476 /* only fake entries are present in the queue */
4477 updatebits |= PTH_RWS_CV_PBIT;
4478 }
4479 if (updatep != NULL)
4480 *updatep = updatebits;
4481}
4482
4483void
4484psynch_zoneinit(void)
4485{
4486 kwq_zone = (zone_t)zinit(sizeof(struct ksyn_wait_queue), 8192 * sizeof(struct ksyn_wait_queue), 4096, "ksyn_waitqueue zone");
4487 kwe_zone = (zone_t)zinit(sizeof(struct ksyn_waitq_element), 8192 * sizeof(struct ksyn_waitq_element), 4096, "ksyn_waitq_element zone");
4488}
b0d623f7 4489#endif /* PSYNCH */
mirror of XNU