[apple/libpthread.git] / src / internal.h

/*
 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991  
 *              All Rights Reserved 
 *  
 * Permission to use, copy, modify, and distribute this software and 
 * its documentation for any purpose and without fee is hereby granted, 
 * provided that the above copyright notice appears in all copies and 
 * that both the copyright notice and this permission notice appear in 
 * supporting documentation. 
 *  
 * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE 
 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
 * FOR A PARTICULAR PURPOSE. 
 *  
 * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR 
 * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 
 * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT, 
 * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION 
 * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 
 * 
 */
/*
 * MkLinux
 */

/*
 * POSIX Threads - IEEE 1003.1c
 */

#ifndef _POSIX_PTHREAD_INTERNALS_H
#define _POSIX_PTHREAD_INTERNALS_H

#define _PTHREAD_BUILDING_PTHREAD_

// suppress pthread_attr_t typedef in sys/signal.h
#define _PTHREAD_ATTR_T
struct _pthread_attr_t; /* forward reference */
typedef struct _pthread_attr_t pthread_attr_t;

#include <_simple.h>
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <limits.h>
#include <errno.h>
#include <TargetConditionals.h>
#include <libkern/OSAtomic.h>
#include <mach/mach.h>
#include <mach/mach_error.h>
#include <sys/queue.h>

#define __OS_EXPOSE_INTERNALS__ 1
#include <os/internal/internal_shared.h>
#include <os/once_private.h>

#define PTHREAD_INTERNAL_CRASH(c, x) do { \
		_os_set_crash_log_cause_and_message((c), \
				"BUG IN LIBPTHREAD: " x); \
		__builtin_trap(); \
	} while (0)

#define PTHREAD_CLIENT_CRASH(c, x) do { \
		_os_set_crash_log_cause_and_message((c), \
				"BUG IN CLIENT OF LIBPTHREAD: " x); \
		__builtin_trap(); \
	} while (0)

#ifndef __POSIX_LIB__
#define __POSIX_LIB__
#endif

#ifndef PTHREAD_LAYOUT_SPI
#define PTHREAD_LAYOUT_SPI 1
#endif

#include "posix_sched.h"
#include "tsd_private.h"
#include "spinlock_private.h"

#define OS_UNFAIR_LOCK_INLINE 1
#include <os/lock_private.h>
typedef os_unfair_lock _pthread_lock;
#define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
#define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER)
#define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
#define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock))
#define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock))
#define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock))

#if TARGET_IPHONE_SIMULATOR
#error Unsupported target
#endif

// List of all pthreads in the process.
TAILQ_HEAD(__pthread_list, _pthread);
extern struct __pthread_list __pthread_head;

// Lock protects access to above list.
extern _pthread_lock _pthread_list_lock;

extern int __is_threaded;

/*
 * Compiled-in limits
 */
#if TARGET_OS_EMBEDDED
#define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
#define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
#define _INTERNAL_POSIX_THREAD_KEYS_END 512
#else
#define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
#define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
#define _INTERNAL_POSIX_THREAD_KEYS_END 768
#endif

#define MAXTHREADNAMESIZE	64
#define _PTHREAD_T
typedef struct _pthread {
	//
	// ABI - These fields are externally known as struct _opaque_pthread_t.
	//
	long sig; // _PTHREAD_SIG
	struct __darwin_pthread_handler_rec *__cleanup_stack;

	//
	// SPI - These fields are private.
	//
	// these fields are globally protected by _pthread_list_lock:
	uint32_t childrun:1,
			parentcheck:1,
			childexit:1,
			pad3:29;
	
	_pthread_lock lock; // protect access to everything below
	uint32_t detached:8,
			inherit:8,
			policy:8,
			kernalloc:1,
			schedset:1,
			wqthread:1,
			wqkillset:1,
			pad:4;

#if defined(__LP64__)
	uint32_t pad0;
#endif

	void *(*fun)(void*);	// thread start routine
	void *arg;		// thread start routine argument
	void *exit_value;	// thread exit value storage

	semaphore_t joiner_notify;	// pthread_join notification

	int max_tsd_key;
	int cancel_state;	// whether the thread can be cancelled
	int cancel_error;

	int err_no;		// thread-local errno

	struct _pthread *joiner;

	struct sched_param param;	// [aligned]

	TAILQ_ENTRY(_pthread) plist;	// global thread list [aligned]

	char pthread_name[MAXTHREADNAMESIZE];	// includes NUL [aligned]

	void *stackaddr;	// base of the stack (page aligned)
	size_t stacksize;	// size of stack (page multiple and >= PTHREAD_STACK_MIN)

	void* freeaddr;		// stack/thread allocation base address
	size_t freesize;	// stack/thread allocation size
	size_t guardsize;	// guard page size in bytes

	// tsd-base relative accessed elements
	__attribute__((aligned(8)))
	uint64_t thread_id;	// 64-bit unique thread id

	/* Thread Specific Data slots
	 *
	 * The offset of this field from the start of the structure is difficult to
	 * change on OS X because of a thorny bitcompat issue: mono has hard coded
	 * the value into their source.  Newer versions of mono will fall back to
	 * scanning to determine it at runtime, but there's lots of software built
	 * with older mono that won't.  We will have to break them someday...
	 */
	__attribute__ ((aligned (16)))
	void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
} *pthread_t;


struct _pthread_attr_t {
	long sig;
	_pthread_lock lock;
	uint32_t detached:8,
		inherit:8,
		policy:8,
		fastpath:1,
		schedset:1,
		qosset:1,
		unused:5;
	struct sched_param param; // [aligned]
	void *stackaddr; // stack base; vm_page_size aligned
	size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
	size_t guardsize; // size in bytes of stack overflow guard area
	unsigned long qosclass;
#if defined(__LP64__)
	uint32_t _reserved[2];
#else
	uint32_t _reserved[1];
#endif
};

/*
 * Mutex attributes
 */
#define _PTHREAD_MUTEX_POLICY_NONE		0
#define _PTHREAD_MUTEX_POLICY_FAIRSHARE		1
#define _PTHREAD_MUTEX_POLICY_FIRSTFIT		2
#define _PTHREAD_MUTEX_POLICY_REALTIME		3
#define _PTHREAD_MUTEX_POLICY_ADAPTIVE		4
#define _PTHREAD_MUTEX_POLICY_PRIPROTECT	5
#define _PTHREAD_MUTEX_POLICY_PRIINHERIT	6

#define _PTHREAD_MUTEXATTR_T
typedef struct {
	long sig;
	int prioceiling;
	uint32_t protocol:2,
		type:2,
		pshared:2,
		policy:3,
		unused:23;
} pthread_mutexattr_t;

struct _pthread_mutex_options {
	uint32_t protocol:2,
		type:2,
		pshared:2,
		policy:3,
		hold:2,
		misalign:1,
		notify:1,
		mutex:1,
		unused:2,
		lock_count:16;
};

typedef struct {
	long sig;
	_pthread_lock lock;
	union {
		uint32_t value;
		struct _pthread_mutex_options options;
	} mtxopts;
	int16_t prioceiling;
	int16_t priority;
#if defined(__LP64__)
	uint32_t _pad;
#endif
	uint32_t m_tid[2]; // thread id of thread that has mutex locked
	uint32_t m_seq[2]; // mutex sequence id
	uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
#if defined(__LP64__)
	uint32_t _reserved[4];
#else
	uint32_t _reserved[1];
#endif
} _pthread_mutex;


#define _PTHREAD_CONDATTR_T
typedef struct {
	long sig;
	uint32_t pshared:2,
		unsupported:30;
} pthread_condattr_t;


typedef struct {
	long sig;
	_pthread_lock lock;
	uint32_t unused:29,
		misalign:1,
		pshared:2;
	_pthread_mutex *busy;
	uint32_t c_seq[3];
#if defined(__LP64__)
	uint32_t _reserved[3];
#endif
} _pthread_cond;


#define _PTHREAD_ONCE_T
typedef struct {
	long sig;
	os_once_t once;
} pthread_once_t;


#define _PTHREAD_RWLOCKATTR_T
typedef struct {
	long sig;
	int pshared;
#if defined(__LP64__)
	uint32_t _reserved[3];
#else
	uint32_t _reserved[2];
#endif
} pthread_rwlockattr_t;


typedef struct {
	long sig;
	_pthread_lock lock;
	uint32_t unused:29,
		misalign:1,
		pshared:2;
	uint32_t rw_flags;
#if defined(__LP64__)
	uint32_t _pad;
#endif
	volatile uint32_t rw_seq[4];
	struct _pthread *rw_owner;
	volatile uint32_t *rw_lcntaddr;
	volatile uint32_t *rw_seqaddr;
	volatile uint32_t *rw_ucntaddr;
#if defined(__LP64__)
	uint32_t _reserved[31];
#else
	uint32_t _reserved[19];
#endif
} _pthread_rwlock;

#include "pthread.h"
#include "pthread_spis.h"

_Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
		"Incorrect _pthread_mutex structure size");

// Internal references to pthread_self() use TSD slot 0 directly.
inline static pthread_t __attribute__((__pure__))
_pthread_self_direct(void)
{
	return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
}
#define pthread_self() _pthread_self_direct()

inline static pthread_t __attribute__((__pure__))
_pthread_selfid_direct(void)
{
	return (_pthread_self_direct())->thread_id;
}

#define _PTHREAD_DEFAULT_INHERITSCHED	PTHREAD_INHERIT_SCHED
#define _PTHREAD_DEFAULT_PROTOCOL	PTHREAD_PRIO_NONE
#define _PTHREAD_DEFAULT_PRIOCEILING	0
#define _PTHREAD_DEFAULT_POLICY		SCHED_OTHER
#define _PTHREAD_DEFAULT_STACKSIZE	0x80000	  /* 512K */
#define _PTHREAD_DEFAULT_PSHARED	PTHREAD_PROCESS_PRIVATE

#define _PTHREAD_NO_SIG			0x00000000
#define _PTHREAD_MUTEX_ATTR_SIG		0x4D545841  /* 'MTXA' */
#define _PTHREAD_MUTEX_SIG		0x4D555458  /* 'MUTX' */
#define _PTHREAD_MUTEX_SIG_fast		0x4D55545A  /* 'MUTZ' */
#define _PTHREAD_MUTEX_SIG_MASK		0xfffffffd
#define _PTHREAD_MUTEX_SIG_CMP		0x4D555458  /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
#define _PTHREAD_MUTEX_SIG_init		0x32AAABA7  /* [almost] ~'MUTX' */
#define _PTHREAD_ERRORCHECK_MUTEX_SIG_init      0x32AAABA1
#define _PTHREAD_RECURSIVE_MUTEX_SIG_init       0x32AAABA2
#define _PTHREAD_FIRSTFIT_MUTEX_SIG_init        0x32AAABA3
#define _PTHREAD_MUTEX_SIG_init_MASK            0xfffffff0
#define _PTHREAD_MUTEX_SIG_init_CMP             0x32AAABA0
#define _PTHREAD_COND_ATTR_SIG		0x434E4441  /* 'CNDA' */
#define _PTHREAD_COND_SIG		0x434F4E44  /* 'COND' */
#define _PTHREAD_COND_SIG_init		0x3CB0B1BB  /* [almost] ~'COND' */
#define _PTHREAD_ATTR_SIG		0x54484441  /* 'THDA' */
#define _PTHREAD_ONCE_SIG		0x4F4E4345  /* 'ONCE' */
#define _PTHREAD_ONCE_SIG_init		0x30B1BCBA  /* [almost] ~'ONCE' */
#define _PTHREAD_SIG			0x54485244  /* 'THRD' */
#define _PTHREAD_RWLOCK_ATTR_SIG	0x52574C41  /* 'RWLA' */
#define _PTHREAD_RWLOCK_SIG		0x52574C4B  /* 'RWLK' */
#define _PTHREAD_RWLOCK_SIG_init	0x2DA8B3B4  /* [almost] ~'RWLK' */


#define _PTHREAD_KERN_COND_SIG		0x12345678  /*  */
#define _PTHREAD_KERN_MUTEX_SIG		0x34567812  /*  */
#define _PTHREAD_KERN_RWLOCK_SIG	0x56781234  /*  */

#define _PTHREAD_CREATE_PARENT		4
#define _PTHREAD_EXITED			8
// 4597450: begin
#define _PTHREAD_WASCANCEL		0x10
// 4597450: end

#if defined(DEBUG)
#define _PTHREAD_MUTEX_OWNER_SELF	pthread_self()
#else
#define _PTHREAD_MUTEX_OWNER_SELF	(pthread_t)0x12141968
#endif
#define _PTHREAD_MUTEX_OWNER_SWITCHING	(pthread_t)(~0)

#define _PTHREAD_CANCEL_STATE_MASK   0x01
#define _PTHREAD_CANCEL_TYPE_MASK    0x02
#define _PTHREAD_CANCEL_PENDING	     0x10  /* pthread_cancel() has been called for this thread */

extern boolean_t swtch_pri(int);

#define PTHREAD_EXPORT extern __attribute__((visibility("default")))
#define PTHREAD_EXTERN extern
#define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
#define PTHREAD_NORETURN __attribute__((__noreturn__))
#define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
#define PTHREAD_NOINLINE __attribute__((noinline))
#define PTHREAD_WEAK __attribute__((weak))
#define PTHREAD_USED __attribute__((used))

#include "kern/kern_internal.h"

/* Prototypes. */

/* Internal globals. */
PTHREAD_NOEXPORT extern int __pthread_supported_features;

/* Functions defined in machine-dependent files. */
PTHREAD_NOEXPORT void _pthread_setup(pthread_t th, void (*f)(pthread_t), void *sp, int suspended, int needresume);

PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);

PTHREAD_NOEXPORT int __mtx_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);

/* internally redirected upcalls. */
PTHREAD_NOEXPORT void* malloc(size_t);
PTHREAD_NOEXPORT void free(void*);

/* syscall interfaces */
extern uint32_t __psynch_mutexwait(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);
extern uint32_t __psynch_mutexdrop(pthread_mutex_t * mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags);

extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid);
extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex,  uint64_t mugen, uint64_t tid, uint32_t flags);
extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);

PTHREAD_EXTERN
int
__proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);

PTHREAD_NOEXPORT int _pthread_lookup_thread(pthread_t thread, mach_port_t * port, int only_joinable);
PTHREAD_NOEXPORT int _pthread_join_cleanup(pthread_t thread, void ** value_ptr, int conforming);

PTHREAD_NORETURN PTHREAD_NOEXPORT
void
__pthread_abort(void);

PTHREAD_NORETURN PTHREAD_NOEXPORT
void
__pthread_abort_reason(const char *fmt, ...);

PTHREAD_NOEXPORT
void
_pthread_set_main_qos(pthread_priority_t qos);

PTHREAD_NOEXPORT
void
_pthread_key_global_init(const char *envp[]);

PTHREAD_EXPORT
void
_pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);

PTHREAD_EXPORT
void
_pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents);

PTHREAD_NOEXPORT
void
__pthread_fork_child_internal(pthread_t p);

PTHREAD_EXPORT
void
_pthread_clear_qos_tsd(mach_port_t thread_port);

PTHREAD_EXPORT
void
_pthread_testcancel(pthread_t thread, int isconforming);

PTHREAD_EXPORT
void
_pthread_exit_if_canceled(int error);

PTHREAD_ALWAYS_INLINE
static inline mach_port_t
_pthread_kernel_thread(pthread_t t)
{
	return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
}

PTHREAD_ALWAYS_INLINE
static inline void
_pthread_set_kernel_thread(pthread_t t, mach_port_t p)
{
	if (os_slowpath(!MACH_PORT_VALID(p))) {
		PTHREAD_INTERNAL_CRASH(t, "Invalid thread port");
	}
	t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
}

#define PTHREAD_ABORT(f,...) __pthread_abort_reason("%s:%s:%u: " f, __FILE__, __func__, __LINE__, ## __VA_ARGS__)

#define PTHREAD_ASSERT(b) do { if (!(b)) PTHREAD_ABORT("failed assertion `%s'", #b); } while (0)

#include <os/semaphore_private.h>
#include <os/alloc_once_private.h>

struct pthread_atfork_entry {
	void (*prepare)(void);
	void (*parent)(void);
	void (*child)(void);
};

#define PTHREAD_ATFORK_INLINE_MAX 10
#define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))

struct pthread_globals_s {
	// atfork.c
	pthread_t psaved_self;
	_pthread_lock psaved_self_global_lock;
	_pthread_lock pthread_atfork_lock;

	size_t atfork_count;
	struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
	struct pthread_atfork_entry *atfork;
};
typedef struct pthread_globals_s *pthread_globals_t;

__attribute__((__pure__))
static inline pthread_globals_t
_pthread_globals(void)
{
	return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
			     sizeof(struct pthread_globals_s),
			     NULL);
}

#pragma mark _pthread_mutex_check_signature

PTHREAD_ALWAYS_INLINE
static inline bool
_pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
{
	return os_fastpath(mutex->sig == _PTHREAD_MUTEX_SIG_fast);
}

PTHREAD_ALWAYS_INLINE
static inline bool
_pthread_mutex_check_signature(_pthread_mutex *mutex)
{
	return os_fastpath((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
}

PTHREAD_ALWAYS_INLINE
static inline bool
_pthread_mutex_check_signature_init(_pthread_mutex *mutex)
{
	return os_fastpath((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) == _PTHREAD_MUTEX_SIG_init_CMP);
}

#endif /* _POSIX_PTHREAD_INTERNALS_H */
Commit	Line	Data
f1a1da6c A	1	/*
	2	* Copyright (c) 2000-2013 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
	12	*
	13	* The Original Code and all software distributed under the License are
	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*/
	23	/*
	24	* Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
	25	* All Rights Reserved
	26	*
	27	* Permission to use, copy, modify, and distribute this software and
	28	* its documentation for any purpose and without fee is hereby granted,
	29	* provided that the above copyright notice appears in all copies and
	30	* that both the copyright notice and this permission notice appear in
	31	* supporting documentation.
	32	*
	33	* OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
	34	* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	35	* FOR A PARTICULAR PURPOSE.
	36	*
	37	* IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
	38	* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
	39	* LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
	40	* NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
	41	* WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	42	*
	43	*/
	44	/*
	45	* MkLinux
	46	*/
	47
	48	/*
	49	* POSIX Threads - IEEE 1003.1c
	50	*/
	51
	52	#ifndef _POSIX_PTHREAD_INTERNALS_H
	53	#define _POSIX_PTHREAD_INTERNALS_H
	54
	55	#define _PTHREAD_BUILDING_PTHREAD_
	56
	57	// suppress pthread_attr_t typedef in sys/signal.h
	58	#define _PTHREAD_ATTR_T
	59	struct _pthread_attr_t; /* forward reference */
	60	typedef struct _pthread_attr_t pthread_attr_t;
	61
964d3577	62	#include <_simple.h>
f1a1da6c A	63	#include <assert.h>
	64	#include <stddef.h>
	65	#include <stdint.h>
	66	#include <stdlib.h>
	67	#include <limits.h>
	68	#include <errno.h>
	69	#include <TargetConditionals.h>
	70	#include <libkern/OSAtomic.h>
	71	#include <mach/mach.h>
	72	#include <mach/mach_error.h>
f1a1da6c A	73	#include <sys/queue.h>
f1a1da6c A	74
2546420a A	75	#define __OS_EXPOSE_INTERNALS__ 1
	76	#include <os/internal/internal_shared.h>
	77	#include <os/once_private.h>
	78
	79	#define PTHREAD_INTERNAL_CRASH(c, x) do { \
	80	_os_set_crash_log_cause_and_message((c), \
	81	"BUG IN LIBPTHREAD: " x); \
	82	__builtin_trap(); \
	83	} while (0)
	84
	85	#define PTHREAD_CLIENT_CRASH(c, x) do { \
	86	_os_set_crash_log_cause_and_message((c), \
	87	"BUG IN CLIENT OF LIBPTHREAD: " x); \
	88	__builtin_trap(); \
	89	} while (0)
	90
f1a1da6c A	91	#ifndef __POSIX_LIB__
	92	#define __POSIX_LIB__
	93	#endif
	94
	95	#ifndef PTHREAD_LAYOUT_SPI
	96	#define PTHREAD_LAYOUT_SPI 1
	97	#endif
	98
	99	#include "posix_sched.h"
	100	#include "tsd_private.h"
	101	#include "spinlock_private.h"
	102
2546420a A	103	#define OS_UNFAIR_LOCK_INLINE 1
	104	#include <os/lock_private.h>
	105	typedef os_unfair_lock _pthread_lock;
	106	#define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
	107	#define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER)
	108	#define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION)
	109	#define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock))
	110	#define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock))
	111	#define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock))
	112
f1a1da6c A	113	#if TARGET_IPHONE_SIMULATOR
	114	#error Unsupported target
	115	#endif
	116
	117	// List of all pthreads in the process.
	118	TAILQ_HEAD(__pthread_list, _pthread);
	119	extern struct __pthread_list __pthread_head;
	120
	121	// Lock protects access to above list.
2546420a	122	extern _pthread_lock _pthread_list_lock;
f1a1da6c A	123
	124	extern int __is_threaded;
	125
	126	/*
	127	* Compiled-in limits
	128	*/
	129	#if TARGET_OS_EMBEDDED
	130	#define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256
	131	#define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
	132	#define _INTERNAL_POSIX_THREAD_KEYS_END 512
	133	#else
	134	#define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512
	135	#define _INTERNAL_POSIX_THREAD_KEYS_MAX 256
	136	#define _INTERNAL_POSIX_THREAD_KEYS_END 768
	137	#endif
	138
	139	#define MAXTHREADNAMESIZE 64
	140	#define _PTHREAD_T
	141	typedef struct _pthread {
	142	//
	143	// ABI - These fields are externally known as struct _opaque_pthread_t.
	144	//
	145	long sig; // _PTHREAD_SIG
	146	struct __darwin_pthread_handler_rec *__cleanup_stack;
	147
	148	//
	149	// SPI - These fields are private.
	150	//
	151	// these fields are globally protected by _pthread_list_lock:
	152	uint32_t childrun:1,
	153	parentcheck:1,
	154	childexit:1,
	155	pad3:29;
	156
2546420a	157	_pthread_lock lock; // protect access to everything below
f1a1da6c A	158	uint32_t detached:8,
	159	inherit:8,
	160	policy:8,
	161	kernalloc:1,
	162	schedset:1,
	163	wqthread:1,
	164	wqkillset:1,
	165	pad:4;
	166
2546420a	167	#if defined(__LP64__)
f1a1da6c A	168	uint32_t pad0;
f1a1da6c A	169	#endif
f1a1da6c A	170
	171	void (fun)(void*); // thread start routine
	172	void *arg; // thread start routine argument
	173	void *exit_value; // thread exit value storage
	174
	175	semaphore_t joiner_notify; // pthread_join notification
	176
	177	int max_tsd_key;
	178	int cancel_state; // whether the thread can be cancelled
	179	int cancel_error;
	180
f1a1da6c A	181	int err_no; // thread-local errno
	182
	183	struct _pthread *joiner;
	184
	185	struct sched_param param; // [aligned]
	186
	187	TAILQ_ENTRY(_pthread) plist; // global thread list [aligned]
	188
	189	char pthread_name[MAXTHREADNAMESIZE]; // includes NUL [aligned]
2546420a	190
f1a1da6c A	191	void *stackaddr; // base of the stack (page aligned)
	192	size_t stacksize; // size of stack (page multiple and >= PTHREAD_STACK_MIN)
	193
	194	void* freeaddr; // stack/thread allocation base address
	195	size_t freesize; // stack/thread allocation size
	196	size_t guardsize; // guard page size in bytes
	197
2546420a A	198	// tsd-base relative accessed elements
	199	__attribute__((aligned(8)))
	200	uint64_t thread_id; // 64-bit unique thread id
	201
	202	/* Thread Specific Data slots
	203	*
	204	* The offset of this field from the start of the structure is difficult to
	205	* change on OS X because of a thorny bitcompat issue: mono has hard coded
	206	* the value into their source. Newer versions of mono will fall back to
	207	* scanning to determine it at runtime, but there's lots of software built
	208	* with older mono that won't. We will have to break them someday...
	209	*/
	210	__attribute__ ((aligned (16)))
	211	void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX];
f1a1da6c A	212	} *pthread_t;
	213
	214
	215	struct _pthread_attr_t {
	216	long sig;
2546420a	217	_pthread_lock lock;
f1a1da6c A	218	uint32_t detached:8,
	219	inherit:8,
	220	policy:8,
	221	fastpath:1,
	222	schedset:1,
	223	qosset:1,
	224	unused:5;
	225	struct sched_param param; // [aligned]
	226	void *stackaddr; // stack base; vm_page_size aligned
	227	size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN
	228	size_t guardsize; // size in bytes of stack overflow guard area
	229	unsigned long qosclass;
	230	#if defined(__LP64__)
	231	uint32_t _reserved[2];
	232	#else
	233	uint32_t _reserved[1];
	234	#endif
	235	};
	236
	237	/*
	238	* Mutex attributes
	239	*/
	240	#define _PTHREAD_MUTEX_POLICY_NONE 0
	241	#define _PTHREAD_MUTEX_POLICY_FAIRSHARE 1
	242	#define _PTHREAD_MUTEX_POLICY_FIRSTFIT 2
	243	#define _PTHREAD_MUTEX_POLICY_REALTIME 3
	244	#define _PTHREAD_MUTEX_POLICY_ADAPTIVE 4
	245	#define _PTHREAD_MUTEX_POLICY_PRIPROTECT 5
	246	#define _PTHREAD_MUTEX_POLICY_PRIINHERIT 6
	247
	248	#define _PTHREAD_MUTEXATTR_T
	249	typedef struct {
	250	long sig;
	251	int prioceiling;
	252	uint32_t protocol:2,
	253	type:2,
	254	pshared:2,
	255	policy:3,
	256	unused:23;
	257	} pthread_mutexattr_t;
	258
	259	struct _pthread_mutex_options {
	260	uint32_t protocol:2,
	261	type:2,
	262	pshared:2,
	263	policy:3,
	264	hold:2,
	265	misalign:1,
	266	notify:1,
	267	mutex:1,
	268	unused:2,
	269	lock_count:16;
	270	};
	271
	272	typedef struct {
	273	long sig;
2546420a	274	_pthread_lock lock;
f1a1da6c A	275	union {
	276	uint32_t value;
	277	struct _pthread_mutex_options options;
	278	} mtxopts;
	279	int16_t prioceiling;
	280	int16_t priority;
	281	#if defined(__LP64__)
	282	uint32_t _pad;
	283	#endif
3a6437e6 A	284	uint32_t m_tid[2]; // thread id of thread that has mutex locked
	285	uint32_t m_seq[2]; // mutex sequence id
	286	uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here
f1a1da6c	287	#if defined(__LP64__)
3a6437e6 A	288	uint32_t _reserved[4];
	289	#else
	290	uint32_t _reserved[1];
f1a1da6c	291	#endif
f1a1da6c A	292	} _pthread_mutex;
	293
	294
	295	#define _PTHREAD_CONDATTR_T
	296	typedef struct {
	297	long sig;
	298	uint32_t pshared:2,
	299	unsupported:30;
	300	} pthread_condattr_t;
	301
	302
	303	typedef struct {
	304	long sig;
2546420a	305	_pthread_lock lock;
f1a1da6c A	306	uint32_t unused:29,
	307	misalign:1,
	308	pshared:2;
	309	_pthread_mutex *busy;
	310	uint32_t c_seq[3];
	311	#if defined(__LP64__)
	312	uint32_t _reserved[3];
	313	#endif
	314	} _pthread_cond;
	315
	316
	317	#define _PTHREAD_ONCE_T
	318	typedef struct {
	319	long sig;
	320	os_once_t once;
	321	} pthread_once_t;
	322
	323
	324	#define _PTHREAD_RWLOCKATTR_T
	325	typedef struct {
	326	long sig;
	327	int pshared;
	328	#if defined(__LP64__)
	329	uint32_t _reserved[3];
	330	#else
	331	uint32_t _reserved[2];
	332	#endif
	333	} pthread_rwlockattr_t;
	334
	335
	336	typedef struct {
	337	long sig;
2546420a	338	_pthread_lock lock;
f1a1da6c A	339	uint32_t unused:29,
	340	misalign:1,
	341	pshared:2;
	342	uint32_t rw_flags;
	343	#if defined(__LP64__)
	344	uint32_t _pad;
	345	#endif
	346	volatile uint32_t rw_seq[4];
	347	struct _pthread *rw_owner;
	348	volatile uint32_t *rw_lcntaddr;
	349	volatile uint32_t *rw_seqaddr;
	350	volatile uint32_t *rw_ucntaddr;
	351	#if defined(__LP64__)
	352	uint32_t _reserved[31];
	353	#else
	354	uint32_t _reserved[19];
	355	#endif
	356	} _pthread_rwlock;
	357
3a6437e6	358	#include "pthread.h"
f1a1da6c A	359	#include "pthread_spis.h"
f1a1da6c A	360
3a6437e6 A	361	_Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t),
	362	"Incorrect _pthread_mutex structure size");
	363
f1a1da6c A	364	// Internal references to pthread_self() use TSD slot 0 directly.
	365	inline static pthread_t __attribute__((__pure__))
	366	_pthread_self_direct(void)
	367	{
	368	return _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF);
	369	}
	370	#define pthread_self() _pthread_self_direct()
	371
	372	inline static pthread_t __attribute__((__pure__))
	373	_pthread_selfid_direct(void)
	374	{
	375	return (_pthread_self_direct())->thread_id;
	376	}
	377
	378	#define _PTHREAD_DEFAULT_INHERITSCHED PTHREAD_INHERIT_SCHED
	379	#define _PTHREAD_DEFAULT_PROTOCOL PTHREAD_PRIO_NONE
	380	#define _PTHREAD_DEFAULT_PRIOCEILING 0
	381	#define _PTHREAD_DEFAULT_POLICY SCHED_OTHER
	382	#define _PTHREAD_DEFAULT_STACKSIZE 0x80000 /* 512K */
	383	#define _PTHREAD_DEFAULT_PSHARED PTHREAD_PROCESS_PRIVATE
	384
	385	#define _PTHREAD_NO_SIG 0x00000000
	386	#define _PTHREAD_MUTEX_ATTR_SIG 0x4D545841 /* 'MTXA' */
	387	#define _PTHREAD_MUTEX_SIG 0x4D555458 /* 'MUTX' */
964d3577 A	388	#define _PTHREAD_MUTEX_SIG_fast 0x4D55545A /* 'MUTZ' */
	389	#define _PTHREAD_MUTEX_SIG_MASK 0xfffffffd
	390	#define _PTHREAD_MUTEX_SIG_CMP 0x4D555458 /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */
f1a1da6c A	391	#define _PTHREAD_MUTEX_SIG_init 0x32AAABA7 /* [almost] ~'MUTX' */
	392	#define _PTHREAD_ERRORCHECK_MUTEX_SIG_init 0x32AAABA1
	393	#define _PTHREAD_RECURSIVE_MUTEX_SIG_init 0x32AAABA2
964d3577	394	#define _PTHREAD_FIRSTFIT_MUTEX_SIG_init 0x32AAABA3
f1a1da6c	395	#define _PTHREAD_MUTEX_SIG_init_MASK 0xfffffff0
964d3577	396	#define _PTHREAD_MUTEX_SIG_init_CMP 0x32AAABA0
f1a1da6c A	397	#define _PTHREAD_COND_ATTR_SIG 0x434E4441 /* 'CNDA' */
	398	#define _PTHREAD_COND_SIG 0x434F4E44 /* 'COND' */
	399	#define _PTHREAD_COND_SIG_init 0x3CB0B1BB /* [almost] ~'COND' */
	400	#define _PTHREAD_ATTR_SIG 0x54484441 /* 'THDA' */
	401	#define _PTHREAD_ONCE_SIG 0x4F4E4345 /* 'ONCE' */
	402	#define _PTHREAD_ONCE_SIG_init 0x30B1BCBA /* [almost] ~'ONCE' */
	403	#define _PTHREAD_SIG 0x54485244 /* 'THRD' */
	404	#define _PTHREAD_RWLOCK_ATTR_SIG 0x52574C41 /* 'RWLA' */
	405	#define _PTHREAD_RWLOCK_SIG 0x52574C4B /* 'RWLK' */
	406	#define _PTHREAD_RWLOCK_SIG_init 0x2DA8B3B4 /* [almost] ~'RWLK' */
	407
	408
	409	#define _PTHREAD_KERN_COND_SIG 0x12345678 /* */
	410	#define _PTHREAD_KERN_MUTEX_SIG 0x34567812 /* */
	411	#define _PTHREAD_KERN_RWLOCK_SIG 0x56781234 /* */
	412
	413	#define _PTHREAD_CREATE_PARENT 4
	414	#define _PTHREAD_EXITED 8
	415	// 4597450: begin
	416	#define _PTHREAD_WASCANCEL 0x10
	417	// 4597450: end
	418
	419	#if defined(DEBUG)
	420	#define _PTHREAD_MUTEX_OWNER_SELF pthread_self()
	421	#else
	422	#define _PTHREAD_MUTEX_OWNER_SELF (pthread_t)0x12141968
	423	#endif
	424	#define _PTHREAD_MUTEX_OWNER_SWITCHING (pthread_t)(~0)
	425
	426	#define _PTHREAD_CANCEL_STATE_MASK 0x01
	427	#define _PTHREAD_CANCEL_TYPE_MASK 0x02
	428	#define _PTHREAD_CANCEL_PENDING 0x10 /* pthread_cancel() has been called for this thread */
	429
	430	extern boolean_t swtch_pri(int);
	431
	432	#define PTHREAD_EXPORT extern __attribute__((visibility("default")))
	433	#define PTHREAD_EXTERN extern
	434	#define PTHREAD_NOEXPORT __attribute__((visibility("hidden")))
	435	#define PTHREAD_NORETURN __attribute__((__noreturn__))
	436	#define PTHREAD_ALWAYS_INLINE __attribute__((always_inline))
	437	#define PTHREAD_NOINLINE __attribute__((noinline))
964d3577 A	438	#define PTHREAD_WEAK __attribute__((weak))
964d3577 A	439	#define PTHREAD_USED __attribute__((used))
f1a1da6c A	440
	441	#include "kern/kern_internal.h"
	442
	443	/* Prototypes. */
	444
	445	/* Internal globals. */
	446	PTHREAD_NOEXPORT extern int __pthread_supported_features;
	447
	448	/* Functions defined in machine-dependent files. */
	449	PTHREAD_NOEXPORT void _pthread_setup(pthread_t th, void (f)(pthread_t), void sp, int suspended, int needresume);
	450
	451	PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self);
	452
	453	PTHREAD_NOEXPORT int __mtx_droplock(_pthread_mutex mutex, uint32_t flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp);
	454
	455	/* internally redirected upcalls. */
	456	PTHREAD_NOEXPORT void* malloc(size_t);
	457	PTHREAD_NOEXPORT void free(void*);
	458
	459	/* syscall interfaces */
	460	extern uint32_t __psynch_mutexwait(pthread_mutex_t * mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags);
	461	extern uint32_t __psynch_mutexdrop(pthread_mutex_t * mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags);
	462
	463	extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex, uint64_t mugen, uint64_t tid);
	464	extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex, uint64_t mugen, uint64_t tid, uint32_t flags);
	465	extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex, uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec);
	466	extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags);
	467	extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
	468	extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
	469	extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
	470	extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
	471	extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
	472	extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
	473	extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
	474	extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags);
	475	extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
	476
	477	PTHREAD_EXTERN
	478	int
	479	__proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize);
	480
	481	PTHREAD_NOEXPORT int _pthread_lookup_thread(pthread_t thread, mach_port_t * port, int only_joinable);
	482	PTHREAD_NOEXPORT int _pthread_join_cleanup(pthread_t thread, void ** value_ptr, int conforming);
	483
	484	PTHREAD_NORETURN PTHREAD_NOEXPORT
	485	void
	486	__pthread_abort(void);
	487
	488	PTHREAD_NORETURN PTHREAD_NOEXPORT
	489	void
	490	__pthread_abort_reason(const char *fmt, ...);
	491
	492	PTHREAD_NOEXPORT
	493	void
	494	_pthread_set_main_qos(pthread_priority_t qos);
	495
964d3577 A	496	PTHREAD_NOEXPORT
	497	void
	498	_pthread_key_global_init(const char *envp[]);
	499
f1a1da6c A	500	PTHREAD_EXPORT
	501	void
	502	_pthread_start(pthread_t self, mach_port_t kport, void (fun)(void ), void funarg, size_t stacksize, unsigned int flags);
	503
	504	PTHREAD_EXPORT
	505	void
964d3577	506	_pthread_wqthread(pthread_t self, mach_port_t kport, void stackaddr, void keventlist, int flags, int nkevents);
f1a1da6c A	507
	508	PTHREAD_NOEXPORT
	509	void
	510	__pthread_fork_child_internal(pthread_t p);
	511
	512	PTHREAD_EXPORT
	513	void
	514	_pthread_clear_qos_tsd(mach_port_t thread_port);
	515
	516	PTHREAD_EXPORT
	517	void
	518	_pthread_testcancel(pthread_t thread, int isconforming);
	519
	520	PTHREAD_EXPORT
	521	void
	522	_pthread_exit_if_canceled(int error);
	523
	524	PTHREAD_ALWAYS_INLINE
	525	static inline mach_port_t
	526	_pthread_kernel_thread(pthread_t t)
	527	{
	528	return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF];
	529	}
	530
	531	PTHREAD_ALWAYS_INLINE
	532	static inline void
	533	_pthread_set_kernel_thread(pthread_t t, mach_port_t p)
	534	{
2546420a A	535	if (os_slowpath(!MACH_PORT_VALID(p))) {
	536	PTHREAD_INTERNAL_CRASH(t, "Invalid thread port");
	537	}
f1a1da6c A	538	t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p;
	539	}
	540
	541	#define PTHREAD_ABORT(f,...) __pthread_abort_reason("%s:%s:%u: " f, __FILE__, __func__, __LINE__, ## __VA_ARGS__)
	542
	543	#define PTHREAD_ASSERT(b) do { if (!(b)) PTHREAD_ABORT("failed assertion `%s'", #b); } while (0)
	544
	545	#include <os/semaphore_private.h>
	546	#include <os/alloc_once_private.h>
	547
	548	struct pthread_atfork_entry {
	549	void (*prepare)(void);
	550	void (*parent)(void);
	551	void (*child)(void);
	552	};
	553
	554	#define PTHREAD_ATFORK_INLINE_MAX 10
	555	#define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry))
	556
	557	struct pthread_globals_s {
	558	// atfork.c
	559	pthread_t psaved_self;
2546420a A	560	_pthread_lock psaved_self_global_lock;
2546420a A	561	_pthread_lock pthread_atfork_lock;
f1a1da6c A	562
	563	size_t atfork_count;
	564	struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX];
	565	struct pthread_atfork_entry *atfork;
	566	};
	567	typedef struct pthread_globals_s *pthread_globals_t;
	568
	569	__attribute__((__pure__))
	570	static inline pthread_globals_t
	571	_pthread_globals(void)
	572	{
	573	return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD,
	574	sizeof(struct pthread_globals_s),
	575	NULL);
	576	}
	577
964d3577 A	578	#pragma mark _pthread_mutex_check_signature
	579
	580	PTHREAD_ALWAYS_INLINE
	581	static inline bool
	582	_pthread_mutex_check_signature_fast(_pthread_mutex *mutex)
	583	{
	584	return os_fastpath(mutex->sig == _PTHREAD_MUTEX_SIG_fast);
	585	}
	586
	587	PTHREAD_ALWAYS_INLINE
	588	static inline bool
	589	_pthread_mutex_check_signature(_pthread_mutex *mutex)
	590	{
	591	return os_fastpath((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP);
	592	}
	593
	594	PTHREAD_ALWAYS_INLINE
	595	static inline bool
	596	_pthread_mutex_check_signature_init(_pthread_mutex *mutex)
	597	{
	598	return os_fastpath((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) == _PTHREAD_MUTEX_SIG_init_CMP);
	599	}
	600
f1a1da6c	601	#endif /* _POSIX_PTHREAD_INTERNALS_H */