[apple/xnu.git] / bsd / pthread / priority_private.h

/*
 * Copyright (c) 2000-2017 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef _PTHREAD_PRIORITY_PRIVATE_H_
#define _PTHREAD_PRIORITY_PRIVATE_H_

/*!
 * @typedef pthread_priority_t
 *
 * @abstract
 * pthread_priority_t is an on opaque integer that is guaranteed to be ordered
 * such that combations of QoS classes and relative priorities are ordered
 * numerically, according to their combined priority.
 *
 * <b>xnu, pthread & libdispatch flags</b>
 *
 * @const _PTHREAD_PRIORITY_OVERCOMMIT_FLAG
 * The thread this priority is applied to is overcommit (affects the workqueue
 * creation policy for this priority).
 *
 * @const _PTHREAD_PRIORITY_FALLBACK_FLAG
 * Indicates that this priority is is used only when incoming events have no
 * priority at all. It is merely used as a fallback (hence the name) instead of
 * a floor.
 *
 * This is usually used with QOS_CLASS_DEFAULT and a 0 relative priority.
 *
 * @const _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG
 * The event manager flag indicates that this thread/request is for a event
 * manager thread.  There can only ever be one event manager thread at a time
 * and it is brought up at the highest of all event manager priorities pthread
 * knows about.
 *
 * <b>pthread & dispatch only flags</b>
 *
 * @const _PTHREAD_PRIORITY_SCHED_PRI_FLAG
 * @const _PTHREAD_PRIORITY_SCHED_PRI_MASK
 * This flag indicates that the bits extracted using
 * _PTHREAD_PRIORITY_SCHED_PRI_MASK represent a scheduler priority instead of
 * a {qos, relative priority} pair.
 *
 * This flag is only used by the pthread kext to indicate libdispatch that the
 * event manager queue priority is a scheduling priority and not a QoS. This
 * flag is never used as an input by anything else and is why it can perform
 * a double duty with _PTHREAD_PRIORITY_ROOTQUEUE_FLAG.
 *
 * @const _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG
 * This flag is used for the priority of event delivery threads to indicate
 * to libdispatch that this thread is bound to a kqueue.
 *
 * <b>dispatch only flags</b>
 *
 * @const _PTHREAD_PRIORITY_INHERIT_FLAG
 * This flag is meaningful to libdispatch only and has no meanting for the
 * kernel and/or pthread.
 *
 * @const _PTHREAD_PRIORITY_ROOTQUEUE_FLAG
 * This flag is meaningful to libdispatch only and has no meanting for the
 * kernel and/or pthread.
 *
 * @const _PTHREAD_PRIORITY_ENFORCE_FLAG
 * This flag is used to indicate that this priority should be prefered for work
 * submited asynchronously over the intrinsic priority of the queue/thread the
 * work is submitted to.
 *
 * @const _PTHREAD_PRIORITY_OVERRIDE_FLAG
 * No longer used
 */
typedef unsigned long pthread_priority_t;

#define _PTHREAD_PRIORITY_FLAGS_MASK                    0xff000000u
#define _PTHREAD_PRIORITY_FLAGS_SHIFT                   (24ull)

#define _PTHREAD_PRIORITY_OVERCOMMIT_FLAG               0x80000000u
#define _PTHREAD_PRIORITY_INHERIT_FLAG                  0x40000000u /* dispatch only */
#define _PTHREAD_PRIORITY_ROOTQUEUE_FLAG                0x20000000u /* dispatch only */
#define _PTHREAD_PRIORITY_SCHED_PRI_FLAG                0x20000000u
#define _PTHREAD_PRIORITY_SCHED_PRI_MASK                0x0000ffffu
#define _PTHREAD_PRIORITY_ENFORCE_FLAG                  0x10000000u /* dispatch only */
#define _PTHREAD_PRIORITY_OVERRIDE_FLAG                 0x08000000u /* unused */
#define _PTHREAD_PRIORITY_FALLBACK_FLAG                 0x04000000u
#define _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG            0x02000000u
#define _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG             0x01000000u
#define _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG             _PTHREAD_PRIORITY_FALLBACK_FLAG // compat

#define _PTHREAD_PRIORITY_ENCODING_MASK                 0x00a00000u
#define _PTHREAD_PRIORITY_ENCODING_SHIFT                (22ull)
#define _PTHREAD_PRIORITY_ENCODING_V0                   0x00000000u
#define _PTHREAD_PRIORITY_ENCODING_V1                   0x00400000u /* unused */
#define _PTHREAD_PRIORITY_ENCODING_V2                   0x00800000u /* unused */
#define _PTHREAD_PRIORITY_ENCODING_V3                   0x00a00000u /* unused */

#define _PTHREAD_PRIORITY_QOS_CLASS_MASK                0x003fff00u
#define _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK          0x00003f00u
#define _PTHREAD_PRIORITY_QOS_CLASS_SHIFT               (8ull)

#define _PTHREAD_PRIORITY_PRIORITY_MASK                 0x000000ffu
#define _PTHREAD_PRIORITY_PRIORITY_SHIFT                (0)

#if PRIVATE
#if XNU_KERNEL_PRIVATE && !defined(__PTHREAD_EXPOSE_INTERNALS__)
#define __PTHREAD_EXPOSE_INTERNALS__ 1
#endif // XNU_KERNEL_PRIVATE
#ifdef __PTHREAD_EXPOSE_INTERNALS__
/*
 * This exposes the encoding used for pthread_priority_t
 * and is meant to be used by pthread and XNU only
 */
#include <mach/thread_policy.h> // THREAD_QOS_*
#include <stdbool.h>

// pthread_priority_t's type is unfortunately 64bits on LP64
// so we use this type for people who need to store it in structs
typedef unsigned int pthread_priority_compact_t;

__attribute__((always_inline, const))
static inline bool
_pthread_priority_has_qos(pthread_priority_t pp)
{
	return (pp & (_PTHREAD_PRIORITY_SCHED_PRI_FLAG |
	       _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) == 0 &&
	       (pp & (_PTHREAD_PRIORITY_QOS_CLASS_MASK &
	       ~_PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK)) == 0 &&
	       (pp & _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK) != 0;
}

__attribute__((always_inline, const))
static inline pthread_priority_compact_t
_pthread_priority_make_from_thread_qos(thread_qos_t qos, int relpri,
    unsigned long flags)
{
	pthread_priority_compact_t pp = (flags & _PTHREAD_PRIORITY_FLAGS_MASK);
	if (qos && qos < THREAD_QOS_LAST) {
		pp |= (1 << (_PTHREAD_PRIORITY_QOS_CLASS_SHIFT + qos - 1));
		pp |= ((uint8_t)relpri - 1) & _PTHREAD_PRIORITY_PRIORITY_MASK;
	}
	return pp;
}

__attribute__((always_inline, const))
static inline pthread_priority_compact_t
_pthread_event_manager_priority(void)
{
	return _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG;
}

__attribute__((always_inline, const))
static inline pthread_priority_compact_t
_pthread_unspecified_priority(void)
{
	return _pthread_priority_make_from_thread_qos(THREAD_QOS_UNSPECIFIED, 0, 0);
}

__attribute__((always_inline, const))
static inline pthread_priority_compact_t
_pthread_default_priority(unsigned long flags)
{
	return _pthread_priority_make_from_thread_qos(THREAD_QOS_LEGACY, 0, flags);
}

__attribute__((always_inline, const))
static inline thread_qos_t
_pthread_priority_thread_qos_fast(pthread_priority_t pp)
{
	pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
	pp >>= _PTHREAD_PRIORITY_QOS_CLASS_SHIFT;
	return (thread_qos_t)__builtin_ffs((int)pp);
}

__attribute__((always_inline, const))
static inline thread_qos_t
_pthread_priority_thread_qos(pthread_priority_t pp)
{
	if (_pthread_priority_has_qos(pp)) {
		return _pthread_priority_thread_qos_fast(pp);
	}
	return THREAD_QOS_UNSPECIFIED;
}

__attribute__((always_inline, const))
static inline int
_pthread_priority_relpri(pthread_priority_t pp)
{
	if (_pthread_priority_has_qos(pp)) {
		pp &= _PTHREAD_PRIORITY_PRIORITY_MASK;
		pp >>= _PTHREAD_PRIORITY_PRIORITY_SHIFT;
		return (int8_t)pp + 1;
	}
	return 0;
}

#if KERNEL
// Interfaces only used by the kernel and not implemented in userspace.

/*
 * Keep managerness, overcomitness and fallback, discard other flags.
 * Normalize and validate QoS/relpri
 */
__attribute__((const))
pthread_priority_compact_t
_pthread_priority_normalize(pthread_priority_t pp);

/*
 * Keep managerness, discard other flags.
 * Normalize and validate QoS/relpri
 */
__attribute__((const))
pthread_priority_compact_t
_pthread_priority_normalize_for_ipc(pthread_priority_t pp);

/*
 * Keep the flags from base_pp and return the priority with the maximum priority
 * of base_pp and _pthread_priority_make_from_thread_qos(qos, 0, 0)
 */
__attribute__((const))
pthread_priority_compact_t
_pthread_priority_combine(pthread_priority_t base_pp, thread_qos_t qos);

#endif // KERNEL
#endif // __PTHREAD_EXPOSE_INTERNALS__
#endif // PRIVATE
#endif // _PTHREAD_PRIORITY_PRIVATE_H_
Commit	Line	Data
d9a64523 A	1	/*
	2	* Copyright (c) 2000-2017 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#ifndef _PTHREAD_PRIORITY_PRIVATE_H_
	30	#define _PTHREAD_PRIORITY_PRIVATE_H_
	31
	32	/*!
	33	* @typedef pthread_priority_t
	34	*
	35	* @abstract
	36	* pthread_priority_t is an on opaque integer that is guaranteed to be ordered
	37	* such that combations of QoS classes and relative priorities are ordered
	38	* numerically, according to their combined priority.
	39	*
	40	* <b>xnu, pthread & libdispatch flags</b>
	41	*
	42	* @const _PTHREAD_PRIORITY_OVERCOMMIT_FLAG
	43	* The thread this priority is applied to is overcommit (affects the workqueue
	44	* creation policy for this priority).
	45	*
	46	* @const _PTHREAD_PRIORITY_FALLBACK_FLAG
	47	* Indicates that this priority is is used only when incoming events have no
	48	* priority at all. It is merely used as a fallback (hence the name) instead of
	49	* a floor.
	50	*
	51	* This is usually used with QOS_CLASS_DEFAULT and a 0 relative priority.
	52	*
	53	* @const _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG
	54	* The event manager flag indicates that this thread/request is for a event
	55	* manager thread. There can only ever be one event manager thread at a time
	56	* and it is brought up at the highest of all event manager priorities pthread
	57	* knows about.
	58	*
	59	* <b>pthread & dispatch only flags</b>
	60	*
	61	* @const _PTHREAD_PRIORITY_SCHED_PRI_FLAG
	62	* @const _PTHREAD_PRIORITY_SCHED_PRI_MASK
	63	* This flag indicates that the bits extracted using
	64	* _PTHREAD_PRIORITY_SCHED_PRI_MASK represent a scheduler priority instead of
65	* a {qos, relative priority} pair.
66	*
67	* This flag is only used by the pthread kext to indicate libdispatch that the
68	* event manager queue priority is a scheduling priority and not a QoS. This
69	* flag is never used as an input by anything else and is why it can perform
70	* a double duty with _PTHREAD_PRIORITY_ROOTQUEUE_FLAG.
71	*
72	* @const _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG
73	* This flag is used for the priority of event delivery threads to indicate
74	* to libdispatch that this thread is bound to a kqueue.
75	*
76	* <b>dispatch only flags</b>
77	*
78	* @const _PTHREAD_PRIORITY_INHERIT_FLAG
79	* This flag is meaningful to libdispatch only and has no meanting for the
80	* kernel and/or pthread.
81	*
82	* @const _PTHREAD_PRIORITY_ROOTQUEUE_FLAG
83	* This flag is meaningful to libdispatch only and has no meanting for the
84	* kernel and/or pthread.
85	*
86	* @const _PTHREAD_PRIORITY_ENFORCE_FLAG
87	* This flag is used to indicate that this priority should be prefered for work
88	* submited asynchronously over the intrinsic priority of the queue/thread the
89	* work is submitted to.
90	*
91	* @const _PTHREAD_PRIORITY_OVERRIDE_FLAG
92	* No longer used
93	*/
94	typedef unsigned long pthread_priority_t;
95
f427ee49	96	#define _PTHREAD_PRIORITY_FLAGS_MASK 0xff000000u
0a7de745	97	#define _PTHREAD_PRIORITY_FLAGS_SHIFT (24ull)
d9a64523	98
f427ee49 A	99	#define _PTHREAD_PRIORITY_OVERCOMMIT_FLAG 0x80000000u
	100	#define _PTHREAD_PRIORITY_INHERIT_FLAG 0x40000000u /* dispatch only */
	101	#define _PTHREAD_PRIORITY_ROOTQUEUE_FLAG 0x20000000u /* dispatch only */
	102	#define _PTHREAD_PRIORITY_SCHED_PRI_FLAG 0x20000000u
	103	#define _PTHREAD_PRIORITY_SCHED_PRI_MASK 0x0000ffffu
	104	#define _PTHREAD_PRIORITY_ENFORCE_FLAG 0x10000000u /* dispatch only */
	105	#define _PTHREAD_PRIORITY_OVERRIDE_FLAG 0x08000000u /* unused */
	106	#define _PTHREAD_PRIORITY_FALLBACK_FLAG 0x04000000u
	107	#define _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG 0x02000000u
	108	#define _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG 0x01000000u
0a7de745	109	#define _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG _PTHREAD_PRIORITY_FALLBACK_FLAG // compat
d9a64523	110
f427ee49	111	#define _PTHREAD_PRIORITY_ENCODING_MASK 0x00a00000u
0a7de745	112	#define _PTHREAD_PRIORITY_ENCODING_SHIFT (22ull)
f427ee49 A	113	#define _PTHREAD_PRIORITY_ENCODING_V0 0x00000000u
	114	#define _PTHREAD_PRIORITY_ENCODING_V1 0x00400000u /* unused */
	115	#define _PTHREAD_PRIORITY_ENCODING_V2 0x00800000u /* unused */
	116	#define _PTHREAD_PRIORITY_ENCODING_V3 0x00a00000u /* unused */
d9a64523	117
f427ee49 A	118	#define _PTHREAD_PRIORITY_QOS_CLASS_MASK 0x003fff00u
f427ee49 A	119	#define _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK 0x00003f00u
0a7de745	120	#define _PTHREAD_PRIORITY_QOS_CLASS_SHIFT (8ull)
d9a64523	121
f427ee49	122	#define _PTHREAD_PRIORITY_PRIORITY_MASK 0x000000ffu
0a7de745	123	#define _PTHREAD_PRIORITY_PRIORITY_SHIFT (0)
d9a64523 A	124
	125	#if PRIVATE
	126	#if XNU_KERNEL_PRIVATE && !defined(__PTHREAD_EXPOSE_INTERNALS__)
	127	#define __PTHREAD_EXPOSE_INTERNALS__ 1
	128	#endif // XNU_KERNEL_PRIVATE
	129	#ifdef __PTHREAD_EXPOSE_INTERNALS__
	130	/*
	131	* This exposes the encoding used for pthread_priority_t
	132	* and is meant to be used by pthread and XNU only
	133	*/
	134	#include <mach/thread_policy.h> // THREAD_QOS_*
	135	#include <stdbool.h>
	136
f427ee49 A	137	// pthread_priority_t's type is unfortunately 64bits on LP64
	138	// so we use this type for people who need to store it in structs
	139	typedef unsigned int pthread_priority_compact_t;
	140
d9a64523 A	141	__attribute__((always_inline, const))
	142	static inline bool
	143	_pthread_priority_has_qos(pthread_priority_t pp)
	144	{
	145	return (pp & (_PTHREAD_PRIORITY_SCHED_PRI_FLAG \|
0a7de745 A	146	_PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) == 0 &&
	147	(pp & (_PTHREAD_PRIORITY_QOS_CLASS_MASK &
	148	~_PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK)) == 0 &&
	149	(pp & _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK) != 0;
d9a64523 A	150	}
	151
	152	__attribute__((always_inline, const))
f427ee49	153	static inline pthread_priority_compact_t
d9a64523	154	_pthread_priority_make_from_thread_qos(thread_qos_t qos, int relpri,
0a7de745	155	unsigned long flags)
d9a64523	156	{
f427ee49	157	pthread_priority_compact_t pp = (flags & _PTHREAD_PRIORITY_FLAGS_MASK);
d9a64523 A	158	if (qos && qos < THREAD_QOS_LAST) {
	159	pp \|= (1 << (_PTHREAD_PRIORITY_QOS_CLASS_SHIFT + qos - 1));
	160	pp \|= ((uint8_t)relpri - 1) & _PTHREAD_PRIORITY_PRIORITY_MASK;
	161	}
	162	return pp;
	163	}
	164
	165	__attribute__((always_inline, const))
f427ee49	166	static inline pthread_priority_compact_t
d9a64523 A	167	_pthread_event_manager_priority(void)
	168	{
	169	return _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG;
	170	}
	171
	172	__attribute__((always_inline, const))
f427ee49	173	static inline pthread_priority_compact_t
d9a64523 A	174	_pthread_unspecified_priority(void)
	175	{
	176	return _pthread_priority_make_from_thread_qos(THREAD_QOS_UNSPECIFIED, 0, 0);
	177	}
	178
	179	__attribute__((always_inline, const))
f427ee49	180	static inline pthread_priority_compact_t
d9a64523 A	181	_pthread_default_priority(unsigned long flags)
	182	{
	183	return _pthread_priority_make_from_thread_qos(THREAD_QOS_LEGACY, 0, flags);
	184	}
	185
cb323159 A	186	__attribute__((always_inline, const))
	187	static inline thread_qos_t
	188	_pthread_priority_thread_qos_fast(pthread_priority_t pp)
	189	{
	190	pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
	191	pp >>= _PTHREAD_PRIORITY_QOS_CLASS_SHIFT;
	192	return (thread_qos_t)__builtin_ffs((int)pp);
	193	}
	194
d9a64523 A	195	__attribute__((always_inline, const))
	196	static inline thread_qos_t
	197	_pthread_priority_thread_qos(pthread_priority_t pp)
	198	{
	199	if (_pthread_priority_has_qos(pp)) {
cb323159	200	return _pthread_priority_thread_qos_fast(pp);
d9a64523 A	201	}
	202	return THREAD_QOS_UNSPECIFIED;
	203	}
	204
	205	__attribute__((always_inline, const))
	206	static inline int
	207	_pthread_priority_relpri(pthread_priority_t pp)
	208	{
	209	if (_pthread_priority_has_qos(pp)) {
	210	pp &= _PTHREAD_PRIORITY_PRIORITY_MASK;
	211	pp >>= _PTHREAD_PRIORITY_PRIORITY_SHIFT;
	212	return (int8_t)pp + 1;
	213	}
	214	return 0;
	215	}
	216
	217	#if KERNEL
	218	// Interfaces only used by the kernel and not implemented in userspace.
	219
	220	/*
	221	* Keep managerness, overcomitness and fallback, discard other flags.
	222	* Normalize and validate QoS/relpri
	223	*/
	224	__attribute__((const))
f427ee49	225	pthread_priority_compact_t
d9a64523 A	226	_pthread_priority_normalize(pthread_priority_t pp);
	227
	228	/*
	229	* Keep managerness, discard other flags.
	230	* Normalize and validate QoS/relpri
	231	*/
	232	__attribute__((const))
f427ee49	233	pthread_priority_compact_t
d9a64523 A	234	_pthread_priority_normalize_for_ipc(pthread_priority_t pp);
	235
	236	/*
	237	* Keep the flags from base_pp and return the priority with the maximum priority
	238	* of base_pp and _pthread_priority_make_from_thread_qos(qos, 0, 0)
	239	*/
	240	__attribute__((const))
f427ee49	241	pthread_priority_compact_t
d9a64523 A	242	_pthread_priority_combine(pthread_priority_t base_pp, thread_qos_t qos);
	243
	244	#endif // KERNEL
	245	#endif // __PTHREAD_EXPOSE_INTERNALS__
	246	#endif // PRIVATE
	247	#endif // _PTHREAD_PRIORITY_PRIVATE_H_