git.saurik.com Git - apple/xnu.git/blob

1 /*

3 *

4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

5 *

6 * This file contains Original Code and/or Modifications of Original Code

7 * as defined in and that are subject to the Apple Public Source License

8 * Version 2.0 (the 'License'). You may not use this file except in

9 * compliance with the License. The rights granted to you under the License

10 * may not be used to create, or enable the creation or redistribution of,

11 * unlawful or unlicensed copies of an Apple operating system, or to

12 * circumvent, violate, or enable the circumvention or violation of, any

13 * terms of an Apple operating system software license agreement.

14 *

15 * Please obtain a copy of the License at

16 * http://www.opensource.apple.com/apsl/ and read it before using this file.

17 *

18 * The Original Code and all software distributed under the License are

19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,

21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.

23 * Please see the License for the specific language governing rights and

24 * limitations under the License.

25 *

26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

27 */

28 /*

29 * @OSF_COPYRIGHT@

30 */

31 /*

32 * Mach Operating System

35 *

36 * Permission to use, copy, modify and distribute this software and its

37 * documentation is hereby granted, provided that both the copyright

38 * notice and this permission notice appear in all copies of the

39 * software, derivative works or modified versions, and any portions

40 * thereof, and that both notices appear in supporting documentation.

41 *

42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"

43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR

44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

45 *

46 * Carnegie Mellon requests users of this software to return to

47 *

48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU

49 * School of Computer Science

50 * Carnegie Mellon University

51 * Pittsburgh PA 15213-3890

52 *

53 * any improvements or extensions that they make and grant Carnegie Mellon

54 * the rights to redistribute these changes.

55 */

56 /*

57 */

59 /*

60 * processor.h: Processor and processor-related definitions.

61 */

63 #ifndef _KERN_PROCESSOR_H_

64 #define _KERN_PROCESSOR_H_

66 #include <mach/boolean.h>

67 #include <mach/kern_return.h>

68 #include <kern/kern_types.h>

70 #include <sys/cdefs.h>

72 #ifdef MACH_KERNEL_PRIVATE

74 #include <mach/mach_types.h>

75 #include <kern/ast.h>

76 #include <kern/cpu_number.h>

77 #include <kern/smp.h>

78 #include <kern/simple_lock.h>

79 #include <kern/locks.h>

80 #include <kern/percpu.h>

81 #include <kern/queue.h>

82 #include <kern/sched.h>

83 #include <kern/sched_urgency.h>

84 #include <kern/timer.h>

85 #include <mach/sfi_class.h>

86 #include <kern/sched_clutch.h>

87 #include <kern/timer_call.h>

88 #include <kern/assert.h>

89 #include <machine/limits.h>

91 /*

92 * Processor state is accessed by locking the scheduling lock

93 * for the assigned processor set.

94 *

95 * -------------------- SHUTDOWN

96 * / ^ ^

97 * _/ | \

98 * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING

99 * \_________________^ ^ ^______/ /

100 * \__________________/

101 *

102 * Most of these state transitions are externally driven as a

103 * a directive (for instance telling an IDLE processor to start

104 * coming out of the idle state to run a thread). However these

105 * are typically paired with a handshake by the processor itself

106 * to indicate that it has completed a transition of indeterminate

107 * length (for example, the DISPATCHING->RUNNING or START->RUNNING

108 * transitions must occur on the processor itself).

109 *

110 * The boot processor has some special cases, and skips the START state,

111 * since it has already bootstrapped and is ready to context switch threads.

112 *

113 * When a processor is in DISPATCHING or RUNNING state, the current_pri,

114 * current_thmode, and deadline fields should be set, so that other

115 * processors can evaluate if it is an appropriate candidate for preemption.

116 */

117 #if defined(CONFIG_SCHED_DEFERRED_AST)

118 /*

119 * -------------------- SHUTDOWN

120 * / ^ ^

121 * _/ | \

122 * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING

123 * \_________________^ ^ ^______/ ^_____ / /

124 * \__________________/

125 *

126 * A DISPATCHING processor may be put back into IDLE, if another

127 * processor determines that the target processor will have nothing to do

128 * upon reaching the RUNNING state. This is racy, but if the target

129 * responds and becomes RUNNING, it will not break the processor state

130 * machine.

131 *

132 * This change allows us to cancel an outstanding signal/AST on a processor

133 * (if such an operation is supported through hardware or software), and

134 * push the processor back into the IDLE state as a power optimization.

135 */

136 #endif

137

138 typedef enum {

139 PROCESSOR_OFF_LINE = 0, /* Not available */

140 PROCESSOR_SHUTDOWN = 1, /* Going off-line */

141 PROCESSOR_START = 2, /* Being started */

142 PROCESSOR_UNUSED = 3, /* Formerly Inactive (unavailable) */

143 PROCESSOR_IDLE = 4, /* Idle (available) */

144 PROCESSOR_DISPATCHING = 5, /* Dispatching (idle -> active) */

145 PROCESSOR_RUNNING = 6, /* Normal execution */

146 PROCESSOR_STATE_LEN = (PROCESSOR_RUNNING + 1)

147 } processor_state_t;

148

149 typedef enum {

150 PSET_SMP,

151 #if __AMP__

152 PSET_AMP_E,

153 PSET_AMP_P,

154 #endif

155 } pset_cluster_type_t;

156

157 #if __AMP__

158

159 typedef enum {

160 SCHED_PERFCTL_POLICY_DEFAULT, /* static policy: set at boot */

161 SCHED_PERFCTL_POLICY_FOLLOW_GROUP, /* dynamic policy: perfctl_class follows thread group across amp clusters */

162 SCHED_PERFCTL_POLICY_RESTRICT_E, /* dynamic policy: limits perfctl_class to amp e cluster */

163 } sched_perfctl_class_policy_t;

164

165 extern _Atomic sched_perfctl_class_policy_t sched_perfctl_policy_util;

166 extern _Atomic sched_perfctl_class_policy_t sched_perfctl_policy_bg;

167

168 #endif /* __AMP__ */

169

170 typedef bitmap_t cpumap_t;

171

172 #if __arm64__

173

174 /*

175 * pset_execution_time_t

176 *

177 * The pset_execution_time_t type is used to maintain the average

178 * execution time of threads on a pset. Since the avg. execution time is

179 * updated from contexts where the pset lock is not held, it uses a

180 * double-wide RMW loop to update these values atomically.

181 */

182 typedef union {

183 struct {

184 uint64_t pset_avg_thread_execution_time;

185 uint64_t pset_execution_time_last_update;

186 };

187 unsigned __int128 pset_execution_time_packed;

188 } pset_execution_time_t;

189

190 #endif /* __arm64__ */

191

192 struct processor_set {

193 int pset_id;

194 int online_processor_count;

195 int cpu_set_low, cpu_set_hi;

196 int cpu_set_count;

197 int last_chosen;

198

199 uint64_t load_average;

200 uint64_t pset_load_average[TH_BUCKET_SCHED_MAX];

201 uint64_t pset_load_last_update;

202 cpumap_t cpu_bitmask;

203 cpumap_t recommended_bitmask;

204 cpumap_t cpu_state_map[PROCESSOR_STATE_LEN];

205 cpumap_t primary_map;

206 cpumap_t realtime_map;

207 cpumap_t cpu_running_foreign;

208 sched_bucket_t cpu_running_buckets[MAX_CPUS];

209

210 #define SCHED_PSET_TLOCK (1)

211 #if defined(SCHED_PSET_TLOCK)

212 /* TODO: reorder struct for temporal cache locality */

         __attribute__((aligned(128))) lck_ticket_t      sched_lock;

214 #else /* SCHED_PSET_TLOCK*/

         __attribute__((aligned(128))) lck_spin_t        sched_lock;     /* lock for above */

216 #endif /* SCHED_PSET_TLOCK*/

217

218 #if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ)

219 struct run_queue pset_runq; /* runq for this processor set */

220 #endif

221 struct rt_queue rt_runq; /* realtime runq for this processor set */

222 #if CONFIG_SCHED_CLUTCH

223 struct sched_clutch_root pset_clutch_root; /* clutch hierarchy root */

224 #endif /* CONFIG_SCHED_CLUTCH */

225

226 #if defined(CONFIG_SCHED_TRADITIONAL)

227 int pset_runq_bound_count;

228 /* # of threads in runq bound to any processor in pset */

229 #endif

230

231 /* CPUs that have been sent an unacknowledged remote AST for scheduling purposes */

232 cpumap_t pending_AST_URGENT_cpu_mask;

233 cpumap_t pending_AST_PREEMPT_cpu_mask;

234 #if defined(CONFIG_SCHED_DEFERRED_AST)

235 /*

236 * A separate mask, for ASTs that we may be able to cancel. This is dependent on

237 * some level of support for requesting an AST on a processor, and then quashing

238 * that request later.

239 *

240 * The purpose of this field (and the associated codepaths) is to infer when we

241 * no longer need a processor that is DISPATCHING to come up, and to prevent it

242 * from coming out of IDLE if possible. This should serve to decrease the number

243 * of spurious ASTs in the system, and let processors spend longer periods in

244 * IDLE.

245 */

246 cpumap_t pending_deferred_AST_cpu_mask;

247 #endif

248 cpumap_t pending_spill_cpu_mask;

249

250 struct ipc_port * pset_self; /* port for operations */

251 struct ipc_port * pset_name_self; /* port for information */

252

253 processor_set_t pset_list; /* chain of associated psets */

254 pset_node_t node;

255 uint32_t pset_cluster_id;

256

257 /*

258 * Currently the scheduler uses a mix of pset_cluster_type_t & cluster_type_t

259 * for recommendations etc. It might be useful to unify these as a single type.

260 */

261 pset_cluster_type_t pset_cluster_type;

262 cluster_type_t pset_type;

263

264 #if CONFIG_SCHED_EDGE

265 bitmap_t foreign_psets[BITMAP_LEN(MAX_PSETS)];

266 sched_clutch_edge sched_edges[MAX_PSETS];

267 pset_execution_time_t pset_execution_time[TH_BUCKET_SCHED_MAX];

268 #endif /* CONFIG_SCHED_EDGE */

269 bool is_SMT; /* pset contains SMT processors */

270 };

271

272 extern struct processor_set pset0;

273

274 typedef bitmap_t pset_map_t;

275

276 struct pset_node {

277 processor_set_t psets; /* list of associated psets */

278

279 pset_node_t nodes; /* list of associated subnodes */

280 pset_node_t node_list; /* chain of associated nodes */

281

282 pset_node_t parent;

283

284 pset_map_t pset_map; /* map of associated psets */

285 _Atomic pset_map_t pset_idle_map; /* psets with at least one IDLE CPU */

286 _Atomic pset_map_t pset_idle_primary_map; /* psets with at least one IDLE primary CPU */

287 _Atomic pset_map_t pset_non_rt_map; /* psets with at least one available CPU not running a realtime thread */

288 _Atomic pset_map_t pset_non_rt_primary_map;/* psets with at least one available primary CPU not running a realtime thread */

289 };

290

291 extern struct pset_node pset_node0;

292

293 extern queue_head_t tasks, threads, corpse_tasks;

 extern int tasks_count, terminated_tasks_count, threads_count, terminated_threads_count;

 decl_lck_mtx_data(extern, tasks_threads_lock);

 decl_lck_mtx_data(extern, tasks_corpse_lock);

297

298 /*

299 * The terminated tasks queue should only be inspected elsewhere by stackshot.

300 */

301 extern queue_head_t terminated_tasks;

302

303 extern queue_head_t terminated_threads;

304

305 struct processor {

306 processor_state_t state; /* See above */

307 bool is_SMT;

308 bool is_recommended;

309 bool current_is_NO_SMT; /* cached TH_SFLAG_NO_SMT of current thread */

310 bool current_is_bound; /* current thread is bound to this processor */

311 bool current_is_eagerpreempt;/* current thread is TH_SFLAG_EAGERPREEMPT */

312 struct thread *active_thread; /* thread running on processor */

313 struct thread *idle_thread; /* this processor's idle thread. */

314 struct thread *startup_thread;

315

316 processor_set_t processor_set; /* assigned set */

317

318 /*

319 * XXX All current_* fields should be grouped together, as they're

320 * updated at the same time.

321 */

322 int current_pri; /* priority of current thread */

323 sfi_class_id_t current_sfi_class; /* SFI class of current thread */

324 perfcontrol_class_t current_perfctl_class; /* Perfcontrol class for current thread */

325 /*

326 * The cluster type recommended for the current thread.

327 */

328 pset_cluster_type_t current_recommended_pset_type;

329 thread_urgency_t current_urgency; /* cached urgency of current thread */

330

331 #if CONFIG_SCHED_TRADITIONAL

332 int runq_bound_count; /* # of threads bound to this processor */

333 #endif /* CONFIG_SCHED_TRADITIONAL */

334

335 #if CONFIG_THREAD_GROUPS

336 struct thread_group *current_thread_group; /* thread_group of current thread */

337 #endif

338 int starting_pri; /* priority of current thread as it was when scheduled */

339 int cpu_id; /* platform numeric id */

340

341 uint64_t quantum_end; /* time when current quantum ends */

342 uint64_t last_dispatch; /* time of last dispatch */

343

344 #if KPERF

345 uint64_t kperf_last_sample_time; /* time of last kperf sample */

346 #endif /* KPERF */

347

348 uint64_t deadline; /* for next realtime thread */

349 bool first_timeslice; /* has the quantum expired since context switch */

350

351 bool processor_offlined; /* has the processor been explicitly processor_offline'ed */

352 bool must_idle; /* Needs to be forced idle as next selected thread is allowed on this processor */

353

354 bool running_timers_active; /* whether the running timers should fire */

355 struct timer_call running_timers[RUNNING_TIMER_MAX];

356

357 #if CONFIG_SCHED_TRADITIONAL || CONFIG_SCHED_MULTIQ

358 struct run_queue runq; /* runq for this processor */

359 #endif /* CONFIG_SCHED_TRADITIONAL || CONFIG_SCHED_MULTIQ */

360

361 #if CONFIG_SCHED_GRRR

362 struct grrr_run_queue grrr_runq; /* Group Ratio Round-Robin runq */

363 #endif /* CONFIG_SCHED_GRRR */

364

365 /*

366 * Pointer to primary processor for secondary SMT processors, or a

367 * pointer to ourselves for primaries or non-SMT.

368 */

369 processor_t processor_primary;

370 processor_t processor_secondary;

371 struct ipc_port *processor_self; /* port for operations */

372

373 processor_t processor_list; /* all existing processors */

374

375 /* Processor state statistics */

376 timer_data_t idle_state;

377 timer_data_t system_state;

378 timer_data_t user_state;

379

380 timer_t current_state; /* points to processor's idle, system, or user state timer */

381

382 /* Thread execution timers */

383 timer_t thread_timer; /* points to current thread's user or system timer */

384 timer_t kernel_timer; /* points to current thread's system_timer */

385

386 uint64_t timer_call_ttd; /* current timer call time-to-deadline */

387 };

388

389 extern processor_t processor_list;

 decl_simple_lock_data(extern, processor_list_lock);

391

392 /*

393 * Maximum number of CPUs supported by the scheduler. bits.h bitmap macros

394 * need to be used to support greater than 64.

395 */

396 #define MAX_SCHED_CPUS 64

397 extern processor_t processor_array[MAX_SCHED_CPUS]; /* array indexed by cpuid */

398 extern processor_set_t pset_array[MAX_PSETS]; /* array indexed by pset_id */

399

400 extern uint32_t processor_avail_count;

401 extern uint32_t processor_avail_count_user;

402 extern uint32_t primary_processor_avail_count;

403 extern uint32_t primary_processor_avail_count_user;

404

405 #define master_processor PERCPU_GET_MASTER(processor)

 PERCPU_DECL(struct processor, processor);

407

 extern processor_t      current_processor(void);

409

410 /* Lock macros, always acquired and released with interrupts disabled (splsched()) */

411

412 extern lck_grp_t pset_lck_grp;

413

414 #if defined(SCHED_PSET_TLOCK)

415 #define pset_lock_init(p) lck_ticket_init(&(p)->sched_lock, &pset_lck_grp)

416 #define pset_lock(p) lck_ticket_lock(&(p)->sched_lock, &pset_lck_grp)

417 #define pset_unlock(p) lck_ticket_unlock(&(p)->sched_lock)

418 #define pset_assert_locked(p) lck_ticket_assert_owned(&(p)->sched_lock)

419 #else /* SCHED_PSET_TLOCK*/

420 #define pset_lock_init(p) lck_spin_init(&(p)->sched_lock, &pset_lck_grp, NULL)

421 #define pset_lock(p) lck_spin_lock_grp(&(p)->sched_lock, &pset_lck_grp)

422 #define pset_unlock(p) lck_spin_unlock(&(p)->sched_lock)

423 #define pset_assert_locked(p) LCK_SPIN_ASSERT(&(p)->sched_lock, LCK_ASSERT_OWNED)

424 #endif /*!SCHED_PSET_TLOCK*/

425

 extern void             processor_bootstrap(void);

427

428 extern void processor_init(

429 processor_t processor,

430 int cpu_id,

431 processor_set_t processor_set);

432

433 extern void processor_set_primary(

434 processor_t processor,

435 processor_t primary);

436

437 extern kern_return_t processor_shutdown(

438 processor_t processor);

439

440 extern kern_return_t processor_start_from_user(

441 processor_t processor);

442 extern kern_return_t processor_exit_from_user(

443 processor_t processor);

444

445 extern kern_return_t sched_processor_enable(

446 processor_t processor,

447 boolean_t enable);

448

449 extern void processor_queue_shutdown(

450 processor_t processor);

451

452 extern void processor_queue_shutdown(

453 processor_t processor);

454

455 extern processor_set_t processor_pset(

456 processor_t processor);

457

 extern pset_node_t      pset_node_root(void);

459

460 extern processor_set_t pset_create(

461 pset_node_t node);

462

463 extern void pset_init(

464 processor_set_t pset,

465 pset_node_t node);

466

467 extern processor_set_t pset_find(

468 uint32_t cluster_id,

469 processor_set_t default_pset);

470

471 #if !defined(RC_HIDE_XNU_FIRESTORM) && (MAX_CPU_CLUSTERS > 2)

472

473 /*

474 * Find the first processor_set for the given pset_cluster_type.

475 * Should be removed with rdar://57340304, as it's only

476 * useful for the workaround described in rdar://57306691.

477 */

478

479 extern processor_set_t pset_find_first_by_cluster_type(

480 pset_cluster_type_t pset_cluster_type);

481

482 #endif /* !defined(RC_HIDE_XNU_FIRESTORM) && (MAX_CPU_CLUSTERS > 2) */

483

484 extern kern_return_t processor_info_count(

485 processor_flavor_t flavor,

486 mach_msg_type_number_t *count);

487

488 #define pset_deallocate(x)

489 #define pset_reference(x)

490

491 extern void machine_run_count(

492 uint32_t count);

493

494 extern processor_t machine_choose_processor(

495 processor_set_t pset,

496 processor_t processor);

497

498 #define next_pset(p) (((p)->pset_list != PROCESSOR_SET_NULL)? (p)->pset_list: (p)->node->psets)

499

500 #define PSET_THING_TASK 0

501 #define PSET_THING_THREAD 1

502

503 extern pset_cluster_type_t recommended_pset_type(

504 thread_t thread);

505 #if CONFIG_THREAD_GROUPS

506 extern pset_cluster_type_t thread_group_pset_recommendation(

507 struct thread_group *tg,

508 cluster_type_t recommendation);

509 #endif /* CONFIG_THREAD_GROUPS */

510

511 inline static bool

512 pset_is_recommended(processor_set_t pset)

513 {

         return (pset->recommended_bitmask & pset->cpu_bitmask) != 0;

515 }

516

517 extern void processor_state_update_idle(

518 processor_t processor);

519

520 extern void processor_state_update_from_thread(

521 processor_t processor,

522 thread_t thread);

523

524 extern void processor_state_update_explicit(

525 processor_t processor,

526 int pri,

527 sfi_class_id_t sfi_class,

528 pset_cluster_type_t pset_type,

529 perfcontrol_class_t perfctl_class,

530 thread_urgency_t urgency,

531 sched_bucket_t bucket);

532

533 #define PSET_LOAD_NUMERATOR_SHIFT 16

534 #define PSET_LOAD_FRACTIONAL_SHIFT 4

535

536 #if CONFIG_SCHED_EDGE

537

 extern cluster_type_t pset_type_for_id(uint32_t cluster_id);

539

540 /*

541 * The Edge scheduler uses average scheduling latency as the metric for making

542 * thread migration decisions. One component of avg scheduling latency is the load

543 * average on the cluster.

544 *

545 * Load Average Fixed Point Arithmetic

546 *

547 * The load average is maintained as a 24.8 fixed point arithmetic value for precision.

548 * When multiplied by the average execution time, it needs to be rounded up (based on

549 * the most significant bit of the fractional part) for better accuracy. After rounding

550 * up, the whole number part of the value is used as the actual load value for

551 * migrate/steal decisions.

552 */

553 #define SCHED_PSET_LOAD_EWMA_FRACTION_BITS 8

554 #define SCHED_PSET_LOAD_EWMA_ROUND_BIT (1 << (SCHED_PSET_LOAD_EWMA_FRACTION_BITS - 1))

555 #define SCHED_PSET_LOAD_EWMA_FRACTION_MASK ((1 << SCHED_PSET_LOAD_EWMA_FRACTION_BITS) - 1)

556

557 inline static int

558 sched_get_pset_load_average(processor_set_t pset, sched_bucket_t sched_bucket)

559 {

         return (int)(((pset->pset_load_average[sched_bucket] + SCHED_PSET_LOAD_EWMA_ROUND_BIT) >> SCHED_PSET_LOAD_EWMA_FRACTION_BITS) *

561 pset->pset_execution_time[sched_bucket].pset_avg_thread_execution_time);

562 }

563

564 #else /* CONFIG_SCHED_EDGE */

565 inline static int

566 sched_get_pset_load_average(processor_set_t pset, __unused sched_bucket_t sched_bucket)

567 {

         return (int)pset->load_average >> (PSET_LOAD_NUMERATOR_SHIFT - PSET_LOAD_FRACTIONAL_SHIFT);

569 }

570 #endif /* CONFIG_SCHED_EDGE */

571

 extern void sched_update_pset_load_average(processor_set_t pset, uint64_t curtime);

 extern void sched_update_pset_avg_execution_time(processor_set_t pset, uint64_t delta, uint64_t curtime, sched_bucket_t sched_bucket);

574

575 inline static void

 pset_update_processor_state(processor_set_t pset, processor_t processor, uint new_state)

577 {

578 pset_assert_locked(pset);

579

580 uint old_state = processor->state;

581 uint cpuid = (uint)processor->cpu_id;

582

         assert(processor->processor_set == pset);

         assert(bit_test(pset->cpu_bitmask, cpuid));

585

586 assert(old_state < PROCESSOR_STATE_LEN);

587 assert(new_state < PROCESSOR_STATE_LEN);

588

589 processor->state = new_state;

590

         bit_clear(pset->cpu_state_map[old_state], cpuid);

         bit_set(pset->cpu_state_map[new_state], cpuid);

593

         if ((old_state == PROCESSOR_RUNNING) || (new_state == PROCESSOR_RUNNING)) {

                 sched_update_pset_load_average(pset, 0);

596 if (new_state == PROCESSOR_RUNNING) {

                         assert(processor == current_processor());

598 }

599 }

         if ((old_state == PROCESSOR_IDLE) || (new_state == PROCESSOR_IDLE)) {

601 if (new_state == PROCESSOR_IDLE) {

                         bit_clear(pset->realtime_map, cpuid);

603 }

604

605 pset_node_t node = pset->node;

606

                 if (bit_count(node->pset_map) == 1) {

608 /* Node has only a single pset, so skip node pset map updates */

609 return;

610 }

611

612 if (new_state == PROCESSOR_IDLE) {

                         if (processor->processor_primary == processor) {

                                 if (!bit_test(atomic_load(&node->pset_non_rt_primary_map), pset->pset_id)) {

                                         atomic_bit_set(&node->pset_non_rt_primary_map, pset->pset_id, memory_order_relaxed);

616 }

                                 if (!bit_test(atomic_load(&node->pset_idle_primary_map), pset->pset_id)) {

                                         atomic_bit_set(&node->pset_idle_primary_map, pset->pset_id, memory_order_relaxed);

619 }

620 }

                         if (!bit_test(atomic_load(&node->pset_non_rt_map), pset->pset_id)) {

                                 atomic_bit_set(&node->pset_non_rt_map, pset->pset_id, memory_order_relaxed);

623 }

                         if (!bit_test(atomic_load(&node->pset_idle_map), pset->pset_id)) {

                                 atomic_bit_set(&node->pset_idle_map, pset->pset_id, memory_order_relaxed);

626 }

627 } else {

628 cpumap_t idle_map = pset->cpu_state_map[PROCESSOR_IDLE];

                         if (idle_map == 0) {

630 /* No more IDLE CPUs */

                                 if (bit_test(atomic_load(&node->pset_idle_map), pset->pset_id)) {

                                         atomic_bit_clear(&node->pset_idle_map, pset->pset_id, memory_order_relaxed);

633 }

634 }

                         if (processor->processor_primary == processor) {

636 idle_map &= pset->primary_map;

                                 if (idle_map == 0) {

638 /* No more IDLE primary CPUs */

                                         if (bit_test(atomic_load(&node->pset_idle_primary_map), pset->pset_id)) {

                                                 atomic_bit_clear(&node->pset_idle_primary_map, pset->pset_id, memory_order_relaxed);

641 }

642 }

643 }

644 }

645 }

646 }

647

648 #else /* MACH_KERNEL_PRIVATE */

649

650 __BEGIN_DECLS

651

652 extern void pset_deallocate(

653 processor_set_t pset);

654

655 extern void pset_reference(

656 processor_set_t pset);

657

658 __END_DECLS

659

660 #endif /* MACH_KERNEL_PRIVATE */

661

662 #ifdef KERNEL_PRIVATE

663 __BEGIN_DECLS

664 extern unsigned int processor_count;

 extern processor_t      cpu_to_processor(int cpu);

666

 extern kern_return_t    enable_smt_processors(bool enable);

668

669 __END_DECLS

670

671 #endif /* KERNEL_PRIVATE */

672

673 #endif /* _KERN_PROCESSOR_H_ */