]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (c) 2017 Apple Inc. All rights reserved. | |
3 | * | |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ | |
5 | * | |
6 | * This file contains Original Code and/or Modifications of Original Code | |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
14 | * | |
15 | * Please obtain a copy of the License at | |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
17 | * | |
18 | * The Original Code and all software distributed under the License are | |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, | |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. | |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
25 | * | |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ | |
27 | */ | |
28 | ||
29 | #include <kern/monotonic.h> | |
30 | #include <machine/machine_routines.h> | |
31 | #include <machine/monotonic.h> | |
32 | #include <pexpert/pexpert.h> | |
33 | #include <sys/param.h> /* NULL */ | |
34 | #include <sys/stat.h> /* dev_t */ | |
35 | #include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */ | |
36 | #include <sys/conf.h> /* must come after sys/stat.h */ | |
37 | #include <sys/sysctl.h> | |
38 | #include <sys/sysproto.h> | |
39 | #include <sys/systm.h> | |
40 | #include <sys/types.h> | |
41 | #include <sys/monotonic.h> | |
42 | ||
43 | static int mt_cdev_open(dev_t dev, int flags, int devtype, proc_t p); | |
44 | static int mt_cdev_close(dev_t dev, int flags, int devtype, proc_t p); | |
45 | static int mt_cdev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag, | |
46 | proc_t p); | |
47 | ||
48 | #define MT_NODE "monotonic" | |
49 | ||
50 | static const struct cdevsw mt_cdevsw = { | |
51 | .d_open = mt_cdev_open, | |
52 | .d_close = mt_cdev_close, | |
53 | .d_ioctl = mt_cdev_ioctl, | |
54 | ||
55 | .d_read = eno_rdwrt, .d_write = eno_rdwrt, .d_stop = eno_stop, | |
56 | .d_reset = eno_reset, .d_ttys = NULL, .d_select = eno_select, | |
57 | .d_mmap = eno_mmap, .d_strategy = eno_strat, .d_type = 0 | |
58 | }; | |
59 | ||
60 | /* | |
61 | * Written at initialization, read-only thereafter. | |
62 | */ | |
63 | lck_grp_t *mt_lock_grp = NULL; | |
64 | static int mt_dev_major; | |
65 | ||
66 | static mt_device_t | |
67 | mt_get_device(dev_t devnum) | |
68 | { | |
69 | return &mt_devices[minor(devnum)]; | |
70 | } | |
71 | ||
72 | static void | |
73 | mt_device_lock(mt_device_t dev) | |
74 | { | |
75 | lck_mtx_lock(&dev->mtd_lock); | |
76 | } | |
77 | ||
78 | static void | |
79 | mt_device_unlock(mt_device_t dev) | |
80 | { | |
81 | lck_mtx_unlock(&dev->mtd_lock); | |
82 | } | |
83 | ||
84 | static void | |
85 | mt_device_assert_lock_held(__assert_only mt_device_t dev) | |
86 | { | |
87 | LCK_MTX_ASSERT(&dev->mtd_lock, LCK_MTX_ASSERT_OWNED); | |
88 | } | |
89 | ||
90 | static void | |
91 | mt_device_assert_inuse(__assert_only mt_device_t dev) | |
92 | { | |
93 | assert(dev->mtd_inuse == true); | |
94 | } | |
95 | ||
96 | int | |
97 | mt_dev_init(void) | |
98 | { | |
99 | mt_lock_grp = lck_grp_alloc_init(MT_NODE, LCK_GRP_ATTR_NULL); | |
100 | assert(mt_lock_grp != NULL); | |
101 | ||
102 | mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw); | |
103 | if (mt_dev_major < 0) { | |
104 | panic("monotonic: cdevsw_add failed: %d", mt_dev_major); | |
105 | __builtin_unreachable(); | |
106 | } | |
107 | ||
108 | for (int i = 0; i < MT_NDEVS; i++) { | |
109 | if (mt_devices[i].mtd_init(&mt_devices[i])) { | |
110 | continue; | |
111 | } | |
112 | ||
113 | assert(mt_devices[i].mtd_ncounters > 0); | |
114 | ||
115 | dev_t dev = makedev(mt_dev_major, i); | |
116 | char name[128]; | |
117 | snprintf(name, sizeof(name), MT_NODE "/%s", mt_devices[i].mtd_name); | |
118 | void *node = devfs_make_node(dev, DEVFS_CHAR, UID_ROOT, | |
119 | GID_WINDOWSERVER, 0666, name); | |
120 | if (!node) { | |
121 | panic("monotonic: devfs_make_node failed for '%s'", | |
122 | mt_devices[i].mtd_name); | |
123 | __builtin_unreachable(); | |
124 | } | |
125 | ||
126 | lck_mtx_init(&mt_devices[i].mtd_lock, mt_lock_grp, LCK_ATTR_NULL); | |
127 | } | |
128 | ||
129 | return 0; | |
130 | } | |
131 | ||
132 | static int | |
133 | mt_cdev_open(dev_t devnum, __unused int flags, __unused int devtype, | |
134 | __unused proc_t p) | |
135 | { | |
136 | int error = 0; | |
137 | ||
138 | mt_device_t dev = mt_get_device(devnum); | |
139 | mt_device_lock(dev); | |
140 | if (dev->mtd_inuse) { | |
141 | error = EBUSY; | |
142 | } else { | |
143 | dev->mtd_inuse = true; | |
144 | } | |
145 | mt_device_unlock(dev); | |
146 | ||
147 | return error; | |
148 | } | |
149 | ||
150 | static int | |
151 | mt_cdev_close(dev_t devnum, __unused int flags, __unused int devtype, | |
152 | __unused struct proc *p) | |
153 | { | |
154 | mt_device_t dev = mt_get_device(devnum); | |
155 | ||
156 | mt_device_lock(dev); | |
157 | mt_device_assert_inuse(dev); | |
158 | dev->mtd_inuse = false; | |
159 | dev->mtd_reset(); | |
160 | mt_device_unlock(dev); | |
161 | ||
162 | return 0; | |
163 | } | |
164 | ||
165 | static int | |
166 | mt_ctl_add(mt_device_t dev, user_addr_t uptr) | |
167 | { | |
168 | int error; | |
169 | uint32_t ctr; | |
170 | union monotonic_ctl_add ctl; | |
171 | ||
172 | mt_device_assert_lock_held(dev); | |
173 | ||
174 | error = copyin(uptr, &ctl, sizeof(ctl.in)); | |
175 | if (error) { | |
176 | return error; | |
177 | } | |
178 | ||
179 | error = dev->mtd_add(&ctl.in.config, &ctr); | |
180 | if (error) { | |
181 | return error; | |
182 | } | |
183 | ||
184 | ctl.out.ctr = ctr; | |
185 | ||
186 | error = copyout(&ctl, uptr, sizeof(ctl.out)); | |
187 | if (error) { | |
188 | return error; | |
189 | } | |
190 | ||
191 | return 0; | |
192 | } | |
193 | ||
194 | static int | |
195 | mt_ctl_counts(mt_device_t dev, user_addr_t uptr) | |
196 | { | |
197 | int error; | |
198 | union monotonic_ctl_counts ctl; | |
199 | ||
200 | mt_device_assert_lock_held(dev); | |
201 | ||
202 | error = copyin(uptr, &ctl, sizeof(ctl.in)); | |
203 | if (error) { | |
204 | return error; | |
205 | } | |
206 | ||
207 | if (ctl.in.ctr_mask == 0) { | |
208 | return EINVAL; | |
209 | } | |
210 | ||
211 | { | |
212 | uint64_t counts[dev->mtd_nmonitors][dev->mtd_ncounters]; | |
213 | memset(counts, 0, | |
214 | dev->mtd_ncounters * dev->mtd_nmonitors * sizeof(counts[0][0])); | |
215 | error = dev->mtd_read(ctl.in.ctr_mask, (uint64_t *)counts); | |
216 | if (error) { | |
217 | return error; | |
218 | } | |
219 | ||
220 | error = copyout(&counts, uptr, sizeof(counts)); | |
221 | if (error) { | |
222 | return error; | |
223 | } | |
224 | } | |
225 | ||
226 | return 0; | |
227 | } | |
228 | ||
229 | static int | |
230 | mt_ctl_enable(mt_device_t dev, user_addr_t uptr) | |
231 | { | |
232 | int error; | |
233 | union monotonic_ctl_enable ctl; | |
234 | ||
235 | mt_device_assert_lock_held(dev); | |
236 | ||
237 | error = copyin(uptr, &ctl, sizeof(ctl)); | |
238 | if (error) { | |
239 | return error; | |
240 | } | |
241 | ||
242 | dev->mtd_enable(ctl.in.enable); | |
243 | ||
244 | return 0; | |
245 | } | |
246 | ||
247 | static int | |
248 | mt_ctl_reset(mt_device_t dev) | |
249 | { | |
250 | mt_device_assert_lock_held(dev); | |
251 | dev->mtd_reset(); | |
252 | return 0; | |
253 | } | |
254 | ||
255 | static int | |
256 | mt_cdev_ioctl(dev_t devnum, unsigned long cmd, char *arg, __unused int flags, | |
257 | __unused proc_t p) | |
258 | { | |
259 | int error = ENODEV; | |
260 | user_addr_t uptr = *(user_addr_t *)(void *)arg; | |
261 | ||
262 | mt_device_t dev = mt_get_device(devnum); | |
263 | mt_device_lock(dev); | |
264 | ||
265 | switch (cmd) { | |
266 | case MT_IOC_RESET: | |
267 | error = mt_ctl_reset(dev); | |
268 | break; | |
269 | ||
270 | case MT_IOC_ADD: | |
271 | error = mt_ctl_add(dev, uptr); | |
272 | break; | |
273 | ||
274 | case MT_IOC_ENABLE: | |
275 | error = mt_ctl_enable(dev, uptr); | |
276 | break; | |
277 | ||
278 | case MT_IOC_COUNTS: | |
279 | error = mt_ctl_counts(dev, uptr); | |
280 | break; | |
281 | ||
282 | case MT_IOC_GET_INFO: { | |
283 | union monotonic_ctl_info info = { | |
284 | .out = { | |
285 | .nmonitors = dev->mtd_nmonitors, | |
286 | .ncounters = dev->mtd_ncounters, | |
287 | }, | |
288 | }; | |
289 | error = copyout(&info, uptr, sizeof(info)); | |
290 | break; | |
291 | } | |
292 | ||
293 | default: | |
294 | error = ENODEV; | |
295 | break; | |
296 | } | |
297 | ||
298 | mt_device_unlock(dev); | |
299 | ||
300 | return error; | |
301 | } | |
302 | ||
303 | int | |
304 | thread_selfcounts(__unused struct proc *p, | |
305 | struct thread_selfcounts_args *uap, __unused int *ret_out) | |
306 | { | |
307 | switch (uap->type) { | |
308 | case 1: { | |
309 | uint64_t counts[2] = { 0 }; | |
310 | uint64_t thread_counts[MT_CORE_NFIXED] = { 0 }; | |
311 | ||
312 | mt_cur_thread_fixed_counts(thread_counts); | |
313 | ||
314 | #ifdef MT_CORE_INSTRS | |
315 | counts[0] = thread_counts[MT_CORE_INSTRS]; | |
316 | #endif /* defined(MT_CORE_INSTRS) */ | |
317 | counts[1] = thread_counts[MT_CORE_CYCLES]; | |
318 | ||
319 | return copyout(counts, uap->buf, MIN(sizeof(counts), uap->nbytes)); | |
320 | } | |
321 | default: | |
322 | return EINVAL; | |
323 | } | |
324 | } | |
325 | ||
326 | enum mt_sysctl { | |
327 | MT_SUPPORTED, | |
328 | MT_PMIS, | |
329 | MT_RETROGRADE, | |
330 | MT_TASK_THREAD, | |
331 | MT_DEBUG, | |
332 | MT_KDBG_TEST, | |
333 | MT_FIX_CPU_PERF, | |
334 | MT_FIX_THREAD_PERF, | |
335 | MT_FIX_TASK_PERF, | |
336 | }; | |
337 | ||
338 | static int | |
339 | mt_sysctl SYSCTL_HANDLER_ARGS | |
340 | { | |
341 | #pragma unused(oidp, arg2) | |
342 | uint64_t start[MT_CORE_NFIXED] = { 0 }, end[MT_CORE_NFIXED] = { 0 }; | |
343 | uint64_t counts[2] = { 0 }; | |
344 | ||
345 | switch ((enum mt_sysctl)arg1) { | |
346 | case MT_SUPPORTED: | |
347 | return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL); | |
348 | case MT_PMIS: | |
349 | return sysctl_io_number(req, mt_count_pmis(), sizeof(uint64_t), NULL, NULL); | |
350 | case MT_RETROGRADE: { | |
351 | uint64_t value = os_atomic_load_wide(&mt_retrograde, relaxed); | |
352 | return sysctl_io_number(req, value, sizeof(mt_retrograde), NULL, NULL); | |
353 | } | |
354 | case MT_TASK_THREAD: | |
355 | return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL); | |
356 | case MT_DEBUG: { | |
357 | int value = mt_debug; | |
358 | ||
359 | int r = sysctl_io_number(req, value, sizeof(value), &value, NULL); | |
360 | if (r) { | |
361 | return r; | |
362 | } | |
363 | mt_debug = value; | |
364 | ||
365 | return 0; | |
366 | } | |
367 | case MT_KDBG_TEST: { | |
368 | if (req->newptr == USER_ADDR_NULL) { | |
369 | return EINVAL; | |
370 | } | |
371 | ||
372 | int intrs_en = ml_set_interrupts_enabled(FALSE); | |
373 | MT_KDBG_TMPCPU_START(0x3fff); | |
374 | MT_KDBG_TMPCPU_END(0x3fff); | |
375 | ||
376 | MT_KDBG_TMPTH_START(0x3fff); | |
377 | MT_KDBG_TMPTH_END(0x3fff); | |
378 | ml_set_interrupts_enabled(intrs_en); | |
379 | ||
380 | return 0; | |
381 | } | |
382 | case MT_FIX_CPU_PERF: { | |
383 | int intrs_en = ml_set_interrupts_enabled(FALSE); | |
384 | mt_fixed_counts(start); | |
385 | mt_fixed_counts(end); | |
386 | ml_set_interrupts_enabled(intrs_en); | |
387 | ||
388 | goto copyout_counts; | |
389 | } | |
390 | case MT_FIX_THREAD_PERF: { | |
391 | int intrs_en = ml_set_interrupts_enabled(FALSE); | |
392 | mt_cur_thread_fixed_counts(start); | |
393 | mt_cur_thread_fixed_counts(end); | |
394 | ml_set_interrupts_enabled(intrs_en); | |
395 | ||
396 | goto copyout_counts; | |
397 | } | |
398 | case MT_FIX_TASK_PERF: { | |
399 | int intrs_en = ml_set_interrupts_enabled(FALSE); | |
400 | mt_cur_task_fixed_counts(start); | |
401 | mt_cur_task_fixed_counts(end); | |
402 | ml_set_interrupts_enabled(intrs_en); | |
403 | ||
404 | goto copyout_counts; | |
405 | } | |
406 | default: | |
407 | return ENOENT; | |
408 | } | |
409 | ||
410 | copyout_counts: | |
411 | ||
412 | #ifdef MT_CORE_INSTRS | |
413 | counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS]; | |
414 | #endif /* defined(MT_CORE_INSTRS) */ | |
415 | counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES]; | |
416 | ||
417 | return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts))); | |
418 | } | |
419 | ||
420 | SYSCTL_DECL(_kern_monotonic); | |
421 | SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW | CTLFLAG_LOCKED, 0, | |
422 | "monotonic"); | |
423 | ||
424 | #define MT_SYSCTL(NAME, ARG, FLAGS, SIZE, SIZESTR, DESC) \ | |
425 | SYSCTL_PROC(_kern_monotonic, OID_AUTO, NAME, \ | |
426 | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | (FLAGS), \ | |
427 | (void *)(ARG), SIZE, mt_sysctl, SIZESTR, DESC) | |
428 | ||
429 | MT_SYSCTL(supported, MT_SUPPORTED, 0, sizeof(int), "I", | |
430 | "whether monotonic is supported"); | |
431 | MT_SYSCTL(debug, MT_DEBUG, CTLFLAG_MASKED, sizeof(int), "I", | |
432 | "whether monotonic is printing debug messages"); | |
433 | MT_SYSCTL(pmis, MT_PMIS, 0, sizeof(uint64_t), "Q", | |
434 | "number of PMIs seen"); | |
435 | MT_SYSCTL(retrograde_updates, MT_RETROGRADE, 0, sizeof(uint64_t), "Q", | |
436 | "number of times a counter appeared to go backwards"); | |
437 | MT_SYSCTL(task_thread_counting, MT_TASK_THREAD, 0, sizeof(int), "I", | |
438 | "whether task and thread counting is enabled"); | |
439 | MT_SYSCTL(kdebug_test, MT_KDBG_TEST, CTLFLAG_MASKED, sizeof(int), "O", | |
440 | "whether task and thread counting is enabled"); | |
441 | MT_SYSCTL(fixed_cpu_perf, MT_FIX_CPU_PERF, CTLFLAG_MASKED, | |
442 | sizeof(uint64_t) * 2, "O", | |
443 | "overhead of accessing the current CPU's counters"); | |
444 | MT_SYSCTL(fixed_thread_perf, MT_FIX_THREAD_PERF, CTLFLAG_MASKED, | |
445 | sizeof(uint64_t) * 2, "O", | |
446 | "overhead of accessing the current thread's counters"); | |
447 | MT_SYSCTL(fixed_task_perf, MT_FIX_TASK_PERF, CTLFLAG_MASKED, | |
448 | sizeof(uint64_t) * 2, "O", | |
449 | "overhead of accessing the current task's counters"); |