+#endif /* x86_64 */
+
+ switch (size) {
+ case 1:
+ s1 = *(volatile unsigned char *)vaddr;
+ result = s1;
+ break;
+ case 2:
+ s2 = *(volatile unsigned short *)vaddr;
+ result = s2;
+ break;
+ case 4:
+ result = *(volatile unsigned int *)vaddr;
+ break;
+ case 8:
+ result = *(volatile unsigned long long *)vaddr;
+ break;
+ default:
+ panic("Invalid size %d for ml_io_read(%p)", size, (void *)vaddr);
+ break;
+ }
+
+#if defined(__x86_64__)
+ if (__improbable(timeread == TRUE)) {
+ eabs = mach_absolute_time();
+
+#if DEVELOPMENT || DEBUG
+ iotrace(IOTRACE_IO_READ, vaddr, paddr, size, result, sabs, eabs - sabs);
+#endif
+
+ if (__improbable((eabs - sabs) > reportphyreaddelayabs)) {
+#if !(DEVELOPMENT || DEBUG)
+ uintptr_t paddr = kvtophys(vaddr);
+#endif
+
+ (void)ml_set_interrupts_enabled(istate);
+
+ if (phyreadpanic && (machine_timeout_suspended() == FALSE)) {
+ panic_notify();
+ panic("Read from IO vaddr 0x%lx paddr 0x%lx took %llu ns, "
+ "result: 0x%llx (start: %llu, end: %llu), ceiling: %llu",
+ vaddr, paddr, (eabs - sabs), result, sabs, eabs,
+ reportphyreaddelayabs);
+ }
+
+ if (reportphyreadosbt) {
+ OSReportWithBacktrace("ml_io_read(v=%p, p=%p) size %d result 0x%llx "
+ "took %lluus",
+ (void *)vaddr, (void *)paddr, size, result,
+ (eabs - sabs) / NSEC_PER_USEC);
+ }
+#if CONFIG_DTRACE
+ DTRACE_PHYSLAT5(physioread, uint64_t, (eabs - sabs),
+ uint64_t, vaddr, uint32_t, size, uint64_t, paddr, uint64_t, result);
+#endif /* CONFIG_DTRACE */
+ } else if (__improbable(tracephyreaddelayabs > 0 && (eabs - sabs) > tracephyreaddelayabs)) {
+#if !(DEVELOPMENT || DEBUG)
+ uintptr_t paddr = kvtophys(vaddr);
+#endif
+
+ KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_MMIO_READ),
+ (eabs - sabs), VM_KERNEL_UNSLIDE_OR_PERM(vaddr), paddr, result);
+
+ (void)ml_set_interrupts_enabled(istate);
+ } else {
+ (void)ml_set_interrupts_enabled(istate);
+ }
+ }
+#endif /* x86_64 */
+ return result;
+}
+
+unsigned int
+ml_io_read8(uintptr_t vaddr)
+{
+ return (unsigned) ml_io_read(vaddr, 1);
+}
+
+unsigned int
+ml_io_read16(uintptr_t vaddr)
+{
+ return (unsigned) ml_io_read(vaddr, 2);
+}
+
+unsigned int
+ml_io_read32(uintptr_t vaddr)
+{
+ return (unsigned) ml_io_read(vaddr, 4);
+}
+
+unsigned long long
+ml_io_read64(uintptr_t vaddr)
+{
+ return ml_io_read(vaddr, 8);
+}
+
+/* ml_io_write* */
+
+void
+ml_io_write(uintptr_t vaddr, uint64_t val, int size)
+{
+#if defined(__x86_64__)
+ uint64_t sabs, eabs;
+ boolean_t istate, timewrite = FALSE;
+#if DEVELOPMENT || DEBUG
+ extern uint64_t simulate_stretched_io;
+ uintptr_t paddr = pmap_verify_noncacheable(vaddr);
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+ if (__improbable(reportphywritedelayabs != 0)) {
+ istate = ml_set_interrupts_enabled(FALSE);
+ sabs = mach_absolute_time();
+ timewrite = TRUE;
+ }
+
+#if DEVELOPMENT || DEBUG
+ if (__improbable(timewrite && simulate_stretched_io)) {
+ sabs -= simulate_stretched_io;
+ }
+#endif /* x86_64 DEVELOPMENT || DEBUG */
+#endif /* x86_64 */
+
+ switch (size) {
+ case 1:
+ *(volatile uint8_t *)vaddr = (uint8_t)val;
+ break;
+ case 2:
+ *(volatile uint16_t *)vaddr = (uint16_t)val;
+ break;
+ case 4:
+ *(volatile uint32_t *)vaddr = (uint32_t)val;
+ break;
+ case 8:
+ *(volatile uint64_t *)vaddr = (uint64_t)val;
+ break;
+ default:
+ panic("Invalid size %d for ml_io_write(%p, 0x%llx)", size, (void *)vaddr, val);
+ break;
+ }
+
+#if defined(__x86_64__)
+ if (__improbable(timewrite == TRUE)) {
+ eabs = mach_absolute_time();
+
+#if DEVELOPMENT || DEBUG
+ iotrace(IOTRACE_IO_WRITE, vaddr, paddr, size, val, sabs, eabs - sabs);
+#endif
+
+ if (__improbable((eabs - sabs) > reportphywritedelayabs)) {
+#if !(DEVELOPMENT || DEBUG)
+ uintptr_t paddr = kvtophys(vaddr);
+#endif
+
+ (void)ml_set_interrupts_enabled(istate);
+
+ if (phywritepanic && (machine_timeout_suspended() == FALSE)) {
+ panic_notify();
+ panic("Write to IO vaddr %p paddr %p val 0x%llx took %llu ns,"
+ " (start: %llu, end: %llu), ceiling: %llu",
+ (void *)vaddr, (void *)paddr, val, (eabs - sabs), sabs, eabs,
+ reportphywritedelayabs);
+ }
+
+ if (reportphywriteosbt) {
+ OSReportWithBacktrace("ml_io_write size %d (v=%p, p=%p, 0x%llx) "
+ "took %lluus",
+ size, (void *)vaddr, (void *)paddr, val, (eabs - sabs) / NSEC_PER_USEC);
+ }
+#if CONFIG_DTRACE
+ DTRACE_PHYSLAT5(physiowrite, uint64_t, (eabs - sabs),
+ uint64_t, vaddr, uint32_t, size, uint64_t, paddr, uint64_t, val);
+#endif /* CONFIG_DTRACE */
+ } else if (__improbable(tracephywritedelayabs > 0 && (eabs - sabs) > tracephywritedelayabs)) {
+#if !(DEVELOPMENT || DEBUG)
+ uintptr_t paddr = kvtophys(vaddr);
+#endif
+
+ KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_MMIO_WRITE),
+ (eabs - sabs), VM_KERNEL_UNSLIDE_OR_PERM(vaddr), paddr, val);
+
+ (void)ml_set_interrupts_enabled(istate);
+ } else {
+ (void)ml_set_interrupts_enabled(istate);
+ }
+ }
+#endif /* x86_64 */
+}
+
+void
+ml_io_write8(uintptr_t vaddr, uint8_t val)
+{
+ ml_io_write(vaddr, val, 1);
+}
+
+void
+ml_io_write16(uintptr_t vaddr, uint16_t val)
+{
+ ml_io_write(vaddr, val, 2);
+}
+
+void
+ml_io_write32(uintptr_t vaddr, uint32_t val)
+{
+ ml_io_write(vaddr, val, 4);
+}
+
+void
+ml_io_write64(uintptr_t vaddr, uint64_t val)
+{
+ ml_io_write(vaddr, val, 8);
+}
+
+struct cpu_callback_chain_elem {
+ cpu_callback_t fn;
+ void *param;
+ struct cpu_callback_chain_elem *next;
+};
+
+static struct cpu_callback_chain_elem *cpu_callback_chain;
+static LCK_GRP_DECLARE(cpu_callback_chain_lock_grp, "cpu_callback_chain");
+static LCK_SPIN_DECLARE(cpu_callback_chain_lock, &cpu_callback_chain_lock_grp);
+
+void
+cpu_event_register_callback(cpu_callback_t fn, void *param)
+{
+ struct cpu_callback_chain_elem *new_elem;
+
+ new_elem = zalloc_permanent_type(struct cpu_callback_chain_elem);
+ if (!new_elem) {
+ panic("can't allocate cpu_callback_chain_elem");
+ }
+
+ lck_spin_lock(&cpu_callback_chain_lock);
+ new_elem->next = cpu_callback_chain;
+ new_elem->fn = fn;
+ new_elem->param = param;
+ os_atomic_store(&cpu_callback_chain, new_elem, release);
+ lck_spin_unlock(&cpu_callback_chain_lock);
+}
+
+__attribute__((noreturn))
+void
+cpu_event_unregister_callback(__unused cpu_callback_t fn)
+{
+ panic("Unfortunately, cpu_event_unregister_callback is unimplemented.");
+}
+
+void
+ml_broadcast_cpu_event(enum cpu_event event, unsigned int cpu_or_cluster)
+{
+ struct cpu_callback_chain_elem *cursor;
+
+ cursor = os_atomic_load(&cpu_callback_chain, dependency);
+ for (; cursor != NULL; cursor = cursor->next) {
+ cursor->fn(cursor->param, event, cpu_or_cluster);
+ }
projects / apple / xnu.git / blobdiff