[apple/xnu.git] / iokit / Kernel / IOMapper.cpp

/*
 * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#include <IOKit/IOLib.h>
#include <IOKit/IOMapper.h>
#include <IOKit/IODMACommand.h>
#include <libkern/c++/OSData.h>
#include <libkern/OSDebug.h>
#include <mach_debug/zone_info.h>
#include "IOKitKernelInternal.h"

__BEGIN_DECLS
extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
__END_DECLS

#define super IOService
OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);

OSMetaClassDefineReservedUnused(IOMapper, 0);
OSMetaClassDefineReservedUnused(IOMapper, 1);
OSMetaClassDefineReservedUnused(IOMapper, 2);
OSMetaClassDefineReservedUnused(IOMapper, 3);
OSMetaClassDefineReservedUnused(IOMapper, 4);
OSMetaClassDefineReservedUnused(IOMapper, 5);
OSMetaClassDefineReservedUnused(IOMapper, 6);
OSMetaClassDefineReservedUnused(IOMapper, 7);
OSMetaClassDefineReservedUnused(IOMapper, 8);
OSMetaClassDefineReservedUnused(IOMapper, 9);
OSMetaClassDefineReservedUnused(IOMapper, 10);
OSMetaClassDefineReservedUnused(IOMapper, 11);
OSMetaClassDefineReservedUnused(IOMapper, 12);
OSMetaClassDefineReservedUnused(IOMapper, 13);
OSMetaClassDefineReservedUnused(IOMapper, 14);
OSMetaClassDefineReservedUnused(IOMapper, 15);

IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown;

class IOMapperLock {
    IOLock *fWaitLock;
public:
    IOMapperLock() { fWaitLock = IOLockAlloc(); }
    ~IOMapperLock() { IOLockFree(fWaitLock); }

    void lock()   { IOLockLock(fWaitLock); }
    void unlock() { IOLockUnlock(fWaitLock); }
    void sleep(void *event)  { IOLockSleep(fWaitLock, event, THREAD_UNINT); }
    void wakeup(void *event) { IOLockWakeup(fWaitLock, event, false); }
};

static IOMapperLock sMapperLock;

bool IOMapper::start(IOService *provider)
{
    OSObject * obj;
    if (!super::start(provider))
        return false;

    if (!initHardware(provider))
        return false;

    fPageSize = getPageSize();

    if (fIsSystem) { 
        sMapperLock.lock();
        IOMapper::gSystem = this;
        sMapperLock.wakeup(&IOMapper::gSystem);
        sMapperLock.unlock();
    }

    if (provider)
    {
    	obj = provider->getProperty("iommu-id");
	if (!obj)
	    obj = provider->getProperty("AAPL,phandle");
	if (obj)
	    setProperty(gIOMapperIDKey, obj);
    }
    return true;
}

void IOMapper::free()
{
    super::free();
}

void IOMapper::setMapperRequired(bool hasMapper)
{
    if (hasMapper)
        IOMapper::gSystem = (IOMapper *) kHasMapper;
    else {
        sMapperLock.lock();
        IOMapper::gSystem = (IOMapper *) kNoMapper;
        sMapperLock.unlock();
        sMapperLock.wakeup(&IOMapper::gSystem);
    }
}

void IOMapper::waitForSystemMapper()
{
    sMapperLock.lock();
    while ((uintptr_t) IOMapper::gSystem & kWaitMask)
    {
		OSReportWithBacktrace("waitForSystemMapper");
        sMapperLock.sleep(&IOMapper::gSystem);
    }
    sMapperLock.unlock();
}

IOMapper * IOMapper::copyMapperForDevice(IOService * device)
{
    return copyMapperForDeviceWithIndex(device, 0);
}

IOMapper * IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index)
{
    OSData *data;
    OSObject * obj;
    IOMapper * mapper = NULL;
    OSDictionary * matching;
    
    obj = device->copyProperty("iommu-parent");
    if (!obj) return (NULL);

    if ((mapper = OSDynamicCast(IOMapper, obj))) goto found;

    if ((data = OSDynamicCast(OSData, obj)))
    {
        if (index >= data->getLength() / sizeof(UInt32)) goto done;
        
        data = OSData::withBytesNoCopy((UInt32 *)data->getBytesNoCopy() + index, sizeof(UInt32));
        if (!data) goto done;

        matching = IOService::propertyMatching(gIOMapperIDKey, data);
        data->release();
    }
    else
        matching = IOService::propertyMatching(gIOMapperIDKey, obj);

    if (matching)
    {
        mapper = OSDynamicCast(IOMapper, IOService::waitForMatchingService(matching));
        matching->release();
    }

done:
    if (obj) obj->release();
found:
    if (mapper)
    {
        if (!mapper->fAllocName)
        {
            char name[MACH_ZONE_NAME_MAX_LEN];
            char kmodname[KMOD_MAX_NAME];
            vm_tag_t tag;
            uint32_t kmodid;

            tag = IOMemoryTag(kernel_map);
            if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME)))
            {
                snprintf(kmodname, sizeof(kmodname), "%d", tag);
            }
            snprintf(name, sizeof(name), "%s.DMA.%s", kmodname, device->getName());
            mapper->fAllocName = kern_allocation_name_allocate(name, 16);
        }
    }

    return (mapper);
}

__BEGIN_DECLS

// These are C accessors to the system mapper for non-IOKit clients
ppnum_t IOMapperIOVMAlloc(unsigned pages)
{
    IOReturn ret;
    uint64_t dmaAddress, dmaLength;

    IOMapper::checkForSystemMapper();

    ret = kIOReturnUnsupported;
    if (IOMapper::gSystem)
    {
        ret = IOMapper::gSystem->iovmMapMemory(
        		NULL, 0, ptoa_64(pages), 
        		(kIODMAMapReadAccess | kIODMAMapWriteAccess),
        		NULL, NULL, NULL,
        		&dmaAddress, &dmaLength);
    }

    if (kIOReturnSuccess == ret) return (atop_64(dmaAddress));
    return (0);
}

void IOMapperIOVMFree(ppnum_t addr, unsigned pages)
{
    if (IOMapper::gSystem)
    {
        IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages));
    }
}

ppnum_t IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page)
{
    if (!IOMapper::gSystem) return (page);
    if (!addr) panic("!addr");
    IOMapper::gSystem->iovmInsert((kIODMAMapReadAccess | kIODMAMapWriteAccess),
				  ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1));
    return (addr + offset);
}

/////////////////////////////////////////////////////////////////////////////
//
//
//	IOLib.h APIs
//
//
/////////////////////////////////////////////////////////////////////////////

#include <machine/machine_routines.h>

UInt8 IOMappedRead8(IOPhysicalAddress address)
{
    IOMapper::checkForSystemMapper();

    if (IOMapper::gSystem) {
        addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
        return (UInt8) ml_phys_read_byte_64(addr);
    }
    else
        return (UInt8) ml_phys_read_byte((vm_offset_t) address);
}

UInt16 IOMappedRead16(IOPhysicalAddress address)
{
    IOMapper::checkForSystemMapper();

    if (IOMapper::gSystem) {
        addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
        return (UInt16) ml_phys_read_half_64(addr);
    }
    else
        return (UInt16) ml_phys_read_half((vm_offset_t) address);
}

UInt32 IOMappedRead32(IOPhysicalAddress address)
{
    IOMapper::checkForSystemMapper();

    if (IOMapper::gSystem) {
        addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
	return (UInt32) ml_phys_read_word_64(addr);
    }
    else
        return (UInt32) ml_phys_read_word((vm_offset_t) address);
}

UInt64 IOMappedRead64(IOPhysicalAddress address)
{
    IOMapper::checkForSystemMapper();

    if (IOMapper::gSystem) {
        addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
        return (UInt64) ml_phys_read_double_64(addr);
    }
    else
        return (UInt64) ml_phys_read_double((vm_offset_t) address);
}

void IOMappedWrite8(IOPhysicalAddress address, UInt8 value)
{
    IOMapper::checkForSystemMapper();

    if (IOMapper::gSystem) {
        addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
        ml_phys_write_byte_64(addr, value);
    }
    else
        ml_phys_write_byte((vm_offset_t) address, value);
}

void IOMappedWrite16(IOPhysicalAddress address, UInt16 value)
{
    IOMapper::checkForSystemMapper();

    if (IOMapper::gSystem) {
        addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
        ml_phys_write_half_64(addr, value);
    }
    else
        ml_phys_write_half((vm_offset_t) address, value);
}

void IOMappedWrite32(IOPhysicalAddress address, UInt32 value)
{
    IOMapper::checkForSystemMapper();

    if (IOMapper::gSystem) {
        addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
        ml_phys_write_word_64(addr, value);
    }
    else
        ml_phys_write_word((vm_offset_t) address, value);
}

void IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
{
    IOMapper::checkForSystemMapper();

    if (IOMapper::gSystem) {
        addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
        ml_phys_write_double_64(addr, value);
    }
    else
        ml_phys_write_double((vm_offset_t) address, value);
}

__END_DECLS
Commit	Line	Data
55e303ae	1	/*
39037602	2	* Copyright (c) 1998-2016 Apple Inc. All rights reserved.
55e303ae	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
55e303ae	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
55e303ae A	27	*/
	28	#include <IOKit/IOLib.h>
	29	#include <IOKit/IOMapper.h>
b0d623f7	30	#include <IOKit/IODMACommand.h>
55e303ae	31	#include <libkern/c++/OSData.h>
99c3a104	32	#include <libkern/OSDebug.h>
5ba3f43e	33	#include <mach_debug/zone_info.h>
3e170ce0	34	#include "IOKitKernelInternal.h"
55e303ae	35
0c530ab8 A	36	__BEGIN_DECLS
	37	extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
	38	__END_DECLS
	39
55e303ae A	40	#define super IOService
	41	OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);
	42
3e170ce0 A	43	OSMetaClassDefineReservedUnused(IOMapper, 0);
	44	OSMetaClassDefineReservedUnused(IOMapper, 1);
	45	OSMetaClassDefineReservedUnused(IOMapper, 2);
	46	OSMetaClassDefineReservedUnused(IOMapper, 3);
55e303ae A	47	OSMetaClassDefineReservedUnused(IOMapper, 4);
	48	OSMetaClassDefineReservedUnused(IOMapper, 5);
	49	OSMetaClassDefineReservedUnused(IOMapper, 6);
	50	OSMetaClassDefineReservedUnused(IOMapper, 7);
	51	OSMetaClassDefineReservedUnused(IOMapper, 8);
	52	OSMetaClassDefineReservedUnused(IOMapper, 9);
	53	OSMetaClassDefineReservedUnused(IOMapper, 10);
	54	OSMetaClassDefineReservedUnused(IOMapper, 11);
	55	OSMetaClassDefineReservedUnused(IOMapper, 12);
	56	OSMetaClassDefineReservedUnused(IOMapper, 13);
	57	OSMetaClassDefineReservedUnused(IOMapper, 14);
	58	OSMetaClassDefineReservedUnused(IOMapper, 15);
	59
	60	IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown;
	61
	62	class IOMapperLock {
	63	IOLock *fWaitLock;
	64	public:
39037602 A	65	IOMapperLock() { fWaitLock = IOLockAlloc(); }
39037602 A	66	~IOMapperLock() { IOLockFree(fWaitLock); }
55e303ae	67
39037602 A	68	void lock() { IOLockLock(fWaitLock); }
	69	void unlock() { IOLockUnlock(fWaitLock); }
	70	void sleep(void *event) { IOLockSleep(fWaitLock, event, THREAD_UNINT); }
	71	void wakeup(void *event) { IOLockWakeup(fWaitLock, event, false); }
55e303ae A	72	};
	73
	74	static IOMapperLock sMapperLock;
	75
	76	bool IOMapper::start(IOService *provider)
	77	{
b0d623f7	78	OSObject * obj;
55e303ae A	79	if (!super::start(provider))
	80	return false;
	81
	82	if (!initHardware(provider))
	83	return false;
	84
3e170ce0 A	85	fPageSize = getPageSize();
3e170ce0 A	86
55e303ae A	87	if (fIsSystem) {
	88	sMapperLock.lock();
	89	IOMapper::gSystem = this;
	90	sMapperLock.wakeup(&IOMapper::gSystem);
	91	sMapperLock.unlock();
	92	}
	93
b0d623f7 A	94	if (provider)
	95	{
	96	obj = provider->getProperty("iommu-id");
	97	if (!obj)
	98	obj = provider->getProperty("AAPL,phandle");
	99	if (obj)
	100	setProperty(gIOMapperIDKey, obj);
	101	}
55e303ae A	102	return true;
	103	}
	104
55e303ae A	105	void IOMapper::free()
55e303ae A	106	{
55e303ae A	107	super::free();
	108	}
	109
	110	void IOMapper::setMapperRequired(bool hasMapper)
	111	{
	112	if (hasMapper)
	113	IOMapper::gSystem = (IOMapper *) kHasMapper;
	114	else {
	115	sMapperLock.lock();
	116	IOMapper::gSystem = (IOMapper *) kNoMapper;
	117	sMapperLock.unlock();
	118	sMapperLock.wakeup(&IOMapper::gSystem);
	119	}
	120	}
	121
	122	void IOMapper::waitForSystemMapper()
	123	{
	124	sMapperLock.lock();
b0d623f7	125	while ((uintptr_t) IOMapper::gSystem & kWaitMask)
99c3a104 A	126	{
99c3a104 A	127	OSReportWithBacktrace("waitForSystemMapper");
55e303ae	128	sMapperLock.sleep(&IOMapper::gSystem);
99c3a104	129	}
55e303ae A	130	sMapperLock.unlock();
	131	}
	132
b0d623f7 A	133	IOMapper * IOMapper::copyMapperForDevice(IOService * device)
b0d623f7 A	134	{
39236c6e A	135	return copyMapperForDeviceWithIndex(device, 0);
	136	}
	137
	138	IOMapper * IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index)
	139	{
	140	OSData *data;
b0d623f7	141	OSObject * obj;
39236c6e	142	IOMapper * mapper = NULL;
b0d623f7 A	143	OSDictionary * matching;
	144
	145	obj = device->copyProperty("iommu-parent");
5ba3f43e	146	if (!obj) return (NULL);
b0d623f7	147
5ba3f43e	148	if ((mapper = OSDynamicCast(IOMapper, obj))) goto found;
b0d623f7	149
39236c6e	150	if ((data = OSDynamicCast(OSData, obj)))
b0d623f7	151	{
5ba3f43e	152	if (index >= data->getLength() / sizeof(UInt32)) goto done;
39236c6e A	153
39236c6e A	154	data = OSData::withBytesNoCopy((UInt32 *)data->getBytesNoCopy() + index, sizeof(UInt32));
5ba3f43e	155	if (!data) goto done;
39236c6e A	156
	157	matching = IOService::propertyMatching(gIOMapperIDKey, data);
	158	data->release();
b0d623f7	159	}
b0d623f7	160	else
39236c6e A	161	matching = IOService::propertyMatching(gIOMapperIDKey, obj);
	162
	163	if (matching)
	164	{
	165	mapper = OSDynamicCast(IOMapper, IOService::waitForMatchingService(matching));
5ba3f43e	166	matching->release();
39236c6e A	167	}
	168
	169	done:
5ba3f43e A	170	if (obj) obj->release();
	171	found:
	172	if (mapper)
	173	{
	174	if (!mapper->fAllocName)
	175	{
	176	char name[MACH_ZONE_NAME_MAX_LEN];
	177	char kmodname[KMOD_MAX_NAME];
	178	vm_tag_t tag;
	179	uint32_t kmodid;
	180
	181	tag = IOMemoryTag(kernel_map);
	182	if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME)))
	183	{
	184	snprintf(kmodname, sizeof(kmodname), "%d", tag);
	185	}
	186	snprintf(name, sizeof(name), "%s.DMA.%s", kmodname, device->getName());
	187	mapper->fAllocName = kern_allocation_name_allocate(name, 16);
	188	}
	189	}
	190
b0d623f7 A	191	return (mapper);
	192	}
	193
55e303ae A	194	__BEGIN_DECLS
	195
	196	// These are C accessors to the system mapper for non-IOKit clients
	197	ppnum_t IOMapperIOVMAlloc(unsigned pages)
	198	{
3e170ce0 A	199	IOReturn ret;
	200	uint64_t dmaAddress, dmaLength;
	201
55e303ae A	202	IOMapper::checkForSystemMapper();
55e303ae A	203
3e170ce0	204	ret = kIOReturnUnsupported;
55e303ae	205	if (IOMapper::gSystem)
3e170ce0 A	206	{
	207	ret = IOMapper::gSystem->iovmMapMemory(
	208	NULL, 0, ptoa_64(pages),
	209	(kIODMAMapReadAccess \| kIODMAMapWriteAccess),
	210	NULL, NULL, NULL,
	211	&dmaAddress, &dmaLength);
	212	}
	213
	214	if (kIOReturnSuccess == ret) return (atop_64(dmaAddress));
	215	return (0);
55e303ae A	216	}
	217
	218	void IOMapperIOVMFree(ppnum_t addr, unsigned pages)
	219	{
	220	if (IOMapper::gSystem)
3e170ce0 A	221	{
3e170ce0 A	222	IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages));
55e303ae	223	}
55e303ae A	224	}
55e303ae A	225
3e170ce0	226	ppnum_t IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page)
55e303ae	227	{
3e170ce0 A	228	if (!IOMapper::gSystem) return (page);
	229	if (!addr) panic("!addr");
	230	IOMapper::gSystem->iovmInsert((kIODMAMapReadAccess \| kIODMAMapWriteAccess),
	231	ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1));
	232	return (addr + offset);
55e303ae A	233	}
	234
	235	/////////////////////////////////////////////////////////////////////////////
	236	//
	237	//
	238	// IOLib.h APIs
	239	//
	240	//
	241	/////////////////////////////////////////////////////////////////////////////
	242
	243	#include <machine/machine_routines.h>
	244
	245	UInt8 IOMappedRead8(IOPhysicalAddress address)
	246	{
	247	IOMapper::checkForSystemMapper();
	248
	249	if (IOMapper::gSystem) {
3e170ce0	250	addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
55e303ae A	251	return (UInt8) ml_phys_read_byte_64(addr);
	252	}
	253	else
	254	return (UInt8) ml_phys_read_byte((vm_offset_t) address);
	255	}
	256
	257	UInt16 IOMappedRead16(IOPhysicalAddress address)
	258	{
	259	IOMapper::checkForSystemMapper();
	260
	261	if (IOMapper::gSystem) {
3e170ce0	262	addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
55e303ae A	263	return (UInt16) ml_phys_read_half_64(addr);
	264	}
	265	else
	266	return (UInt16) ml_phys_read_half((vm_offset_t) address);
	267	}
	268
	269	UInt32 IOMappedRead32(IOPhysicalAddress address)
	270	{
	271	IOMapper::checkForSystemMapper();
	272
	273	if (IOMapper::gSystem) {
3e170ce0	274	addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
55e303ae A	275	return (UInt32) ml_phys_read_word_64(addr);
	276	}
	277	else
	278	return (UInt32) ml_phys_read_word((vm_offset_t) address);
	279	}
	280
	281	UInt64 IOMappedRead64(IOPhysicalAddress address)
	282	{
	283	IOMapper::checkForSystemMapper();
	284
	285	if (IOMapper::gSystem) {
3e170ce0	286	addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
55e303ae A	287	return (UInt64) ml_phys_read_double_64(addr);
	288	}
	289	else
	290	return (UInt64) ml_phys_read_double((vm_offset_t) address);
	291	}
	292
	293	void IOMappedWrite8(IOPhysicalAddress address, UInt8 value)
	294	{
	295	IOMapper::checkForSystemMapper();
	296
	297	if (IOMapper::gSystem) {
3e170ce0	298	addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
55e303ae A	299	ml_phys_write_byte_64(addr, value);
	300	}
	301	else
	302	ml_phys_write_byte((vm_offset_t) address, value);
	303	}
	304
	305	void IOMappedWrite16(IOPhysicalAddress address, UInt16 value)
	306	{
	307	IOMapper::checkForSystemMapper();
	308
	309	if (IOMapper::gSystem) {
3e170ce0	310	addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
55e303ae A	311	ml_phys_write_half_64(addr, value);
	312	}
	313	else
	314	ml_phys_write_half((vm_offset_t) address, value);
	315	}
	316
	317	void IOMappedWrite32(IOPhysicalAddress address, UInt32 value)
	318	{
	319	IOMapper::checkForSystemMapper();
	320
	321	if (IOMapper::gSystem) {
3e170ce0	322	addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
55e303ae A	323	ml_phys_write_word_64(addr, value);
	324	}
	325	else
	326	ml_phys_write_word((vm_offset_t) address, value);
	327	}
	328
	329	void IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
	330	{
	331	IOMapper::checkForSystemMapper();
	332
	333	if (IOMapper::gSystem) {
3e170ce0	334	addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
55e303ae A	335	ml_phys_write_double_64(addr, value);
	336	}
	337	else
	338	ml_phys_write_double((vm_offset_t) address, value);
	339	}
	340
	341	__END_DECLS