X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/fe8ab488e9161c46dd9885d58fc52996dc0249ff..a39ff7e25e19b3a8c3020042a3872ca9ec9659f1:/iokit/Kernel/IODataQueue.cpp diff --git a/iokit/Kernel/IODataQueue.cpp b/iokit/Kernel/IODataQueue.cpp index e489cd9a8..3f718a754 100644 --- a/iokit/Kernel/IODataQueue.cpp +++ b/iokit/Kernel/IODataQueue.cpp @@ -37,6 +37,12 @@ #include #include +struct IODataQueueInternal +{ + mach_msg_header_t msg; + UInt32 queueSize; +}; + #ifdef enqueue #undef enqueue #endif @@ -95,21 +101,23 @@ Boolean IODataQueue::initWithCapacity(UInt32 size) return false; } + assert(!notifyMsg); + notifyMsg = IONew(IODataQueueInternal, 1); + if (!notifyMsg) { + return false; + } + bzero(notifyMsg, sizeof(IODataQueueInternal)); + ((IODataQueueInternal *)notifyMsg)->queueSize = size; + dataQueue = (IODataQueueMemory *)IOMallocAligned(allocSize, PAGE_SIZE); if (dataQueue == 0) { return false; } + bzero(dataQueue, allocSize); dataQueue->queueSize = size; - dataQueue->head = 0; - dataQueue->tail = 0; - - if (!notifyMsg) { - notifyMsg = IOMalloc(sizeof(mach_msg_header_t)); - if (!notifyMsg) - return false; - } - bzero(notifyMsg, sizeof(mach_msg_header_t)); +// dataQueue->head = 0; +// dataQueue->tail = 0; return true; } @@ -131,14 +139,14 @@ Boolean IODataQueue::initWithEntries(UInt32 numEntries, UInt32 entrySize) void IODataQueue::free() { - if (dataQueue) { - IOFreeAligned(dataQueue, round_page(dataQueue->queueSize + DATA_QUEUE_MEMORY_HEADER_SIZE)); - dataQueue = NULL; - - if (notifyMsg) { - IOFree(notifyMsg, sizeof(mach_msg_header_t)); - notifyMsg = NULL; - } + if (notifyMsg) { + if (dataQueue) { + IOFreeAligned(dataQueue, round_page(((IODataQueueInternal *)notifyMsg)->queueSize + DATA_QUEUE_MEMORY_HEADER_SIZE)); + dataQueue = NULL; + } + + IODelete(notifyMsg, IODataQueueInternal, 1); + notifyMsg = NULL; } super::free(); @@ -148,17 +156,25 @@ void IODataQueue::free() Boolean IODataQueue::enqueue(void * data, UInt32 dataSize) { - const UInt32 head = dataQueue->head; // volatile - const UInt32 tail = dataQueue->tail; + UInt32 head; + UInt32 tail; + UInt32 newTail; const UInt32 entrySize = dataSize + DATA_QUEUE_ENTRY_HEADER_SIZE; + UInt32 queueSize; IODataQueueEntry * entry; // Check for overflow of entrySize if (dataSize > UINT32_MAX - DATA_QUEUE_ENTRY_HEADER_SIZE) { return false; } + + // Force a single read of head and tail + head = __c11_atomic_load((_Atomic UInt32 *)&dataQueue->head, __ATOMIC_RELAXED); + tail = __c11_atomic_load((_Atomic UInt32 *)&dataQueue->tail, __ATOMIC_RELAXED); + // Check for underflow of (dataQueue->queueSize - tail) - if (dataQueue->queueSize < tail) { + queueSize = ((IODataQueueInternal *) notifyMsg)->queueSize; + if ((queueSize < tail) || (queueSize < head)) { return false; } @@ -166,7 +182,7 @@ Boolean IODataQueue::enqueue(void * data, UInt32 dataSize) { // Is there enough room at the end for the entry? if ((entrySize <= UINT32_MAX - tail) && - ((tail + entrySize) <= dataQueue->queueSize) ) + ((tail + entrySize) <= queueSize) ) { entry = (IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail); @@ -176,8 +192,8 @@ Boolean IODataQueue::enqueue(void * data, UInt32 dataSize) // The tail can be out of bound when the size of the new entry // exactly matches the available space at the end of the queue. // The tail can range from 0 to dataQueue->queueSize inclusive. - - OSAddAtomic(entrySize, (SInt32 *)&dataQueue->tail); + + newTail = tail + entrySize; } else if ( head > entrySize ) // Is there enough room at the beginning? { @@ -190,13 +206,13 @@ Boolean IODataQueue::enqueue(void * data, UInt32 dataSize) // doing this. The user client checks for this and will look for the size // at the beginning if there isn't room for it at the end. - if ( ( dataQueue->queueSize - tail ) >= DATA_QUEUE_ENTRY_HEADER_SIZE ) + if ( ( queueSize - tail ) >= DATA_QUEUE_ENTRY_HEADER_SIZE ) { ((IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail))->size = dataSize; } memcpy(&dataQueue->queue->data, data, dataSize); - OSCompareAndSwap(dataQueue->tail, entrySize, &dataQueue->tail); + newTail = entrySize; } else { @@ -214,7 +230,7 @@ Boolean IODataQueue::enqueue(void * data, UInt32 dataSize) entry->size = dataSize; memcpy(&entry->data, data, dataSize); - OSAddAtomic(entrySize, (SInt32 *)&dataQueue->tail); + newTail = tail + entrySize; } else { @@ -222,10 +238,13 @@ Boolean IODataQueue::enqueue(void * data, UInt32 dataSize) } } + // Store tail with a release memory barrier + __c11_atomic_store((_Atomic UInt32 *)&dataQueue->tail, newTail, __ATOMIC_RELEASE); + // Send notification (via mach message) that data is available. - if ( ( head == tail ) /* queue was empty prior to enqueue() */ - || ( dataQueue->head == tail ) ) /* queue was emptied during enqueue() */ + if ( ( head == tail ) /* queue was empty prior to enqueue() */ + || ( tail == __c11_atomic_load((_Atomic UInt32 *)&dataQueue->head, __ATOMIC_RELAXED) ) ) /* queue was emptied during enqueue() */ { sendDataAvailableNotification(); } @@ -235,14 +254,13 @@ Boolean IODataQueue::enqueue(void * data, UInt32 dataSize) void IODataQueue::setNotificationPort(mach_port_t port) { - mach_msg_header_t * msgh = (mach_msg_header_t *) notifyMsg; + mach_msg_header_t * msgh; - if (msgh) { - bzero(msgh, sizeof(mach_msg_header_t)); - msgh->msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0); - msgh->msgh_size = sizeof(mach_msg_header_t); - msgh->msgh_remote_port = port; - } + msgh = &((IODataQueueInternal *) notifyMsg)->msg; + bzero(msgh, sizeof(mach_msg_header_t)); + msgh->msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0); + msgh->msgh_size = sizeof(mach_msg_header_t); + msgh->msgh_remote_port = port; } void IODataQueue::sendDataAvailableNotification() @@ -250,8 +268,8 @@ void IODataQueue::sendDataAvailableNotification() kern_return_t kr; mach_msg_header_t * msgh; - msgh = (mach_msg_header_t *) notifyMsg; - if (msgh && msgh->msgh_remote_port) { + msgh = &((IODataQueueInternal *) notifyMsg)->msg; + if (msgh->msgh_remote_port) { kr = mach_msg_send_from_kernel_with_options(msgh, msgh->msgh_size, MACH_SEND_TIMEOUT, MACH_MSG_TIMEOUT_NONE); switch(kr) { case MACH_SEND_TIMED_OUT: // Notification already sent @@ -268,9 +286,11 @@ void IODataQueue::sendDataAvailableNotification() IOMemoryDescriptor *IODataQueue::getMemoryDescriptor() { IOMemoryDescriptor *descriptor = 0; + UInt32 queueSize; + queueSize = ((IODataQueueInternal *) notifyMsg)->queueSize; if (dataQueue != 0) { - descriptor = IOMemoryDescriptor::withAddress(dataQueue, dataQueue->queueSize + DATA_QUEUE_MEMORY_HEADER_SIZE, kIODirectionOutIn); + descriptor = IOMemoryDescriptor::withAddress(dataQueue, queueSize + DATA_QUEUE_MEMORY_HEADER_SIZE, kIODirectionOutIn); } return descriptor;