- const UInt32 head = dataQueue->head; // volatile
- const UInt32 tail = dataQueue->tail;
- const UInt32 entrySize = dataSize + DATA_QUEUE_ENTRY_HEADER_SIZE;
- IODataQueueEntry * entry;
-
- // Check for overflow of entrySize
- if (dataSize > UINT32_MAX - DATA_QUEUE_ENTRY_HEADER_SIZE) {
- return false;
- }
- // Check for underflow of (getQueueSize() - tail)
- if (getQueueSize() < tail || getQueueSize() < head) {
- return false;
- }
-
- if ( tail >= head )
- {
- // Is there enough room at the end for the entry?
- if ((entrySize <= UINT32_MAX - tail) &&
- ((tail + entrySize) <= getQueueSize()) )
- {
- entry = (IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail);
-
- entry->size = dataSize;
- memcpy(&entry->data, data, dataSize);
-
- // The tail can be out of bound when the size of the new entry
- // exactly matches the available space at the end of the queue.
- // The tail can range from 0 to dataQueue->queueSize inclusive.
-
- OSAddAtomic(entrySize, (SInt32 *)&dataQueue->tail);
- }
- else if ( head > entrySize ) // Is there enough room at the beginning?
- {
- // Wrap around to the beginning, but do not allow the tail to catch
- // up to the head.
-
- dataQueue->queue->size = dataSize;
-
- // We need to make sure that there is enough room to set the size before
- // doing this. The user client checks for this and will look for the size
- // at the beginning if there isn't room for it at the end.
-
- if ( ( getQueueSize() - tail ) >= DATA_QUEUE_ENTRY_HEADER_SIZE )
- {
- ((IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail))->size = dataSize;
- }
-
- memcpy(&dataQueue->queue->data, data, dataSize);
- OSCompareAndSwap(dataQueue->tail, entrySize, &dataQueue->tail);
- }
- else
- {
- return false; // queue is full
- }
- }
- else
- {
- // Do not allow the tail to catch up to the head when the queue is full.
- // That's why the comparison uses a '>' rather than '>='.
-
- if ( (head - tail) > entrySize )
- {
- entry = (IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail);
-
- entry->size = dataSize;
- memcpy(&entry->data, data, dataSize);
- OSAddAtomic(entrySize, (SInt32 *)&dataQueue->tail);
- }
- else
- {
- return false; // queue is full
- }
- }
-
- // Send notification (via mach message) that data is available.
-
- if ( ( head == tail ) /* queue was empty prior to enqueue() */
- || ( dataQueue->head == tail ) ) /* queue was emptied during enqueue() */
- {
- sendDataAvailableNotification();
- }
-
- return true;
+ UInt32 head;
+ UInt32 tail;
+ UInt32 newTail;
+ const UInt32 entrySize = dataSize + DATA_QUEUE_ENTRY_HEADER_SIZE;
+ IODataQueueEntry * entry;
+
+ // Force a single read of head and tail
+ // See rdar://problem/40780584 for an explanation of relaxed/acquire barriers
+ tail = __c11_atomic_load((_Atomic UInt32 *)&dataQueue->tail, __ATOMIC_RELAXED);
+ head = __c11_atomic_load((_Atomic UInt32 *)&dataQueue->head, __ATOMIC_ACQUIRE);
+
+ // Check for overflow of entrySize
+ if (dataSize > UINT32_MAX - DATA_QUEUE_ENTRY_HEADER_SIZE) {
+ return false;
+ }
+ // Check for underflow of (getQueueSize() - tail)
+ if (getQueueSize() < tail || getQueueSize() < head) {
+ return false;
+ }
+
+ if (tail >= head) {
+ // Is there enough room at the end for the entry?
+ if ((entrySize <= UINT32_MAX - tail) &&
+ ((tail + entrySize) <= getQueueSize())) {
+ entry = (IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail);
+
+ entry->size = dataSize;
+ __nochk_memcpy(&entry->data, data, dataSize);
+
+ // The tail can be out of bound when the size of the new entry
+ // exactly matches the available space at the end of the queue.
+ // The tail can range from 0 to dataQueue->queueSize inclusive.
+
+ newTail = tail + entrySize;
+ } else if (head > entrySize) { // Is there enough room at the beginning?
+ // Wrap around to the beginning, but do not allow the tail to catch
+ // up to the head.
+
+ dataQueue->queue->size = dataSize;
+
+ // We need to make sure that there is enough room to set the size before
+ // doing this. The user client checks for this and will look for the size
+ // at the beginning if there isn't room for it at the end.
+
+ if ((getQueueSize() - tail) >= DATA_QUEUE_ENTRY_HEADER_SIZE) {
+ ((IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail))->size = dataSize;
+ }
+
+ __nochk_memcpy(&dataQueue->queue->data, data, dataSize);
+ newTail = entrySize;
+ } else {
+ return false; // queue is full
+ }
+ } else {
+ // Do not allow the tail to catch up to the head when the queue is full.
+ // That's why the comparison uses a '>' rather than '>='.
+
+ if ((head - tail) > entrySize) {
+ entry = (IODataQueueEntry *)((UInt8 *)dataQueue->queue + tail);
+
+ entry->size = dataSize;
+ __nochk_memcpy(&entry->data, data, dataSize);
+ newTail = tail + entrySize;
+ } else {
+ return false; // queue is full
+ }
+ }
+
+ // Publish the data we just enqueued
+ __c11_atomic_store((_Atomic UInt32 *)&dataQueue->tail, newTail, __ATOMIC_RELEASE);
+
+ if (tail != head) {
+ //
+ // The memory barrier below paris with the one in ::dequeue
+ // so that either our store to the tail cannot be missed by
+ // the next dequeue attempt, or we will observe the dequeuer
+ // making the queue empty.
+ //
+ // Of course, if we already think the queue is empty,
+ // there's no point paying this extra cost.
+ //
+ __c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
+ head = __c11_atomic_load((_Atomic UInt32 *)&dataQueue->head, __ATOMIC_RELAXED);
+ }
+
+ if (tail == head) {
+ // Send notification (via mach message) that data is now available.
+ sendDataAvailableNotification();
+ }
+ return true;
projects / apple / xnu.git / blobdiff