+/*
+ * journal_trim_extent_overlap
+ *
+ * Return 1 if there are any pending TRIMs that overlap with the given offset and length
+ * Return 0 otherwise.
+ */
+
+int journal_trim_extent_overlap (journal *jnl, uint64_t offset, uint64_t length, uint64_t *end) {
+ transaction *tr = NULL;
+ int overlap = 0;
+
+ uint64_t overlap_start;
+ uint64_t overlap_len;
+ tr = jnl->active_tr;
+ CHECK_TRANSACTION(tr);
+
+ /*
+ * There are two lists that need to be examined for potential overlaps:
+ *
+ * The first is the current transaction. Since this function requires that
+ * a transaction be active when this is called, this is the "active_tr"
+ * pointer in the journal struct. This has a trimlist pointer which needs
+ * to be searched.
+ */
+ overlap = trim_search_extent (&tr->trim, offset, length, &overlap_start, &overlap_len);
+ if (overlap == 0) {
+ /*
+ * The second is the async trim list, which is only done if the current
+ * transaction group (active transaction) did not overlap with our target
+ * extent. This async trim list is the set of all previously
+ * committed transaction groups whose I/Os are now in-flight. We need to hold the
+ * trim lock in order to search this list. If we grab the list before the
+ * TRIM has completed, then we will compare it. If it is grabbed AFTER the
+ * TRIM has completed, then the pointer will be zeroed out and we won't have
+ * to check anything.
+ */
+ lck_rw_lock_shared (&jnl->trim_lock);
+ if (jnl->async_trim != NULL) {
+ overlap = trim_search_extent(jnl->async_trim, offset, length, &overlap_start, &overlap_len);
+ }
+ lck_rw_unlock_shared (&jnl->trim_lock);
+ }
+
+ if (overlap) {
+ /* compute the end (min) of the overlapping range */
+ if ( (overlap_start + overlap_len) < (offset + length)) {
+ *end = (overlap_start + overlap_len);
+ }
+ else {
+ *end = (offset + length);
+ }
+ }
+
+
+ return overlap;
+}
+
+/*
+ * journal_request_immediate_flush
+ *
+ * FS requests that the journal flush immediately upon the
+ * active transaction's completion.
+ *
+ * Returns 0 if operation succeeds
+ * Returns EPERM if we failed to leave hint
+ */
+int
+journal_request_immediate_flush (journal *jnl) {
+
+ transaction *tr = NULL;
+ /*
+ * Is a transaction still in process? You must do
+ * this while there are txns open
+ */
+ tr = jnl->active_tr;
+ if (tr != NULL) {
+ CHECK_TRANSACTION(tr);
+ tr->flush_on_completion = TRUE;
+ }
+ else {
+ return EPERM;
+ }
+ return 0;
+}
+
+
+
+/*
+;________________________________________________________________________________
+;
+; Routine: trim_remove_extent
+;
+; Function: Indicate that a range of bytes, some of which may have previously
+; been passed to journal_trim_add_extent, is now allocated.
+; Any overlapping ranges currently in the journal's trim list will
+; be removed. If the underlying device supports TRIM (UNMAP), then
+; these extents will not be trimmed/unmapped when the transaction
+; is written to the journal.
+;
+; HFS also uses this to prevent newly allocated space from being
+; added to its free extent cache (if some portion of the newly
+; allocated space was recently freed).
+;
+; Input Arguments:
+; trim - The trim list to update.
+; offset - The first byte of the range to be trimmed.
+; length - The number of bytes of the extent being trimmed.
+;________________________________________________________________________________
+*/
+static int
+trim_remove_extent(struct jnl_trim_list *trim, uint64_t offset, uint64_t length)
+{
+ u_int64_t end;
+ dk_extent_t *extent;
+ u_int32_t keep_before;
+ u_int32_t keep_after;
+
+ end = offset + length;
+
+ /*
+ * Find any existing extents that start before or end after the input
+ * extent. These extents will be modified if they overlap the input
+ * extent. Other extents between them will be deleted.
+ */
+ extent = trim->extents;
+ keep_before = 0;
+ while (keep_before < trim->extent_count && extent->offset < offset) {
+ ++keep_before;
+ ++extent;
+ }
+ keep_after = keep_before;
+ if (keep_after > 0) {
+ /* See if previous extent extends beyond both ends of input extent. */
+ --keep_after;
+ --extent;
+ }
+ while (keep_after < trim->extent_count && (extent->offset + extent->length) <= end) {
+ ++keep_after;
+ ++extent;
+ }
+
+ /*
+ * When we get here, the first keep_before extents (0 .. keep_before-1)
+ * start before the input extent, and extents (keep_after .. extent_count-1)
+ * end after the input extent. We'll need to keep, all of those extents,
+ * but possibly modify #(keep_before-1) and #keep_after to remove the portion
+ * that overlaps with the input extent.
+ */
+
+ /*
+ * Does the input extent start after and end before the same existing
+ * extent? If so, we have to "punch a hole" in that extent and convert
+ * it to two separate extents.
+ */
+ if (keep_before > keep_after) {
+ /* If the list was already full, we need to grow it. */
+ if (trim->extent_count == trim->allocated_count) {
+ if (trim_realloc(trim) != 0) {
+ printf("jnl: trim_remove_extent: out of memory!");
+ return ENOMEM;
+ }
+ }
+
+ /*
+ * Make room for a new extent by shifting extents #keep_after and later
+ * down by one extent. When we're done, extents #keep_before and
+ * #keep_after will be identical, and we can fall through to removing
+ * the portion that overlaps the input extent.
+ */
+ memmove(&trim->extents[keep_before],
+ &trim->extents[keep_after],
+ (trim->extent_count - keep_after) * sizeof(dk_extent_t));
+ ++trim->extent_count;
+ ++keep_after;
+
+ /*
+ * Fall through. We now have the case where the length of extent
+ * #(keep_before - 1) needs to be updated, and the start of extent
+ * #(keep_after) needs to be updated.
+ */
+ }
+
+ /*
+ * May need to truncate the end of extent #(keep_before - 1) if it overlaps
+ * the input extent.
+ */
+ if (keep_before > 0) {
+ extent = &trim->extents[keep_before - 1];
+ if (extent->offset + extent->length > offset) {
+ extent->length = offset - extent->offset;
+ }
+ }
+
+ /*
+ * May need to update the start of extent #(keep_after) if it overlaps the
+ * input extent.
+ */
+ if (keep_after < trim->extent_count) {
+ extent = &trim->extents[keep_after];
+ if (extent->offset < end) {
+ extent->length = extent->offset + extent->length - end;
+ extent->offset = end;
+ }
+ }
+
+ /*
+ * If there were whole extents that overlapped the input extent, get rid
+ * of them by shifting any following extents, and updating the count.
+ */
+ if (keep_after > keep_before && keep_after < trim->extent_count) {
+ memmove(&trim->extents[keep_before],
+ &trim->extents[keep_after],
+ (trim->extent_count - keep_after) * sizeof(dk_extent_t));
+ }
+ trim->extent_count -= keep_after - keep_before;
+
+ return 0;
+}
+
+/*
+ ;________________________________________________________________________________
+ ;
+ ; Routine: journal_trim_remove_extent
+ ;
+ ; Function: Make note of a range of bytes, some of which may have previously
+ ; been passed to journal_trim_add_extent, is now in use on the
+ ; volume. The given bytes will be not be trimmed as part of
+ ; this transaction, or a pending trim of a transaction being
+ ; asynchronously flushed.
+ ;
+ ; Input Arguments:
+ ; jnl - The journal for the volume containing the byte range.
+ ; offset - The first byte of the range to be trimmed.
+ ; length - The number of bytes of the extent being trimmed.
+ ;________________________________________________________________________________
+ */
+__private_extern__ int
+journal_trim_remove_extent(journal *jnl, uint64_t offset, uint64_t length)
+{
+ int error = 0;
+ transaction *tr;
+
+ CHECK_JOURNAL(jnl);
+
+ /* TODO: Is it OK to manipulate the trim list even if JOURNAL_INVALID is set? I think so... */
+ if (jnl->flags & JOURNAL_INVALID) {
+ return EINVAL;
+ }
+
+ tr = jnl->active_tr;
+ CHECK_TRANSACTION(tr);
+
+ if (jnl_kdebug)
+ KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE | DBG_FUNC_START, jnl, offset, length, tr->trim.extent_count, 0);
+
+ if (jnl->owner != current_thread()) {
+ panic("jnl: trim_remove_extent: called w/out a transaction! jnl %p, owner %p, curact %p\n",
+ jnl, jnl->owner, current_thread());
+ }
+
+ free_old_stuff(jnl);
+
+ error = trim_remove_extent(&tr->trim, offset, length);
+ if (error == 0) {
+ int found = FALSE;
+
+ /*
+ * See if a pending trim has any extents that overlap with the
+ * one we were given.
+ */
+ lck_rw_lock_shared(&jnl->trim_lock);
+ if (jnl->async_trim != NULL)
+ found = trim_search_extent(jnl->async_trim, offset, length, NULL, NULL);
+ lck_rw_unlock_shared(&jnl->trim_lock);
+
+ if (found) {
+ /*
+ * There was an overlap, so avoid trimming the extent we
+ * just allocated. (Otherwise, it might get trimmed after
+ * we've written to it, which will cause that data to be
+ * corrupted.)
+ */
+ uint32_t async_extent_count = 0;
+
+ if (jnl_kdebug)
+ KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE_PENDING | DBG_FUNC_START, jnl, offset, length, 0, 0);
+ lck_rw_lock_exclusive(&jnl->trim_lock);
+ if (jnl->async_trim != NULL) {
+ error = trim_remove_extent(jnl->async_trim, offset, length);
+ async_extent_count = jnl->async_trim->extent_count;
+ }
+ lck_rw_unlock_exclusive(&jnl->trim_lock);
+ if (jnl_kdebug)
+ KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE_PENDING | DBG_FUNC_END, error, 0, 0, async_extent_count, 0);
+ }
+ }
+
+ if (jnl_kdebug)
+ KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE | DBG_FUNC_END, error, 0, 0, tr->trim.extent_count, 0);
+ return error;
+}
+
+
+static int
+journal_trim_flush(journal *jnl, transaction *tr)
+{
+ int errno = 0;
+
+ if (jnl_kdebug)
+ KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_FLUSH | DBG_FUNC_START, jnl, tr, 0, tr->trim.extent_count, 0);
+
+ lck_rw_lock_shared(&jnl->trim_lock);
+ if (tr->trim.extent_count > 0) {
+ dk_unmap_t unmap;
+
+ bzero(&unmap, sizeof(unmap));
+ if (CONFIG_HFS_TRIM && (jnl->flags & JOURNAL_USE_UNMAP)) {
+ unmap.extents = tr->trim.extents;
+ unmap.extentsCount = tr->trim.extent_count;
+ if (jnl_kdebug)
+ KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_UNMAP | DBG_FUNC_START, jnl, tr, 0, tr->trim.extent_count, 0);
+ errno = VNOP_IOCTL(jnl->fsdev, DKIOCUNMAP, (caddr_t)&unmap, FWRITE, vfs_context_kernel());
+ if (jnl_kdebug)
+ KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_UNMAP | DBG_FUNC_END, errno, 0, 0, 0, 0);
+ }
+
+ /*
+ * Call back into the file system to tell them that we have
+ * trimmed some extents and that they can now be reused.
+ *
+ * CAUTION: If the journal becomes invalid (eg., due to an I/O
+ * error when trying to write to the journal), this callback
+ * will stop getting called, even if extents got freed before
+ * the journal became invalid!
+ */
+ if (jnl->trim_callback)
+ jnl->trim_callback(jnl->trim_callback_arg, tr->trim.extent_count, tr->trim.extents);
+ }
+ lck_rw_unlock_shared(&jnl->trim_lock);
+
+ /*
+ * If the transaction we're flushing was the async transaction, then
+ * tell the current transaction that there is no pending trim
+ * any more.
+ *
+ * NOTE: Since we released the lock, another thread could have
+ * removed one or more extents from our list. That's not a
+ * problem since any writes to the re-allocated blocks
+ * would get sent to the device after the DKIOCUNMAP.
+ */
+ lck_rw_lock_exclusive(&jnl->trim_lock);
+ if (jnl->async_trim == &tr->trim)
+ jnl->async_trim = NULL;
+ lck_rw_unlock_exclusive(&jnl->trim_lock);
+
+ /*
+ * By the time we get here, no other thread can discover the address
+ * of "tr", so it is safe for us to manipulate tr->trim without
+ * holding any locks.
+ */
+ if (tr->trim.extents) {
+ kfree(tr->trim.extents, tr->trim.allocated_count * sizeof(dk_extent_t));
+ tr->trim.allocated_count = 0;
+ tr->trim.extent_count = 0;
+ tr->trim.extents = NULL;
+ }
+
+ if (jnl_kdebug)
+ KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_FLUSH | DBG_FUNC_END, errno, 0, 0, 0, 0);
+
+ return errno;
+}
projects / apple / xnu.git / blobdiff