+ /*
+ * Delete from old vnode list, if on one.
+ */
+ if (bp->b_vnbufs.le_next != NOLIST)
+ bufremvn(bp);
+ /*
+ * If dirty, put on list of dirty buffers;
+ * otherwise insert onto list of clean buffers.
+ */
+ if (ISSET(bp->b_flags, B_DELWRI))
+ listheadp = &newvp->v_dirtyblkhd;
+ else
+ listheadp = &newvp->v_cleanblkhd;
+ bufinsvn(bp, listheadp);
+
+ lck_mtx_unlock(buf_mtxp);
+}
+
+static __inline__ void
+bufhdrinit(buf_t bp)
+{
+ bzero((char *)bp, sizeof *bp);
+ bp->b_dev = NODEV;
+ bp->b_rcred = NOCRED;
+ bp->b_wcred = NOCRED;
+ bp->b_vnbufs.le_next = NOLIST;
+ bp->b_flags = B_INVAL;
+
+ return;
+}
+
+/*
+ * Initialize buffers and hash links for buffers.
+ */
+__private_extern__ void
+bufinit(void)
+{
+ buf_t bp;
+ struct bqueues *dp;
+ int i;
+
+ nbuf_headers = 0;
+ /* Initialize the buffer queues ('freelists') and the hash table */
+ for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
+ TAILQ_INIT(dp);
+ bufhashtbl = hashinit(nbuf_hashelements, M_CACHE, &bufhash);
+
+ buf_busycount = 0;
+
+ /* Initialize the buffer headers */
+ for (i = 0; i < max_nbuf_headers; i++) {
+ nbuf_headers++;
+ bp = &buf_headers[i];
+ bufhdrinit(bp);
+
+ BLISTNONE(bp);
+ dp = &bufqueues[BQ_EMPTY];
+ bp->b_whichq = BQ_EMPTY;
+ bp->b_timestamp = buf_timestamp();
+ binsheadfree(bp, dp, BQ_EMPTY);
+ binshash(bp, &invalhash);
+ }
+ boot_nbuf_headers = nbuf_headers;
+
+ TAILQ_INIT(&iobufqueue);
+ TAILQ_INIT(&delaybufqueue);
+
+ for (; i < nbuf_headers + niobuf_headers; i++) {
+ bp = &buf_headers[i];
+ bufhdrinit(bp);
+ bp->b_whichq = -1;
+ binsheadfree(bp, &iobufqueue, -1);
+ }
+
+ /*
+ * allocate lock group attribute and group
+ */
+ buf_mtx_grp_attr = lck_grp_attr_alloc_init();
+ buf_mtx_grp = lck_grp_alloc_init("buffer cache", buf_mtx_grp_attr);
+
+ /*
+ * allocate the lock attribute
+ */
+ buf_mtx_attr = lck_attr_alloc_init();
+
+ /*
+ * allocate and initialize mutex's for the buffer and iobuffer pools
+ */
+ buf_mtxp = lck_mtx_alloc_init(buf_mtx_grp, buf_mtx_attr);
+ iobuffer_mtxp = lck_mtx_alloc_init(buf_mtx_grp, buf_mtx_attr);
+
+ if (iobuffer_mtxp == NULL)
+ panic("couldn't create iobuffer mutex");
+
+ if (buf_mtxp == NULL)
+ panic("couldn't create buf mutex");
+
+ /*
+ * allocate and initialize cluster specific global locks...
+ */
+ cluster_init();
+
+ printf("using %d buffer headers and %d cluster IO buffer headers\n",
+ nbuf_headers, niobuf_headers);
+
+ /* Set up zones used by the buffer cache */
+ bufzoneinit();
+
+ /* start the bcleanbuf() thread */
+ bcleanbuf_thread_init();
+
+ /* Register a callout for relieving vm pressure */
+ if (vm_set_buffer_cleanup_callout(buffer_cache_gc) != KERN_SUCCESS) {
+ panic("Couldn't register buffer cache callout for vm pressure!\n");
+ }
+
+#if BALANCE_QUEUES
+ {
+ static void bufq_balance_thread_init(void) __attribute__((section("__TEXT, initcode")));
+ /* create a thread to do dynamic buffer queue balancing */
+ bufq_balance_thread_init();
+ }
+#endif /* notyet */
+}
+
+
+
+/*
+ * Zones for the meta data buffers
+ */
+
+#define MINMETA 512
+#define MAXMETA 8192
+
+struct meta_zone_entry {
+ zone_t mz_zone;
+ vm_size_t mz_size;
+ vm_size_t mz_max;
+ const char *mz_name;
+};
+
+struct meta_zone_entry meta_zones[] = {
+ {NULL, (MINMETA * 1), 128 * (MINMETA * 1), "buf.512" },
+ {NULL, (MINMETA * 2), 64 * (MINMETA * 2), "buf.1024" },
+ {NULL, (MINMETA * 4), 16 * (MINMETA * 4), "buf.2048" },
+ {NULL, (MINMETA * 8), 512 * (MINMETA * 8), "buf.4096" },
+ {NULL, (MINMETA * 16), 512 * (MINMETA * 16), "buf.8192" },
+ {NULL, 0, 0, "" } /* End */
+};
+
+/*
+ * Initialize the meta data zones
+ */
+static void
+bufzoneinit(void)
+{
+ int i;
+
+ for (i = 0; meta_zones[i].mz_size != 0; i++) {
+ meta_zones[i].mz_zone =
+ zinit(meta_zones[i].mz_size,
+ meta_zones[i].mz_max,
+ PAGE_SIZE,
+ meta_zones[i].mz_name);
+ zone_change(meta_zones[i].mz_zone, Z_CALLERACCT, FALSE);
+ }
+ buf_hdr_zone = zinit(sizeof(struct buf), 32, PAGE_SIZE, "buf headers");
+ zone_change(buf_hdr_zone, Z_CALLERACCT, FALSE);
+}
+
+static __inline__ zone_t
+getbufzone(size_t size)
+{
+ int i;
+
+ if ((size % 512) || (size < MINMETA) || (size > MAXMETA))
+ panic("getbufzone: incorect size = %lu", size);
+
+ for (i = 0; meta_zones[i].mz_size != 0; i++) {
+ if (meta_zones[i].mz_size >= size)
+ break;
+ }
+
+ return (meta_zones[i].mz_zone);
+}
+
+
+
+static struct buf *
+bio_doread(vnode_t vp, daddr64_t blkno, int size, kauth_cred_t cred, int async, int queuetype)
+{
+ buf_t bp;
+
+ bp = buf_getblk(vp, blkno, size, 0, 0, queuetype);
+
+ /*
+ * If buffer does not have data valid, start a read.
+ * Note that if buffer is B_INVAL, buf_getblk() won't return it.
+ * Therefore, it's valid if it's I/O has completed or been delayed.
+ */
+ if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) {
+ struct proc *p;
+
+ p = current_proc();
+
+ /* Start I/O for the buffer (keeping credentials). */
+ SET(bp->b_flags, B_READ | async);
+ if (IS_VALID_CRED(cred) && !IS_VALID_CRED(bp->b_rcred)) {
+ kauth_cred_ref(cred);
+ bp->b_rcred = cred;
+ }
+
+ VNOP_STRATEGY(bp);
+
+ trace(TR_BREADMISS, pack(vp, size), blkno);
+
+ /* Pay for the read. */
+ if (p && p->p_stats)
+ OSIncrementAtomicLong(&p->p_stats->p_ru.ru_inblock); /* XXX */
+
+ if (async) {
+ /*
+ * since we asked for an ASYNC I/O
+ * the biodone will do the brelse
+ * we don't want to pass back a bp
+ * that we don't 'own'
+ */
+ bp = NULL;
+ }
+ } else if (async) {
+ buf_brelse(bp);
+ bp = NULL;
+ }
+
+ trace(TR_BREADHIT, pack(vp, size), blkno);
+
+ return (bp);
+}
+
+/*
+ * Perform the reads for buf_breadn() and buf_meta_breadn().
+ * Trivial modification to the breada algorithm presented in Bach (p.55).
+ */
+static errno_t
+do_breadn_for_type(vnode_t vp, daddr64_t blkno, int size, daddr64_t *rablks, int *rasizes,
+ int nrablks, kauth_cred_t cred, buf_t *bpp, int queuetype)
+{
+ buf_t bp;
+ int i;
+
+ bp = *bpp = bio_doread(vp, blkno, size, cred, 0, queuetype);
+
+ /*
+ * For each of the read-ahead blocks, start a read, if necessary.
+ */
+ for (i = 0; i < nrablks; i++) {
+ /* If it's in the cache, just go on to next one. */
+ if (incore(vp, rablks[i]))
+ continue;
+
+ /* Get a buffer for the read-ahead block */
+ (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC, queuetype);
+ }
+
+ /* Otherwise, we had to start a read for it; wait until it's valid. */
+ return (buf_biowait(bp));
+}
+
+
+/*
+ * Read a disk block.
+ * This algorithm described in Bach (p.54).
+ */
+errno_t
+buf_bread(vnode_t vp, daddr64_t blkno, int size, kauth_cred_t cred, buf_t *bpp)
+{
+ buf_t bp;
+
+ /* Get buffer for block. */
+ bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_READ);
+
+ /* Wait for the read to complete, and return result. */
+ return (buf_biowait(bp));
+}
+
+/*
+ * Read a disk block. [bread() for meta-data]
+ * This algorithm described in Bach (p.54).
+ */
+errno_t
+buf_meta_bread(vnode_t vp, daddr64_t blkno, int size, kauth_cred_t cred, buf_t *bpp)
+{
+ buf_t bp;
+
+ /* Get buffer for block. */
+ bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_META);
+
+ /* Wait for the read to complete, and return result. */
+ return (buf_biowait(bp));
+}
+
+/*
+ * Read-ahead multiple disk blocks. The first is sync, the rest async.
+ */
+errno_t
+buf_breadn(vnode_t vp, daddr64_t blkno, int size, daddr64_t *rablks, int *rasizes, int nrablks, kauth_cred_t cred, buf_t *bpp)
+{
+ return (do_breadn_for_type(vp, blkno, size, rablks, rasizes, nrablks, cred, bpp, BLK_READ));
+}
+
+/*
+ * Read-ahead multiple disk blocks. The first is sync, the rest async.
+ * [buf_breadn() for meta-data]
+ */
+errno_t
+buf_meta_breadn(vnode_t vp, daddr64_t blkno, int size, daddr64_t *rablks, int *rasizes, int nrablks, kauth_cred_t cred, buf_t *bpp)
+{
+ return (do_breadn_for_type(vp, blkno, size, rablks, rasizes, nrablks, cred, bpp, BLK_META));
+}
+
+/*
+ * Block write. Described in Bach (p.56)
+ */
+errno_t
+buf_bwrite(buf_t bp)
+{
+ int sync, wasdelayed;
+ errno_t rv;
+ proc_t p = current_proc();
+ vnode_t vp = bp->b_vp;
+
+ if (bp->b_datap == 0) {
+ if (brecover_data(bp) == 0)
+ return (0);
+ }
+ /* Remember buffer type, to switch on it later. */
+ sync = !ISSET(bp->b_flags, B_ASYNC);
+ wasdelayed = ISSET(bp->b_flags, B_DELWRI);
+ CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI));
+
+ if (wasdelayed)
+ OSAddAtomicLong(-1, &nbdwrite);
+
+ if (!sync) {
+ /*
+ * If not synchronous, pay for the I/O operation and make
+ * sure the buf is on the correct vnode queue. We have
+ * to do this now, because if we don't, the vnode may not
+ * be properly notified that its I/O has completed.
+ */
+ if (wasdelayed)
+ buf_reassign(bp, vp);
+ else
+ if (p && p->p_stats)
+ OSIncrementAtomicLong(&p->p_stats->p_ru.ru_oublock); /* XXX */
+ }
+ trace(TR_BUFWRITE, pack(vp, bp->b_bcount), bp->b_lblkno);
+
+ /* Initiate disk write. Make sure the appropriate party is charged. */
+
+ OSAddAtomic(1, &vp->v_numoutput);
+
+ VNOP_STRATEGY(bp);
+
+ if (sync) {
+ /*
+ * If I/O was synchronous, wait for it to complete.
+ */
+ rv = buf_biowait(bp);
+
+ /*
+ * Pay for the I/O operation, if it's not been paid for, and
+ * make sure it's on the correct vnode queue. (async operatings
+ * were payed for above.)
+ */
+ if (wasdelayed)
+ buf_reassign(bp, vp);
+ else
+ if (p && p->p_stats)
+ OSIncrementAtomicLong(&p->p_stats->p_ru.ru_oublock); /* XXX */
+
+ /* Release the buffer. */
+ // XXXdbg - only if the unused bit is set
+ if (!ISSET(bp->b_flags, B_NORELSE)) {
+ buf_brelse(bp);
+ } else {
+ CLR(bp->b_flags, B_NORELSE);
+ }
+
+ return (rv);
+ } else {
+ return (0);
+ }
+}
+
+int
+vn_bwrite(struct vnop_bwrite_args *ap)
+{
+ return (buf_bwrite(ap->a_bp));
+}
+
+/*
+ * Delayed write.
+ *
+ * The buffer is marked dirty, but is not queued for I/O.
+ * This routine should be used when the buffer is expected
+ * to be modified again soon, typically a small write that
+ * partially fills a buffer.
+ *
+ * NB: magnetic tapes cannot be delayed; they must be
+ * written in the order that the writes are requested.
+ *
+ * Described in Leffler, et al. (pp. 208-213).
+ *
+ * Note: With the ability to allocate additional buffer
+ * headers, we can get in to the situation where "too" many
+ * buf_bdwrite()s can create situation where the kernel can create
+ * buffers faster than the disks can service. Doing a buf_bawrite() in
+ * cases where we have "too many" outstanding buf_bdwrite()s avoids that.
+ */
+__private_extern__ int
+bdwrite_internal(buf_t bp, int return_error)
+{
+ proc_t p = current_proc();
+ vnode_t vp = bp->b_vp;
+
+ /*
+ * If the block hasn't been seen before:
+ * (1) Mark it as having been seen,
+ * (2) Charge for the write.
+ * (3) Make sure it's on its vnode's correct block list,
+ */
+ if (!ISSET(bp->b_flags, B_DELWRI)) {
+ SET(bp->b_flags, B_DELWRI);
+ if (p && p->p_stats)
+ OSIncrementAtomicLong(&p->p_stats->p_ru.ru_oublock); /* XXX */
+ OSAddAtomicLong(1, &nbdwrite);
+ buf_reassign(bp, vp);
+ }
+
+ /*
+ * if we're not LOCKED, but the total number of delayed writes
+ * has climbed above 75% of the total buffers in the system
+ * return an error if the caller has indicated that it can
+ * handle one in this case, otherwise schedule the I/O now
+ * this is done to prevent us from allocating tons of extra
+ * buffers when dealing with virtual disks (i.e. DiskImages),
+ * because additional buffers are dynamically allocated to prevent
+ * deadlocks from occurring
+ *
+ * however, can't do a buf_bawrite() if the LOCKED bit is set because the
+ * buffer is part of a transaction and can't go to disk until
+ * the LOCKED bit is cleared.
+ */
+ if (!ISSET(bp->b_flags, B_LOCKED) && nbdwrite > ((nbuf_headers/4)*3)) {
+ if (return_error)
+ return (EAGAIN);
+ /*
+ * If the vnode has "too many" write operations in progress
+ * wait for them to finish the IO
+ */
+ (void)vnode_waitforwrites(vp, VNODE_ASYNC_THROTTLE, 0, 0, "buf_bdwrite");
+
+ return (buf_bawrite(bp));
+ }
+
+ /* Otherwise, the "write" is done, so mark and release the buffer. */
+ SET(bp->b_flags, B_DONE);
+ buf_brelse(bp);
+ return (0);
+}
+
+errno_t
+buf_bdwrite(buf_t bp)
+{
+ return (bdwrite_internal(bp, 0));
+}
+
+
+/*
+ * Asynchronous block write; just an asynchronous buf_bwrite().
+ *
+ * Note: With the abilitty to allocate additional buffer
+ * headers, we can get in to the situation where "too" many
+ * buf_bawrite()s can create situation where the kernel can create
+ * buffers faster than the disks can service.
+ * We limit the number of "in flight" writes a vnode can have to
+ * avoid this.
+ */
+static int
+bawrite_internal(buf_t bp, int throttle)
+{
+ vnode_t vp = bp->b_vp;
+
+ if (vp) {
+ if (throttle)
+ /*
+ * If the vnode has "too many" write operations in progress
+ * wait for them to finish the IO
+ */
+ (void)vnode_waitforwrites(vp, VNODE_ASYNC_THROTTLE, 0, 0, (const char *)"buf_bawrite");
+ else if (vp->v_numoutput >= VNODE_ASYNC_THROTTLE)
+ /*
+ * return to the caller and
+ * let him decide what to do
+ */
+ return (EWOULDBLOCK);
+ }
+ SET(bp->b_flags, B_ASYNC);
+
+ return (VNOP_BWRITE(bp));
+}
+
+errno_t
+buf_bawrite(buf_t bp)
+{
+ return (bawrite_internal(bp, 1));
+}
+
+
+
+static void
+buf_free_meta_store(buf_t bp)
+{
+ if (bp->b_bufsize) {
+ if (ISSET(bp->b_flags, B_ZALLOC)) {
+ zone_t z;
+
+ z = getbufzone(bp->b_bufsize);
+ zfree(z, (void *)bp->b_datap);
+ } else
+ kmem_free(kernel_map, bp->b_datap, bp->b_bufsize);
+
+ bp->b_datap = (uintptr_t)NULL;
+ bp->b_bufsize = 0;
+ }
+}
+
+
+static buf_t
+buf_brelse_shadow(buf_t bp)
+{
+ buf_t bp_head;
+ buf_t bp_temp;
+ buf_t bp_return = NULL;
+#ifdef BUF_MAKE_PRIVATE
+ buf_t bp_data;
+ int data_ref = 0;
+#endif
+ lck_mtx_lock_spin(buf_mtxp);
+
+ bp_head = (buf_t)bp->b_orig;
+
+ if (bp_head->b_whichq != -1)
+ panic("buf_brelse_shadow: bp_head on freelist %d\n", bp_head->b_whichq);
+
+#ifdef BUF_MAKE_PRIVATE
+ if (bp_data = bp->b_data_store) {
+ bp_data->b_data_ref--;
+ /*
+ * snapshot the ref count so that we can check it
+ * outside of the lock... we only want the guy going
+ * from 1 -> 0 to try and release the storage
+ */
+ data_ref = bp_data->b_data_ref;
+ }
+#endif
+ KERNEL_DEBUG(0xbbbbc008 | DBG_FUNC_START, bp, bp_head, bp_head->b_shadow_ref, 0, 0);
+
+ bp_head->b_shadow_ref--;
+
+ for (bp_temp = bp_head; bp_temp && bp != bp_temp->b_shadow; bp_temp = bp_temp->b_shadow);
+
+ if (bp_temp == NULL)
+ panic("buf_brelse_shadow: bp not on list %p", bp_head);
+
+ bp_temp->b_shadow = bp_temp->b_shadow->b_shadow;
+
+#ifdef BUF_MAKE_PRIVATE
+ /*
+ * we're about to free the current 'owner' of the data buffer and
+ * there is at least one other shadow buf_t still pointing at it
+ * so transfer it to the first shadow buf left in the chain
+ */
+ if (bp == bp_data && data_ref) {
+ if ((bp_data = bp_head->b_shadow) == NULL)
+ panic("buf_brelse_shadow: data_ref mismatch bp(%p)", bp);
+
+ for (bp_temp = bp_data; bp_temp; bp_temp = bp_temp->b_shadow)
+ bp_temp->b_data_store = bp_data;
+ bp_data->b_data_ref = data_ref;
+ }
+#endif
+ if (bp_head->b_shadow_ref == 0 && bp_head->b_shadow)
+ panic("buf_relse_shadow: b_shadow != NULL && b_shadow_ref == 0 bp(%p)", bp);
+ if (bp_head->b_shadow_ref && bp_head->b_shadow == 0)
+ panic("buf_relse_shadow: b_shadow == NULL && b_shadow_ref != 0 bp(%p)", bp);
+
+ if (bp_head->b_shadow_ref == 0) {
+ if (!ISSET(bp_head->b_lflags, BL_BUSY)) {
+
+ CLR(bp_head->b_flags, B_AGE);
+ bp_head->b_timestamp = buf_timestamp();
+
+ if (ISSET(bp_head->b_flags, B_LOCKED)) {
+ bp_head->b_whichq = BQ_LOCKED;
+ binstailfree(bp_head, &bufqueues[BQ_LOCKED], BQ_LOCKED);
+ } else {
+ bp_head->b_whichq = BQ_META;
+ binstailfree(bp_head, &bufqueues[BQ_META], BQ_META);
+ }
+ } else if (ISSET(bp_head->b_lflags, BL_WAITSHADOW)) {
+ CLR(bp_head->b_lflags, BL_WAITSHADOW);
+
+ bp_return = bp_head;
+ }
+ }
+ lck_mtx_unlock(buf_mtxp);
+#ifdef BUF_MAKE_PRIVATE
+ if (bp == bp_data && data_ref == 0)
+ buf_free_meta_store(bp);
+
+ bp->b_data_store = NULL;
+#endif
+ KERNEL_DEBUG(0xbbbbc008 | DBG_FUNC_END, bp, 0, 0, 0, 0);
+
+ return (bp_return);
+}
+
+
+/*
+ * Release a buffer on to the free lists.
+ * Described in Bach (p. 46).
+ */
+void
+buf_brelse(buf_t bp)
+{
+ struct bqueues *bufq;
+ long whichq;
+ upl_t upl;
+ int need_wakeup = 0;
+ int need_bp_wakeup = 0;
+
+
+ if (bp->b_whichq != -1 || !(bp->b_lflags & BL_BUSY))
+ panic("buf_brelse: bad buffer = %p\n", bp);
+
+#ifdef JOE_DEBUG
+ (void) OSBacktrace(&bp->b_stackbrelse[0], 6);
+
+ bp->b_lastbrelse = current_thread();
+ bp->b_tag = 0;
+#endif
+ if (bp->b_lflags & BL_IOBUF) {
+ buf_t shadow_master_bp = NULL;
+
+ if (ISSET(bp->b_lflags, BL_SHADOW))
+ shadow_master_bp = buf_brelse_shadow(bp);
+ else if (ISSET(bp->b_lflags, BL_IOBUF_ALLOC))
+ buf_free_meta_store(bp);
+ free_io_buf(bp);
+
+ if (shadow_master_bp) {
+ bp = shadow_master_bp;
+ goto finish_shadow_master;
+ }
+ return;
+ }
+
+ KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 388)) | DBG_FUNC_START,
+ bp->b_lblkno * PAGE_SIZE, bp, bp->b_datap,
+ bp->b_flags, 0);
+
+ trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno);
+
+ /*
+ * if we're invalidating a buffer that has the B_FILTER bit
+ * set then call the b_iodone function so it gets cleaned
+ * up properly.
+ *
+ * the HFS journal code depends on this
+ */
+ if (ISSET(bp->b_flags, B_META) && ISSET(bp->b_flags, B_INVAL)) {
+ if (ISSET(bp->b_flags, B_FILTER)) { /* if necessary, call out */
+ void (*iodone_func)(struct buf *, void *) = bp->b_iodone;
+ void *arg = bp->b_transaction;
+
+ CLR(bp->b_flags, B_FILTER); /* but note callout done */
+ bp->b_iodone = NULL;
+ bp->b_transaction = NULL;
+
+ if (iodone_func == NULL) {
+ panic("brelse: bp @ %p has NULL b_iodone!\n", bp);
+ }
+ (*iodone_func)(bp, arg);
+ }
+ }
+ /*
+ * I/O is done. Cleanup the UPL state
+ */
+ upl = bp->b_upl;
+
+ if ( !ISSET(bp->b_flags, B_META) && UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) {
+ kern_return_t kret;
+ int upl_flags;
+
+ if (upl == NULL) {
+ if ( !ISSET(bp->b_flags, B_INVAL)) {
+ kret = ubc_create_upl(bp->b_vp,
+ ubc_blktooff(bp->b_vp, bp->b_lblkno),
+ bp->b_bufsize,
+ &upl,
+ NULL,
+ UPL_PRECIOUS);
+
+ if (kret != KERN_SUCCESS)
+ panic("brelse: Failed to create UPL");
+#if UPL_DEBUG
+ upl_ubc_alias_set(upl, (uintptr_t) bp, (uintptr_t) 5);
+#endif /* UPL_DEBUG */
+ }
+ } else {
+ if (bp->b_datap) {
+ kret = ubc_upl_unmap(upl);
+
+ if (kret != KERN_SUCCESS)
+ panic("ubc_upl_unmap failed");
+ bp->b_datap = (uintptr_t)NULL;
+ }
+ }
+ if (upl) {
+ if (bp->b_flags & (B_ERROR | B_INVAL)) {
+ if (bp->b_flags & (B_READ | B_INVAL))
+ upl_flags = UPL_ABORT_DUMP_PAGES;
+ else
+ upl_flags = 0;
+
+ ubc_upl_abort(upl, upl_flags);
+ } else {
+ if (ISSET(bp->b_flags, B_DELWRI | B_WASDIRTY))
+ upl_flags = UPL_COMMIT_SET_DIRTY ;
+ else
+ upl_flags = UPL_COMMIT_CLEAR_DIRTY ;
+
+ ubc_upl_commit_range(upl, 0, bp->b_bufsize, upl_flags |
+ UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
+ }
+ bp->b_upl = NULL;
+ }
+ } else {
+ if ( (upl) )
+ panic("brelse: UPL set for non VREG; vp=%p", bp->b_vp);
+ }
+
+ /*
+ * If it's locked, don't report an error; try again later.
+ */
+ if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR))
+ CLR(bp->b_flags, B_ERROR);
+ /*
+ * If it's not cacheable, or an error, mark it invalid.
+ */
+ if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR)))
+ SET(bp->b_flags, B_INVAL);
+
+ if ((bp->b_bufsize <= 0) ||
+ ISSET(bp->b_flags, B_INVAL) ||
+ (ISSET(bp->b_lflags, BL_WANTDEALLOC) && !ISSET(bp->b_flags, B_DELWRI))) {
+
+ boolean_t delayed_buf_free_meta_store = FALSE;
+
+ /*
+ * If it's invalid or empty, dissociate it from its vnode,
+ * release its storage if B_META, and
+ * clean it up a bit and put it on the EMPTY queue
+ */
+ if (ISSET(bp->b_flags, B_DELWRI))
+ OSAddAtomicLong(-1, &nbdwrite);
+
+ if (ISSET(bp->b_flags, B_META)) {
+ if (bp->b_shadow_ref)
+ delayed_buf_free_meta_store = TRUE;
+ else
+ buf_free_meta_store(bp);
+ }
+ /*
+ * nuke any credentials we were holding
+ */
+ buf_release_credentials(bp);
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ if (bp->b_shadow_ref) {
+ SET(bp->b_lflags, BL_WAITSHADOW);
+
+ lck_mtx_unlock(buf_mtxp);
+
+ return;
+ }
+ if (delayed_buf_free_meta_store == TRUE) {
+
+ lck_mtx_unlock(buf_mtxp);
+finish_shadow_master:
+ buf_free_meta_store(bp);
+
+ lck_mtx_lock_spin(buf_mtxp);
+ }
+ CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE | B_FUA));
+
+ if (bp->b_vp)
+ brelvp_locked(bp);
+
+ bremhash(bp);
+ BLISTNONE(bp);
+ binshash(bp, &invalhash);
+
+ bp->b_whichq = BQ_EMPTY;
+ binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY);
+ } else {
+
+ /*
+ * It has valid data. Put it on the end of the appropriate
+ * queue, so that it'll stick around for as long as possible.
+ */
+ if (ISSET(bp->b_flags, B_LOCKED))
+ whichq = BQ_LOCKED; /* locked in core */
+ else if (ISSET(bp->b_flags, B_META))
+ whichq = BQ_META; /* meta-data */
+ else if (ISSET(bp->b_flags, B_AGE))
+ whichq = BQ_AGE; /* stale but valid data */
+ else
+ whichq = BQ_LRU; /* valid data */
+ bufq = &bufqueues[whichq];
+
+ bp->b_timestamp = buf_timestamp();
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ /*
+ * the buf_brelse_shadow routine doesn't take 'ownership'
+ * of the parent buf_t... it updates state that is protected by
+ * the buf_mtxp, and checks for BL_BUSY to determine whether to
+ * put the buf_t back on a free list. b_shadow_ref is protected
+ * by the lock, and since we have not yet cleared B_BUSY, we need
+ * to check it while holding the lock to insure that one of us
+ * puts this buf_t back on a free list when it is safe to do so
+ */
+ if (bp->b_shadow_ref == 0) {
+ CLR(bp->b_flags, (B_AGE | B_ASYNC | B_NOCACHE));
+ bp->b_whichq = whichq;
+ binstailfree(bp, bufq, whichq);
+ } else {
+ /*
+ * there are still cloned buf_t's pointing
+ * at this guy... need to keep it off the
+ * freelists until a buf_brelse is done on
+ * the last clone
+ */
+ CLR(bp->b_flags, (B_ASYNC | B_NOCACHE));
+ }
+ }
+ if (needbuffer) {
+ /*
+ * needbuffer is a global
+ * we're currently using buf_mtxp to protect it
+ * delay doing the actual wakeup until after
+ * we drop buf_mtxp
+ */
+ needbuffer = 0;
+ need_wakeup = 1;
+ }
+ if (ISSET(bp->b_lflags, BL_WANTED)) {
+ /*
+ * delay the actual wakeup until after we
+ * clear BL_BUSY and we've dropped buf_mtxp
+ */
+ need_bp_wakeup = 1;
+ }
+ /*
+ * Unlock the buffer.
+ */
+ CLR(bp->b_lflags, (BL_BUSY | BL_WANTED));
+ buf_busycount--;
+
+ lck_mtx_unlock(buf_mtxp);
+
+ if (need_wakeup) {
+ /*
+ * Wake up any processes waiting for any buffer to become free.
+ */
+ wakeup(&needbuffer);
+ }
+ if (need_bp_wakeup) {
+ /*
+ * Wake up any proceeses waiting for _this_ buffer to become free.
+ */
+ wakeup(bp);
+ }
+ KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 388)) | DBG_FUNC_END,
+ bp, bp->b_datap, bp->b_flags, 0, 0);
+}
+
+/*
+ * Determine if a block is in the cache.
+ * Just look on what would be its hash chain. If it's there, return
+ * a pointer to it, unless it's marked invalid. If it's marked invalid,
+ * we normally don't return the buffer, unless the caller explicitly
+ * wants us to.
+ */
+static boolean_t
+incore(vnode_t vp, daddr64_t blkno)
+{
+ boolean_t retval;
+ struct bufhashhdr *dp;
+
+ dp = BUFHASH(vp, blkno);
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ if (incore_locked(vp, blkno, dp))
+ retval = TRUE;
+ else
+ retval = FALSE;
+ lck_mtx_unlock(buf_mtxp);
+
+ return (retval);
+}
+
+
+static buf_t
+incore_locked(vnode_t vp, daddr64_t blkno, struct bufhashhdr *dp)
+{
+ struct buf *bp;
+
+ /* Search hash chain */
+ for (bp = dp->lh_first; bp != NULL; bp = bp->b_hash.le_next) {
+ if (bp->b_lblkno == blkno && bp->b_vp == vp &&
+ !ISSET(bp->b_flags, B_INVAL)) {
+ return (bp);
+ }
+ }
+ return (NULL);
+}
+
+
+/* XXX FIXME -- Update the comment to reflect the UBC changes (please) -- */
+/*
+ * Get a block of requested size that is associated with
+ * a given vnode and block offset. If it is found in the
+ * block cache, mark it as having been found, make it busy
+ * and return it. Otherwise, return an empty block of the
+ * correct size. It is up to the caller to insure that the
+ * cached blocks be of the correct size.
+ */
+buf_t
+buf_getblk(vnode_t vp, daddr64_t blkno, int size, int slpflag, int slptimeo, int operation)
+{
+ buf_t bp;
+ int err;
+ upl_t upl;
+ upl_page_info_t *pl;
+ kern_return_t kret;
+ int ret_only_valid;
+ struct timespec ts;
+ int upl_flags;
+ struct bufhashhdr *dp;
+
+ KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 386)) | DBG_FUNC_START,
+ (uintptr_t)(blkno * PAGE_SIZE), size, operation, 0, 0);
+
+ ret_only_valid = operation & BLK_ONLYVALID;
+ operation &= ~BLK_ONLYVALID;
+ dp = BUFHASH(vp, blkno);
+start:
+ lck_mtx_lock_spin(buf_mtxp);
+
+ if ((bp = incore_locked(vp, blkno, dp))) {
+ /*
+ * Found in the Buffer Cache
+ */
+ if (ISSET(bp->b_lflags, BL_BUSY)) {
+ /*
+ * but is busy
+ */
+ switch (operation) {
+ case BLK_READ:
+ case BLK_WRITE:
+ case BLK_META:
+ SET(bp->b_lflags, BL_WANTED);
+ bufstats.bufs_busyincore++;
+
+ /*
+ * don't retake the mutex after being awakened...
+ * the time out is in msecs
+ */
+ ts.tv_sec = (slptimeo/1000);
+ ts.tv_nsec = (slptimeo % 1000) * 10 * NSEC_PER_USEC * 1000;
+
+ KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 396)) | DBG_FUNC_NONE,
+ (uintptr_t)blkno, size, operation, 0, 0);
+
+ err = msleep(bp, buf_mtxp, slpflag | PDROP | (PRIBIO + 1), "buf_getblk", &ts);
+
+ /*
+ * Callers who call with PCATCH or timeout are
+ * willing to deal with the NULL pointer
+ */
+ if (err && ((slpflag & PCATCH) || ((err == EWOULDBLOCK) && slptimeo)))
+ return (NULL);
+ goto start;
+ /*NOTREACHED*/
+ break;
+
+ default:
+ /*
+ * unknown operation requested
+ */
+ panic("getblk: paging or unknown operation for incore busy buffer - %x\n", operation);
+ /*NOTREACHED*/
+ break;
+ }
+ } else {
+ /*
+ * buffer in core and not busy
+ */
+ SET(bp->b_lflags, BL_BUSY);
+ SET(bp->b_flags, B_CACHE);
+ buf_busycount++;
+
+ bremfree_locked(bp);
+ bufstats.bufs_incore++;
+
+ lck_mtx_unlock(buf_mtxp);
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 1;
+#endif
+ if ( (bp->b_upl) )
+ panic("buffer has UPL, but not marked BUSY: %p", bp);
+
+ if ( !ret_only_valid && bp->b_bufsize != size)
+ allocbuf(bp, size);
+
+ upl_flags = 0;
+ switch (operation) {
+ case BLK_WRITE:
+ /*
+ * "write" operation: let the UPL subsystem
+ * know that we intend to modify the buffer
+ * cache pages we're gathering.
+ */
+ upl_flags |= UPL_WILL_MODIFY;
+ case BLK_READ:
+ upl_flags |= UPL_PRECIOUS;
+ if (UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) {
+ kret = ubc_create_upl(vp,
+ ubc_blktooff(vp, bp->b_lblkno),
+ bp->b_bufsize,
+ &upl,
+ &pl,
+ upl_flags);
+ if (kret != KERN_SUCCESS)
+ panic("Failed to create UPL");
+
+ bp->b_upl = upl;
+
+ if (upl_valid_page(pl, 0)) {
+ if (upl_dirty_page(pl, 0))
+ SET(bp->b_flags, B_WASDIRTY);
+ else
+ CLR(bp->b_flags, B_WASDIRTY);
+ } else
+ CLR(bp->b_flags, (B_DONE | B_CACHE | B_WASDIRTY | B_DELWRI));
+
+ kret = ubc_upl_map(upl, (vm_offset_t*)&(bp->b_datap));
+
+ if (kret != KERN_SUCCESS)
+ panic("getblk: ubc_upl_map() failed with (%d)", kret);
+ }
+ break;
+
+ case BLK_META:
+ /*
+ * VM is not involved in IO for the meta data
+ * buffer already has valid data
+ */
+ break;
+
+ default:
+ panic("getblk: paging or unknown operation for incore buffer- %d\n", operation);
+ /*NOTREACHED*/
+ break;
+ }
+ }
+ } else { /* not incore() */
+ int queue = BQ_EMPTY; /* Start with no preference */
+
+ if (ret_only_valid) {
+ lck_mtx_unlock(buf_mtxp);
+ return (NULL);
+ }
+ if ((vnode_isreg(vp) == 0) || (UBCINFOEXISTS(vp) == 0) /*|| (vnode_issystem(vp) == 1)*/)
+ operation = BLK_META;
+
+ if ((bp = getnewbuf(slpflag, slptimeo, &queue)) == NULL)
+ goto start;
+
+ /*
+ * getnewbuf may block for a number of different reasons...
+ * if it does, it's then possible for someone else to
+ * create a buffer for the same block and insert it into
+ * the hash... if we see it incore at this point we dump
+ * the buffer we were working on and start over
+ */
+ if (incore_locked(vp, blkno, dp)) {
+ SET(bp->b_flags, B_INVAL);
+ binshash(bp, &invalhash);
+
+ lck_mtx_unlock(buf_mtxp);
+
+ buf_brelse(bp);
+ goto start;
+ }
+ /*
+ * NOTE: YOU CAN NOT BLOCK UNTIL binshash() HAS BEEN
+ * CALLED! BE CAREFUL.
+ */
+
+ /*
+ * mark the buffer as B_META if indicated
+ * so that when buffer is released it will goto META queue
+ */
+ if (operation == BLK_META)
+ SET(bp->b_flags, B_META);
+
+ bp->b_blkno = bp->b_lblkno = blkno;
+ bp->b_vp = vp;
+
+ /*
+ * Insert in the hash so that incore() can find it
+ */
+ binshash(bp, BUFHASH(vp, blkno));
+
+ bgetvp_locked(vp, bp);
+
+ lck_mtx_unlock(buf_mtxp);
+
+ allocbuf(bp, size);
+
+ upl_flags = 0;
+ switch (operation) {
+ case BLK_META:
+ /*
+ * buffer data is invalid...
+ *
+ * I don't want to have to retake buf_mtxp,
+ * so the miss and vmhits counters are done
+ * with Atomic updates... all other counters
+ * in bufstats are protected with either
+ * buf_mtxp or iobuffer_mtxp
+ */
+ OSAddAtomicLong(1, &bufstats.bufs_miss);
+ break;
+
+ case BLK_WRITE:
+ /*
+ * "write" operation: let the UPL subsystem know
+ * that we intend to modify the buffer cache pages
+ * we're gathering.
+ */
+ upl_flags |= UPL_WILL_MODIFY;
+ case BLK_READ:
+ { off_t f_offset;
+ size_t contig_bytes;
+ int bmap_flags;
+
+ if ( (bp->b_upl) )
+ panic("bp already has UPL: %p",bp);
+
+ f_offset = ubc_blktooff(vp, blkno);
+
+ upl_flags |= UPL_PRECIOUS;
+ kret = ubc_create_upl(vp,
+ f_offset,
+ bp->b_bufsize,
+ &upl,
+ &pl,
+ upl_flags);
+
+ if (kret != KERN_SUCCESS)
+ panic("Failed to create UPL");
+#if UPL_DEBUG
+ upl_ubc_alias_set(upl, (uintptr_t) bp, (uintptr_t) 4);
+#endif /* UPL_DEBUG */
+ bp->b_upl = upl;
+
+ if (upl_valid_page(pl, 0)) {
+
+ if (operation == BLK_READ)
+ bmap_flags = VNODE_READ;
+ else
+ bmap_flags = VNODE_WRITE;
+
+ SET(bp->b_flags, B_CACHE | B_DONE);
+
+ OSAddAtomicLong(1, &bufstats.bufs_vmhits);
+
+ bp->b_validoff = 0;
+ bp->b_dirtyoff = 0;
+
+ if (upl_dirty_page(pl, 0)) {
+ /* page is dirty */
+ SET(bp->b_flags, B_WASDIRTY);
+
+ bp->b_validend = bp->b_bcount;
+ bp->b_dirtyend = bp->b_bcount;
+ } else {
+ /* page is clean */
+ bp->b_validend = bp->b_bcount;
+ bp->b_dirtyend = 0;
+ }
+ /*
+ * try to recreate the physical block number associated with
+ * this buffer...
+ */
+ if (VNOP_BLOCKMAP(vp, f_offset, bp->b_bcount, &bp->b_blkno, &contig_bytes, NULL, bmap_flags, NULL))
+ panic("getblk: VNOP_BLOCKMAP failed");
+ /*
+ * if the extent represented by this buffer
+ * is not completely physically contiguous on
+ * disk, than we can't cache the physical mapping
+ * in the buffer header
+ */
+ if ((long)contig_bytes < bp->b_bcount)
+ bp->b_blkno = bp->b_lblkno;
+ } else {
+ OSAddAtomicLong(1, &bufstats.bufs_miss);
+ }
+ kret = ubc_upl_map(upl, (vm_offset_t *)&(bp->b_datap));
+
+ if (kret != KERN_SUCCESS)
+ panic("getblk: ubc_upl_map() failed with (%d)", kret);
+ break;
+ }
+ default:
+ panic("getblk: paging or unknown operation - %x", operation);
+ /*NOTREACHED*/
+ break;
+ }
+ }
+ KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 386)) | DBG_FUNC_END,
+ bp, bp->b_datap, bp->b_flags, 3, 0);
+
+#ifdef JOE_DEBUG
+ (void) OSBacktrace(&bp->b_stackgetblk[0], 6);
+#endif
+ return (bp);
+}
+
+/*
+ * Get an empty, disassociated buffer of given size.
+ */
+buf_t
+buf_geteblk(int size)
+{
+ buf_t bp = NULL;
+ int queue = BQ_EMPTY;
+
+ do {
+ lck_mtx_lock_spin(buf_mtxp);
+
+ bp = getnewbuf(0, 0, &queue);
+ } while (bp == NULL);
+
+ SET(bp->b_flags, (B_META|B_INVAL));
+
+#if DIAGNOSTIC
+ assert(queue == BQ_EMPTY);
+#endif /* DIAGNOSTIC */
+ /* XXX need to implement logic to deal with other queues */
+
+ binshash(bp, &invalhash);
+ bufstats.bufs_eblk++;
+
+ lck_mtx_unlock(buf_mtxp);
+
+ allocbuf(bp, size);
+
+ return (bp);
+}
+
+uint32_t
+buf_redundancy_flags(buf_t bp)
+{
+ return bp->b_redundancy_flags;
+}
+
+void
+buf_set_redundancy_flags(buf_t bp, uint32_t flags)
+{
+ SET(bp->b_redundancy_flags, flags);
+}
+
+void
+buf_clear_redundancy_flags(buf_t bp, uint32_t flags)
+{
+ CLR(bp->b_redundancy_flags, flags);
+}
+
+/*
+ * With UBC, there is no need to expand / shrink the file data
+ * buffer. The VM uses the same pages, hence no waste.
+ * All the file data buffers can have one size.
+ * In fact expand / shrink would be an expensive operation.
+ *
+ * Only exception to this is meta-data buffers. Most of the
+ * meta data operations are smaller than PAGE_SIZE. Having the
+ * meta-data buffers grow and shrink as needed, optimizes use
+ * of the kernel wired memory.
+ */
+
+int
+allocbuf(buf_t bp, int size)
+{
+ vm_size_t desired_size;
+
+ desired_size = roundup(size, CLBYTES);
+
+ if (desired_size < PAGE_SIZE)
+ desired_size = PAGE_SIZE;
+ if (desired_size > MAXBSIZE)
+ panic("allocbuf: buffer larger than MAXBSIZE requested");
+
+ if (ISSET(bp->b_flags, B_META)) {
+ zone_t zprev, z;
+ int nsize = roundup(size, MINMETA);
+
+ if (bp->b_datap) {
+ vm_offset_t elem = (vm_offset_t)bp->b_datap;
+
+ if (ISSET(bp->b_flags, B_ZALLOC)) {
+ if (bp->b_bufsize < nsize) {
+ /* reallocate to a bigger size */
+
+ zprev = getbufzone(bp->b_bufsize);
+ if (nsize <= MAXMETA) {
+ desired_size = nsize;
+ z = getbufzone(nsize);
+ /* b_datap not really a ptr */
+ *(void **)(&bp->b_datap) = zalloc(z);
+ } else {
+ bp->b_datap = (uintptr_t)NULL;
+ kmem_alloc_kobject(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
+ CLR(bp->b_flags, B_ZALLOC);
+ }
+ bcopy((void *)elem, (caddr_t)bp->b_datap, bp->b_bufsize);
+ zfree(zprev, (void *)elem);
+ } else {
+ desired_size = bp->b_bufsize;
+ }
+
+ } else {
+ if ((vm_size_t)bp->b_bufsize < desired_size) {
+ /* reallocate to a bigger size */
+ bp->b_datap = (uintptr_t)NULL;
+ kmem_alloc_kobject(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
+ bcopy((const void *)elem, (caddr_t)bp->b_datap, bp->b_bufsize);
+ kmem_free(kernel_map, elem, bp->b_bufsize);
+ } else {
+ desired_size = bp->b_bufsize;
+ }
+ }
+ } else {
+ /* new allocation */
+ if (nsize <= MAXMETA) {
+ desired_size = nsize;
+ z = getbufzone(nsize);
+ /* b_datap not really a ptr */
+ *(void **)(&bp->b_datap) = zalloc(z);
+ SET(bp->b_flags, B_ZALLOC);
+ } else
+ kmem_alloc_kobject(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
+ }
+
+ if (bp->b_datap == 0)
+ panic("allocbuf: NULL b_datap");
+ }
+ bp->b_bufsize = desired_size;
+ bp->b_bcount = size;
+
+ return (0);
+}
+
+/*
+ * Get a new buffer from one of the free lists.
+ *
+ * Request for a queue is passes in. The queue from which the buffer was taken
+ * from is returned. Out of range queue requests get BQ_EMPTY. Request for
+ * BQUEUE means no preference. Use heuristics in that case.
+ * Heuristics is as follows:
+ * Try BQ_AGE, BQ_LRU, BQ_EMPTY, BQ_META in that order.
+ * If none available block till one is made available.
+ * If buffers available on both BQ_AGE and BQ_LRU, check the timestamps.
+ * Pick the most stale buffer.
+ * If found buffer was marked delayed write, start the async. write
+ * and restart the search.
+ * Initialize the fields and disassociate the buffer from the vnode.
+ * Remove the buffer from the hash. Return the buffer and the queue
+ * on which it was found.
+ *
+ * buf_mtxp is held upon entry
+ * returns with buf_mtxp locked if new buf available
+ * returns with buf_mtxp UNlocked if new buf NOT available
+ */
+
+static buf_t
+getnewbuf(int slpflag, int slptimeo, int * queue)
+{
+ buf_t bp;
+ buf_t lru_bp;
+ buf_t age_bp;
+ buf_t meta_bp;
+ int age_time, lru_time, bp_time, meta_time;
+ int req = *queue; /* save it for restarts */
+ struct timespec ts;
+
+start:
+ /*
+ * invalid request gets empty queue
+ */
+ if ((*queue >= BQUEUES) || (*queue < 0)
+ || (*queue == BQ_LAUNDRY) || (*queue == BQ_LOCKED))
+ *queue = BQ_EMPTY;
+
+
+ if (*queue == BQ_EMPTY && (bp = bufqueues[*queue].tqh_first))
+ goto found;
+
+ /*
+ * need to grow number of bufs, add another one rather than recycling
+ */
+ if (nbuf_headers < max_nbuf_headers) {
+ /*
+ * Increment count now as lock
+ * is dropped for allocation.
+ * That avoids over commits
+ */
+ nbuf_headers++;
+ goto add_newbufs;
+ }
+ /* Try for the requested queue first */
+ bp = bufqueues[*queue].tqh_first;
+ if (bp)
+ goto found;
+
+ /* Unable to use requested queue */
+ age_bp = bufqueues[BQ_AGE].tqh_first;
+ lru_bp = bufqueues[BQ_LRU].tqh_first;
+ meta_bp = bufqueues[BQ_META].tqh_first;
+
+ if (!age_bp && !lru_bp && !meta_bp) {
+ /*
+ * Unavailble on AGE or LRU or META queues
+ * Try the empty list first
+ */
+ bp = bufqueues[BQ_EMPTY].tqh_first;
+ if (bp) {
+ *queue = BQ_EMPTY;
+ goto found;
+ }
+ /*
+ * We have seen is this is hard to trigger.
+ * This is an overcommit of nbufs but needed
+ * in some scenarios with diskiamges
+ */
+
+add_newbufs:
+ lck_mtx_unlock(buf_mtxp);
+
+ /* Create a new temporary buffer header */
+ bp = (struct buf *)zalloc(buf_hdr_zone);
+
+ if (bp) {
+ bufhdrinit(bp);
+ bp->b_whichq = BQ_EMPTY;
+ bp->b_timestamp = buf_timestamp();
+ BLISTNONE(bp);
+ SET(bp->b_flags, B_HDRALLOC);
+ *queue = BQ_EMPTY;
+ }
+ lck_mtx_lock_spin(buf_mtxp);
+
+ if (bp) {
+ binshash(bp, &invalhash);
+ binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY);
+ buf_hdr_count++;
+ goto found;
+ }
+ /* subtract already accounted bufcount */
+ nbuf_headers--;
+
+ bufstats.bufs_sleeps++;
+
+ /* wait for a free buffer of any kind */
+ needbuffer = 1;
+ /* hz value is 100 */
+ ts.tv_sec = (slptimeo/1000);
+ /* the hz value is 100; which leads to 10ms */
+ ts.tv_nsec = (slptimeo % 1000) * NSEC_PER_USEC * 1000 * 10;
+
+ msleep(&needbuffer, buf_mtxp, slpflag | PDROP | (PRIBIO+1), "getnewbuf", &ts);
+ return (NULL);
+ }
+
+ /* Buffer available either on AGE or LRU or META */
+ bp = NULL;
+ *queue = -1;
+
+ /* Buffer available either on AGE or LRU */
+ if (!age_bp) {
+ bp = lru_bp;
+ *queue = BQ_LRU;
+ } else if (!lru_bp) {
+ bp = age_bp;
+ *queue = BQ_AGE;
+ } else { /* buffer available on both AGE and LRU */
+ int t = buf_timestamp();
+
+ age_time = t - age_bp->b_timestamp;
+ lru_time = t - lru_bp->b_timestamp;
+ if ((age_time < 0) || (lru_time < 0)) { /* time set backwards */
+ bp = age_bp;
+ *queue = BQ_AGE;
+ /*
+ * we should probably re-timestamp eveything in the
+ * queues at this point with the current time
+ */
+ } else {
+ if ((lru_time >= lru_is_stale) && (age_time < age_is_stale)) {
+ bp = lru_bp;
+ *queue = BQ_LRU;
+ } else {
+ bp = age_bp;
+ *queue = BQ_AGE;
+ }
+ }
+ }
+
+ if (!bp) { /* Neither on AGE nor on LRU */
+ bp = meta_bp;
+ *queue = BQ_META;
+ } else if (meta_bp) {
+ int t = buf_timestamp();
+
+ bp_time = t - bp->b_timestamp;
+ meta_time = t - meta_bp->b_timestamp;
+
+ if (!(bp_time < 0) && !(meta_time < 0)) {
+ /* time not set backwards */
+ int bp_is_stale;
+ bp_is_stale = (*queue == BQ_LRU) ?
+ lru_is_stale : age_is_stale;
+
+ if ((meta_time >= meta_is_stale) &&
+ (bp_time < bp_is_stale)) {
+ bp = meta_bp;
+ *queue = BQ_META;
+ }
+ }
+ }
+found:
+ if (ISSET(bp->b_flags, B_LOCKED) || ISSET(bp->b_lflags, BL_BUSY))
+ panic("getnewbuf: bp @ %p is LOCKED or BUSY! (flags 0x%x)\n", bp, bp->b_flags);
+
+ /* Clean it */
+ if (bcleanbuf(bp, FALSE)) {
+ /*
+ * moved to the laundry thread, buffer not ready
+ */
+ *queue = req;
+ goto start;
+ }
+ return (bp);
+}
+
+
+/*
+ * Clean a buffer.
+ * Returns 0 if buffer is ready to use,
+ * Returns 1 if issued a buf_bawrite() to indicate
+ * that the buffer is not ready.
+ *
+ * buf_mtxp is held upon entry
+ * returns with buf_mtxp locked
+ */
+int
+bcleanbuf(buf_t bp, boolean_t discard)
+{
+ /* Remove from the queue */
+ bremfree_locked(bp);
+
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 2;
+#endif
+ /*
+ * If buffer was a delayed write, start the IO by queuing
+ * it on the LAUNDRY queue, and return 1
+ */
+ if (ISSET(bp->b_flags, B_DELWRI)) {
+ if (discard) {
+ SET(bp->b_lflags, BL_WANTDEALLOC);
+ }
+
+ bmovelaundry(bp);
+
+ lck_mtx_unlock(buf_mtxp);
+
+ wakeup(&bufqueues[BQ_LAUNDRY]);
+ /*
+ * and give it a chance to run
+ */
+ (void)thread_block(THREAD_CONTINUE_NULL);
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ return (1);
+ }
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 8;
+#endif
+ /*
+ * Buffer is no longer on any free list... we own it
+ */
+ SET(bp->b_lflags, BL_BUSY);
+ buf_busycount++;
+
+ bremhash(bp);
+
+ /*
+ * disassociate us from our vnode, if we had one...
+ */
+ if (bp->b_vp)
+ brelvp_locked(bp);
+
+ lck_mtx_unlock(buf_mtxp);
+
+ BLISTNONE(bp);
+
+ if (ISSET(bp->b_flags, B_META))
+ buf_free_meta_store(bp);
+
+ trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno);
+
+ buf_release_credentials(bp);
+
+ /* If discarding, just move to the empty queue */
+ if (discard) {
+ lck_mtx_lock_spin(buf_mtxp);
+ CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE | B_FUA));
+ bp->b_whichq = BQ_EMPTY;
+ binshash(bp, &invalhash);
+ binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY);
+ CLR(bp->b_lflags, BL_BUSY);
+ buf_busycount--;
+ } else {
+ /* Not discarding: clean up and prepare for reuse */
+ bp->b_bufsize = 0;
+ bp->b_datap = (uintptr_t)NULL;
+ bp->b_upl = (void *)NULL;
+ /*
+ * preserve the state of whether this buffer
+ * was allocated on the fly or not...
+ * the only other flag that should be set at
+ * this point is BL_BUSY...
+ */
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 3;
+#endif
+ bp->b_lflags = BL_BUSY;
+ bp->b_flags = (bp->b_flags & B_HDRALLOC);
+ bp->b_dev = NODEV;
+ bp->b_blkno = bp->b_lblkno = 0;
+ bp->b_iodone = NULL;
+ bp->b_error = 0;
+ bp->b_resid = 0;
+ bp->b_bcount = 0;
+ bp->b_dirtyoff = bp->b_dirtyend = 0;
+ bp->b_validoff = bp->b_validend = 0;
+#ifdef CONFIG_PROTECT
+ bp->b_cpentry = 0;
+#endif
+
+ lck_mtx_lock_spin(buf_mtxp);
+ }
+ return (0);
+}
+
+
+
+errno_t
+buf_invalblkno(vnode_t vp, daddr64_t lblkno, int flags)
+{
+ buf_t bp;
+ errno_t error;
+ struct bufhashhdr *dp;
+
+ dp = BUFHASH(vp, lblkno);
+
+relook:
+ lck_mtx_lock_spin(buf_mtxp);
+
+ if ((bp = incore_locked(vp, lblkno, dp)) == (struct buf *)0) {
+ lck_mtx_unlock(buf_mtxp);
+ return (0);
+ }
+ if (ISSET(bp->b_lflags, BL_BUSY)) {
+ if ( !ISSET(flags, BUF_WAIT)) {
+ lck_mtx_unlock(buf_mtxp);
+ return (EBUSY);
+ }
+ SET(bp->b_lflags, BL_WANTED);
+
+ error = msleep((caddr_t)bp, buf_mtxp, PDROP | (PRIBIO + 1), "buf_invalblkno", NULL);
+
+ if (error) {
+ return (error);
+ }
+ goto relook;
+ }
+ bremfree_locked(bp);
+ SET(bp->b_lflags, BL_BUSY);
+ SET(bp->b_flags, B_INVAL);
+ buf_busycount++;
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 4;
+#endif
+ lck_mtx_unlock(buf_mtxp);
+ buf_brelse(bp);
+
+ return (0);
+}
+
+
+void
+buf_drop(buf_t bp)
+{
+ int need_wakeup = 0;
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ if (ISSET(bp->b_lflags, BL_WANTED)) {
+ /*
+ * delay the actual wakeup until after we
+ * clear BL_BUSY and we've dropped buf_mtxp
+ */
+ need_wakeup = 1;
+ }
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 9;
+#endif
+ /*
+ * Unlock the buffer.
+ */
+ CLR(bp->b_lflags, (BL_BUSY | BL_WANTED));
+ buf_busycount--;
+
+ lck_mtx_unlock(buf_mtxp);
+
+ if (need_wakeup) {
+ /*
+ * Wake up any proceeses waiting for _this_ buffer to become free.
+ */
+ wakeup(bp);
+ }
+}
+
+
+errno_t
+buf_acquire(buf_t bp, int flags, int slpflag, int slptimeo) {
+ errno_t error;
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ error = buf_acquire_locked(bp, flags, slpflag, slptimeo);
+
+ lck_mtx_unlock(buf_mtxp);
+
+ return (error);
+}
+
+
+static errno_t
+buf_acquire_locked(buf_t bp, int flags, int slpflag, int slptimeo)
+{
+ errno_t error;
+ struct timespec ts;
+
+ if (ISSET(bp->b_flags, B_LOCKED)) {
+ if ((flags & BAC_SKIP_LOCKED))
+ return (EDEADLK);
+ } else {
+ if ((flags & BAC_SKIP_NONLOCKED))
+ return (EDEADLK);
+ }
+ if (ISSET(bp->b_lflags, BL_BUSY)) {
+ /*
+ * since the lck_mtx_lock may block, the buffer
+ * may become BUSY, so we need to
+ * recheck for a NOWAIT request
+ */
+ if (flags & BAC_NOWAIT)
+ return (EBUSY);
+ SET(bp->b_lflags, BL_WANTED);
+
+ /* the hz value is 100; which leads to 10ms */
+ ts.tv_sec = (slptimeo/100);
+ ts.tv_nsec = (slptimeo % 100) * 10 * NSEC_PER_USEC * 1000;
+ error = msleep((caddr_t)bp, buf_mtxp, slpflag | (PRIBIO + 1), "buf_acquire", &ts);
+
+ if (error)
+ return (error);
+ return (EAGAIN);
+ }
+ if (flags & BAC_REMOVE)
+ bremfree_locked(bp);
+ SET(bp->b_lflags, BL_BUSY);
+ buf_busycount++;
+
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 5;
+#endif
+ return (0);
+}
+
+
+/*
+ * Wait for operations on the buffer to complete.
+ * When they do, extract and return the I/O's error value.
+ */
+errno_t
+buf_biowait(buf_t bp)
+{
+ while (!ISSET(bp->b_flags, B_DONE)) {
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ if (!ISSET(bp->b_flags, B_DONE)) {
+ DTRACE_IO1(wait__start, buf_t, bp);
+ (void) msleep(bp, buf_mtxp, PDROP | (PRIBIO+1), "buf_biowait", NULL);
+ DTRACE_IO1(wait__done, buf_t, bp);
+ } else
+ lck_mtx_unlock(buf_mtxp);
+ }
+ /* check for interruption of I/O (e.g. via NFS), then errors. */
+ if (ISSET(bp->b_flags, B_EINTR)) {
+ CLR(bp->b_flags, B_EINTR);
+ return (EINTR);
+ } else if (ISSET(bp->b_flags, B_ERROR))
+ return (bp->b_error ? bp->b_error : EIO);
+ else
+ return (0);
+}
+
+
+/*
+ * Mark I/O complete on a buffer.
+ *
+ * If a callback has been requested, e.g. the pageout
+ * daemon, do so. Otherwise, awaken waiting processes.
+ *
+ * [ Leffler, et al., says on p.247:
+ * "This routine wakes up the blocked process, frees the buffer
+ * for an asynchronous write, or, for a request by the pagedaemon
+ * process, invokes a procedure specified in the buffer structure" ]
+ *
+ * In real life, the pagedaemon (or other system processes) wants
+ * to do async stuff to, and doesn't want the buffer buf_brelse()'d.
+ * (for swap pager, that puts swap buffers on the free lists (!!!),
+ * for the vn device, that puts malloc'd buffers on the free lists!)
+ */
+extern struct timeval priority_IO_timestamp_for_root;
+extern int hard_throttle_on_root;
+
+void
+buf_biodone(buf_t bp)
+{
+ mount_t mp;
+
+ KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 387)) | DBG_FUNC_START,
+ bp, bp->b_datap, bp->b_flags, 0, 0);
+
+ if (ISSET(bp->b_flags, B_DONE))
+ panic("biodone already");
+
+ if (ISSET(bp->b_flags, B_ERROR)) {
+ fslog_io_error(bp);
+ }
+
+ if (bp->b_vp && bp->b_vp->v_mount) {
+ mp = bp->b_vp->v_mount;
+ } else {
+ mp = NULL;
+ }
+
+ if (mp && (bp->b_flags & B_READ) == 0) {
+ update_last_io_time(mp);
+ INCR_PENDING_IO(-(pending_io_t)buf_count(bp), mp->mnt_pending_write_size);
+ } else if (mp) {
+ INCR_PENDING_IO(-(pending_io_t)buf_count(bp), mp->mnt_pending_read_size);
+ }
+
+ if (kdebug_enable) {
+ int code = DKIO_DONE;
+
+ if (bp->b_flags & B_READ)
+ code |= DKIO_READ;
+ if (bp->b_flags & B_ASYNC)
+ code |= DKIO_ASYNC;
+
+ if (bp->b_flags & B_META)
+ code |= DKIO_META;
+ else if (bp->b_flags & B_PAGEIO)
+ code |= DKIO_PAGING;
+
+ if (bp->b_flags & B_THROTTLED_IO)
+ code |= DKIO_THROTTLE;
+ else if (bp->b_flags & B_PASSIVE)
+ code |= DKIO_PASSIVE;
+
+ KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_DKRW, code) | DBG_FUNC_NONE,
+ bp, (uintptr_t)bp->b_vp,
+ bp->b_resid, bp->b_error, 0);
+ }
+ if ((bp->b_vp != NULLVP) &&
+ ((bp->b_flags & (B_IOSTREAMING | B_PAGEIO | B_READ)) == (B_PAGEIO | B_READ)) &&
+ (bp->b_vp->v_mount->mnt_kern_flag & MNTK_ROOTDEV)) {
+ microuptime(&priority_IO_timestamp_for_root);
+ hard_throttle_on_root = 0;
+ }
+
+ /*
+ * I/O was done, so don't believe
+ * the DIRTY state from VM anymore...
+ * and we need to reset the THROTTLED/PASSIVE
+ * indicators
+ */
+ CLR(bp->b_flags, (B_WASDIRTY | B_THROTTLED_IO | B_PASSIVE));
+ DTRACE_IO1(done, buf_t, bp);
+
+ if (!ISSET(bp->b_flags, B_READ) && !ISSET(bp->b_flags, B_RAW))
+ /*
+ * wake up any writer's blocked
+ * on throttle or waiting for I/O
+ * to drain
+ */
+ vnode_writedone(bp->b_vp);
+
+ if (ISSET(bp->b_flags, (B_CALL | B_FILTER))) { /* if necessary, call out */
+ void (*iodone_func)(struct buf *, void *) = bp->b_iodone;
+ void *arg = bp->b_transaction;
+ int callout = ISSET(bp->b_flags, B_CALL);
+
+ if (iodone_func == NULL)
+ panic("biodone: bp @ %p has NULL b_iodone!\n", bp);
+
+ CLR(bp->b_flags, (B_CALL | B_FILTER)); /* filters and callouts are one-shot */
+ bp->b_iodone = NULL;
+ bp->b_transaction = NULL;
+
+ if (callout)
+ SET(bp->b_flags, B_DONE); /* note that it's done */
+
+ (*iodone_func)(bp, arg);
+
+ if (callout) {
+ /*
+ * assumes that the callback function takes
+ * ownership of the bp and deals with releasing it if necessary
+ */
+ goto biodone_done;
+ }
+ /*
+ * in this case the call back function is acting
+ * strictly as a filter... it does not take
+ * ownership of the bp and is expecting us
+ * to finish cleaning up... this is currently used
+ * by the HFS journaling code
+ */
+ }
+ if (ISSET(bp->b_flags, B_ASYNC)) { /* if async, release it */
+ SET(bp->b_flags, B_DONE); /* note that it's done */
+
+ buf_brelse(bp);
+ } else { /* or just wakeup the buffer */
+ /*
+ * by taking the mutex, we serialize
+ * the buf owner calling buf_biowait so that we'll
+ * only see him in one of 2 states...
+ * state 1: B_DONE wasn't set and he's
+ * blocked in msleep
+ * state 2: he's blocked trying to take the
+ * mutex before looking at B_DONE
+ * BL_WANTED is cleared in case anyone else
+ * is blocked waiting for the buffer... note
+ * that we haven't cleared B_BUSY yet, so if
+ * they do get to run, their going to re-set
+ * BL_WANTED and go back to sleep
+ */
+ lck_mtx_lock_spin(buf_mtxp);
+
+ CLR(bp->b_lflags, BL_WANTED);
+ SET(bp->b_flags, B_DONE); /* note that it's done */
+
+ lck_mtx_unlock(buf_mtxp);
+
+ wakeup(bp);
+ }
+biodone_done:
+ KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 387)) | DBG_FUNC_END,
+ (uintptr_t)bp, (uintptr_t)bp->b_datap, bp->b_flags, 0, 0);
+}
+
+/*
+ * Return a count of buffers on the "locked" queue.
+ */
+int
+count_lock_queue(void)
+{
+ buf_t bp;
+ int n = 0;
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ for (bp = bufqueues[BQ_LOCKED].tqh_first; bp;
+ bp = bp->b_freelist.tqe_next)
+ n++;
+ lck_mtx_unlock(buf_mtxp);
+
+ return (n);
+}
+
+/*
+ * Return a count of 'busy' buffers. Used at the time of shutdown.
+ */
+int
+count_busy_buffers(void)
+{
+ return buf_busycount + bufstats.bufs_iobufinuse;
+}
+
+#if DIAGNOSTIC
+/*
+ * Print out statistics on the current allocation of the buffer pool.
+ * Can be enabled to print out on every ``sync'' by setting "syncprt"
+ * in vfs_syscalls.c using sysctl.
+ */
+void
+vfs_bufstats()
+{
+ int i, j, count;
+ struct buf *bp;
+ struct bqueues *dp;
+ int counts[MAXBSIZE/CLBYTES+1];
+ static char *bname[BQUEUES] =
+ { "LOCKED", "LRU", "AGE", "EMPTY", "META", "LAUNDRY" };
+
+ for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
+ count = 0;
+ for (j = 0; j <= MAXBSIZE/CLBYTES; j++)
+ counts[j] = 0;
+
+ lck_mtx_lock(buf_mtxp);
+
+ for (bp = dp->tqh_first; bp; bp = bp->b_freelist.tqe_next) {
+ counts[bp->b_bufsize/CLBYTES]++;
+ count++;
+ }
+ lck_mtx_unlock(buf_mtxp);
+
+ printf("%s: total-%d", bname[i], count);
+ for (j = 0; j <= MAXBSIZE/CLBYTES; j++)
+ if (counts[j] != 0)
+ printf(", %d-%d", j * CLBYTES, counts[j]);
+ printf("\n");
+ }
+}
+#endif /* DIAGNOSTIC */
+
+#define NRESERVEDIOBUFS 128
+
+
+buf_t
+alloc_io_buf(vnode_t vp, int priv)
+{
+ buf_t bp;
+
+ lck_mtx_lock_spin(iobuffer_mtxp);
+
+ while (((niobuf_headers - NRESERVEDIOBUFS < bufstats.bufs_iobufinuse) && !priv) ||
+ (bp = iobufqueue.tqh_first) == NULL) {
+ bufstats.bufs_iobufsleeps++;
+
+ need_iobuffer = 1;
+ (void) msleep(&need_iobuffer, iobuffer_mtxp, PSPIN | (PRIBIO+1), (const char *)"alloc_io_buf", NULL);
+ }
+ TAILQ_REMOVE(&iobufqueue, bp, b_freelist);
+
+ bufstats.bufs_iobufinuse++;
+ if (bufstats.bufs_iobufinuse > bufstats.bufs_iobufmax)
+ bufstats.bufs_iobufmax = bufstats.bufs_iobufinuse;
+
+ lck_mtx_unlock(iobuffer_mtxp);
+
+ /*
+ * initialize various fields
+ * we don't need to hold the mutex since the buffer
+ * is now private... the vp should have a reference
+ * on it and is not protected by this mutex in any event
+ */
+ bp->b_timestamp = 0;
+ bp->b_proc = NULL;
+
+ bp->b_datap = 0;
+ bp->b_flags = 0;
+ bp->b_lflags = BL_BUSY | BL_IOBUF;
+ bp->b_redundancy_flags = 0;
+ bp->b_blkno = bp->b_lblkno = 0;
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 6;
+#endif
+ bp->b_iodone = NULL;
+ bp->b_error = 0;
+ bp->b_resid = 0;
+ bp->b_bcount = 0;
+ bp->b_bufsize = 0;
+ bp->b_upl = NULL;
+ bp->b_vp = vp;
+#ifdef CONFIG_PROTECT
+ bp->b_cpentry = 0;
+#endif
+
+ if (vp && (vp->v_type == VBLK || vp->v_type == VCHR))
+ bp->b_dev = vp->v_rdev;
+ else
+ bp->b_dev = NODEV;
+
+ return (bp);
+}
+
+
+void
+free_io_buf(buf_t bp)
+{
+ int need_wakeup = 0;
+
+ /*
+ * put buffer back on the head of the iobufqueue
+ */
+ bp->b_vp = NULL;
+ bp->b_flags = B_INVAL;
+
+ lck_mtx_lock_spin(iobuffer_mtxp);
+
+ binsheadfree(bp, &iobufqueue, -1);
+
+ if (need_iobuffer) {
+ /*
+ * Wake up any processes waiting because they need an io buffer
+ *
+ * do the wakeup after we drop the mutex... it's possible that the
+ * wakeup will be superfluous if need_iobuffer gets set again and
+ * another thread runs this path, but it's highly unlikely, doesn't
+ * hurt, and it means we don't hold up I/O progress if the wakeup blocks
+ * trying to grab a task related lock...
+ */
+ need_iobuffer = 0;
+ need_wakeup = 1;
+ }
+ if (bufstats.bufs_iobufinuse <= 0)
+ panic("free_io_buf: bp(%p) - bufstats.bufs_iobufinuse < 0", bp);
+
+ bufstats.bufs_iobufinuse--;
+
+ lck_mtx_unlock(iobuffer_mtxp);
+
+ if (need_wakeup)
+ wakeup(&need_iobuffer);
+}
+
+
+void
+buf_list_lock(void)
+{
+ lck_mtx_lock_spin(buf_mtxp);
+}
+
+void
+buf_list_unlock(void)
+{
+ lck_mtx_unlock(buf_mtxp);
+}
+
+/*
+ * If getnewbuf() calls bcleanbuf() on the same thread
+ * there is a potential for stack overrun and deadlocks.
+ * So we always handoff the work to a worker thread for completion
+ */
+
+
+static void
+bcleanbuf_thread_init(void)
+{
+ thread_t thread = THREAD_NULL;
+
+ /* create worker thread */
+ kernel_thread_start((thread_continue_t)bcleanbuf_thread, NULL, &thread);
+ thread_deallocate(thread);
+}
+
+typedef int (*bcleanbufcontinuation)(int);
+
+static void
+bcleanbuf_thread(void)
+{
+ struct buf *bp;
+ int error = 0;
+ int loopcnt = 0;
+
+ for (;;) {
+ lck_mtx_lock_spin(buf_mtxp);
+
+ while ( (bp = TAILQ_FIRST(&bufqueues[BQ_LAUNDRY])) == NULL) {
+ (void)msleep0(&bufqueues[BQ_LAUNDRY], buf_mtxp, PRIBIO|PDROP, "blaundry", 0, (bcleanbufcontinuation)bcleanbuf_thread);
+ }
+
+ /*
+ * Remove from the queue
+ */
+ bremfree_locked(bp);
+
+ /*
+ * Buffer is no longer on any free list
+ */
+ SET(bp->b_lflags, BL_BUSY);
+ buf_busycount++;
+
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 10;
+#endif
+
+ lck_mtx_unlock(buf_mtxp);
+ /*
+ * do the IO
+ */
+ error = bawrite_internal(bp, 0);
+
+ if (error) {
+ bp->b_whichq = BQ_LAUNDRY;
+ bp->b_timestamp = buf_timestamp();
+
+ lck_mtx_lock_spin(buf_mtxp);
+
+ binstailfree(bp, &bufqueues[BQ_LAUNDRY], BQ_LAUNDRY);
+ blaundrycnt++;
+
+ /* we never leave a busy page on the laundry queue */
+ CLR(bp->b_lflags, BL_BUSY);
+ buf_busycount--;
+#ifdef JOE_DEBUG
+ bp->b_owner = current_thread();
+ bp->b_tag = 11;
+#endif
+
+ lck_mtx_unlock(buf_mtxp);
+
+ if (loopcnt > MAXLAUNDRY) {
+ /*
+ * bawrite_internal() can return errors if we're throttled. If we've
+ * done several I/Os and failed, give the system some time to unthrottle
+ * the vnode
+ */
+ (void)tsleep((void *)&bufqueues[BQ_LAUNDRY], PRIBIO, "blaundry", 1);
+ loopcnt = 0;
+ } else {
+ /* give other threads a chance to run */
+ (void)thread_block(THREAD_CONTINUE_NULL);
+ loopcnt++;
+ }
+ }
+ }
+}
+
+
+static int
+brecover_data(buf_t bp)
+{
+ int upl_offset;
+ upl_t upl;
+ upl_page_info_t *pl;
+ kern_return_t kret;
+ vnode_t vp = bp->b_vp;
+ int upl_flags;
+
+
+ if ( !UBCINFOEXISTS(vp) || bp->b_bufsize == 0)
+ goto dump_buffer;
+
+ upl_flags = UPL_PRECIOUS;
+ if (! (buf_flags(bp) & B_READ)) {
+ /*
+ * "write" operation: let the UPL subsystem know
+ * that we intend to modify the buffer cache pages we're
+ * gathering.
+ */
+ upl_flags |= UPL_WILL_MODIFY;
+ }
+
+ kret = ubc_create_upl(vp,
+ ubc_blktooff(vp, bp->b_lblkno),
+ bp->b_bufsize,
+ &upl,
+ &pl,
+ upl_flags);
+ if (kret != KERN_SUCCESS)
+ panic("Failed to create UPL");
+
+ for (upl_offset = 0; upl_offset < bp->b_bufsize; upl_offset += PAGE_SIZE) {
+
+ if (!upl_valid_page(pl, upl_offset / PAGE_SIZE) || !upl_dirty_page(pl, upl_offset / PAGE_SIZE)) {
+ ubc_upl_abort(upl, 0);
+ goto dump_buffer;
+ }
+ }
+ bp->b_upl = upl;
+
+ kret = ubc_upl_map(upl, (vm_offset_t *)&(bp->b_datap));
projects / apple / xnu.git / blobdiff