/*
- * Copyright (c) 2000-2015 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* @(#)nfs_bio.c 8.9 (Berkeley) 3/30/95
* FreeBSD-Id: nfs_bio.c,v 1.44 1997/09/10 19:52:25 phk Exp $
*/
+
+#include <nfs/nfs_conf.h>
+#if CONFIG_NFS_CLIENT
+
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/resourcevar.h>
#include <nfs/nfsnode.h>
#include <sys/buf_internal.h>
#include <libkern/OSAtomic.h>
+#include <os/refcnt.h>
#define NFS_BIO_DBG(...) NFS_DBG(NFS_FAC_BIO, 7, ## __VA_ARGS__)
int nfs_buf_timer_on = 0;
thread_t nfsbufdelwrithd = NULL;
+ZONE_DECLARE(nfsbuf_zone, "NFS bio", sizeof(struct nfsbuf), ZC_NONE);
+
lck_grp_t *nfs_buf_lck_grp;
lck_mtx_t *nfs_buf_mutex;
/* fraction of total nfsbufs that nfsbuffreemetacnt should exceed before bothering to call nfs_buf_freeup() */
#define META_FREEUP_MIN_FRAC 2
+#define NFS_ROUND_BLOCK(p, blksize) ((((uint64_t)(p) + blksize - 1) & ~((uint64_t)blksize - 1)) / blksize)
+
#define NFS_BUF_FREEUP() \
do { \
/* only call nfs_buf_freeup() if it has work to do: */ \
nfs_buf_freeup(0); \
} while (0)
+void
+nfs_buf_pgs_get_page_mask(nfsbufpgs *nfsbp, off_t page)
+{
+ off_t page_pos = page / NBPGS_ELEMENT_PAGES;
+ off_t max_page = NBPGS_STRUCT_SIZE * 8;
+ NBPGS_ERASE(nfsbp);
+
+ if (page >= max_page) {
+ nfs_buf_pgs_bit_not(nfsbp);
+ return;
+ }
+
+ NBPGS_SET(nfsbp, page);
+ nfsbp->pages[page_pos]--;
+ for (off_t i = page_pos - 1; i >= 0; i--) {
+ nfsbp->pages[i] = ~0;
+ }
+}
+
+void
+nfs_buf_pgs_bit_not(nfsbufpgs *nfsbp)
+{
+ for (uint32_t i = 0; i < NBPGS_ELEMENTS; i++) {
+ nfsbp->pages[i] = ~nfsbp->pages[i];
+ }
+}
+
+void
+nfs_buf_pgs_bit_and(nfsbufpgs *nfsbp_src1, nfsbufpgs *nfsbp_src2, nfsbufpgs *nfsbp_dst)
+{
+ for (uint32_t i = 0; i < NBPGS_ELEMENTS; i++) {
+ nfsbp_dst->pages[i] = nfsbp_src1->pages[i] & nfsbp_src2->pages[i];
+ }
+}
+
+void
+nfs_buf_pgs_set_pages_between(nfsbufpgs *nfsbp, off_t firstpg, off_t lastpg)
+{
+ nfsbufpgs pagemaskfirst, pagemasklast;
+
+ nfs_buf_pgs_get_page_mask(&pagemasklast, lastpg);
+ nfs_buf_pgs_get_page_mask(&pagemaskfirst, firstpg);
+ nfs_buf_pgs_bit_not(&pagemaskfirst);
+ nfs_buf_pgs_bit_and(&pagemaskfirst, &pagemasklast, nfsbp);
+}
+
+int
+nfs_buf_pgs_is_set(nfsbufpgs *nfsbp)
+{
+ for (uint32_t i = 0; i < NBPGS_ELEMENTS; i++) {
+ if (nfsbp->pages[i] != 0) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
/*
* Initialize nfsbuf lists
*/
nfsbuffreecnt = nfsbuffreemetacnt = nfsbufdelwricnt = 0;
nfsbufmin = 128;
/* size nfsbufmax to cover at most half sane_size (w/default buf size) */
- nfsbufmax = (sane_size >> PAGE_SHIFT) / (2 * (NFS_RWSIZE >> PAGE_SHIFT));
+ nfsbufmax = (int)(sane_size >> PAGE_SHIFT) / (2 * (NFS_RWSIZE >> PAGE_SHIFT));
nfsbufmetamax = nfsbufmax / 4;
nfsneedbuffer = 0;
nfs_nbdwrite = 0;
- nfsbufhashtbl = hashinit(nfsbufmax / 4, M_TEMP, &nfsbufhash);
+ nfsbufhashtbl = hashinit(nfsbufmax / 4, M_NFSBIO, &nfsbufhash);
TAILQ_INIT(&nfsbuffree);
TAILQ_INIT(&nfsbuffreemeta);
TAILQ_INIT(&nfsbufdelwri);
if (!fbp) {
break;
}
- if (fbp->nb_refs) {
+ if (os_ref_get_count(&fbp->nb_refs) > 1) {
break;
}
if (NBUFSTAMPVALID(fbp) &&
if (!fbp) {
break;
}
- if (fbp->nb_refs) {
+ if (os_ref_get_count(&fbp->nb_refs) > 1) {
break;
}
if (NBUFSTAMPVALID(fbp) &&
}
/* if buf was NB_META, dump buffer */
if (ISSET(fbp->nb_flags, NB_META) && fbp->nb_data) {
- kfree(fbp->nb_data, fbp->nb_bufsize);
+ kheap_free(KHEAP_DATA_BUFFERS, fbp->nb_data, fbp->nb_bufsize);
}
- FREE(fbp, M_TEMP);
+ NFS_ZFREE(nfsbuf_zone, fbp);
}
}
* If it does, we can't let the pager drop the page.
*/
if (bp->nb_dirtyend > 0) {
- int start = offset - NBOFF(bp);
+ off_t start = offset - NBOFF(bp);
if ((bp->nb_dirtyend > start) &&
(bp->nb_dirtyoff < (start + PAGE_SIZE))) {
/*
}
pl = ubc_upl_pageinfo(bp->nb_pagelist);
- bp->nb_valid = bp->nb_dirty = 0;
+ NBPGS_ERASE(&bp->nb_valid);
+ NBPGS_ERASE(&bp->nb_dirty);
for (i = 0; i < npages; i++, fileoffset += PAGE_SIZE_64) {
/* anything beyond the end of the file is not valid or dirty */
void
nfs_buf_normalize_valid_range(nfsnode_t np, struct nfsbuf *bp)
{
- int pg, npg;
+ off_t pg, npg;
/* pull validoff back to start of contiguous valid page range */
pg = bp->nb_validoff / PAGE_SIZE;
while (pg >= 0 && NBPGVALID(bp, pg)) {
void
nfs_buf_delwri_thread(__unused void *arg, __unused wait_result_t wr)
{
- struct timespec ts = { 30, 0 };
+ struct timespec ts = { .tv_sec = 30, .tv_nsec = 0 };
int error = 0;
lck_mtx_lock(nfs_buf_mutex);
} else if (!lrubp) {
bp = metabp;
} else {
- int32_t lru_stale_time, meta_stale_time;
+ time_t lru_stale_time, meta_stale_time;
lru_stale_time = lrubp->nb_timestamp + NFSBUF_LRU_STALE;
meta_stale_time = metabp->nb_timestamp + NFSBUF_META_STALE;
if (lru_stale_time <= meta_stale_time) {
}
} else if (ISSET(bp->nb_flags, NB_META)) {
if (bp->nb_data) {
- kfree(bp->nb_data, bp->nb_bufsize);
+ kheap_free(KHEAP_DATA_BUFFERS, bp->nb_data, bp->nb_bufsize);
bp->nb_data = NULL;
}
nfsbufmetacnt--;
bp->nb_error = 0;
bp->nb_validoff = bp->nb_validend = -1;
bp->nb_dirtyoff = bp->nb_dirtyend = 0;
- bp->nb_valid = 0;
- bp->nb_dirty = 0;
+ NBPGS_ERASE(&bp->nb_valid);
+ NBPGS_ERASE(&bp->nb_dirty);
bp->nb_verf = 0;
} else {
/* no buffer to reuse */
if ((nfsbufcnt < nfsbufmax) &&
((operation != NBLK_META) || (nfsbufmetacnt < nfsbufmetamax))) {
/* just alloc a new one */
- MALLOC(bp, struct nfsbuf *, sizeof(struct nfsbuf), M_TEMP, M_WAITOK);
- if (!bp) {
- lck_mtx_unlock(nfs_buf_mutex);
- FSDBG_BOT(541, np, blkno, 0, error);
- return ENOMEM;
- }
+ bp = zalloc(nfsbuf_zone);
nfsbufcnt++;
/*
NFSBUFCNTCHK();
/* init nfsbuf */
bzero(bp, sizeof(*bp));
+ os_ref_init(&bp->nb_refs, NULL);
+
bp->nb_free.tqe_next = NFSNOLIST;
bp->nb_validoff = bp->nb_validend = -1;
FSDBG(545, np, blkno, bp, 0);
case NBLK_META:
SET(bp->nb_flags, NB_META);
if ((bp->nb_bufsize != bufsize) && bp->nb_data) {
- kfree(bp->nb_data, bp->nb_bufsize);
+ kheap_free(KHEAP_DATA_BUFFERS, bp->nb_data, bp->nb_bufsize);
bp->nb_data = NULL;
bp->nb_validoff = bp->nb_validend = -1;
bp->nb_dirtyoff = bp->nb_dirtyend = 0;
- bp->nb_valid = 0;
- bp->nb_dirty = 0;
+ NBPGS_ERASE(&bp->nb_valid);
+ NBPGS_ERASE(&bp->nb_dirty);
CLR(bp->nb_flags, NB_CACHE);
}
if (!bp->nb_data) {
- bp->nb_data = kalloc(bufsize);
+ bp->nb_data = kheap_alloc(KHEAP_DATA_BUFFERS,
+ bufsize, Z_WAITOK);
}
if (!bp->nb_data) {
/* Ack! couldn't allocate the data buffer! */
* Abort the pages on error or: if this is an invalid or
* non-needcommit nocache buffer AND no pages are dirty.
*/
- if (ISSET(bp->nb_flags, NB_ERROR) || (!bp->nb_dirty && (ISSET(bp->nb_flags, NB_INVAL) ||
+ if (ISSET(bp->nb_flags, NB_ERROR) || (!nfs_buf_pgs_is_set(&bp->nb_dirty) && (ISSET(bp->nb_flags, NB_INVAL) ||
(ISSET(bp->nb_flags, NB_NOCACHE) && !ISSET(bp->nb_flags, (NB_NEEDCOMMIT | NB_DELWRI)))))) {
if (ISSET(bp->nb_flags, (NB_READ | NB_INVAL | NB_NOCACHE))) {
upl_flags = UPL_ABORT_DUMP_PAGES;
void
nfs_buf_refget(struct nfsbuf *bp)
{
- bp->nb_refs++;
+ os_ref_retain_locked(&bp->nb_refs);
}
/*
* release a reference on a buffer
void
nfs_buf_refrele(struct nfsbuf *bp)
{
- bp->nb_refs--;
+ (void) os_ref_release_locked(&bp->nb_refs);
}
/*
/* update valid range */
bp->nb_validoff = 0;
bp->nb_validend = bp->nb_endio;
- if (bp->nb_endio < (int)bp->nb_bufsize) {
+ if (bp->nb_endio < bp->nb_bufsize) {
/*
* The read may be short because we have unflushed writes
* that are extending the file size and the reads hit the
((NBOFF(bp) + bp->nb_validend) > 0x100000000LL)) {
bp->nb_validend = 0x100000000LL - NBOFF(bp);
}
- bp->nb_valid = (1 << (round_page_32(bp->nb_validend) / PAGE_SIZE)) - 1;
+ nfs_buf_pgs_get_page_mask(&bp->nb_valid, round_page_64(bp->nb_validend) / PAGE_SIZE);
if (bp->nb_validend & PAGE_MASK) {
/* zero-fill remainder of last page */
bzero(bp->nb_data + bp->nb_validend, PAGE_SIZE - (bp->nb_validend & PAGE_MASK));
struct nfsmount *nmp;
nfsnode_t np = bp->nb_np;
int error = 0, nfsvers, async;
- int offset, nrpcs;
- uint32_t nmrsize, length, len;
+ int offset;
+ uint64_t length, nrpcs;
+ uint32_t nmrsize;
+ size_t len;
off_t boff;
struct nfsreq *req;
struct nfsreq_cbinfo cb;
error = bp->nb_error;
break;
}
- len = (length > nmrsize) ? nmrsize : length;
- cb.rcb_args[0] = offset;
- cb.rcb_args[1] = len;
+ len = (length > nmrsize) ? nmrsize : (uint32_t)length;
+ cb.rcb_args.offset = offset;
+ cb.rcb_args.length = len;
+#if CONFIG_NFS4
if (nmp->nm_vers >= NFS_VER4) {
- cb.rcb_args[2] = nmp->nm_stategenid;
+ cb.rcb_args.stategenid = nmp->nm_stategenid;
}
+#endif
req = NULL;
error = nmp->nm_funcs->nf_read_rpc_async(np, boff + offset, len, thd, cred, &cb, &req);
if (error) {
nfs_buf_read_rpc_finish(struct nfsreq *req)
{
struct nfsmount *nmp;
- size_t rlen;
+ size_t rlen, length;
struct nfsreq_cbinfo cb;
struct nfsbuf *bp;
- int error = 0, nfsvers, offset, length, eof = 0, multasyncrpc, finished;
+ int error = 0, nfsvers, eof = 0, multasyncrpc, finished;
+ off_t offset;
void *wakeme = NULL;
struct nfsreq *rreq = NULL;
nfsnode_t np;
}
nfsvers = nmp->nm_vers;
- offset = cb.rcb_args[0];
- rlen = length = cb.rcb_args[1];
+ offset = cb.rcb_args.offset;
+ rlen = length = cb.rcb_args.length;
auio = uio_createwithbuffer(1, NBOFF(bp) + offset, UIO_SYSSPACE,
UIO_READ, &uio_buf, sizeof(uio_buf));
}
return;
}
+#if CONFIG_NFS4
if ((nmp->nm_vers >= NFS_VER4) && nfs_mount_state_error_should_restart(error) && !ISSET(bp->nb_flags, NB_ERROR)) {
lck_mtx_lock(&nmp->nm_lock);
- if ((error != NFSERR_OLD_STATEID) && (error != NFSERR_GRACE) && (cb.rcb_args[2] == nmp->nm_stategenid)) {
+ if ((error != NFSERR_OLD_STATEID) && (error != NFSERR_GRACE) && (cb.rcb_args.stategenid == nmp->nm_stategenid)) {
NP(np, "nfs_buf_read_rpc_finish: error %d @ 0x%llx, 0x%x 0x%x, initiating recovery",
- error, NBOFF(bp) + offset, cb.rcb_args[2], nmp->nm_stategenid);
+ error, NBOFF(bp) + offset, cb.rcb_args.stategenid, nmp->nm_stategenid);
nfs_need_recover(nmp, error);
}
lck_mtx_unlock(&nmp->nm_lock);
}
}
}
+#endif
if (error) {
SET(bp->nb_flags, NB_ERROR);
bp->nb_error = error;
eofrem = np->n_size - (NBOFF(bp) + offset + rlen);
rem = (rpcrem < eofrem) ? rpcrem : eofrem;
if (rem > 0) {
- bzero(bp->nb_data + offset + rlen, rem);
+ NFS_BZERO(bp->nb_data + offset + rlen, rem);
}
- } else if (((int)rlen < length) && !ISSET(bp->nb_flags, NB_ERROR)) {
+ } else if ((rlen < length) && !ISSET(bp->nb_flags, NB_ERROR)) {
/*
* short read
*
* requested, so we need to issue another read for the rest.
* (Don't bother if the buffer already hit an error.)
*/
+#if CONFIG_NFS4
readagain:
+#endif
offset += rlen;
length -= rlen;
- cb.rcb_args[0] = offset;
- cb.rcb_args[1] = length;
+ cb.rcb_args.offset = offset;
+ cb.rcb_args.length = length;
+#if CONFIG_NFS4
if (nmp->nm_vers >= NFS_VER4) {
- cb.rcb_args[2] = nmp->nm_stategenid;
+ cb.rcb_args.stategenid = nmp->nm_stategenid;
}
+#endif
error = nmp->nm_funcs->nf_read_rpc_async(np, NBOFF(bp) + offset, length, thd, cred, &cb, &rreq);
if (!error) {
if (IS_VALID_CRED(cred)) {
continue;
}
if ((ioflag & IO_NOCACHE) && ISSET(bp->nb_flags, NB_CACHE) &&
- !bp->nb_dirty && !ISSET(bp->nb_flags, (NB_DELWRI | NB_NCRDAHEAD))) {
+ !nfs_buf_pgs_is_set(&bp->nb_dirty) && !ISSET(bp->nb_flags, (NB_DELWRI | NB_NCRDAHEAD))) {
CLR(bp->nb_flags, NB_CACHE);
- bp->nb_valid = 0;
+ NBPGS_ERASE(&bp->nb_valid);
bp->nb_validoff = bp->nb_validend = -1;
}
- if ((bp->nb_dirtyend <= 0) && !bp->nb_dirty &&
+ if ((bp->nb_dirtyend <= 0) && !nfs_buf_pgs_is_set(&bp->nb_dirty) &&
!ISSET(bp->nb_flags, (NB_CACHE | NB_DELWRI))) {
SET(bp->nb_flags, (NB_READ | NB_ASYNC));
if (ioflag & IO_NOCACHE) {
struct nfsbuf *bp = NULL;
struct nfsmount *nmp = VTONMP(vp);
daddr64_t lbn, rabn = 0, lastrabn, maxrabn = -1;
- off_t diff;
- int error = 0, n = 0, on = 0;
+ off_t diff, on = 0, n = 0;
+ int error = 0, n32;
int nfsvers, biosize, modified, readaheads = 0;
thread_t thd;
kauth_cred_t cred;
io_resid = diff;
}
if (io_resid > 0) {
- int count = (io_resid > INT_MAX) ? INT_MAX : io_resid;
+ int count = (io_resid > INT_MAX) ? INT_MAX : (int)io_resid;
error = cluster_copy_ubc_data(vp, uio, &count, 0);
if (error) {
nfs_data_unlock(np);
}
/* count any biocache reads that we just copied directly */
if (lbn != (uio_offset(uio) / biosize)) {
- OSAddAtomic64((uio_offset(uio) / biosize) - lbn, &nfsstats.biocache_reads);
+ OSAddAtomic64(NFS_ROUND_BLOCK(uio_offset(uio), biosize) - lbn, &nfsstats.biocache_reads);
FSDBG(514, np, 0xcacefeed, uio_offset(uio), error);
}
}
return error;
}
readaheads = 1;
+ OSAddAtomic64(rabn - lbn, &nfsstats.biocache_reads);
+ } else {
+ OSAddAtomic64(1, &nfsstats.biocache_reads);
}
- OSAddAtomic64(1, &nfsstats.biocache_reads);
-
/*
* If the block is in the cache and has the required data
* in a valid region, just copy it out.
* Invalidate the data if it wasn't just read
* in as part of a "nocache readahead".
*/
- if (bp->nb_dirty || (bp->nb_dirtyend > 0)) {
+ if (nfs_buf_pgs_is_set(&bp->nb_dirty) || (bp->nb_dirtyend > 0)) {
/* so write the buffer out and try again */
SET(bp->nb_flags, NB_NOCACHE);
goto flushbuffer;
}
/* if any pages are valid... */
- if (bp->nb_valid) {
+ if (nfs_buf_pgs_is_set(&bp->nb_valid)) {
/* ...check for any invalid pages in the read range */
- int pg, firstpg, lastpg, dirtypg;
+ off_t pg, firstpg, lastpg, dirtypg;
dirtypg = firstpg = lastpg = -1;
pg = on / PAGE_SIZE;
while (pg <= (on + n - 1) / PAGE_SIZE) {
if (bp->nb_validoff < 0) {
/* valid range isn't set up, so */
/* set it to what we know is valid */
- bp->nb_validoff = trunc_page(on);
- bp->nb_validend = round_page(on + n);
+ bp->nb_validoff = trunc_page_64(on);
+ bp->nb_validend = round_page_64(on + n);
nfs_buf_normalize_valid_range(np, bp);
}
goto buffer_ready;
}
goto again;
}
- if (!bp->nb_dirty && bp->nb_dirtyend <= 0 &&
+ if (!nfs_buf_pgs_is_set(&bp->nb_dirty) && bp->nb_dirtyend <= 0 &&
(lastpg - firstpg + 1) > (biosize / PAGE_SIZE) / 2) {
/* we need to read in more than half the buffer and the */
/* buffer's not dirty, so just fetch the whole buffer */
- bp->nb_valid = 0;
+ NBPGS_ERASE(&bp->nb_valid);
} else {
/* read the page range in */
uio_t auio;
if (!auio) {
error = ENOMEM;
} else {
- uio_addiov(auio, CAST_USER_ADDR_T(bp->nb_data + (firstpg * PAGE_SIZE)),
+ NFS_UIO_ADDIOV(auio, CAST_USER_ADDR_T(bp->nb_data + (firstpg * PAGE_SIZE)),
((lastpg - firstpg + 1) * PAGE_SIZE));
error = nfs_read_rpc(np, auio, ctx);
}
return error;
}
/* Make sure that the valid range is set to cover this read. */
- bp->nb_validoff = trunc_page_32(on);
- bp->nb_validend = round_page_32(on + n);
+ bp->nb_validoff = trunc_page_64(on);
+ bp->nb_validend = round_page_64(on + n);
nfs_buf_normalize_valid_range(np, bp);
if (uio_resid(auio) > 0) {
/* if short read, must have hit EOF, */
}
}
/* if no pages are valid, read the whole block */
- if (!bp->nb_valid) {
+ if (!nfs_buf_pgs_is_set(&bp->nb_valid)) {
if (!IS_VALID_CRED(bp->nb_rcred) && IS_VALID_CRED(cred)) {
kauth_cred_ref(cred);
bp->nb_rcred = cred;
}
if (n > 0) {
NFS_BUF_MAP(bp);
- error = uiomove(bp->nb_data + on, n, uio);
+ n32 = n > INT_MAX ? INT_MAX : (int)n;
+ error = uiomove(bp->nb_data + on, n32, uio);
+ if (!error && n > n32) {
+ error = uiomove(bp->nb_data + on + n32, (int)(n - n32), uio);
+ }
}
+
nfs_buf_release(bp, 1);
nfs_data_unlock(np);
nfs_node_lock_force(np);
nfs_async_write_start(struct nfsmount *nmp)
{
int error = 0, slpflag = NMFLAG(nmp, INTR) ? PCATCH : 0;
- struct timespec ts = {1, 0};
+ struct timespec ts = { .tv_sec = 1, .tv_nsec = 0 };
if (nfs_max_async_writes <= 0) {
return 0;
thread_t thd;
kauth_cred_t cred;
proc_t p = current_proc();
- int iomode, doff, dend, firstpg, lastpg;
- uint32_t pagemask;
+ int iomode;
+ off_t doff, dend, firstpg, lastpg;
FSDBG_TOP(553, bp, NBOFF(bp), bp->nb_flags, 0);
}
if (!error && (bp->nb_dirtyend > 0)) {
/* there's a dirty range that needs to be written out */
+ nfsbufpgs pagemask, pagemaskand;
NFS_BUF_MAP(bp);
doff = bp->nb_dirtyoff;
}
/* if dend page is dirty, move dend to start of next page */
if ((dend & PAGE_MASK) && NBPGDIRTY(bp, dend / PAGE_SIZE)) {
- dend = round_page_32(dend);
+ dend = round_page_64(dend);
}
/* try to expand write range to include trailing dirty pages */
if (!(dend & PAGE_MASK)) {
dend = np->n_size - NBOFF(bp);
}
/* calculate range of complete pages being written */
- firstpg = round_page_32(doff) / PAGE_SIZE;
- lastpg = (trunc_page_32(dend) - 1) / PAGE_SIZE;
- /* calculate mask for that page range */
- pagemask = ((1 << (lastpg + 1)) - 1) & ~((1 << firstpg) - 1);
+ if (dend > doff) {
+ firstpg = doff / PAGE_SIZE;
+ lastpg = (dend - 1) / PAGE_SIZE;
+ /* calculate mask for that page range */
+ nfs_buf_pgs_set_pages_between(&pagemask, firstpg, lastpg + 1);
+ } else {
+ NBPGS_ERASE(&pagemask);
+ }
/*
* compare page mask to nb_dirty; if there are other dirty pages
* then write FILESYNC; otherwise, write UNSTABLE if async and
* not needcommit/stable; otherwise write FILESYNC
*/
- if (bp->nb_dirty & ~pagemask) {
+ nfs_buf_pgs_bit_not(&pagemask);
+ nfs_buf_pgs_bit_and(&bp->nb_dirty, &pagemask, &pagemaskand);
+ if (nfs_buf_pgs_is_set(&pagemaskand)) {
iomode = NFS_WRITE_FILESYNC;
} else if ((bp->nb_flags & (NB_ASYNC | NB_NEEDCOMMIT | NB_STABLE)) == NB_ASYNC) {
iomode = NFS_WRITE_UNSTABLE;
* pages pushed out.
*/
} else {
- if (!error && bp->nb_dirty) { /* write out any dirty pages */
+ if (!error && nfs_buf_pgs_is_set(&bp->nb_dirty)) { /* write out any dirty pages */
error = nfs_buf_write_dirty_pages(bp, thd, cred);
}
nfs_buf_iodone(bp);
{
nfsnode_t np = bp->nb_np;
int error = (bp->nb_flags & NB_ERROR) ? bp->nb_error : 0;
- int firstpg, lastpg;
- uint32_t pagemask;
+ off_t firstpg, lastpg;
if ((error == EINTR) || (error == ERESTART)) {
CLR(bp->nb_flags, NB_ERROR);
}
if (!error) {
+ nfsbufpgs pagemask;
/* calculate range of complete pages being written */
- firstpg = round_page_32(bp->nb_offio) / PAGE_SIZE;
- lastpg = (trunc_page_32(bp->nb_endio) - 1) / PAGE_SIZE;
- /* calculate mask for that page range written */
- pagemask = ((1 << (lastpg + 1)) - 1) & ~((1 << firstpg) - 1);
+ if (bp->nb_endio > bp->nb_offio) {
+ firstpg = bp->nb_offio / PAGE_SIZE;
+ lastpg = (bp->nb_endio - 1) / PAGE_SIZE;
+ /* calculate mask for that page range written */
+ nfs_buf_pgs_set_pages_between(&pagemask, firstpg, lastpg + 1);
+ } else {
+ NBPGS_ERASE(&pagemask);
+ }
/* clear dirty bits for pages we've written */
- bp->nb_dirty &= ~pagemask;
+ nfs_buf_pgs_bit_not(&pagemask);
+ nfs_buf_pgs_bit_and(&bp->nb_dirty, &pagemask, &bp->nb_dirty);
}
/* manage needcommit state */
bp->nb_dirtyoff = bp->nb_dirtyend = 0;
}
- if (!error && bp->nb_dirty) {
+ if (!error && nfs_buf_pgs_is_set(&bp->nb_dirty)) {
nfs_buf_write_dirty_pages(bp, thd, cred);
}
nfs_buf_iodone(bp);
nfsnode_t np = bp->nb_np;
struct nfsmount *nmp = NFSTONMP(np);
int error = 0, commit, iomode, iomode2, len, pg, count, npages, off;
- uint32_t dirty = bp->nb_dirty;
+ nfsbufpgs dirty;
uint64_t wverf;
uio_t auio;
char uio_buf[UIO_SIZEOF(1)];
- if (!bp->nb_dirty) {
+ if (!nfs_buf_pgs_is_set(&bp->nb_dirty)) {
return 0;
}
&uio_buf, sizeof(uio_buf));
again:
- dirty = bp->nb_dirty;
+ NBPGS_COPY(&dirty, &bp->nb_dirty);
wverf = bp->nb_verf;
commit = NFS_WRITE_FILESYNC;
for (pg = 0; pg < npages; pg++) {
}
/* clear dirty bits */
while (count--) {
- dirty &= ~(1 << pg);
+ NBPGS_UNSET(&dirty, pg);
if (count) { /* leave pg on last page */
pg++;
}
}
}
if (!error) {
- bp->nb_dirty = dirty;
+ NBPGS_COPY(&bp->nb_dirty, &dirty);
} else {
SET(bp->nb_flags, NB_ERROR);
bp->nb_error = error;
struct nfsmount *nmp;
nfsnode_t np = bp->nb_np;
int error = 0, nfsvers, async;
- int offset, nrpcs;
- uint32_t nmwsize, length, len;
+ int64_t nrpcs;
+ size_t len;
+ uint32_t nmwsize;
struct nfsreq *req;
struct nfsreq_cbinfo cb;
uio_t auio;
char uio_buf[UIO_SIZEOF(1)];
+ off_t offset, length;
nmp = NFSTONMP(np);
if (nfs_mount_gone(nmp)) {
return error;
}
+ if (length == 0) {
+ /* We should never get here */
+#if DEVELOPMENT
+ printf("nfs_buf_write_rpc: Got request with zero length. np %p, bp %p, offset %lld\n", np, bp, offset);
+#else
+ printf("nfs_buf_write_rpc: Got request with zero length.\n");
+#endif /* DEVELOPMENT */
+ nfs_buf_iodone(bp);
+ return 0;
+ }
+
auio = uio_createwithbuffer(1, NBOFF(bp) + offset, UIO_SYSSPACE,
UIO_WRITE, &uio_buf, sizeof(uio_buf));
- uio_addiov(auio, CAST_USER_ADDR_T(bp->nb_data + offset), length);
+ NFS_UIO_ADDIOV(auio, CAST_USER_ADDR_T(bp->nb_data + offset), length);
bp->nb_rpcs = nrpcs = (length + nmwsize - 1) / nmwsize;
if (async && (nrpcs > 1)) {
error = bp->nb_error;
break;
}
- len = (length > nmwsize) ? nmwsize : length;
- cb.rcb_args[0] = offset;
- cb.rcb_args[1] = len;
+ len = (length > nmwsize) ? nmwsize : (uint32_t)length;
+ cb.rcb_args.offset = offset;
+ cb.rcb_args.length = len;
+#if CONFIG_NFS4
if (nmp->nm_vers >= NFS_VER4) {
- cb.rcb_args[2] = nmp->nm_stategenid;
+ cb.rcb_args.stategenid = nmp->nm_stategenid;
}
+#endif
if (async && ((error = nfs_async_write_start(nmp)))) {
break;
}
void
nfs_buf_write_rpc_finish(struct nfsreq *req)
{
- int error = 0, nfsvers, offset, length, multasyncrpc, finished;
+ int error = 0, nfsvers, multasyncrpc, finished;
int committed = NFS_WRITE_FILESYNC;
uint64_t wverf = 0;
- size_t rlen;
+ off_t offset;
+ size_t rlen, length;
void *wakeme = NULL;
struct nfsreq_cbinfo cb;
struct nfsreq *wreq = NULL;
}
nfsvers = nmp->nm_vers;
- offset = cb.rcb_args[0];
- rlen = length = cb.rcb_args[1];
+ offset = cb.rcb_args.offset;
+ rlen = length = cb.rcb_args.length;
/* finish the RPC */
error = nmp->nm_funcs->nf_write_rpc_async_finish(np, req, &committed, &rlen, &wverf);
}
return;
}
+#if CONFIG_NFS4
if ((nmp->nm_vers >= NFS_VER4) && nfs_mount_state_error_should_restart(error) && !ISSET(bp->nb_flags, NB_ERROR)) {
lck_mtx_lock(&nmp->nm_lock);
- if ((error != NFSERR_OLD_STATEID) && (error != NFSERR_GRACE) && (cb.rcb_args[2] == nmp->nm_stategenid)) {
+ if ((error != NFSERR_OLD_STATEID) && (error != NFSERR_GRACE) && (cb.rcb_args.stategenid == nmp->nm_stategenid)) {
NP(np, "nfs_buf_write_rpc_finish: error %d @ 0x%llx, 0x%x 0x%x, initiating recovery",
- error, NBOFF(bp) + offset, cb.rcb_args[2], nmp->nm_stategenid);
+ error, NBOFF(bp) + offset, cb.rcb_args.stategenid, nmp->nm_stategenid);
nfs_need_recover(nmp, error);
}
lck_mtx_unlock(&nmp->nm_lock);
}
}
}
+#endif
if (error) {
SET(bp->nb_flags, NB_ERROR);
bp->nb_error = error;
bp->nb_verf = wverf;
}
+ if (!ISSET(bp->nb_flags, NB_STALEWVERF) && rlen > 0 && (bp->nb_offio < (offset + (int)rlen))) {
+ bp->nb_offio = offset + rlen;
+ }
+
/*
* check for a short write
*
* need to issue another write for the rest of it.
* (Don't bother if the buffer hit an error or stale wverf.)
*/
- if (((int)rlen < length) && !(bp->nb_flags & (NB_STALEWVERF | NB_ERROR))) {
+ if ((rlen < length) && !(bp->nb_flags & (NB_STALEWVERF | NB_ERROR))) {
+#if CONFIG_NFS4
writeagain:
+#endif
offset += rlen;
length -= rlen;
UIO_WRITE, &uio_buf, sizeof(uio_buf));
uio_addiov(auio, CAST_USER_ADDR_T(bp->nb_data + offset), length);
- cb.rcb_args[0] = offset;
- cb.rcb_args[1] = length;
+ cb.rcb_args.offset = offset;
+ cb.rcb_args.length = length;
+#if CONFIG_NFS4
if (nmp->nm_vers >= NFS_VER4) {
- cb.rcb_args[2] = nmp->nm_stategenid;
+ cb.rcb_args.stategenid = nmp->nm_stategenid;
}
-
+#endif
// XXX iomode should really match the original request
error = nmp->nm_funcs->nf_write_rpc_async(np, auio, length, thd, cred,
NFS_WRITE_FILESYNC, &cb, &wreq);
if (IS_VALID_CRED(cred)) {
kauth_cred_unref(&cred);
}
+
+ if (cb.rcb_func && np->n_needcommitcnt >= NFS_A_LOT_OF_NEEDCOMMITS) {
+ nfs_flushcommits(np, 1);
+ }
}
/*
struct nfsmount *nmp;
struct nfsbuf *bp, *prevlbp, *lbp;
struct nfsbuflists blist, commitlist;
- int error = 0, retv, wcred_set, flags, dirty;
+ int error = 0, retv, wcred_set, flags;
u_quad_t off, endoff, toff;
- uint64_t wverf;
- u_int32_t count;
+ uint64_t wverf, count;
kauth_cred_t wcred = NULL;
+ nfsbufpgs dirty;
FSDBG_TOP(557, np, 0, 0, 0);
if (retv) {
/* Unable to create the UPL, the VM object probably no longer exists. */
printf("nfs_flushcommits: upl create failed %d\n", retv);
- bp->nb_valid = bp->nb_dirty = 0;
+ NBPGS_ERASE(&bp->nb_valid);
+ NBPGS_ERASE(&bp->nb_dirty);
}
}
nfs_buf_upl_check(bp);
CLR(bp->nb_flags, (NB_READ | NB_DONE | NB_ERROR | NB_DELWRI));
/* if block still has dirty pages, we don't want it to */
/* be released in nfs_buf_iodone(). So, don't set NB_ASYNC. */
- if (!(dirty = bp->nb_dirty)) {
+ NBPGS_COPY(&dirty, &bp->nb_dirty);
+ if (!nfs_buf_pgs_is_set(&dirty)) {
SET(bp->nb_flags, NB_ASYNC);
} else {
CLR(bp->nb_flags, NB_ASYNC);
bp->nb_dirtyoff = bp->nb_dirtyend = 0;
nfs_buf_iodone(bp);
- if (dirty) {
+ if (nfs_buf_pgs_is_set(&dirty)) {
/* throw it back in as a delayed write buffer */
CLR(bp->nb_flags, NB_DONE);
nfs_buf_write_delayed(bp);
(NBOFF(bp) < (off_t)np->n_size)) {
/* extra paranoia: make sure we're not */
/* somehow leaving any dirty data around */
+ nfsbufpgs pagemask;
int mustwrite = 0;
- int end = (NBOFF(bp) + bp->nb_bufsize > (off_t)np->n_size) ?
- ((off_t)np->n_size - NBOFF(bp)) : bp->nb_bufsize;
+ off_t end = (NBOFF(bp) + bp->nb_bufsize > (off_t)np->n_size) ?
+ (np->n_size - NBOFF(bp)) : bp->nb_bufsize;
if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
error = nfs_buf_upl_setup(bp);
if (error == EINVAL) {
} else if (error) {
printf("nfs_vinvalbuf: upl setup failed %d\n", error);
}
- bp->nb_valid = bp->nb_dirty = 0;
+ NBPGS_ERASE(&bp->nb_valid);
+ NBPGS_ERASE(&bp->nb_dirty);
}
nfs_buf_upl_check(bp);
/* check for any dirty data before the EOF */
mustwrite++;
}
}
- bp->nb_dirty &= (1 << (round_page_32(end) / PAGE_SIZE)) - 1;
- if (bp->nb_dirty) {
+ nfs_buf_pgs_get_page_mask(&pagemask, round_page_64(end) / PAGE_SIZE);
+ nfs_buf_pgs_bit_and(&bp->nb_dirty, &pagemask, &bp->nb_dirty);
+ if (nfs_buf_pgs_is_set(&bp->nb_dirty)) {
mustwrite++;
}
/* also make sure we'll have a credential to do the write */
struct nfsmount *nmp = VTONMP(vp);
int error, slpflag, slptimeo, nflags, retry = 0;
int ubcflags = UBC_PUSHALL | UBC_SYNC | UBC_INVALIDATE;
- struct timespec ts = { 2, 0 };
+ struct timespec ts = { .tv_sec = 2, .tv_nsec = 0 };
off_t size;
FSDBG_TOP(554, np, flags, intrflg, 0);
lck_mtx_lock(&req->r_mtx);
if (req->r_flags & R_RESENDQ) {
lck_mtx_lock(&nmp->nm_lock);
- if (req->r_rchain.tqe_next != NFSREQNOLIST) {
+ if ((req->r_flags & R_RESENDQ) && req->r_rchain.tqe_next != NFSREQNOLIST) {
NFS_BIO_DBG("Proccessing async request on resendq. Removing");
TAILQ_REMOVE(&nmp->nm_resendq, req, r_rchain);
+ req->r_flags &= ~R_RESENDQ;
req->r_rchain.tqe_next = NFSREQNOLIST;
assert(req->r_refs > 1);
/* Remove resendq reference */
req->r_refs--;
}
lck_mtx_unlock(&nmp->nm_lock);
- req->r_flags &= ~R_RESENDQ;
}
lck_mtx_unlock(&req->r_mtx);
/*
* queue up async I/O request for resend
+ * Must be called with req->r_mtx locked.
*/
void
nfs_asyncio_resend(struct nfsreq *req)
return;
}
+#if CONFIG_NFS_GSS
nfs_gss_clnt_rpcdone(req);
+#endif
lck_mtx_lock(&nmp->nm_lock);
if (!(req->r_flags & R_RESENDQ)) {
TAILQ_INSERT_TAIL(&nmp->nm_resendq, req, r_rchain);
if (nmp->nm_vers < NFS_VER4) {
error = nfs3_readdir_rpc(np, bp, ctx);
- } else {
+ }
+#if CONFIG_NFS4
+ else {
error = nfs4_readdir_rpc(np, bp, ctx);
}
-
+#endif
if (error && (error != NFSERR_DIRBUFDROPPED)) {
SET(bp->nb_flags, NB_ERROR);
bp->nb_error = error;
}
return error;
}
+
+#endif /* CONFIG_NFS_CLIENT */
projects / apple / xnu.git / blobdiff