X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/b4c24cb9d3df001f2892dc4ed451bc769ff28a9f..c0fea4742e91338fffdcf79f86a7c1d5e2b97eb1:/bsd/vfs/vfs_journal.c diff --git a/bsd/vfs/vfs_journal.c b/bsd/vfs/vfs_journal.c index 2acb4fab2..dba9c0b3c 100644 --- a/bsd/vfs/vfs_journal.c +++ b/bsd/vfs/vfs_journal.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1995-2002 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1995-2004 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -34,19 +34,18 @@ #include #include #include -#include +#include #include -#include -#include -#include +#include +#include +#include #include -#include +#include #include #include #include #include -#include -#include +#include #include #include @@ -81,6 +80,31 @@ static int end_transaction(transaction *tr, int force_it); static void abort_transaction(journal *jnl, transaction *tr); static void dump_journal(journal *jnl); +static __inline__ void lock_journal(journal *jnl); +static __inline__ void unlock_journal(journal *jnl); +static __inline__ void lock_oldstart(journal *jnl); +static __inline__ void unlock_oldstart(journal *jnl); + + + + +// +// 3105942 - Coalesce writes to the same block on journal replay +// + +typedef struct bucket { + off_t block_num; + size_t jnl_offset; + size_t block_size; +} bucket; + +#define STARTING_BUCKETS 256 + +static int add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr); +static int grow_table(struct bucket **buf_ptr, int num_buckets, int new_size); +static int lookup_bucket(struct bucket **buf_ptr, off_t block_num, int num_full); +static int do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr); +static int insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr, int overwriting); #define CHECK_JOURNAL(jnl) \ do { \ @@ -99,17 +123,17 @@ static void dump_journal(journal *jnl); }\ if ( jnl->jhdr->start <= 0 \ || jnl->jhdr->start > jnl->jhdr->size\ - || jnl->jhdr->start > 128*1024*1024) {\ + || jnl->jhdr->start > 1024*1024*1024) {\ panic("%s:%d: jhdr start looks bad (0x%llx max size 0x%llx)\n", \ __FILE__, __LINE__, jnl->jhdr->start, jnl->jhdr->size);\ }\ if ( jnl->jhdr->end <= 0 \ || jnl->jhdr->end > jnl->jhdr->size\ - || jnl->jhdr->end > 128*1024*1024) {\ + || jnl->jhdr->end > 1024*1024*1024) {\ panic("%s:%d: jhdr end looks bad (0x%llx max size 0x%llx)\n", \ __FILE__, __LINE__, jnl->jhdr->end, jnl->jhdr->size);\ }\ - if (jnl->jhdr->size > 128*1024*1024) {\ + if (jnl->jhdr->size > 1024*1024*1024) {\ panic("%s:%d: jhdr size looks bad (0x%llx)\n",\ __FILE__, __LINE__, jnl->jhdr->size);\ } \ @@ -129,13 +153,13 @@ static void dump_journal(journal *jnl); if (tr->total_bytes < 0) {\ panic("%s:%d: tr total_bytes looks bad: %d\n", __FILE__, __LINE__, tr->total_bytes);\ }\ - if (tr->journal_start < 0 || tr->journal_start > 128*1024*1024) {\ + if (tr->journal_start < 0 || tr->journal_start > 1024*1024*1024) {\ panic("%s:%d: tr journal start looks bad: 0x%llx\n", __FILE__, __LINE__, tr->journal_start);\ }\ - if (tr->journal_end < 0 || tr->journal_end > 128*1024*1024) {\ + if (tr->journal_end < 0 || tr->journal_end > 1024*1024*1024) {\ panic("%s:%d: tr journal end looks bad: 0x%llx\n", __FILE__, __LINE__, tr->journal_end);\ }\ - if (tr->blhdr && (tr->blhdr->max_blocks <= 0 || tr->blhdr->max_blocks > 2048)) {\ + if (tr->blhdr && (tr->blhdr->max_blocks <= 0 || tr->blhdr->max_blocks > (tr->jnl->jhdr->size/tr->jnl->jhdr->jhdr_size))) {\ panic("%s:%d: tr blhdr max_blocks looks bad: %d\n", __FILE__, __LINE__, tr->blhdr->max_blocks);\ }\ } while(0) @@ -160,9 +184,50 @@ calc_checksum(char *ptr, int len) return (~cksum); } +// +// Journal Locking +// +lck_grp_attr_t * jnl_group_attr; +lck_attr_t * jnl_lock_attr; +lck_grp_t * jnl_mutex_group; -#define JNL_WRITE 1 -#define JNL_READ 2 +void +journal_init() +{ + jnl_lock_attr = lck_attr_alloc_init(); + jnl_group_attr = lck_grp_attr_alloc_init(); + jnl_mutex_group = lck_grp_alloc_init("jnl-mutex", jnl_group_attr); +} + +static __inline__ void +lock_journal(journal *jnl) +{ + lck_mtx_lock(&jnl->jlock); +} + +static __inline__ void +unlock_journal(journal *jnl) +{ + lck_mtx_unlock(&jnl->jlock); +} + +static __inline__ void +lock_oldstart(journal *jnl) +{ + lck_mtx_lock(&jnl->old_start_lock); +} + +static __inline__ void +unlock_oldstart(journal *jnl) +{ + lck_mtx_unlock(&jnl->old_start_lock); +} + + + +#define JNL_WRITE 0x0001 +#define JNL_READ 0x0002 +#define JNL_HEADER 0x8000 // // This function sets up a fake buf and passes it directly to the @@ -177,29 +242,23 @@ static size_t do_journal_io(journal *jnl, off_t *offset, void *data, size_t len, int direction) { int err, io_sz=0, curlen=len; - struct buf *bp; - int max_iosize=0, max_vectors; + buf_t bp; + int max_iosize = 128 * 1024; + struct vfsioattr ioattr; if (*offset < 0 || *offset > jnl->jhdr->size) { panic("jnl: do_jnl_io: bad offset 0x%llx (max 0x%llx)\n", *offset, jnl->jhdr->size); } + vfs_ioattr(vnode_mount(jnl->jdev), &ioattr); + + if (direction & JNL_WRITE) + max_iosize = ioattr.io_maxwritecnt; + else if (direction & JNL_READ) + max_iosize = ioattr.io_maxreadcnt; again: bp = alloc_io_buf(jnl->jdev, 1); - if (direction == JNL_WRITE) { - bp->b_flags |= 0; // don't have to set any flags (was: B_WRITEINPROG) - jnl->jdev->v_numoutput++; - vfs_io_attributes(jnl->jdev, B_WRITE, &max_iosize, &max_vectors); - } else if (direction == JNL_READ) { - bp->b_flags |= B_READ; - vfs_io_attributes(jnl->jdev, B_READ, &max_iosize, &max_vectors); - } - - if (max_iosize == 0) { - max_iosize = 128 * 1024; - } - if (*offset + (off_t)curlen > jnl->jhdr->size && *offset != 0 && jnl->jhdr->size != 0) { if (*offset == jnl->jhdr->size) { *offset = jnl->jhdr->jhdr_size; @@ -216,21 +275,28 @@ do_journal_io(journal *jnl, off_t *offset, void *data, size_t len, int direction panic("jnl: do_jnl_io: curlen == %d, offset 0x%llx len %d\n", curlen, *offset, len); } - bp->b_bufsize = curlen; - bp->b_bcount = curlen; - bp->b_data = data; - bp->b_blkno = (daddr_t) ((jnl->jdev_offset + *offset) / (off_t)jnl->jhdr->jhdr_size); - bp->b_lblkno = (daddr_t) ((jnl->jdev_offset + *offset) / (off_t)jnl->jhdr->jhdr_size); + if (*offset == 0 && (direction & JNL_HEADER) == 0) { + panic("jnl: request for i/o to jnl-header without JNL_HEADER flag set! (len %d, data %p)\n", curlen, data); + } - err = VOP_STRATEGY(bp); + if (direction & JNL_READ) + buf_setflags(bp, B_READ); + else { + /* + * don't have to set any flags + */ + vnode_startwrite(jnl->jdev); + } + buf_setsize(bp, curlen); + buf_setcount(bp, curlen); + buf_setdataptr(bp, (uintptr_t)data); + buf_setblkno(bp, (daddr64_t) ((jnl->jdev_offset + *offset) / (off_t)jnl->jhdr->jhdr_size)); + buf_setlblkno(bp, (daddr64_t) ((jnl->jdev_offset + *offset) / (off_t)jnl->jhdr->jhdr_size)); + + err = VNOP_STRATEGY(bp); if (!err) { - err = biowait(bp); + err = (int)buf_biowait(bp); } - - bp->b_data = NULL; - bp->b_bufsize = bp->b_bcount = 0; - bp->b_blkno = bp->b_lblkno = -1; - free_io_buf(bp); if (err) { @@ -266,29 +332,67 @@ write_journal_data(journal *jnl, off_t *offset, void *data, size_t len) } +static int +read_journal_header(journal *jnl, void *data, size_t len) +{ + off_t hdr_offset = 0; + + return do_journal_io(jnl, &hdr_offset, data, len, JNL_READ|JNL_HEADER); +} + static int write_journal_header(journal *jnl) { + static int num_err_prints = 0; int ret; off_t jhdr_offset = 0; - + struct vfs_context context; + + context.vc_proc = current_proc(); + context.vc_ucred = NOCRED; // // XXXdbg note: this ioctl doesn't seem to do anything on firewire disks. // - ret = VOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, NOCRED, current_proc()); + ret = VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, &context); if (ret != 0) { - printf("jnl: flushing fs disk buffer returned 0x%x\n", ret); + // + // Only print this error if it's a different error than the + // previous one, or if it's the first time for this device + // or if the total number of printfs is less than 25. We + // allow for up to 25 printfs to insure that some make it + // into the on-disk syslog. Otherwise if we only printed + // one, it's possible it would never make it to the syslog + // for the root volume and that makes debugging hard. + // + if ( ret != jnl->last_flush_err + || (jnl->flags & JOURNAL_FLUSHCACHE_ERR) == 0 + || num_err_prints++ < 25) { + + printf("jnl: flushing fs disk buffer returned 0x%x\n", ret); + + jnl->flags |= JOURNAL_FLUSHCACHE_ERR; + jnl->last_flush_err = ret; + } } - + jnl->jhdr->checksum = 0; jnl->jhdr->checksum = calc_checksum((char *)jnl->jhdr, sizeof(struct journal_header)); - if (write_journal_data(jnl, &jhdr_offset, jnl->header_buf, jnl->jhdr->jhdr_size) != jnl->jhdr->jhdr_size) { - printf("jnl: write_journal_header: error writing the journal header!\n"); - jnl->flags |= JOURNAL_INVALID; - return -1; + if (do_journal_io(jnl, &jhdr_offset, jnl->header_buf, jnl->jhdr->jhdr_size, JNL_WRITE|JNL_HEADER) != jnl->jhdr->jhdr_size) { + printf("jnl: write_journal_header: error writing the journal header!\n"); + jnl->flags |= JOURNAL_INVALID; + return -1; } + // Have to flush after writing the journal header so that + // a future transaction doesn't sneak out to disk before + // the header does and thus overwrite data that the old + // journal header refers to. Saw this exact case happen + // on an IDE bus analyzer with Larry Barras so while it + // may seem obscure, it's not. + // + VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, &context); + return 0; } @@ -307,12 +411,16 @@ free_old_stuff(journal *jnl) { transaction *tr, *next; - for(tr=jnl->tr_freeme; tr; tr=next) { - next = tr->next; - kmem_free(kernel_map, (vm_offset_t)tr, sizeof(transaction)); + lock_oldstart(jnl); + tr = jnl->tr_freeme; + jnl->tr_freeme = NULL; + unlock_oldstart(jnl); + + for(; tr; tr=next) { + next = tr->next; + FREE_ZONE(tr, sizeof(transaction), M_JNL_TR); } - jnl->tr_freeme = NULL; } @@ -324,7 +432,7 @@ free_old_stuff(journal *jnl) // not initiate any new i/o's or allocate/free memory. // static void -buffer_flushed_callback(struct buf *bp) +buffer_flushed_callback(struct buf *bp, void *arg) { transaction *tr; journal *jnl; @@ -332,29 +440,12 @@ buffer_flushed_callback(struct buf *bp) int i, bufsize; - //printf("jnl: buf flush: bp @ 0x%x l/blkno %d/%d vp 0x%x tr @ 0x%x\n", - // bp, bp->b_lblkno, bp->b_blkno, bp->b_vp, bp->b_transaction); + //printf("jnl: buf flush: bp @ 0x%x l/blkno %qd/%qd vp 0x%x tr @ 0x%x\n", + // bp, buf_lblkno(bp), buf_blkno(bp), buf_vnode(bp), arg); // snarf out the bits we want - bufsize = bp->b_bufsize; - tr = bp->b_transaction; - - bp->b_iodone = NULL; // don't call us for this guy again - bp->b_transaction = NULL; - - // - // This is what biodone() would do if it didn't call us. - // NOTE: THIS CODE *HAS* TO BE HERE! - // - if (ISSET(bp->b_flags, B_ASYNC)) { /* if async, release it */ - brelse(bp); - } else { /* or just wakeup the buffer */ - CLR(bp->b_flags, B_WANTED); - wakeup(bp); - } - - // NOTE: from here on out we do *NOT* touch bp anymore. - + bufsize = buf_size(bp); + tr = (transaction *)arg; // then we've already seen it if (tr == NULL) { @@ -373,7 +464,7 @@ buffer_flushed_callback(struct buf *bp) // update the number of blocks that have been flushed. // this buf may represent more than one block so take // that into account. - tr->num_flushed += bufsize; + OSAddAtomic(bufsize, &tr->num_flushed); // if this transaction isn't done yet, just return as @@ -382,11 +473,23 @@ buffer_flushed_callback(struct buf *bp) return; } + // this will single thread checking the transaction + lock_oldstart(jnl); + + if (tr->total_bytes == 0xfbadc0de) { + // then someone beat us to it... + unlock_oldstart(jnl); + return; + } + + // mark this so that we're the owner of dealing with the + // cleanup for this transaction + tr->total_bytes = 0xfbadc0de; + //printf("jnl: tr 0x%x (0x%llx 0x%llx) in jnl 0x%x completed.\n", // tr, tr->journal_start, tr->journal_end, jnl); // find this entry in the old_start[] index and mark it completed - simple_lock(&jnl->old_start_lock); for(i=0; i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0]); i++) { if ((jnl->old_start[i] & ~(0x8000000000000000LL)) == tr->journal_start) { @@ -398,7 +501,7 @@ buffer_flushed_callback(struct buf *bp) panic("jnl: buffer_flushed: did not find tr w/start @ %lld (tr 0x%x, jnl 0x%x)\n", tr->journal_start, tr, jnl); } - simple_unlock(&jnl->old_start_lock); + unlock_oldstart(jnl); // if we are here then we need to update the journal header @@ -420,10 +523,12 @@ buffer_flushed_callback(struct buf *bp) jnl->completed_trs = ctr->next; } + lock_oldstart(jnl); next = jnl->completed_trs; // this starts us over again ctr->next = jnl->tr_freeme; jnl->tr_freeme = ctr; ctr = NULL; + unlock_oldstart(jnl); } else if (tr->journal_end == ctr->journal_start) { ctr->journal_start = tr->journal_start; next = jnl->completed_trs; // this starts us over again @@ -438,9 +543,6 @@ buffer_flushed_callback(struct buf *bp) } } - // at this point no one should be using this guy anymore - tr->total_bytes = 0xfbadc0de; - // if this is true then we didn't merge with anyone // so link ourselves in at the head of the completed // transaction list. @@ -467,71 +569,391 @@ buffer_flushed_callback(struct buf *bp) } else { // if we're here this tr got merged with someone else so // put it on the list to be free'd + lock_oldstart(jnl); tr->next = jnl->tr_freeme; jnl->tr_freeme = tr; + unlock_oldstart(jnl); + } +} + + +#include + +#define SWAP16(x) OSSwapInt16(x) +#define SWAP32(x) OSSwapInt32(x) +#define SWAP64(x) OSSwapInt64(x) + + +static void +swap_journal_header(journal *jnl) +{ + jnl->jhdr->magic = SWAP32(jnl->jhdr->magic); + jnl->jhdr->endian = SWAP32(jnl->jhdr->endian); + jnl->jhdr->start = SWAP64(jnl->jhdr->start); + jnl->jhdr->end = SWAP64(jnl->jhdr->end); + jnl->jhdr->size = SWAP64(jnl->jhdr->size); + jnl->jhdr->blhdr_size = SWAP32(jnl->jhdr->blhdr_size); + jnl->jhdr->checksum = SWAP32(jnl->jhdr->checksum); + jnl->jhdr->jhdr_size = SWAP32(jnl->jhdr->jhdr_size); +} + +static void +swap_block_list_header(journal *jnl, block_list_header *blhdr) +{ + int i; + + blhdr->max_blocks = SWAP16(blhdr->max_blocks); + blhdr->num_blocks = SWAP16(blhdr->num_blocks); + blhdr->bytes_used = SWAP32(blhdr->bytes_used); + blhdr->checksum = SWAP32(blhdr->checksum); + blhdr->pad = SWAP32(blhdr->pad); + + if (blhdr->num_blocks * sizeof(blhdr->binfo[0]) > jnl->jhdr->blhdr_size) { + printf("jnl: blhdr num blocks looks suspicious (%d). not swapping.\n", blhdr->num_blocks); + return; + } + + for(i=0; i < blhdr->num_blocks; i++) { + blhdr->binfo[i].bnum = SWAP64(blhdr->binfo[i].bnum); + blhdr->binfo[i].bsize = SWAP32(blhdr->binfo[i].bsize); + blhdr->binfo[i].bp = (void *)SWAP32((int)blhdr->binfo[i].bp); } } + static int update_fs_block(journal *jnl, void *block_ptr, off_t fs_block, size_t bsize) { - int ret; + int ret; struct buf *oblock_bp=NULL; // first read the block we want. - ret = meta_bread(jnl->fsdev, (daddr_t)fs_block, bsize, NOCRED, &oblock_bp); + ret = buf_meta_bread(jnl->fsdev, (daddr64_t)fs_block, bsize, NOCRED, &oblock_bp); if (ret != 0) { printf("jnl: update_fs_block: error reading fs block # %lld! (ret %d)\n", fs_block, ret); if (oblock_bp) { - brelse(oblock_bp); + buf_brelse(oblock_bp); oblock_bp = NULL; } // let's try to be aggressive here and just re-write the block - oblock_bp = getblk(jnl->fsdev, (daddr_t)fs_block, bsize, 0, 0, BLK_META); + oblock_bp = buf_getblk(jnl->fsdev, (daddr64_t)fs_block, bsize, 0, 0, BLK_META); if (oblock_bp == NULL) { - printf("jnl: update_fs_block: getblk() for %lld failed! failing update.\n", fs_block); + printf("jnl: update_fs_block: buf_getblk() for %lld failed! failing update.\n", fs_block); return -1; } } // make sure it's the correct size. - if (oblock_bp->b_bufsize != bsize) { - brelse(oblock_bp); + if (buf_size(oblock_bp) != bsize) { + buf_brelse(oblock_bp); return -1; } // copy the journal data over top of it - memcpy(oblock_bp->b_data, block_ptr, bsize); + memcpy((void *)buf_dataptr(oblock_bp), block_ptr, bsize); - if ((ret = VOP_BWRITE(oblock_bp)) != 0) { + if ((ret = VNOP_BWRITE(oblock_bp)) != 0) { printf("jnl: update_fs_block: failed to update block %lld (ret %d)\n", fs_block,ret); - brelse(oblock_bp); return ret; } // and now invalidate it so that if someone else wants to read // it in a different size they'll be able to do it. - ret = meta_bread(jnl->fsdev, (daddr_t)fs_block, bsize, NOCRED, &oblock_bp); + ret = buf_meta_bread(jnl->fsdev, (daddr64_t)fs_block, bsize, NOCRED, &oblock_bp); if (oblock_bp) { - oblock_bp->b_flags |= B_INVAL; - brelse(oblock_bp); + buf_markinvalid(oblock_bp); + buf_brelse(oblock_bp); } return 0; } +static int +grow_table(struct bucket **buf_ptr, int num_buckets, int new_size) +{ + struct bucket *newBuf; + int current_size = num_buckets, i; + + // return if newsize is less than the current size + if (new_size < num_buckets) { + return current_size; + } + + if ((MALLOC(newBuf, struct bucket *, new_size*sizeof(struct bucket), M_TEMP, M_WAITOK)) == NULL) { + printf("jnl: grow_table: no memory to expand coalesce buffer!\n"); + return -1; + } + + // printf("jnl: lookup_bucket: expanded co_buf to %d elems\n", new_size); + + // copy existing elements + bcopy(*buf_ptr, newBuf, num_buckets*sizeof(struct bucket)); + + // initialize the new ones + for(i=num_buckets; i < new_size; i++) { + newBuf[i].block_num = (off_t)-1; + } + + // free the old container + FREE(*buf_ptr, M_TEMP); + + // reset the buf_ptr + *buf_ptr = newBuf; + + return new_size; +} + +static int +lookup_bucket(struct bucket **buf_ptr, off_t block_num, int num_full) +{ + int lo, hi, index, matches, i; + + if (num_full == 0) { + return 0; // table is empty, so insert at index=0 + } + + lo = 0; + hi = num_full - 1; + index = -1; + + // perform binary search for block_num + do { + int mid = (hi - lo)/2 + lo; + off_t this_num = (*buf_ptr)[mid].block_num; + + if (block_num == this_num) { + index = mid; + break; + } + + if (block_num < this_num) { + hi = mid; + continue; + } + + if (block_num > this_num) { + lo = mid + 1; + continue; + } + } while(lo < hi); + + // check if lo and hi converged on the match + if (block_num == (*buf_ptr)[hi].block_num) { + index = hi; + } + + // if no existing entry found, find index for new one + if (index == -1) { + index = (block_num < (*buf_ptr)[hi].block_num) ? hi : hi + 1; + } else { + // make sure that we return the right-most index in the case of multiple matches + matches = 0; + i = index + 1; + while(i < num_full && block_num == (*buf_ptr)[i].block_num) { + matches++; + i++; + } + + index += matches; + } + + return index; +} + +static int +insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr, int overwriting) +{ + if (!overwriting) { + // grow the table if we're out of space + if (*num_full_ptr >= *num_buckets_ptr) { + int new_size = *num_buckets_ptr * 2; + int grow_size = grow_table(buf_ptr, *num_buckets_ptr, new_size); + + if (grow_size < new_size) { + printf("jnl: add_block: grow_table returned an error!\n"); + return -1; + } + + *num_buckets_ptr = grow_size; //update num_buckets to reflect the new size + } + + // if we're not inserting at the end, we need to bcopy + if (blk_index != *num_full_ptr) { + bcopy( (*buf_ptr)+(blk_index), (*buf_ptr)+(blk_index+1), (*num_full_ptr-blk_index)*sizeof(struct bucket) ); + } + + (*num_full_ptr)++; // increment only if we're not overwriting + } + + // sanity check the values we're about to add + if (offset >= jnl->jhdr->size) { + offset = jnl->jhdr->jhdr_size + (offset - jnl->jhdr->size); + } + if (size <= 0) { + panic("jnl: insert_block: bad size in insert_block (%d)\n", size); + } + + (*buf_ptr)[blk_index].block_num = num; + (*buf_ptr)[blk_index].block_size = size; + (*buf_ptr)[blk_index].jnl_offset = offset; + + return blk_index; +} + +static int +do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr) +{ + int num_to_remove, index, i, overwrite, err; + size_t jhdr_size = jnl->jhdr->jhdr_size, new_offset; + off_t overlap, block_start, block_end; + + block_start = block_num*jhdr_size; + block_end = block_start + size; + overwrite = (block_num == (*buf_ptr)[blk_index].block_num && size >= (*buf_ptr)[blk_index].block_size); + + // first, eliminate any overlap with the previous entry + if (blk_index != 0 && !overwrite) { + off_t prev_block_start = (*buf_ptr)[blk_index-1].block_num*jhdr_size; + off_t prev_block_end = prev_block_start + (*buf_ptr)[blk_index-1].block_size; + overlap = prev_block_end - block_start; + if (overlap > 0) { + if (overlap % jhdr_size != 0) { + panic("jnl: do_overlap: overlap with previous entry not a multiple of %d\n", jhdr_size); + } + + // if the previous entry completely overlaps this one, we need to break it into two pieces. + if (prev_block_end > block_end) { + off_t new_num = block_end / jhdr_size; + size_t new_size = prev_block_end - block_end; + + new_offset = (*buf_ptr)[blk_index-1].jnl_offset + (block_end - prev_block_start); + + err = insert_block(jnl, buf_ptr, blk_index, new_num, new_size, new_offset, num_buckets_ptr, num_full_ptr, 0); + if (err < 0) { + panic("jnl: do_overlap: error inserting during pre-overlap\n"); + } + } + + // Regardless, we need to truncate the previous entry to the beginning of the overlap + (*buf_ptr)[blk_index-1].block_size = block_start - prev_block_start; + } + } + + // then, bail out fast if there's no overlap with the entries that follow + if (!overwrite && block_end <= (*buf_ptr)[blk_index].block_num*jhdr_size) { + return 0; // no overlap, no overwrite + } else if (overwrite && (blk_index + 1 >= *num_full_ptr || block_end <= (*buf_ptr)[blk_index+1].block_num*jhdr_size)) { + return 1; // simple overwrite + } + + // Otherwise, find all cases of total and partial overlap. We use the special + // block_num of -2 to designate entries that are completely overlapped and must + // be eliminated. The block_num, size, and jnl_offset of partially overlapped + // entries must be adjusted to keep the array consistent. + index = blk_index; + num_to_remove = 0; + while(index < *num_full_ptr && block_end > (*buf_ptr)[index].block_num*jhdr_size) { + if (block_end >= ((*buf_ptr)[index].block_num*jhdr_size + (*buf_ptr)[index].block_size)) { + (*buf_ptr)[index].block_num = -2; // mark this for deletion + num_to_remove++; + } else { + overlap = block_end - (*buf_ptr)[index].block_num*jhdr_size; + if (overlap > 0) { + if (overlap % jhdr_size != 0) { + panic("jnl: do_overlap: overlap of %lld is not multiple of %d\n", overlap, jhdr_size); + } + + // if we partially overlap this entry, adjust its block number, jnl offset, and size + (*buf_ptr)[index].block_num += (overlap / jhdr_size); // make sure overlap is multiple of jhdr_size, or round up + + new_offset = (*buf_ptr)[index].jnl_offset + overlap; // check for wrap-around + if (new_offset >= jnl->jhdr->size) { + new_offset = jhdr_size + (new_offset - jnl->jhdr->size); + } + (*buf_ptr)[index].jnl_offset = new_offset; + + (*buf_ptr)[index].block_size -= overlap; // sanity check for negative value + if ((*buf_ptr)[index].block_size <= 0) { + panic("jnl: do_overlap: after overlap, new block size is invalid (%d)\n", (*buf_ptr)[index].block_size); + // return -1; // if above panic is removed, return -1 for error + } + } + + } + + index++; + } + + // bcopy over any completely overlapped entries, starting at the right (where the above loop broke out) + index--; // start with the last index used within the above loop + while(index >= blk_index) { + if ((*buf_ptr)[index].block_num == -2) { + if (index == *num_full_ptr-1) { + (*buf_ptr)[index].block_num = -1; // it's the last item in the table... just mark as free + } else { + bcopy( (*buf_ptr)+(index+1), (*buf_ptr)+(index), (*num_full_ptr - (index + 1)) * sizeof(struct bucket) ); + } + (*num_full_ptr)--; + } + index--; + } + + // eliminate any stale entries at the end of the table + for(i=*num_full_ptr; i < (*num_full_ptr + num_to_remove); i++) { + (*buf_ptr)[i].block_num = -1; + } + + return 0; // if we got this far, we need to insert the entry into the table (rather than overwrite) +} + +// PR-3105942: Coalesce writes to the same block in journal replay +// We coalesce writes by maintaining a dynamic sorted array of physical disk blocks +// to be replayed and the corresponding location in the journal which contains +// the most recent data for those blocks. The array is "played" once the all the +// blocks in the journal have been coalesced. The code for the case of conflicting/ +// overlapping writes to a single block is the most dense. Because coalescing can +// disrupt the existing time-ordering of blocks in the journal playback, care +// is taken to catch any overlaps and keep the array consistent. +static int +add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr) +{ + int blk_index, overwriting; + + // on return from lookup_bucket(), blk_index is the index into the table where block_num should be + // inserted (or the index of the elem to overwrite). + blk_index = lookup_bucket( buf_ptr, block_num, *num_full_ptr); + + // check if the index is within bounds (if we're adding this block to the end of + // the table, blk_index will be equal to num_full) + if (blk_index < 0 || blk_index > *num_full_ptr) { + //printf("jnl: add_block: trouble adding block to co_buf\n"); + return -1; + } // else printf("jnl: add_block: adding block 0x%llx at i=%d\n", block_num, blk_index); + + // Determine whether we're overwriting an existing entry by checking for overlap + overwriting = do_overlap(jnl, buf_ptr, blk_index, block_num, size, offset, num_buckets_ptr, num_full_ptr); + if (overwriting < 0) { + return -1; // if we got an error, pass it along + } + + // returns the index, or -1 on error + blk_index = insert_block(jnl, buf_ptr, blk_index, block_num, size, offset, num_buckets_ptr, num_full_ptr, overwriting); + + return blk_index; +} static int replay_journal(journal *jnl) { - int i, ret, checksum, max_bsize; - struct buf *oblock_bp; + int i, ret, orig_checksum, checksum, max_bsize; block_list_header *blhdr; off_t offset; - char *buf, *block_ptr=NULL; - + char *buff, *block_ptr=NULL; + struct bucket *co_buf; + int num_buckets = STARTING_BUCKETS, num_full; + // wrap the start ptr if it points to the very end of the journal if (jnl->jhdr->start == jnl->jhdr->size) { jnl->jhdr->start = jnl->jhdr->jhdr_size; @@ -545,118 +967,190 @@ replay_journal(journal *jnl) } // allocate memory for the header_block. we'll read each blhdr into this - if (kmem_alloc(kernel_map, (vm_offset_t *)&buf, jnl->jhdr->blhdr_size)) { + if (kmem_alloc(kernel_map, (vm_offset_t *)&buff, jnl->jhdr->blhdr_size)) { printf("jnl: replay_journal: no memory for block buffer! (%d bytes)\n", jnl->jhdr->blhdr_size); return -1; } - + + // allocate memory for the coalesce buffer + if ((MALLOC(co_buf, struct bucket *, num_buckets*sizeof(struct bucket), M_TEMP, M_WAITOK)) == NULL) { + printf("jnl: replay_journal: no memory for coalesce buffer!\n"); + return -1; + } + + // initialize entries + for(i=0; i < num_buckets; i++) { + co_buf[i].block_num = -1; + } + num_full = 0; // empty at first + printf("jnl: replay_journal: from: %lld to: %lld (joffset 0x%llx)\n", jnl->jhdr->start, jnl->jhdr->end, jnl->jdev_offset); while(jnl->jhdr->start != jnl->jhdr->end) { offset = jnl->jhdr->start; - ret = read_journal_data(jnl, &offset, buf, jnl->jhdr->blhdr_size); + ret = read_journal_data(jnl, &offset, buff, jnl->jhdr->blhdr_size); if (ret != jnl->jhdr->blhdr_size) { printf("jnl: replay_journal: Could not read block list header block @ 0x%llx!\n", offset); goto bad_replay; } - blhdr = (block_list_header *)buf; - checksum = blhdr->checksum; + blhdr = (block_list_header *)buff; + + orig_checksum = blhdr->checksum; blhdr->checksum = 0; - if (checksum != calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE)) { - printf("jnl: replay_journal: bad block list header @ 0x%llx (checksum 0x%x != 0x%x)\n", - offset, checksum, calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE)); - goto bad_replay; + if (jnl->flags & JOURNAL_NEED_SWAP) { + // calculate the checksum based on the unswapped data + // because it is done byte-at-a-time. + orig_checksum = SWAP32(orig_checksum); + checksum = calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE); + swap_block_list_header(jnl, blhdr); + } else { + checksum = calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE); + } + if (checksum != orig_checksum) { + printf("jnl: replay_journal: bad block list header @ 0x%llx (checksum 0x%x != 0x%x)\n", + offset, orig_checksum, checksum); + goto bad_replay; } if ( blhdr->max_blocks <= 0 || blhdr->max_blocks > 2048 || blhdr->num_blocks <= 0 || blhdr->num_blocks > blhdr->max_blocks) { - printf("jnl: replay_journal: bad looking journal entry: max: %d num: %d\n", - blhdr->max_blocks, blhdr->num_blocks); - goto bad_replay; + printf("jnl: replay_journal: bad looking journal entry: max: %d num: %d\n", + blhdr->max_blocks, blhdr->num_blocks); + goto bad_replay; } - for(i=1,max_bsize=0; i < blhdr->num_blocks; i++) { + for(i=1; i < blhdr->num_blocks; i++) { if (blhdr->binfo[i].bnum < 0 && blhdr->binfo[i].bnum != (off_t)-1) { printf("jnl: replay_journal: bogus block number 0x%llx\n", blhdr->binfo[i].bnum); goto bad_replay; } - if (blhdr->binfo[i].bsize > max_bsize) { - max_bsize = blhdr->binfo[i].bsize; - } } - // make sure it's at least one page in size. - if (max_bsize & (PAGE_SIZE - 1)) { - max_bsize = (max_bsize + PAGE_SIZE) & ~(PAGE_SIZE - 1); - } - - if (kmem_alloc(kernel_map, (vm_offset_t *)&block_ptr, max_bsize)) { - goto bad_replay; - } - - //printf("jnl: replay_journal: %d blocks in journal entry @ 0x%llx\n", blhdr->num_blocks-1, - // jnl->jhdr->start); + //printf("jnl: replay_journal: adding %d blocks in journal entry @ 0x%llx to co_buf\n", + // blhdr->num_blocks-1, jnl->jhdr->start); for(i=1; i < blhdr->num_blocks; i++) { - int size; + int size, ret_val; + off_t number; size = blhdr->binfo[i].bsize; - - ret = read_journal_data(jnl, &offset, block_ptr, size); - if (ret != size) { - printf("jnl: replay_journal: Could not read journal entry data @ offset 0x%llx!\n", offset); - goto bad_replay; - } - - // don't replay "killed" blocks - if (blhdr->binfo[i].bnum == (off_t)-1) { - // printf("jnl: replay_journal: skipping killed fs block (slot %d)\n", i); + number = blhdr->binfo[i].bnum; + + // don't add "killed" blocks + if (number == (off_t)-1) { + //printf("jnl: replay_journal: skipping killed fs block (index %d)\n", i); } else { - //printf("jnl: replay_journal: fixing fs block # %lld (%d)\n", - // blhdr->binfo[i].bnum, blhdr->binfo[i].bsize); - - if (update_fs_block(jnl, block_ptr, blhdr->binfo[i].bnum, blhdr->binfo[i].bsize) != 0) { - goto bad_replay; - } + // add this bucket to co_buf, coalescing where possible + // printf("jnl: replay_journal: adding block 0x%llx\n", number); + ret_val = add_block(jnl, &co_buf, number, size, (size_t) offset, &num_buckets, &num_full); + + if (ret_val == -1) { + printf("jnl: replay_journal: trouble adding block to co_buf\n"); + goto bad_replay; + } // else printf("jnl: replay_journal: added block 0x%llx at i=%d\n", number); } - - // check if we need to wrap offset back to the beginning - // (which is just past the journal header) + + // increment offset + offset += size; + + // check if the last block added puts us off the end of the jnl. + // if so, we need to wrap to the beginning and take any remainder + // into account // if (offset >= jnl->jhdr->size) { - offset = jnl->jhdr->jhdr_size; + offset = jnl->jhdr->jhdr_size + (offset - jnl->jhdr->size); } } - kmem_free(kernel_map, (vm_offset_t)block_ptr, max_bsize); - block_ptr = NULL; - + jnl->jhdr->start += blhdr->bytes_used; if (jnl->jhdr->start >= jnl->jhdr->size) { // wrap around and skip the journal header block jnl->jhdr->start = (jnl->jhdr->start % jnl->jhdr->size) + jnl->jhdr->jhdr_size; } + } - // only update the on-disk journal header if we've reached the - // last chunk of updates from this transaction. if binfo[0].bnum - // is zero then we know we're at the end. - if (blhdr->binfo[0].bnum == 0) { - if (write_journal_header(jnl) != 0) { - goto bad_replay; - } - } + + //printf("jnl: replay_journal: replaying %d blocks\n", num_full); + + /* + * make sure it's at least one page in size, so + * start max_bsize at PAGE_SIZE + */ + for (i = 0, max_bsize = PAGE_SIZE; i < num_full; i++) { + + if (co_buf[i].block_num == (off_t)-1) + continue; + + if (co_buf[i].block_size > max_bsize) + max_bsize = co_buf[i].block_size; + } + /* + * round max_bsize up to the nearest PAGE_SIZE multiple + */ + if (max_bsize & (PAGE_SIZE - 1)) { + max_bsize = (max_bsize + PAGE_SIZE) & ~(PAGE_SIZE - 1); } - kmem_free(kernel_map, (vm_offset_t)buf, jnl->jhdr->blhdr_size); + if (kmem_alloc(kernel_map, (vm_offset_t *)&block_ptr, max_bsize)) { + goto bad_replay; + } + + // Replay the coalesced entries in the co-buf + for(i=0; i < num_full; i++) { + size_t size = co_buf[i].block_size; + off_t jnl_offset = (off_t) co_buf[i].jnl_offset; + off_t number = co_buf[i].block_num; + + + // printf("replaying co_buf[%d]: block 0x%llx, size 0x%x, jnl_offset 0x%llx\n", i, co_buf[i].block_num, + // co_buf[i].block_size, co_buf[i].jnl_offset); + + if (number == (off_t)-1) { + // printf("jnl: replay_journal: skipping killed fs block\n"); + } else { + + // do journal read, and set the phys. block + ret = read_journal_data(jnl, &jnl_offset, block_ptr, size); + if (ret != size) { + printf("jnl: replay_journal: Could not read journal entry data @ offset 0x%llx!\n", offset); + goto bad_replay; + } + + if (update_fs_block(jnl, block_ptr, number, size) != 0) { + goto bad_replay; + } + } + } + + + // done replaying; update jnl header + if (write_journal_header(jnl) != 0) { + goto bad_replay; + } + + // free block_ptr + kmem_free(kernel_map, (vm_offset_t)block_ptr, max_bsize); + block_ptr = NULL; + + // free the coalesce buffer + FREE(co_buf, M_TEMP); + co_buf = NULL; + + kmem_free(kernel_map, (vm_offset_t)buff, jnl->jhdr->blhdr_size); return 0; bad_replay: if (block_ptr) { kmem_free(kernel_map, (vm_offset_t)block_ptr, max_bsize); } - kmem_free(kernel_map, (vm_offset_t)buf, jnl->jhdr->blhdr_size); + if (co_buf) { + FREE(co_buf, M_TEMP); + } + kmem_free(kernel_map, (vm_offset_t)buff, jnl->jhdr->blhdr_size); + return -1; } @@ -719,9 +1213,12 @@ size_up_tbuffer(journal *jnl, int tbuffer_size, int phys_blksz) } jnl->jhdr->blhdr_size = (jnl->tbuffer_size / jnl->jhdr->jhdr_size) * sizeof(block_info); - if (jnl->jhdr->blhdr_size < phys_blksz) { - jnl->jhdr->blhdr_size = phys_blksz; - } + if (jnl->jhdr->blhdr_size < phys_blksz) { + jnl->jhdr->blhdr_size = phys_blksz; + } else if ((jnl->jhdr->blhdr_size % phys_blksz) != 0) { + // have to round up so we're an even multiple of the physical block size + jnl->jhdr->blhdr_size = (jnl->jhdr->blhdr_size + (phys_blksz - 1)) & ~(phys_blksz - 1); + } } @@ -738,10 +1235,14 @@ journal_create(struct vnode *jvp, void *arg) { journal *jnl; - int ret, phys_blksz; + int phys_blksz; + struct vfs_context context; + + context.vc_proc = current_proc(); + context.vc_ucred = FSCRED; /* Get the real physical block size. */ - if (VOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, FSCRED, NULL)) { + if (VNOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, &context)) { return NULL; } @@ -757,9 +1258,7 @@ journal_create(struct vnode *jvp, return NULL; } - if (kmem_alloc(kernel_map, (vm_offset_t *)&jnl, sizeof(struct journal))) { - return NULL; - } + MALLOC_ZONE(jnl, struct journal *, sizeof(struct journal), M_JNL_JNL, M_WAITOK); memset(jnl, 0, sizeof(*jnl)); jnl->jdev = jvp; @@ -768,7 +1267,7 @@ journal_create(struct vnode *jvp, jnl->flush = flush; jnl->flush_arg = arg; jnl->flags = (flags & JOURNAL_OPTION_FLAGS_MASK); - simple_lock_init(&jnl->old_start_lock); + lck_mtx_init(&jnl->old_start_lock, jnl_mutex_group, jnl_lock_attr); if (kmem_alloc(kernel_map, (vm_offset_t *)&jnl->header_buf, phys_blksz)) { printf("jnl: create: could not allocate space for header buffer (%d bytes)\n", phys_blksz); @@ -792,10 +1291,7 @@ journal_create(struct vnode *jvp, // jnl->jhdr->start = jnl->jhdr->size - (phys_blksz*3); // jnl->jhdr->end = jnl->jhdr->size - (phys_blksz*3); - if (semaphore_create(kernel_task, &jnl->jsem, SYNC_POLICY_FIFO, 1) != 0) { - printf("jnl: journal_create: failed to create journal semaphore..\n"); - goto bad_sem; - } + lck_mtx_init(&jnl->jlock, jnl_mutex_group, jnl_lock_attr); if (write_journal_header(jnl) != 0) { printf("jnl: journal_create: failed to write journal header.\n"); @@ -806,12 +1302,10 @@ journal_create(struct vnode *jvp, bad_write: - semaphore_destroy(kernel_task, jnl->jsem); - bad_sem: kmem_free(kernel_map, (vm_offset_t)jnl->header_buf, phys_blksz); bad_kmem_alloc: jnl->jhdr = NULL; - kmem_free(kernel_map, (vm_offset_t)jnl, sizeof(struct journal)); + FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL); return NULL; } @@ -828,11 +1322,15 @@ journal_open(struct vnode *jvp, void *arg) { journal *jnl; - int orig_blksz=0, phys_blksz, blhdr_size; - off_t hdr_offset=0; + int orig_blksz=0, phys_blksz; + int orig_checksum, checksum; + struct vfs_context context; + + context.vc_proc = current_proc(); + context.vc_ucred = FSCRED; /* Get the real physical block size. */ - if (VOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, FSCRED, NULL)) { + if (VNOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, &context)) { return NULL; } @@ -848,9 +1346,7 @@ journal_open(struct vnode *jvp, return NULL; } - if (kmem_alloc(kernel_map, (vm_offset_t *)&jnl, sizeof(struct journal))) { - return NULL; - } + MALLOC_ZONE(jnl, struct journal *, sizeof(struct journal), M_JNL_JNL, M_WAITOK); memset(jnl, 0, sizeof(*jnl)); jnl->jdev = jvp; @@ -859,7 +1355,7 @@ journal_open(struct vnode *jvp, jnl->flush = flush; jnl->flush_arg = arg; jnl->flags = (flags & JOURNAL_OPTION_FLAGS_MASK); - simple_lock_init(&jnl->old_start_lock); + lck_mtx_init(&jnl->old_start_lock, jnl_mutex_group, jnl_lock_attr); if (kmem_alloc(kernel_map, (vm_offset_t *)&jnl->header_buf, phys_blksz)) { printf("jnl: create: could not allocate space for header buffer (%d bytes)\n", phys_blksz); @@ -872,12 +1368,25 @@ journal_open(struct vnode *jvp, // we have to set this up here so that do_journal_io() will work jnl->jhdr->jhdr_size = phys_blksz; - if (read_journal_data(jnl, &hdr_offset, jnl->jhdr, phys_blksz) != phys_blksz) { + if (read_journal_header(jnl, jnl->jhdr, phys_blksz) != phys_blksz) { printf("jnl: open: could not read %d bytes for the journal header.\n", phys_blksz); goto bad_journal; } + orig_checksum = jnl->jhdr->checksum; + jnl->jhdr->checksum = 0; + + if (jnl->jhdr->magic == SWAP32(JOURNAL_HEADER_MAGIC)) { + // do this before the swap since it's done byte-at-a-time + orig_checksum = SWAP32(orig_checksum); + checksum = calc_checksum((char *)jnl->jhdr, sizeof(struct journal_header)); + swap_journal_header(jnl); + jnl->flags |= JOURNAL_NEED_SWAP; + } else { + checksum = calc_checksum((char *)jnl->jhdr, sizeof(struct journal_header)); + } + if (jnl->jhdr->magic != JOURNAL_HEADER_MAGIC && jnl->jhdr->magic != OLD_JOURNAL_HEADER_MAGIC) { printf("jnl: open: journal magic is bad (0x%x != 0x%x)\n", jnl->jhdr->magic, JOURNAL_HEADER_MAGIC); @@ -886,12 +1395,11 @@ journal_open(struct vnode *jvp, // only check if we're the current journal header magic value if (jnl->jhdr->magic == JOURNAL_HEADER_MAGIC) { - int orig_checksum = jnl->jhdr->checksum; - jnl->jhdr->checksum = 0; - if (orig_checksum != calc_checksum((char *)jnl->jhdr, sizeof(struct journal_header))) { - printf("jnl: open: journal checksum is bad (0x%x != 0x%x)\n", orig_checksum, - calc_checksum((char *)jnl->jhdr, sizeof(struct journal_header))); + if (orig_checksum != checksum) { + printf("jnl: open: journal checksum is bad (0x%x != 0x%x)\n", + orig_checksum, checksum); + //goto bad_journal; } } @@ -907,7 +1415,7 @@ journal_open(struct vnode *jvp, orig_blksz = phys_blksz; phys_blksz = jnl->jhdr->jhdr_size; - if (VOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&phys_blksz, FWRITE, FSCRED, NULL)) { + if (VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&phys_blksz, FWRITE, &context)) { printf("jnl: could not set block size to %d bytes.\n", phys_blksz); goto bad_journal; } @@ -916,7 +1424,7 @@ journal_open(struct vnode *jvp, if ( jnl->jhdr->start <= 0 || jnl->jhdr->start > jnl->jhdr->size - || jnl->jhdr->start > 128*1024*1024) { + || jnl->jhdr->start > 1024*1024*1024) { printf("jnl: open: jhdr start looks bad (0x%llx max size 0x%llx)\n", jnl->jhdr->start, jnl->jhdr->size); goto bad_journal; @@ -924,13 +1432,13 @@ journal_open(struct vnode *jvp, if ( jnl->jhdr->end <= 0 || jnl->jhdr->end > jnl->jhdr->size - || jnl->jhdr->end > 128*1024*1024) { + || jnl->jhdr->end > 1024*1024*1024) { printf("jnl: open: jhdr end looks bad (0x%llx max size 0x%llx)\n", jnl->jhdr->end, jnl->jhdr->size); goto bad_journal; } - if (jnl->jhdr->size > 128*1024*1024) { + if (jnl->jhdr->size > 1024*1024*1024) { printf("jnl: open: jhdr size looks bad (0x%llx)\n", jnl->jhdr->size); goto bad_journal; } @@ -965,8 +1473,14 @@ journal_open(struct vnode *jvp, } if (orig_blksz != 0) { - VOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, FSCRED, NULL); + VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, &context); phys_blksz = orig_blksz; + if (orig_blksz < jnl->jhdr->jhdr_size) { + printf("jnl: open: jhdr_size is %d but orig phys blk size is %d. switching.\n", + jnl->jhdr->jhdr_size, orig_blksz); + + jnl->jhdr->jhdr_size = orig_blksz; + } } // make sure this is in sync! @@ -975,24 +1489,125 @@ journal_open(struct vnode *jvp, // set this now, after we've replayed the journal size_up_tbuffer(jnl, tbuffer_size, phys_blksz); - if (semaphore_create(kernel_task, &jnl->jsem, SYNC_POLICY_FIFO, 1) != 0) { - printf("jnl: journal_create: failed to create journal semaphore..\n"); - goto bad_journal; - } + lck_mtx_init(&jnl->jlock, jnl_mutex_group, jnl_lock_attr); return jnl; bad_journal: if (orig_blksz != 0) { phys_blksz = orig_blksz; - VOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, FSCRED, NULL); + VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, &context); } kmem_free(kernel_map, (vm_offset_t)jnl->header_buf, phys_blksz); bad_kmem_alloc: - kmem_free(kernel_map, (vm_offset_t)jnl, sizeof(struct journal)); + FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL); return NULL; } + +int +journal_is_clean(struct vnode *jvp, + off_t offset, + off_t journal_size, + struct vnode *fsvp, + size_t min_fs_block_size) +{ + journal jnl; + int phys_blksz, ret; + int orig_checksum, checksum; + struct vfs_context context; + + context.vc_proc = current_proc(); + context.vc_ucred = FSCRED; + + /* Get the real physical block size. */ + if (VNOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, &context)) { + printf("jnl: is_clean: failed to get device block size.\n"); + return EINVAL; + } + + if (phys_blksz > min_fs_block_size) { + printf("jnl: is_clean: error: phys blksize %d bigger than min fs blksize %d\n", + phys_blksz, min_fs_block_size); + return EINVAL; + } + + if ((journal_size % phys_blksz) != 0) { + printf("jnl: is_clean: journal size 0x%llx is not an even multiple of block size 0x%x\n", + journal_size, phys_blksz); + return EINVAL; + } + + memset(&jnl, 0, sizeof(jnl)); + + if (kmem_alloc(kernel_map, (vm_offset_t *)&jnl.header_buf, phys_blksz)) { + printf("jnl: is_clean: could not allocate space for header buffer (%d bytes)\n", phys_blksz); + return ENOMEM; + } + + jnl.jhdr = (journal_header *)jnl.header_buf; + memset(jnl.jhdr, 0, sizeof(journal_header)+4); + + jnl.jdev = jvp; + jnl.jdev_offset = offset; + jnl.fsdev = fsvp; + + // we have to set this up here so that do_journal_io() will work + jnl.jhdr->jhdr_size = phys_blksz; + + if (read_journal_header(&jnl, jnl.jhdr, phys_blksz) != phys_blksz) { + printf("jnl: is_clean: could not read %d bytes for the journal header.\n", + phys_blksz); + ret = EINVAL; + goto get_out; + } + + orig_checksum = jnl.jhdr->checksum; + jnl.jhdr->checksum = 0; + + if (jnl.jhdr->magic == SWAP32(JOURNAL_HEADER_MAGIC)) { + // do this before the swap since it's done byte-at-a-time + orig_checksum = SWAP32(orig_checksum); + checksum = calc_checksum((char *)jnl.jhdr, sizeof(struct journal_header)); + swap_journal_header(&jnl); + jnl.flags |= JOURNAL_NEED_SWAP; + } else { + checksum = calc_checksum((char *)jnl.jhdr, sizeof(struct journal_header)); + } + + if (jnl.jhdr->magic != JOURNAL_HEADER_MAGIC && jnl.jhdr->magic != OLD_JOURNAL_HEADER_MAGIC) { + printf("jnl: is_clean: journal magic is bad (0x%x != 0x%x)\n", + jnl.jhdr->magic, JOURNAL_HEADER_MAGIC); + ret = EINVAL; + goto get_out; + } + + if (orig_checksum != checksum) { + printf("jnl: is_clean: journal checksum is bad (0x%x != 0x%x)\n", orig_checksum, checksum); + ret = EINVAL; + goto get_out; + } + + // + // if the start and end are equal then the journal is clean. + // otherwise it's not clean and therefore an error. + // + if (jnl.jhdr->start == jnl.jhdr->end) { + ret = 0; + } else { + ret = EINVAL; + } + + get_out: + kmem_free(kernel_map, (vm_offset_t)jnl.header_buf, phys_blksz); + + return ret; + + +} + + + void journal_close(journal *jnl) { @@ -1006,16 +1621,8 @@ journal_close(journal *jnl) // jnl->flags |= JOURNAL_CLOSE_PENDING; - if (jnl->owner != current_act()) { - int ret; - - while ((ret = semaphore_wait(jnl->jsem)) == KERN_ABORTED) { - // just keep trying if we've been ^C'ed - } - if (ret != 0) { - printf("jnl: close: sem wait failed.\n"); - return; - } + if (jnl->owner != current_thread()) { + lock_journal(jnl); } // @@ -1044,7 +1651,7 @@ journal_close(journal *jnl) if (jnl->flush) { jnl->flush(jnl->flush_arg); } - + tsleep((caddr_t)jnl, PRIBIO, "jnl_close", 1); } if (*start != *end) { @@ -1083,8 +1690,7 @@ journal_close(journal *jnl) kmem_free(kernel_map, (vm_offset_t)jnl->header_buf, jnl->jhdr->jhdr_size); jnl->jhdr = (void *)0xbeefbabe; - semaphore_destroy(kernel_task, jnl->jsem); - kmem_free(kernel_map, (vm_offset_t)jnl, sizeof(struct journal)); + FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL); } static void @@ -1141,6 +1747,8 @@ check_free_space(journal *jnl, int desired_size) // desired_size, free_space(jnl)); while (1) { + int old_start_empty; + if (counter++ == 5000) { dump_journal(jnl); panic("jnl: check_free_space: buffer flushing isn't working " @@ -1161,7 +1769,8 @@ check_free_space(journal *jnl, int desired_size) // here's where we lazily bump up jnl->jhdr->start. we'll consume // entries until there is enough space for the next transaction. // - simple_lock(&jnl->old_start_lock); + old_start_empty = 1; + lock_oldstart(jnl); for(i=0; i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0]); i++) { int counter; @@ -1172,30 +1781,46 @@ check_free_space(journal *jnl, int desired_size) jnl->old_start[i], jnl); } - simple_unlock(&jnl->old_start_lock); + unlock_oldstart(jnl); if (jnl->flush) { jnl->flush(jnl->flush_arg); } tsleep((caddr_t)jnl, PRIBIO, "check_free_space1", 1); - simple_lock(&jnl->old_start_lock); + lock_oldstart(jnl); } if (jnl->old_start[i] == 0) { continue; } + old_start_empty = 0; jnl->jhdr->start = jnl->old_start[i]; jnl->old_start[i] = 0; if (free_space(jnl) > desired_size) { + unlock_oldstart(jnl); write_journal_header(jnl); + lock_oldstart(jnl); break; } } - simple_unlock(&jnl->old_start_lock); + unlock_oldstart(jnl); // if we bumped the start, loop and try again if (i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0])) { continue; + } else if (old_start_empty) { + // + // if there is nothing in old_start anymore then we can + // bump the jhdr->start to be the same as active_start + // since it is possible there was only one very large + // transaction in the old_start array. if we didn't do + // this then jhdr->start would never get updated and we + // would wind up looping until we hit the panic at the + // start of the loop. + // + jnl->jhdr->start = jnl->active_start; + write_journal_header(jnl); + continue; } @@ -1226,29 +1851,23 @@ journal_start_transaction(journal *jnl) return EINVAL; } - if (jnl->owner == current_act()) { + if (jnl->owner == current_thread()) { if (jnl->active_tr == NULL) { - panic("jnl: start_tr: active_tr is NULL (jnl @ 0x%x, owner 0x%x, current_act 0x%x\n", - jnl, jnl->owner, current_act()); + panic("jnl: start_tr: active_tr is NULL (jnl @ 0x%x, owner 0x%x, current_thread 0x%x\n", + jnl, jnl->owner, current_thread()); } jnl->nested_count++; return 0; } - while ((ret = semaphore_wait(jnl->jsem)) == KERN_ABORTED) { - // just keep looping if we've been ^C'ed - } - if (ret != 0) { - printf("jnl: start_tr: sem wait failed.\n"); - return EINVAL; - } + lock_journal(jnl); if (jnl->owner != NULL || jnl->nested_count != 0 || jnl->active_tr != NULL) { panic("jnl: start_tr: owner 0x%x, nested count 0x%x, active_tr 0x%x jnl @ 0x%x\n", jnl->owner, jnl->nested_count, jnl->active_tr, jnl); } - jnl->owner = current_act(); + jnl->owner = current_thread(); jnl->nested_count = 1; free_old_stuff(jnl); @@ -1268,16 +1887,13 @@ journal_start_transaction(journal *jnl) return 0; } - if (kmem_alloc(kernel_map, (vm_offset_t *)&tr, sizeof(transaction))) { - printf("jnl: start transaction failed: no mem\n"); - ret = ENOMEM; - goto bad_start; - } + MALLOC_ZONE(tr, transaction *, sizeof(transaction), M_JNL_TR, M_WAITOK); memset(tr, 0, sizeof(transaction)); tr->tbuffer_size = jnl->tbuffer_size; + if (kmem_alloc(kernel_map, (vm_offset_t *)&tr->tbuffer, tr->tbuffer_size)) { - kmem_free(kernel_map, (vm_offset_t)tr, sizeof(transaction)); + FREE_ZONE(tr, sizeof(transaction), M_JNL_TR); printf("jnl: start transaction failed: no tbuffer mem\n"); ret = ENOMEM; goto bad_start; @@ -1304,7 +1920,7 @@ journal_start_transaction(journal *jnl) bad_start: jnl->owner = NULL; jnl->nested_count = 0; - semaphore_signal(jnl->jsem); + unlock_journal(jnl); return ret; } @@ -1322,35 +1938,35 @@ journal_modify_block_start(journal *jnl, struct buf *bp) // XXXdbg - for debugging I want this to be true. later it may // not be necessary. - if ((bp->b_flags & B_META) == 0) { + if ((buf_flags(bp) & B_META) == 0) { panic("jnl: modify_block_start: bp @ 0x%x is not a meta-data block! (jnl 0x%x)\n", bp, jnl); } tr = jnl->active_tr; CHECK_TRANSACTION(tr); - if (jnl->owner != current_act()) { + if (jnl->owner != current_thread()) { panic("jnl: modify_block_start: called w/out a transaction! jnl 0x%x, owner 0x%x, curact 0x%x\n", - jnl, jnl->owner, current_act()); + jnl, jnl->owner, current_thread()); } free_old_stuff(jnl); - //printf("jnl: mod block start (bp 0x%x vp 0x%x l/blkno %d/%d bsz %d; total bytes %d)\n", - // bp, bp->b_vp, bp->b_lblkno, bp->b_blkno, bp->b_bufsize, tr->total_bytes); + //printf("jnl: mod block start (bp 0x%x vp 0x%x l/blkno %qd/%qd bsz %d; total bytes %d)\n", + // bp, buf_vnode(bp), buf_lblkno(bp), buf_blkno(bp), buf_size(bp), tr->total_bytes); // can't allow blocks that aren't an even multiple of the // underlying block size. - if ((bp->b_bufsize % jnl->jhdr->jhdr_size) != 0) { + if ((buf_size(bp) % jnl->jhdr->jhdr_size) != 0) { panic("jnl: mod block start: bufsize %d not a multiple of block size %d\n", - bp->b_bufsize, jnl->jhdr->jhdr_size); + buf_size(bp), jnl->jhdr->jhdr_size); return -1; } // make sure that this transaction isn't bigger than the whole journal - if (tr->total_bytes+bp->b_bufsize >= (jnl->jhdr->size - jnl->jhdr->jhdr_size)) { + if (tr->total_bytes+buf_size(bp) >= (jnl->jhdr->size - jnl->jhdr->jhdr_size)) { panic("jnl: transaction too big (%d >= %lld bytes, bufsize %d, tr 0x%x bp 0x%x)\n", - tr->total_bytes, (tr->jnl->jhdr->size - jnl->jhdr->jhdr_size), bp->b_bufsize, tr, bp); + tr->total_bytes, (tr->jnl->jhdr->size - jnl->jhdr->jhdr_size), buf_size(bp), tr, bp); return -1; } @@ -1358,14 +1974,17 @@ journal_modify_block_start(journal *jnl, struct buf *bp) // it out before we muck with it because it has data that belongs // (presumably) to another transaction. // - if ((bp->b_flags & B_DELWRI) && (bp->b_flags & B_LOCKED) == 0) { + if ((buf_flags(bp) & (B_DELWRI | B_LOCKED)) == B_DELWRI) { - // this will cause it to not be brelse()'d - bp->b_flags |= B_NORELSE; - VOP_BWRITE(bp); - } + if (buf_flags(bp) & B_ASYNC) { + panic("modify_block_start: bp @ 0x% has async flag set!\n", bp); + } - bp->b_flags |= B_LOCKED; + // this will cause it to not be buf_brelse()'d + buf_setflags(bp, B_NORELSE); + VNOP_BWRITE(bp); + } + buf_setflags(bp, B_LOCKED); return 0; } @@ -1383,11 +2002,11 @@ journal_modify_block_abort(journal *jnl, struct buf *bp) // // if there's no active transaction then we just want to - // call brelse() and return since this is just a block + // call buf_brelse() and return since this is just a block // that happened to be modified as part of another tr. // if (tr == NULL) { - brelse(bp); + buf_brelse(bp); return 0; } @@ -1397,9 +2016,9 @@ journal_modify_block_abort(journal *jnl, struct buf *bp) CHECK_TRANSACTION(tr); - if (jnl->owner != current_act()) { + if (jnl->owner != current_thread()) { panic("jnl: modify_block_abort: called w/out a transaction! jnl 0x%x, owner 0x%x, curact 0x%x\n", - jnl, jnl->owner, current_act()); + jnl, jnl->owner, current_thread()); } free_old_stuff(jnl); @@ -1410,9 +2029,9 @@ journal_modify_block_abort(journal *jnl, struct buf *bp) for(blhdr=tr->blhdr; blhdr; blhdr=(block_list_header *)((long)blhdr->binfo[0].bnum)) { for(i=1; i < blhdr->num_blocks; i++) { if (bp == blhdr->binfo[i].bp) { - if (bp->b_bufsize != blhdr->binfo[i].bsize) { + if (buf_size(bp) != blhdr->binfo[i].bsize) { panic("jnl: bp @ 0x%x changed size on me! (%d vs. %d, jnl 0x%x)\n", - bp, bp->b_bufsize, blhdr->binfo[i].bsize, jnl); + bp, buf_size(bp), blhdr->binfo[i].bsize, jnl); } break; } @@ -1431,10 +2050,10 @@ journal_modify_block_abort(journal *jnl, struct buf *bp) // on it and so we need to keep it locked in memory. // if (blhdr == NULL) { - bp->b_flags &= ~(B_LOCKED); + buf_clearflags(bp, B_LOCKED); } - brelse(bp); + buf_brelse(bp); return 0; } @@ -1456,19 +2075,18 @@ journal_modify_block_end(journal *jnl, struct buf *bp) tr = jnl->active_tr; CHECK_TRANSACTION(tr); - if (jnl->owner != current_act()) { + if (jnl->owner != current_thread()) { panic("jnl: modify_block_end: called w/out a transaction! jnl 0x%x, owner 0x%x, curact 0x%x\n", - jnl, jnl->owner, current_act()); + jnl, jnl->owner, current_thread()); } free_old_stuff(jnl); - //printf("jnl: mod block end: (bp 0x%x vp 0x%x l/blkno %d/%d bsz %d, total bytes %d)\n", - // bp, bp->b_vp, bp->b_lblkno, bp->b_blkno, bp->b_bufsize, tr->total_bytes); + //printf("jnl: mod block end: (bp 0x%x vp 0x%x l/blkno %qd/%qd bsz %d, total bytes %d)\n", + // bp, buf_vnode(bp), buf_lblkno(bp), buf_blkno(bp), buf_size(bp), tr->total_bytes); - if ((bp->b_flags & B_LOCKED) == 0) { + if ((buf_flags(bp) & B_LOCKED) == 0) { panic("jnl: modify_block_end: bp 0x%x not locked! jnl @ 0x%x\n", bp, jnl); - bp->b_flags |= B_LOCKED; } // first check if it's already part of this transaction @@ -1477,9 +2095,9 @@ journal_modify_block_end(journal *jnl, struct buf *bp) for(i=1; i < blhdr->num_blocks; i++) { if (bp == blhdr->binfo[i].bp) { - if (bp->b_bufsize != blhdr->binfo[i].bsize) { + if (buf_size(bp) != blhdr->binfo[i].bsize) { panic("jnl: bp @ 0x%x changed size on me! (%d vs. %d, jnl 0x%x)\n", - bp, bp->b_bufsize, blhdr->binfo[i].bsize, jnl); + bp, buf_size(bp), blhdr->binfo[i].bsize, jnl); } break; } @@ -1494,7 +2112,7 @@ journal_modify_block_end(journal *jnl, struct buf *bp) if (blhdr == NULL && prev && (prev->num_blocks+1) <= prev->max_blocks - && (prev->bytes_used+bp->b_bufsize) <= tr->tbuffer_size) { + && (prev->bytes_used+buf_size(bp)) <= tr->tbuffer_size) { blhdr = prev; } else if (blhdr == NULL) { block_list_header *nblhdr; @@ -1542,23 +2160,27 @@ journal_modify_block_end(journal *jnl, struct buf *bp) // copy the data into the in-memory transaction buffer blkptr = (char *)&((char *)blhdr)[tbuffer_offset]; - memcpy(blkptr, bp->b_data, bp->b_bufsize); + memcpy(blkptr, buf_dataptr(bp), buf_size(bp)); // if this is true then this is a new block we haven't seen if (i >= blhdr->num_blocks) { - vget(bp->b_vp, 0, current_proc()); + int bsize; + vnode_t vp; + + vp = buf_vnode(bp); + vnode_ref(vp); + bsize = buf_size(bp); - blhdr->binfo[i].bnum = bp->b_blkno; - blhdr->binfo[i].bsize = bp->b_bufsize; + blhdr->binfo[i].bnum = (off_t)(buf_blkno(bp)); + blhdr->binfo[i].bsize = bsize; blhdr->binfo[i].bp = bp; - blhdr->bytes_used += bp->b_bufsize; - tr->total_bytes += bp->b_bufsize; + blhdr->bytes_used += bsize; + tr->total_bytes += bsize; blhdr->num_blocks++; } - - bdwrite(bp); + buf_bdwrite(bp); return 0; } @@ -1567,6 +2189,7 @@ int journal_kill_block(journal *jnl, struct buf *bp) { int i; + int bflags; block_list_header *blhdr; transaction *tr; @@ -1579,44 +2202,49 @@ journal_kill_block(journal *jnl, struct buf *bp) tr = jnl->active_tr; CHECK_TRANSACTION(tr); - if (jnl->owner != current_act()) { + if (jnl->owner != current_thread()) { panic("jnl: modify_block_end: called w/out a transaction! jnl 0x%x, owner 0x%x, curact 0x%x\n", - jnl, jnl->owner, current_act()); + jnl, jnl->owner, current_thread()); } free_old_stuff(jnl); - if ((bp->b_flags & B_LOCKED) == 0) { - panic("jnl: kill block: bp 0x%x not locked! jnl @ 0x%x\n", bp, jnl); - } + bflags = buf_flags(bp); + + if ( !(bflags & B_LOCKED)) + panic("jnl: modify_block_end: called with bp not B_LOCKED"); + /* + * bp must be BL_BUSY and B_LOCKED + */ // first check if it's already part of this transaction for(blhdr=tr->blhdr; blhdr; blhdr=(block_list_header *)((long)blhdr->binfo[0].bnum)) { for(i=1; i < blhdr->num_blocks; i++) { if (bp == blhdr->binfo[i].bp) { - bp->b_flags &= ~B_LOCKED; + vnode_t vp; - // this undoes the vget() in journal_modify_block_end() - vrele(bp->b_vp); + buf_clearflags(bp, B_LOCKED); - // if the block has the DELWRI and CALL bits sets, then + // this undoes the vnode_ref() in journal_modify_block_end() + vp = buf_vnode(bp); + vnode_rele_ext(vp, 0, 1); + + // if the block has the DELWRI and FILTER bits sets, then // things are seriously weird. if it was part of another // transaction then journal_modify_block_start() should // have force it to be written. // - if ((bp->b_flags & B_DELWRI) && (bp->b_flags & B_CALL)) { - panic("jnl: kill block: this defies all logic! bp 0x%x\n", bp); - } else { - tr->num_killed += bp->b_bufsize; - } - - if (bp->b_flags & B_BUSY) { - brelse(bp); - } - + //if ((bflags & B_DELWRI) && (bflags & B_FILTER)) { + // panic("jnl: kill block: this defies all logic! bp 0x%x\n", bp); + //} else { + tr->num_killed += buf_size(bp); + //} blhdr->binfo[i].bp = NULL; blhdr->binfo[i].bnum = (off_t)-1; + + buf_brelse(bp); + break; } } @@ -1633,9 +2261,9 @@ journal_kill_block(journal *jnl, struct buf *bp) static int journal_binfo_cmp(void *a, void *b) { - block_info *bi_a = (struct block_info *)a, - *bi_b = (struct block_info *)b; - daddr_t res; + block_info *bi_a = (struct block_info *)a; + block_info *bi_b = (struct block_info *)b; + daddr64_t res; if (bi_a->bp == NULL) { return 1; @@ -1647,7 +2275,7 @@ journal_binfo_cmp(void *a, void *b) // don't have to worry about negative block // numbers so this is ok to do. // - res = (bi_a->bp->b_blkno - bi_b->bp->b_blkno); + res = (buf_blkno(bi_a->bp) - buf_blkno(bi_b->bp)); return (int)res; } @@ -1657,6 +2285,7 @@ static int end_transaction(transaction *tr, int force_it) { int i, j, ret, amt; + errno_t errno; off_t end; journal *jnl = tr->jnl; struct buf *bp; @@ -1671,7 +2300,7 @@ end_transaction(transaction *tr, int force_it) // just save off the transaction pointer and return. if (tr->total_bytes == jnl->jhdr->blhdr_size) { jnl->cur_tr = tr; - return; + return 0; } // if our transaction buffer isn't very full, just hang @@ -1686,7 +2315,7 @@ end_transaction(transaction *tr, int force_it) && (tr->total_bytes <= ((tr->tbuffer_size*tr->num_blhdrs) - tr->tbuffer_size/8))) { jnl->cur_tr = tr; - return; + return 0; } @@ -1709,10 +2338,10 @@ end_transaction(transaction *tr, int force_it) // file system flush routine until it is (or we panic). // i = 0; - simple_lock(&jnl->old_start_lock); + lock_oldstart(jnl); while ((jnl->old_start[0] & 0x8000000000000000LL) != 0) { if (jnl->flush) { - simple_unlock(&jnl->old_start_lock); + unlock_oldstart(jnl); if (jnl->flush) { jnl->flush(jnl->flush_arg); @@ -1721,9 +2350,9 @@ end_transaction(transaction *tr, int force_it) // yield the cpu so others can get in to clear the lock bit (void)tsleep((void *)jnl, PRIBIO, "jnl-old-start-sleep", 1); - simple_lock(&jnl->old_start_lock); + lock_oldstart(jnl); } - if (i++ >= 100) { + if (i++ >= 500) { panic("jnl: transaction that started at 0x%llx is not completing! jnl 0x%x\n", jnl->old_start[0] & (~0x8000000000000000LL), jnl); } @@ -1736,14 +2365,17 @@ end_transaction(transaction *tr, int force_it) memcpy(&jnl->old_start[0], &jnl->old_start[1], sizeof(jnl->old_start)-sizeof(jnl->old_start[0])); jnl->old_start[sizeof(jnl->old_start)/sizeof(jnl->old_start[0]) - 1] = tr->journal_start | 0x8000000000000000LL; - simple_unlock(&jnl->old_start_lock); + unlock_oldstart(jnl); // for each block, make sure that the physical block # is set for(blhdr=tr->blhdr; blhdr; blhdr=next) { for(i=1; i < blhdr->num_blocks; i++) { - + daddr64_t blkno; + daddr64_t lblkno; + struct vnode *vp; + bp = blhdr->binfo[i].bp; if (bp == NULL) { // only true if a block was "killed" if (blhdr->binfo[i].bnum != (off_t)-1) { @@ -1752,25 +2384,40 @@ end_transaction(transaction *tr, int force_it) } continue; } - - if (bp->b_vp == NULL && bp->b_lblkno == bp->b_blkno) { - panic("jnl: end_tr: DANGER! bp @ 0x%x w/null vp and l/blkno = %d/%d\n", - bp, bp->b_lblkno, bp->b_blkno); + vp = buf_vnode(bp); + blkno = buf_blkno(bp); + lblkno = buf_lblkno(bp); + + if (vp == NULL && lblkno == blkno) { + printf("jnl: end_tr: bad news! bp @ 0x%x w/null vp and l/blkno = %qd/%qd. aborting the transaction (tr 0x%x jnl 0x%x).\n", + bp, lblkno, blkno, tr, jnl); + goto bad_journal; } // if the lblkno is the same as blkno and this bp isn't // associated with the underlying file system device then // we need to call bmap() to get the actual physical block. // - if ((bp->b_lblkno == bp->b_blkno) && (bp->b_vp != jnl->fsdev)) { - if (VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL) != 0) { - printf("jnl: end_tr: can't bmap the bp @ 0x%x, jnl 0x%x\n", bp, jnl); + if ((lblkno == blkno) && (vp != jnl->fsdev)) { + off_t f_offset; + size_t contig_bytes; + + if (VNOP_BLKTOOFF(vp, lblkno, &f_offset)) { + printf("jnl: end_tr: vnop_blktooff failed @ 0x%x, jnl 0x%x\n", bp, jnl); + goto bad_journal; + } + if (VNOP_BLOCKMAP(vp, f_offset, buf_count(bp), &blkno, &contig_bytes, NULL, 0, NULL)) { + printf("jnl: end_tr: can't blockmap the bp @ 0x%x, jnl 0x%x\n", bp, jnl); + goto bad_journal; + } + if ((uint32_t)contig_bytes < buf_count(bp)) { + printf("jnl: end_tr: blk not physically contiguous on disk@ 0x%x, jnl 0x%x\n", bp, jnl); goto bad_journal; } + buf_setblkno(bp, blkno); } - // update this so we write out the correct physical block number! - blhdr->binfo[i].bnum = bp->b_blkno; + blhdr->binfo[i].bnum = (off_t)(blkno); } next = (block_list_header *)((long)blhdr->binfo[0].bnum); @@ -1828,51 +2475,53 @@ end_transaction(transaction *tr, int force_it) continue; } - ret = meta_bread(blhdr->binfo[i].bp->b_vp, - (daddr_t)blhdr->binfo[i].bp->b_lblkno, - blhdr->binfo[i].bp->b_bufsize, + errno = buf_meta_bread(buf_vnode(blhdr->binfo[i].bp), + buf_lblkno(blhdr->binfo[i].bp), + buf_size(blhdr->binfo[i].bp), NOCRED, &bp); - if (ret == 0 && bp != NULL) { + if (errno == 0 && bp != NULL) { struct vnode *save_vp; - + void *cur_filter; + if (bp != blhdr->binfo[i].bp) { panic("jnl: end_tr: got back a different bp! (bp 0x%x should be 0x%x, jnl 0x%x\n", bp, blhdr->binfo[i].bp, jnl); } - if ((bp->b_flags & (B_LOCKED|B_DELWRI)) != (B_LOCKED|B_DELWRI)) { + if ((buf_flags(bp) & (B_LOCKED|B_DELWRI)) != (B_LOCKED|B_DELWRI)) { if (jnl->flags & JOURNAL_CLOSE_PENDING) { - brelse(bp); + buf_clearflags(bp, B_LOCKED); + buf_brelse(bp); continue; } else { - panic("jnl: end_tr: !!!DANGER!!! bp 0x%x flags (0x%x) not LOCKED & DELWRI\n", bp, bp->b_flags); + panic("jnl: end_tr: !!!DANGER!!! bp 0x%x flags (0x%x) not LOCKED & DELWRI\n", bp, buf_flags(bp)); } } + save_vp = buf_vnode(bp); - if (bp->b_iodone != NULL) { - panic("jnl: bp @ 0x%x (blkno %d, vp 0x%x) has non-null iodone (0x%x) buffflushcb 0x%x\n", - bp, bp->b_blkno, bp->b_vp, bp->b_iodone, buffer_flushed_callback); - } - - save_vp = bp->b_vp; + buf_setfilter(bp, buffer_flushed_callback, tr, &cur_filter, NULL); - bp->b_iodone = buffer_flushed_callback; - bp->b_transaction = tr; - bp->b_flags |= B_CALL; - bp->b_flags &= ~(B_LOCKED); + if (cur_filter) { + panic("jnl: bp @ 0x%x (blkno %qd, vp 0x%x) has non-null iodone (0x%x) buffflushcb 0x%x\n", + bp, buf_blkno(bp), save_vp, cur_filter, buffer_flushed_callback); + } + buf_clearflags(bp, B_LOCKED); // kicking off the write here helps performance - bawrite(bp); - // XXXdbg this is good for testing: bdwrite(bp); - //bdwrite(bp); + buf_bawrite(bp); + // XXXdbg this is good for testing: buf_bdwrite(bp); + //buf_bdwrite(bp); - // this undoes the vget() in journal_modify_block_end() - vrele(save_vp); - + // this undoes the vnode_ref() in journal_modify_block_end() + vnode_rele_ext(save_vp, 0, 1); } else { printf("jnl: end_transaction: could not find block %Ld vp 0x%x!\n", blhdr->binfo[i].bnum, blhdr->binfo[i].bp); + if (bp) { + buf_clearflags(bp, B_LOCKED); + buf_brelse(bp); + } } } @@ -1890,6 +2539,7 @@ end_transaction(transaction *tr, int force_it) bad_journal: jnl->flags |= JOURNAL_INVALID; + jnl->old_start[sizeof(jnl->old_start)/sizeof(jnl->old_start[0]) - 1] &= ~0x8000000000000000LL; abort_transaction(jnl, tr); return -1; } @@ -1897,9 +2547,11 @@ end_transaction(transaction *tr, int force_it) static void abort_transaction(journal *jnl, transaction *tr) { - int i, ret; + int i; + errno_t errno; block_list_header *blhdr, *next; struct buf *bp; + struct vnode *save_vp; // for each block list header, iterate over the blocks then // free up the memory associated with the block list. @@ -1912,28 +2564,36 @@ abort_transaction(journal *jnl, transaction *tr) if (blhdr->binfo[i].bp == NULL) { continue; } - - ret = meta_bread(blhdr->binfo[i].bp->b_vp, - (daddr_t)blhdr->binfo[i].bp->b_lblkno, - blhdr->binfo[i].bp->b_bufsize, + if ( (buf_vnode(blhdr->binfo[i].bp) == NULL) || + !(buf_flags(blhdr->binfo[i].bp) & B_LOCKED) ) { + continue; + } + + errno = buf_meta_bread(buf_vnode(blhdr->binfo[i].bp), + buf_lblkno(blhdr->binfo[i].bp), + buf_size(blhdr->binfo[i].bp), NOCRED, &bp); - if (ret == 0 && bp != NULL) { + if (errno == 0) { if (bp != blhdr->binfo[i].bp) { panic("jnl: abort_tr: got back a different bp! (bp 0x%x should be 0x%x, jnl 0x%x\n", bp, blhdr->binfo[i].bp, jnl); } - // clear the locked bit and the delayed-write bit. we - // don't want these blocks going to disk. - bp->b_flags &= ~(B_LOCKED|B_DELWRI); - bp->b_flags |= B_INVAL; + // releasing a bp marked invalid + // also clears the locked and delayed state + buf_markinvalid(bp); + save_vp = buf_vnode(bp); - brelse(bp); + buf_brelse(bp); + vnode_rele_ext(save_vp, 0, 1); } else { printf("jnl: abort_tr: could not find block %Ld vp 0x%x!\n", blhdr->binfo[i].bnum, blhdr->binfo[i].bp); + if (bp) { + buf_brelse(bp); + } } } @@ -1947,7 +2607,7 @@ abort_transaction(journal *jnl, transaction *tr) tr->tbuffer = NULL; tr->blhdr = NULL; tr->total_bytes = 0xdbadc0de; - kmem_free(kernel_map, (vm_offset_t)tr, sizeof(transaction)); + FREE_ZONE(tr, sizeof(transaction), M_JNL_TR); } @@ -1955,7 +2615,7 @@ int journal_end_transaction(journal *jnl) { int ret; - transaction *tr; + transaction *tr; CHECK_JOURNAL(jnl); @@ -1963,9 +2623,9 @@ journal_end_transaction(journal *jnl) return 0; } - if (jnl->owner != current_act()) { + if (jnl->owner != current_thread()) { panic("jnl: end_tr: I'm not the owner! jnl 0x%x, owner 0x%x, curact 0x%x\n", - jnl, jnl->owner, current_act()); + jnl, jnl->owner, current_thread()); } free_old_stuff(jnl); @@ -1979,8 +2639,6 @@ journal_end_transaction(journal *jnl) if (jnl->flags & JOURNAL_INVALID) { if (jnl->active_tr) { - transaction *tr; - if (jnl->cur_tr != NULL) { panic("jnl: journal @ 0x%x has active tr (0x%x) and cur tr (0x%x)\n", jnl, jnl->active_tr, jnl->cur_tr); @@ -1992,7 +2650,7 @@ journal_end_transaction(journal *jnl) } jnl->owner = NULL; - semaphore_signal(jnl->jsem); + unlock_journal(jnl); return EINVAL; } @@ -2009,7 +2667,7 @@ journal_end_transaction(journal *jnl) ret = end_transaction(tr, 0); jnl->owner = NULL; - semaphore_signal(jnl->jsem); + unlock_journal(jnl); return ret; } @@ -2026,16 +2684,10 @@ journal_flush(journal *jnl) return -1; } - if (jnl->owner != current_act()) { + if (jnl->owner != current_thread()) { int ret; - while ((ret = semaphore_wait(jnl->jsem)) == KERN_ABORTED) { - // just keep looping if we've ben ^C'ed - } - if (ret != 0) { - printf("jnl: flush: sem wait failed.\n"); - return -1; - } + lock_journal(jnl); need_signal = 1; } @@ -2050,7 +2702,7 @@ journal_flush(journal *jnl) } if (need_signal) { - semaphore_signal(jnl->jsem); + unlock_journal(jnl); } return 0; @@ -2065,3 +2717,9 @@ journal_active(journal *jnl) return (jnl->active_tr == NULL) ? 0 : 1; } + +void * +journal_owner(journal *jnl) +{ + return jnl->owner; +}