]> git.saurik.com Git - apple/hfs.git/blob - core/hfs_journal.c
bfcc6400b9fdf836c60e5076baf42d04d03468c8
[apple/hfs.git] / core / hfs_journal.c
1 /*
2 * Copyright (c) 2002-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 //
29 // This file implements a simple write-ahead journaling layer.
30 // In theory any file system can make use of it by calling these
31 // functions when the fs wants to modify meta-data blocks. See
32 // hfs_journal.h for a more detailed description of the api and
33 // data structures.
34 //
35 // Dominic Giampaolo (dbg@apple.com)
36 //
37
38 #ifdef KERNEL
39
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/kernel.h>
43 #include <sys/stat.h>
44 #include <sys/namei.h>
45 #include <sys/ioctl.h>
46 #include <sys/ubc.h>
47 #include <sys/malloc.h>
48 #include <kern/task.h>
49 #include <kern/thread.h>
50 #include <sys/disk.h>
51 #include <sys/kdebug.h>
52 #include <sys/kpi_private.h>
53 #include <sys/sysctl.h>
54 #include <sys/systm.h>
55 #include <miscfs/specfs/specdev.h>
56 #include <libkern/OSAtomic.h> /* OSAddAtomic */
57
58 #include "hfs.h"
59
60 kern_return_t thread_terminate(thread_t);
61
62 /*
63 * Set sysctl vfs.generic.jnl.kdebug.trim=1 to enable KERNEL_DEBUG_CONSTANT
64 * logging of trim-related calls within the journal. (They're
65 * disabled by default because there can be a lot of these events,
66 * and we don't want to overwhelm the kernel debug buffer. If you
67 * want to watch these events in particular, just set the sysctl.)
68 */
69 static int jnl_kdebug = 0;
70
71 HFS_SYSCTL(NODE, _vfs_generic_hfs, OID_AUTO, jnl, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Journal")
72 HFS_SYSCTL(NODE, _vfs_generic_hfs_jnl, OID_AUTO, kdebug, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Journal kdebug")
73 HFS_SYSCTL(INT, _vfs_generic_hfs_jnl_kdebug, OID_AUTO, trim, CTLFLAG_RW|CTLFLAG_LOCKED, &jnl_kdebug, 0, "Enable kdebug logging for journal TRIM")
74
75 #define DBG_JOURNAL_FLUSH FSDBG_CODE(DBG_JOURNAL, 1)
76 #define DBG_JOURNAL_TRIM_ADD FSDBG_CODE(DBG_JOURNAL, 2)
77 #define DBG_JOURNAL_TRIM_REMOVE FSDBG_CODE(DBG_JOURNAL, 3)
78 #define DBG_JOURNAL_TRIM_REMOVE_PENDING FSDBG_CODE(DBG_JOURNAL, 4)
79 #define DBG_JOURNAL_TRIM_REALLOC FSDBG_CODE(DBG_JOURNAL, 5)
80 #define DBG_JOURNAL_TRIM_FLUSH FSDBG_CODE(DBG_JOURNAL, 6)
81 #define DBG_JOURNAL_TRIM_UNMAP FSDBG_CODE(DBG_JOURNAL, 7)
82
83 /*
84 * Cap the journal max size to 2GB. On HFS, it will attempt to occupy
85 * a full allocation block if the current size is smaller than the allocation
86 * block on which it resides. Once we hit the exabyte filesystem range, then
87 * it will use 2GB allocation blocks. As a result, make the cap 2GB.
88 */
89 #define MAX_JOURNAL_SIZE 0x80000000U
90
91 #include <mach/machine/sdt.h>
92 #else
93
94 #include <stdio.h>
95 #include <stdlib.h>
96 #include <string.h>
97 #include <limits.h>
98 #include <errno.h>
99 #include <fcntl.h>
100 #include <unistd.h>
101 #include <stdarg.h>
102 #include <sys/types.h>
103 #include "compat.h"
104
105 #endif /* KERNEL */
106
107 #include "hfs_journal.h"
108
109 #include <sys/kdebug.h>
110
111 //
112 // By default, we grow the list of extents to trim by 4K at a time.
113 // We'll opt to flush a transaction if it contains at least
114 // JOURNAL_FLUSH_TRIM_EXTENTS extents to be trimmed (even if the number
115 // of modified blocks is small).
116 //
117 enum {
118 JOURNAL_DEFAULT_TRIM_BYTES = 4096,
119 JOURNAL_DEFAULT_TRIM_EXTENTS = JOURNAL_DEFAULT_TRIM_BYTES / sizeof(dk_extent_t),
120 JOURNAL_FLUSH_TRIM_EXTENTS = JOURNAL_DEFAULT_TRIM_EXTENTS * 15 / 16
121 };
122
123 unsigned int jnl_trim_flush_limit = JOURNAL_FLUSH_TRIM_EXTENTS;
124
125 HFS_SYSCTL(UINT, _vfs_generic_hfs_jnl, OID_AUTO, trim_flush, CTLFLAG_RW, &jnl_trim_flush_limit, 0, "number of trimmed extents to cause a journal flush")
126
127 // number of bytes to checksum in a block_list_header
128 // NOTE: this should be enough to clear out the header
129 // fields as well as the first entry of binfo[]
130 #define BLHDR_CHECKSUM_SIZE 32
131
132 static void lock_condition(journal *jnl, boolean_t *condition, const char *condition_name);
133 static void wait_condition(journal *jnl, boolean_t *condition, const char *condition_name);
134 static void unlock_condition(journal *jnl, boolean_t *condition);
135 static void finish_end_thread(transaction *tr);
136 static void write_header_thread(journal *jnl);
137 static int finish_end_transaction(transaction *tr, errno_t (*callback)(void*), void *callback_arg);
138 static int end_transaction(transaction *tr, int force_it, errno_t (*callback)(void*), void *callback_arg, boolean_t drop_lock, boolean_t must_wait);
139 static void abort_transaction(journal *jnl, transaction *tr);
140 static void dump_journal(journal *jnl);
141
142 static __inline__ void lock_oldstart(journal *jnl);
143 static __inline__ void unlock_oldstart(journal *jnl);
144 static __inline__ void lock_flush(journal *jnl);
145 static __inline__ void unlock_flush(journal *jnl);
146
147
148 //
149 // 3105942 - Coalesce writes to the same block on journal replay
150 //
151
152 typedef struct bucket {
153 off_t block_num;
154 uint32_t jnl_offset;
155 uint32_t block_size;
156 int32_t cksum;
157 } bucket;
158
159 #define STARTING_BUCKETS 256
160
161 static int add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr);
162 static int grow_table(struct bucket **buf_ptr, int num_buckets, int new_size);
163 static int lookup_bucket(struct bucket **buf_ptr, off_t block_num, int num_full);
164 static int do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr);
165 static int insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr, int overwriting);
166
167 #define CHECK_JOURNAL(jnl) \
168 do { \
169 if (jnl == NULL) { \
170 panic("%s:%d: null journal ptr?\n", __FILE__, __LINE__); \
171 } \
172 if (jnl->jdev == NULL) { \
173 panic("%s:%d: jdev is null!\n", __FILE__, __LINE__); \
174 } \
175 if (jnl->fsdev == NULL) { \
176 panic("%s:%d: fsdev is null!\n", __FILE__, __LINE__); \
177 } \
178 if (jnl->jhdr->magic != JOURNAL_HEADER_MAGIC) { \
179 panic("%s:%d: jhdr magic corrupted (0x%x != 0x%x)\n", \
180 __FILE__, __LINE__, jnl->jhdr->magic, JOURNAL_HEADER_MAGIC); \
181 } \
182 if ( jnl->jhdr->start <= 0 \
183 || jnl->jhdr->start > jnl->jhdr->size) { \
184 panic("%s:%d: jhdr start looks bad (0x%llx max size 0x%llx)\n", \
185 __FILE__, __LINE__, jnl->jhdr->start, jnl->jhdr->size); \
186 } \
187 if ( jnl->jhdr->end <= 0 \
188 || jnl->jhdr->end > jnl->jhdr->size) { \
189 panic("%s:%d: jhdr end looks bad (0x%llx max size 0x%llx)\n", \
190 __FILE__, __LINE__, jnl->jhdr->end, jnl->jhdr->size); \
191 } \
192 } while(0)
193
194 #define CHECK_TRANSACTION(tr) \
195 do { \
196 if (tr == NULL) { \
197 panic("%s:%d: null transaction ptr?\n", __FILE__, __LINE__); \
198 } \
199 if (tr->jnl == NULL) { \
200 panic("%s:%d: null tr->jnl ptr?\n", __FILE__, __LINE__); \
201 } \
202 if (tr->blhdr != (block_list_header *)tr->tbuffer) { \
203 panic("%s:%d: blhdr (%p) != tbuffer (%p)\n", __FILE__, __LINE__, tr->blhdr, tr->tbuffer); \
204 } \
205 if (tr->total_bytes < 0) { \
206 panic("%s:%d: tr total_bytes looks bad: %d\n", __FILE__, __LINE__, tr->total_bytes); \
207 } \
208 if (tr->journal_start < 0) { \
209 panic("%s:%d: tr journal start looks bad: 0x%llx\n", __FILE__, __LINE__, tr->journal_start); \
210 } \
211 if (tr->journal_end < 0) { \
212 panic("%s:%d: tr journal end looks bad: 0x%llx\n", __FILE__, __LINE__, tr->journal_end); \
213 } \
214 if (tr->blhdr && (tr->blhdr->max_blocks <= 0 || tr->blhdr->max_blocks > (tr->jnl->jhdr->size/tr->jnl->jhdr->jhdr_size))) { \
215 panic("%s:%d: tr blhdr max_blocks looks bad: %d\n", __FILE__, __LINE__, tr->blhdr->max_blocks); \
216 } \
217 } while(0)
218
219
220
221 //
222 // this isn't a great checksum routine but it will do for now.
223 // we use it to checksum the journal header and the block list
224 // headers that are at the start of each transaction.
225 //
226 static unsigned int
227 calc_checksum(const char *ptr, int len)
228 {
229 int i;
230 unsigned int cksum=0;
231
232 // this is a lame checksum but for now it'll do
233 for(i = 0; i < len; i++, ptr++) {
234 cksum = (cksum << 8) ^ (cksum + *(unsigned char *)ptr);
235 }
236
237 return (~cksum);
238 }
239
240 //
241 // Journal Locking
242 //
243 lck_grp_attr_t * jnl_group_attr;
244 lck_attr_t * jnl_lock_attr;
245 lck_grp_t * jnl_mutex_group;
246
247 void
248 journal_init(void)
249 {
250 jnl_lock_attr = lck_attr_alloc_init();
251 jnl_group_attr = lck_grp_attr_alloc_init();
252 jnl_mutex_group = lck_grp_alloc_init("jnl-mutex", jnl_group_attr);
253 }
254
255 __inline__ void
256 journal_lock(journal *jnl)
257 {
258 lck_mtx_lock(&jnl->jlock);
259 if (jnl->owner) {
260 panic ("jnl: owner is %p, expected NULL\n", jnl->owner);
261 }
262 jnl->owner = current_thread();
263 }
264
265 __inline__ void
266 journal_unlock(journal *jnl)
267 {
268 jnl->owner = NULL;
269 lck_mtx_unlock(&jnl->jlock);
270 }
271
272 static __inline__ void
273 lock_flush(journal *jnl)
274 {
275 lck_mtx_lock(&jnl->flock);
276 }
277
278 static __inline__ void
279 unlock_flush(journal *jnl)
280 {
281 lck_mtx_unlock(&jnl->flock);
282 }
283
284 static __inline__ void
285 lock_oldstart(journal *jnl)
286 {
287 lck_mtx_lock(&jnl->old_start_lock);
288 }
289
290 static __inline__ void
291 unlock_oldstart(journal *jnl)
292 {
293 lck_mtx_unlock(&jnl->old_start_lock);
294 }
295
296
297
298 #define JNL_WRITE 0x0001
299 #define JNL_READ 0x0002
300 #define JNL_HEADER 0x8000
301
302 //
303 // This function sets up a fake buf and passes it directly to the
304 // journal device strategy routine (so that it won't get cached in
305 // the block cache.
306 //
307 // It also handles range checking the i/o so that we don't write
308 // outside the journal boundaries and it will wrap the i/o back
309 // to the beginning if necessary (skipping over the journal header)
310 //
311 static size_t
312 do_journal_io(journal *jnl, off_t *offset, void *data, size_t len, int direction)
313 {
314 int err;
315 off_t curlen = len;
316 size_t io_sz = 0;
317 buf_t bp;
318 off_t max_iosize;
319 bufattr_t bap;
320 boolean_t was_vm_privileged = FALSE;
321 boolean_t need_vm_privilege = FALSE;
322
323 if (vfs_isswapmount(jnl->fsmount))
324 need_vm_privilege = TRUE;
325
326 if (*offset < 0 || *offset > jnl->jhdr->size) {
327 panic("jnl: do_jnl_io: bad offset 0x%llx (max 0x%llx)\n", *offset, jnl->jhdr->size);
328 }
329
330 if (direction & JNL_WRITE)
331 max_iosize = jnl->max_write_size;
332 else if (direction & JNL_READ)
333 max_iosize = jnl->max_read_size;
334 else
335 max_iosize = 128 * 1024;
336
337 again:
338 bp = buf_alloc(jnl->jdev);
339
340 if (*offset + curlen > jnl->jhdr->size && *offset != 0 && jnl->jhdr->size != 0) {
341 if (*offset == jnl->jhdr->size) {
342 *offset = jnl->jhdr->jhdr_size;
343 } else {
344 curlen = jnl->jhdr->size - *offset;
345 }
346 }
347
348 if (curlen > max_iosize) {
349 curlen = max_iosize;
350 }
351
352 if (curlen <= 0) {
353 panic("jnl: do_jnl_io: curlen == %lld, offset 0x%llx len %zd\n", curlen, *offset, len);
354 }
355
356 if (*offset == 0 && (direction & JNL_HEADER) == 0) {
357 panic("jnl: request for i/o to jnl-header without JNL_HEADER flag set! (len %lld, data %p)\n", curlen, data);
358 }
359
360 /*
361 * As alluded to in the block comment at the top of the function, we use a "fake" iobuf
362 * here and issue directly to the disk device that the journal protects since we don't
363 * want this to enter the block cache. As a result, we lose the ability to mark it
364 * as a metadata buf_t for the layers below us that may care. If we were to
365 * simply attach the B_META flag into the b_flags this may confuse things further
366 * since this is an iobuf, not a metadata buffer.
367 *
368 * To address this, we use the extended bufattr struct embedded in the bp.
369 * Explicitly mark the buf here as a metadata buffer in its bufattr flags.
370 */
371 bap = buf_attr(bp);
372 bufattr_markmeta(bap);
373
374 if (direction & JNL_READ)
375 buf_setflags(bp, B_READ);
376 else {
377 /*
378 * don't have to set any flags
379 */
380 vnode_startwrite(jnl->jdev);
381 }
382 buf_setsize(bp, curlen);
383 buf_setcount(bp, curlen);
384 buf_setdataptr(bp, (uintptr_t)data);
385 buf_setblkno(bp, (daddr64_t) ((jnl->jdev_offset + *offset) / (off_t)jnl->jhdr->jhdr_size));
386 buf_setlblkno(bp, (daddr64_t) ((jnl->jdev_offset + *offset) / (off_t)jnl->jhdr->jhdr_size));
387
388 if ((direction & JNL_WRITE) && (jnl->flags & JOURNAL_DO_FUA_WRITES)) {
389 buf_markfua(bp);
390 }
391
392 if (need_vm_privilege == TRUE) {
393 /*
394 * if we block waiting for memory, and there is enough pressure to
395 * cause us to try and create a new swap file, we may end up deadlocking
396 * due to waiting for the journal on the swap file creation path...
397 * by making ourselves vm_privileged, we give ourselves the best chance
398 * of not blocking
399 */
400 was_vm_privileged = set_vm_privilege(TRUE);
401 }
402 DTRACE_IO1(journal__start, buf_t, bp);
403 err = VNOP_STRATEGY(bp);
404 if (!err) {
405 err = (int)buf_biowait(bp);
406 }
407 DTRACE_IO1(journal__done, buf_t, bp);
408
409 if (need_vm_privilege == TRUE && was_vm_privileged == FALSE)
410 set_vm_privilege(FALSE);
411
412 buf_free(bp);
413
414 if (err) {
415 printf("jnl: %s: do_jnl_io: strategy err 0x%x\n", jnl->jdev_name, err);
416 return 0;
417 }
418
419 *offset += curlen;
420 io_sz += curlen;
421
422 if (io_sz != len) {
423 // handle wrap-around
424 data = (char *)data + curlen;
425 curlen = len - io_sz;
426 if (*offset >= jnl->jhdr->size) {
427 *offset = jnl->jhdr->jhdr_size;
428 }
429 goto again;
430 }
431
432 return io_sz;
433 }
434
435 static size_t
436 read_journal_data(journal *jnl, off_t *offset, void *data, size_t len)
437 {
438 return do_journal_io(jnl, offset, data, len, JNL_READ);
439 }
440
441 static size_t
442 write_journal_data(journal *jnl, off_t *offset, void *data, size_t len)
443 {
444 return do_journal_io(jnl, offset, data, len, JNL_WRITE);
445 }
446
447
448 static size_t
449 read_journal_header(journal *jnl, void *data, size_t len)
450 {
451 off_t hdr_offset = 0;
452
453 return do_journal_io(jnl, &hdr_offset, data, len, JNL_READ|JNL_HEADER);
454 }
455
456 static int
457 write_journal_header(journal *jnl, int updating_start, uint32_t sequence_num)
458 {
459 static int num_err_prints = 0;
460 int ret=0;
461 off_t jhdr_offset = 0;
462 //
463 // Flush the track cache if we're not doing force-unit-access
464 // writes.
465 //
466 if (!updating_start && (jnl->flags & JOURNAL_DO_FUA_WRITES) == 0) {
467
468 dk_synchronize_t sync_request = {
469 .options = DK_SYNCHRONIZE_OPTION_BARRIER,
470 };
471
472 /*
473 * If device doesn't support barrier-only flush, or
474 * the journal is on a different device, use full flush.
475 */
476 if (!(jnl->flags & JOURNAL_FEATURE_BARRIER) || (jnl->jdev != jnl->fsdev)) {
477 sync_request.options = 0;
478 jnl->flush_counter++;
479 }
480
481 ret = VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZE, (caddr_t)&sync_request, FWRITE, vfs_context_kernel());
482 }
483 if (ret != 0) {
484 //
485 // Only print this error if it's a different error than the
486 // previous one, or if it's the first time for this device
487 // or if the total number of printfs is less than 25. We
488 // allow for up to 25 printfs to insure that some make it
489 // into the on-disk syslog. Otherwise if we only printed
490 // one, it's possible it would never make it to the syslog
491 // for the root volume and that makes debugging hard.
492 //
493 if ( ret != jnl->last_flush_err
494 || (jnl->flags & JOURNAL_FLUSHCACHE_ERR) == 0
495 || num_err_prints++ < 25) {
496
497 printf("jnl: %s: flushing fs disk buffer returned 0x%x\n", jnl->jdev_name, ret);
498
499 jnl->flags |= JOURNAL_FLUSHCACHE_ERR;
500 jnl->last_flush_err = ret;
501 }
502 }
503
504 jnl->jhdr->sequence_num = sequence_num;
505 jnl->jhdr->checksum = 0;
506 jnl->jhdr->checksum = calc_checksum((char *)jnl->jhdr, JOURNAL_HEADER_CKSUM_SIZE);
507
508 if (do_journal_io(jnl, &jhdr_offset, jnl->header_buf, jnl->jhdr->jhdr_size, JNL_WRITE|JNL_HEADER) != (size_t)jnl->jhdr->jhdr_size) {
509 printf("jnl: %s: write_journal_header: error writing the journal header!\n", jnl->jdev_name);
510 jnl->flags |= JOURNAL_INVALID;
511 return -1;
512 }
513
514 // If we're not doing force-unit-access writes, then we
515 // have to flush after writing the journal header so that
516 // a future transaction doesn't sneak out to disk before
517 // the header does and thus overwrite data that the old
518 // journal header refers to. Saw this exact case happen
519 // on an IDE bus analyzer with Larry Barras so while it
520 // may seem obscure, it's not.
521 //
522 if (updating_start && (jnl->flags & JOURNAL_DO_FUA_WRITES) == 0) {
523
524 dk_synchronize_t sync_request = {
525 .options = DK_SYNCHRONIZE_OPTION_BARRIER,
526 };
527
528 /*
529 * If device doesn't support barrier-only flush, or
530 * the journal is on a different device, use full flush.
531 */
532 if (!(jnl->flags & JOURNAL_FEATURE_BARRIER) || (jnl->jdev != jnl->fsdev)) {
533 sync_request.options = 0;
534 jnl->flush_counter++;
535 }
536
537 VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZE, (caddr_t)&sync_request, FWRITE, vfs_context_kernel());
538 }
539
540 return 0;
541 }
542
543
544
545 //
546 // this is a work function used to free up transactions that
547 // completed. they can't be free'd from buffer_flushed_callback
548 // because it is called from deep with the disk driver stack
549 // and thus can't do something that would potentially cause
550 // paging. it gets called by each of the journal api entry
551 // points so stuff shouldn't hang around for too long.
552 //
553 static void
554 free_old_stuff(journal *jnl)
555 {
556 transaction *tr, *next;
557 block_list_header *blhdr=NULL, *next_blhdr=NULL;
558
559 if (jnl->tr_freeme == NULL)
560 return;
561
562 lock_oldstart(jnl);
563 tr = jnl->tr_freeme;
564 jnl->tr_freeme = NULL;
565 unlock_oldstart(jnl);
566
567 for(; tr; tr=next) {
568 for (blhdr = tr->blhdr; blhdr; blhdr = next_blhdr) {
569 next_blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum);
570 blhdr->binfo[0].bnum = 0xdeadc0de;
571
572 hfs_free(blhdr, tr->tbuffer_size);
573
574 KERNEL_DEBUG(0xbbbbc01c, jnl, tr, tr->tbuffer_size, 0, 0);
575 }
576 next = tr->next;
577 hfs_free(tr, sizeof(*tr));
578 }
579 }
580
581
582
583 //
584 // This is our callback that lets us know when a buffer has been
585 // flushed to disk. It's called from deep within the driver stack
586 // and thus is quite limited in what it can do. Notably, it can
587 // not initiate any new i/o's or allocate/free memory.
588 //
589 static void
590 buffer_flushed_callback(struct buf *bp, void *arg)
591 {
592 transaction *tr;
593 journal *jnl;
594 transaction *ctr, *prev=NULL, *next;
595 size_t i;
596 int bufsize, amt_flushed, total_bytes;
597
598
599 //printf("jnl: buf flush: bp @ 0x%x l/blkno %qd/%qd vp 0x%x tr @ 0x%x\n",
600 // bp, buf_lblkno(bp), buf_blkno(bp), buf_vnode(bp), arg);
601
602 // snarf out the bits we want
603 bufsize = buf_size(bp);
604 tr = (transaction *)arg;
605
606 // then we've already seen it
607 if (tr == NULL) {
608 return;
609 }
610
611 CHECK_TRANSACTION(tr);
612
613 jnl = tr->jnl;
614
615 CHECK_JOURNAL(jnl);
616
617 amt_flushed = tr->num_killed;
618 total_bytes = tr->total_bytes;
619
620 // update the number of blocks that have been flushed.
621 // this buf may represent more than one block so take
622 // that into account.
623 //
624 // OSAddAtomic() returns the value of tr->num_flushed before the add
625 //
626 amt_flushed += OSAddAtomic(bufsize, &tr->num_flushed);
627
628
629 // if this transaction isn't done yet, just return as
630 // there is nothing to do.
631 //
632 // NOTE: we are careful to not reference anything through
633 // the tr pointer after doing the OSAddAtomic(). if
634 // this if statement fails then we are the last one
635 // and then it's ok to dereference "tr".
636 //
637 if ((amt_flushed + bufsize) < total_bytes) {
638 return;
639 }
640
641 // this will single thread checking the transaction
642 lock_oldstart(jnl);
643
644 if (tr->total_bytes == (int)0xfbadc0de) {
645 // then someone beat us to it...
646 unlock_oldstart(jnl);
647 return;
648 }
649
650 // mark this so that we're the owner of dealing with the
651 // cleanup for this transaction
652 tr->total_bytes = 0xfbadc0de;
653
654 if (jnl->flags & JOURNAL_INVALID)
655 goto transaction_done;
656
657 //printf("jnl: tr 0x%x (0x%llx 0x%llx) in jnl 0x%x completed.\n",
658 // tr, tr->journal_start, tr->journal_end, jnl);
659
660 // find this entry in the old_start[] index and mark it completed
661 for(i = 0; i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0]); i++) {
662
663 if ((off_t)(jnl->old_start[i] & ~(0x8000000000000000ULL)) == tr->journal_start) {
664 jnl->old_start[i] &= ~(0x8000000000000000ULL);
665 break;
666 }
667 }
668
669 if (i >= sizeof(jnl->old_start)/sizeof(jnl->old_start[0])) {
670 panic("jnl: buffer_flushed: did not find tr w/start @ %lld (tr %p, jnl %p)\n",
671 tr->journal_start, tr, jnl);
672 }
673
674
675 // if we are here then we need to update the journal header
676 // to reflect that this transaction is complete
677 if (tr->journal_start == jnl->active_start) {
678 jnl->active_start = tr->journal_end;
679 tr->journal_start = tr->journal_end = (off_t)0;
680 }
681
682 // go through the completed_trs list and try to coalesce
683 // entries, restarting back at the beginning if we have to.
684 for (ctr = jnl->completed_trs; ctr; prev=ctr, ctr=next) {
685 if (ctr->journal_start == jnl->active_start) {
686 jnl->active_start = ctr->journal_end;
687 if (prev) {
688 prev->next = ctr->next;
689 }
690 if (ctr == jnl->completed_trs) {
691 jnl->completed_trs = ctr->next;
692 }
693
694 next = jnl->completed_trs; // this starts us over again
695 ctr->next = jnl->tr_freeme;
696 jnl->tr_freeme = ctr;
697 ctr = NULL;
698 } else if (tr->journal_end == ctr->journal_start) {
699 ctr->journal_start = tr->journal_start;
700 next = jnl->completed_trs; // this starts us over again
701 ctr = NULL;
702 tr->journal_start = tr->journal_end = (off_t)0;
703 } else if (tr->journal_start == ctr->journal_end) {
704 ctr->journal_end = tr->journal_end;
705 next = ctr->next;
706 tr->journal_start = tr->journal_end = (off_t)0;
707 } else if (ctr->next && ctr->journal_end == ctr->next->journal_start) {
708 // coalesce the next entry with this one and link the next
709 // entry in at the head of the tr_freeme list
710 next = ctr->next; // temporarily use the "next" variable
711 ctr->journal_end = next->journal_end;
712 ctr->next = next->next;
713 next->next = jnl->tr_freeme; // link in the next guy at the head of the tr_freeme list
714 jnl->tr_freeme = next;
715
716 next = jnl->completed_trs; // this starts us over again
717 ctr = NULL;
718 } else {
719 next = ctr->next;
720 }
721 }
722
723 // if this is true then we didn't merge with anyone
724 // so link ourselves in at the head of the completed
725 // transaction list.
726 if (tr->journal_start != 0) {
727 // put this entry into the correct sorted place
728 // in the list instead of just at the head.
729 //
730
731 prev = NULL;
732 for (ctr = jnl->completed_trs; ctr && tr->journal_start > ctr->journal_start; prev=ctr, ctr=ctr->next) {
733 // just keep looping
734 }
735
736 if (ctr == NULL && prev == NULL) {
737 jnl->completed_trs = tr;
738 tr->next = NULL;
739 } else if (ctr == jnl->completed_trs) {
740 tr->next = jnl->completed_trs;
741 jnl->completed_trs = tr;
742 } else {
743 tr->next = prev->next;
744 prev->next = tr;
745 }
746 } else {
747 // if we're here this tr got merged with someone else so
748 // put it on the list to be free'd
749 tr->next = jnl->tr_freeme;
750 jnl->tr_freeme = tr;
751 }
752 transaction_done:
753 unlock_oldstart(jnl);
754
755 unlock_condition(jnl, &jnl->asyncIO);
756 }
757
758
759 #include <libkern/OSByteOrder.h>
760
761 #define SWAP16(x) OSSwapInt16(x)
762 #define SWAP32(x) OSSwapInt32(x)
763 #define SWAP64(x) OSSwapInt64(x)
764
765
766 static void
767 swap_journal_header(journal *jnl)
768 {
769 jnl->jhdr->magic = SWAP32(jnl->jhdr->magic);
770 jnl->jhdr->endian = SWAP32(jnl->jhdr->endian);
771 jnl->jhdr->start = SWAP64(jnl->jhdr->start);
772 jnl->jhdr->end = SWAP64(jnl->jhdr->end);
773 jnl->jhdr->size = SWAP64(jnl->jhdr->size);
774 jnl->jhdr->blhdr_size = SWAP32(jnl->jhdr->blhdr_size);
775 jnl->jhdr->checksum = SWAP32(jnl->jhdr->checksum);
776 jnl->jhdr->jhdr_size = SWAP32(jnl->jhdr->jhdr_size);
777 jnl->jhdr->sequence_num = SWAP32(jnl->jhdr->sequence_num);
778 }
779
780 static void
781 swap_block_list_header(journal *jnl, block_list_header *blhdr)
782 {
783 int i;
784
785 blhdr->max_blocks = SWAP16(blhdr->max_blocks);
786 blhdr->num_blocks = SWAP16(blhdr->num_blocks);
787 blhdr->bytes_used = SWAP32(blhdr->bytes_used);
788 blhdr->checksum = SWAP32(blhdr->checksum);
789 blhdr->flags = SWAP32(blhdr->flags);
790
791 if (blhdr->num_blocks >= ((jnl->jhdr->blhdr_size / sizeof(block_info)) - 1)) {
792 printf("jnl: %s: blhdr num blocks looks suspicious (%d / blhdr size %d). not swapping.\n", jnl->jdev_name, blhdr->num_blocks, jnl->jhdr->blhdr_size);
793 return;
794 }
795
796 for(i = 0; i < blhdr->num_blocks; i++) {
797 blhdr->binfo[i].bnum = SWAP64(blhdr->binfo[i].bnum);
798 blhdr->binfo[i].u.bi.bsize = SWAP32(blhdr->binfo[i].u.bi.bsize);
799 blhdr->binfo[i].u.bi.b.cksum = SWAP32(blhdr->binfo[i].u.bi.b.cksum);
800 }
801 }
802
803
804 static int
805 update_fs_block(journal *jnl, void *block_ptr, off_t fs_block, size_t bsize)
806 {
807 int ret;
808 struct buf *oblock_bp=NULL;
809 boolean_t was_vm_privileged = FALSE;
810
811
812 // first read the block we want.
813 ret = buf_meta_bread(jnl->fsdev, (daddr64_t)fs_block, bsize, NOCRED, &oblock_bp);
814 if (ret != 0) {
815 printf("jnl: %s: update_fs_block: error reading fs block # %lld! (ret %d)\n", jnl->jdev_name, fs_block, ret);
816
817 if (oblock_bp) {
818 buf_brelse(oblock_bp);
819 oblock_bp = NULL;
820 }
821
822 // let's try to be aggressive here and just re-write the block
823 oblock_bp = buf_getblk(jnl->fsdev, (daddr64_t)fs_block, bsize, 0, 0, BLK_META);
824 if (oblock_bp == NULL) {
825 printf("jnl: %s: update_fs_block: buf_getblk() for %lld failed! failing update.\n", jnl->jdev_name, fs_block);
826 return -1;
827 }
828 }
829
830 // make sure it's the correct size.
831 if (buf_size(oblock_bp) != bsize) {
832 buf_brelse(oblock_bp);
833 return -1;
834 }
835
836 // copy the journal data over top of it
837 memcpy((char *)buf_dataptr(oblock_bp), block_ptr, bsize);
838
839 if (vfs_isswapmount(jnl->fsmount)) {
840 /*
841 * if we block waiting for memory, and there is enough pressure to
842 * cause us to try and create a new swap file, we may end up deadlocking
843 * due to waiting for the journal on the swap file creation path...
844 * by making ourselves vm_privileged, we give ourselves the best chance
845 * of not blocking
846 */
847 was_vm_privileged = set_vm_privilege(TRUE);
848 }
849 ret = VNOP_BWRITE(oblock_bp);
850
851 if (vfs_isswapmount(jnl->fsmount) && (was_vm_privileged == FALSE))
852 set_vm_privilege(FALSE);
853
854 if (ret != 0) {
855 printf("jnl: %s: update_fs_block: failed to update block %lld (ret %d)\n", jnl->jdev_name, fs_block,ret);
856 return ret;
857 }
858 // and now invalidate it so that if someone else wants to read
859 // it in a different size they'll be able to do it.
860 ret = buf_meta_bread(jnl->fsdev, (daddr64_t)fs_block, bsize, NOCRED, &oblock_bp);
861 if (oblock_bp) {
862 buf_markinvalid(oblock_bp);
863 buf_brelse(oblock_bp);
864 }
865
866 return 0;
867 }
868
869 static int
870 grow_table(struct bucket **buf_ptr, int num_buckets, int new_size)
871 {
872 struct bucket *newBuf;
873 int current_size = num_buckets, i;
874
875 // return if newsize is less than the current size
876 if (new_size < num_buckets) {
877 return current_size;
878 }
879
880 newBuf = hfs_malloc(new_size*sizeof(struct bucket));
881
882 // printf("jnl: lookup_bucket: expanded co_buf to %d elems\n", new_size);
883
884 // copy existing elements
885 bcopy(*buf_ptr, newBuf, num_buckets*sizeof(struct bucket));
886
887 // initialize the new ones
888 for(i = num_buckets; i < new_size; i++) {
889 newBuf[i].block_num = (off_t)-1;
890 }
891
892 // free the old container
893 hfs_free(*buf_ptr, num_buckets * sizeof(struct bucket));
894
895 // reset the buf_ptr
896 *buf_ptr = newBuf;
897
898 return new_size;
899 }
900
901 static int
902 lookup_bucket(struct bucket **buf_ptr, off_t block_num, int num_full)
903 {
904 int lo, hi, index, matches, i;
905
906 if (num_full == 0) {
907 return 0; // table is empty, so insert at index=0
908 }
909
910 lo = 0;
911 hi = num_full - 1;
912 index = -1;
913
914 // perform binary search for block_num
915 do {
916 int mid = (hi - lo)/2 + lo;
917 off_t this_num = (*buf_ptr)[mid].block_num;
918
919 if (block_num == this_num) {
920 index = mid;
921 break;
922 }
923
924 if (block_num < this_num) {
925 hi = mid;
926 continue;
927 }
928
929 if (block_num > this_num) {
930 lo = mid + 1;
931 continue;
932 }
933 } while (lo < hi);
934
935 // check if lo and hi converged on the match
936 if (block_num == (*buf_ptr)[hi].block_num) {
937 index = hi;
938 }
939
940 // if no existing entry found, find index for new one
941 if (index == -1) {
942 index = (block_num < (*buf_ptr)[hi].block_num) ? hi : hi + 1;
943 } else {
944 // make sure that we return the right-most index in the case of multiple matches
945 matches = 0;
946 i = index + 1;
947 while (i < num_full && block_num == (*buf_ptr)[i].block_num) {
948 matches++;
949 i++;
950 }
951
952 index += matches;
953 }
954
955 return index;
956 }
957
958 static int
959 insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr, int overwriting)
960 {
961 if (!overwriting) {
962 // grow the table if we're out of space - we may index the table
963 // with *num_full_ptr (lookup_bucket() can return a maximum value ==
964 // *num_full_ptr), so we need to grow when we hit (*num_buckets_ptr - 1)
965 // to prevent out-of-bounds indexing
966 if (*num_full_ptr >= (*num_buckets_ptr - 1)) {
967 int new_size = *num_buckets_ptr * 2;
968 int grow_size = grow_table(buf_ptr, *num_buckets_ptr, new_size);
969
970 if (grow_size < new_size) {
971 printf("jnl: %s: add_block: grow_table returned an error!\n", jnl->jdev_name);
972 return -1;
973 }
974
975 *num_buckets_ptr = grow_size; //update num_buckets to reflect the new size
976 }
977
978 // if we're not inserting at the end, we need to bcopy
979 if (blk_index != *num_full_ptr) {
980 bcopy( (*buf_ptr)+(blk_index), (*buf_ptr)+(blk_index+1), (*num_full_ptr-blk_index)*sizeof(struct bucket) );
981 }
982
983 (*num_full_ptr)++; // increment only if we're not overwriting
984 }
985
986 // sanity check the values we're about to add
987 if ((off_t)offset >= jnl->jhdr->size) {
988 offset = jnl->jhdr->jhdr_size + (offset - jnl->jhdr->size);
989 }
990 if (size <= 0) {
991 panic("jnl: insert_block: bad size in insert_block (%zd)\n", size);
992 }
993
994 (*buf_ptr)[blk_index].block_num = num;
995 (*buf_ptr)[blk_index].block_size = (uint32_t)size;
996 (*buf_ptr)[blk_index].jnl_offset = (uint32_t)offset;
997 (*buf_ptr)[blk_index].cksum = cksum;
998
999 return blk_index;
1000 }
1001
1002 static int
1003 do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num, size_t size, __unused size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr)
1004 {
1005 int num_to_remove, index, i, overwrite, err;
1006 size_t jhdr_size = jnl->jhdr->jhdr_size, new_offset;
1007 off_t overlap, block_start, block_end;
1008
1009 block_start = block_num*jhdr_size;
1010 block_end = block_start + size;
1011 overwrite = (block_num == (*buf_ptr)[blk_index].block_num && size >= (*buf_ptr)[blk_index].block_size);
1012
1013 // first, eliminate any overlap with the previous entry
1014 if (blk_index != 0 && !overwrite) {
1015 off_t prev_block_start = (*buf_ptr)[blk_index-1].block_num*jhdr_size;
1016 off_t prev_block_end = prev_block_start + (*buf_ptr)[blk_index-1].block_size;
1017 overlap = prev_block_end - block_start;
1018 if (overlap > 0) {
1019 if (overlap % jhdr_size != 0) {
1020 panic("jnl: do_overlap: overlap with previous entry not a multiple of %zd\n", jhdr_size);
1021 }
1022
1023 // if the previous entry completely overlaps this one, we need to break it into two pieces.
1024 if (prev_block_end > block_end) {
1025 off_t new_num = block_end / jhdr_size;
1026 size_t new_size = prev_block_end - block_end;
1027
1028 new_offset = (*buf_ptr)[blk_index-1].jnl_offset + (block_end - prev_block_start);
1029
1030 err = insert_block(jnl, buf_ptr, blk_index, new_num, new_size, new_offset, cksum, num_buckets_ptr, num_full_ptr, 0);
1031 if (err < 0) {
1032 panic("jnl: do_overlap: error inserting during pre-overlap\n");
1033 }
1034 }
1035
1036 // Regardless, we need to truncate the previous entry to the beginning of the overlap
1037 (*buf_ptr)[blk_index-1].block_size = (uint32_t)(block_start - prev_block_start);
1038 (*buf_ptr)[blk_index-1].cksum = 0; // have to blow it away because there's no way to check it
1039 }
1040 }
1041
1042 // then, bail out fast if there's no overlap with the entries that follow
1043 if (!overwrite && block_end <= (off_t)((*buf_ptr)[blk_index].block_num*jhdr_size)) {
1044 return 0; // no overlap, no overwrite
1045 } else if (overwrite && (blk_index + 1 >= *num_full_ptr || block_end <= (off_t)((*buf_ptr)[blk_index+1].block_num*jhdr_size))) {
1046
1047 (*buf_ptr)[blk_index].cksum = cksum; // update this
1048 return 1; // simple overwrite
1049 }
1050
1051 // Otherwise, find all cases of total and partial overlap. We use the special
1052 // block_num of -2 to designate entries that are completely overlapped and must
1053 // be eliminated. The block_num, size, and jnl_offset of partially overlapped
1054 // entries must be adjusted to keep the array consistent.
1055 index = blk_index;
1056 num_to_remove = 0;
1057 while (index < *num_full_ptr && block_end > (off_t)((*buf_ptr)[index].block_num*jhdr_size)) {
1058 if (block_end >= (off_t)(((*buf_ptr)[index].block_num*jhdr_size + (*buf_ptr)[index].block_size))) {
1059 (*buf_ptr)[index].block_num = -2; // mark this for deletion
1060 num_to_remove++;
1061 } else {
1062 overlap = block_end - (*buf_ptr)[index].block_num*jhdr_size;
1063 if (overlap > 0) {
1064 if (overlap % jhdr_size != 0) {
1065 panic("jnl: do_overlap: overlap of %lld is not multiple of %zd\n", overlap, jhdr_size);
1066 }
1067
1068 // if we partially overlap this entry, adjust its block number, jnl offset, and size
1069 (*buf_ptr)[index].block_num += (overlap / jhdr_size); // make sure overlap is multiple of jhdr_size, or round up
1070 (*buf_ptr)[index].cksum = 0;
1071
1072 new_offset = (*buf_ptr)[index].jnl_offset + overlap; // check for wrap-around
1073 if ((off_t)new_offset >= jnl->jhdr->size) {
1074 new_offset = jhdr_size + (new_offset - jnl->jhdr->size);
1075 }
1076 (*buf_ptr)[index].jnl_offset = (uint32_t)new_offset;
1077
1078 (*buf_ptr)[index].block_size -= overlap; // sanity check for negative value
1079 if ((*buf_ptr)[index].block_size <= 0) {
1080 panic("jnl: do_overlap: after overlap, new block size is invalid (%u)\n", (*buf_ptr)[index].block_size);
1081 // return -1; // if above panic is removed, return -1 for error
1082 }
1083 }
1084
1085 }
1086
1087 index++;
1088 }
1089
1090 // bcopy over any completely overlapped entries, starting at the right (where the above loop broke out)
1091 index--; // start with the last index used within the above loop
1092 while (index >= blk_index) {
1093 if ((*buf_ptr)[index].block_num == -2) {
1094 if (index == *num_full_ptr-1) {
1095 (*buf_ptr)[index].block_num = -1; // it's the last item in the table... just mark as free
1096 } else {
1097 bcopy( (*buf_ptr)+(index+1), (*buf_ptr)+(index), (*num_full_ptr - (index + 1)) * sizeof(struct bucket) );
1098 }
1099 (*num_full_ptr)--;
1100 }
1101 index--;
1102 }
1103
1104 // eliminate any stale entries at the end of the table
1105 for(i = *num_full_ptr; i < (*num_full_ptr + num_to_remove); i++) {
1106 (*buf_ptr)[i].block_num = -1;
1107 }
1108
1109 return 0; // if we got this far, we need to insert the entry into the table (rather than overwrite)
1110 }
1111
1112 // PR-3105942: Coalesce writes to the same block in journal replay
1113 // We coalesce writes by maintaining a dynamic sorted array of physical disk blocks
1114 // to be replayed and the corresponding location in the journal which contains
1115 // the most recent data for those blocks. The array is "played" once the all the
1116 // blocks in the journal have been coalesced. The code for the case of conflicting/
1117 // overlapping writes to a single block is the most dense. Because coalescing can
1118 // disrupt the existing time-ordering of blocks in the journal playback, care
1119 // is taken to catch any overlaps and keep the array consistent.
1120 static int
1121 add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr)
1122 {
1123 int blk_index, overwriting;
1124
1125 // on return from lookup_bucket(), blk_index is the index into the table where block_num should be
1126 // inserted (or the index of the elem to overwrite).
1127 blk_index = lookup_bucket( buf_ptr, block_num, *num_full_ptr);
1128
1129 // check if the index is within bounds (if we're adding this block to the end of
1130 // the table, blk_index will be equal to num_full)
1131 if (blk_index < 0 || blk_index > *num_full_ptr) {
1132 //printf("jnl: add_block: trouble adding block to co_buf\n");
1133 return -1;
1134 } // else printf("jnl: add_block: adding block 0x%llx at i=%d\n", block_num, blk_index);
1135
1136 // Determine whether we're overwriting an existing entry by checking for overlap
1137 overwriting = do_overlap(jnl, buf_ptr, blk_index, block_num, size, offset, cksum, num_buckets_ptr, num_full_ptr);
1138 if (overwriting < 0) {
1139 return -1; // if we got an error, pass it along
1140 }
1141
1142 // returns the index, or -1 on error
1143 blk_index = insert_block(jnl, buf_ptr, blk_index, block_num, size, offset, cksum, num_buckets_ptr, num_full_ptr, overwriting);
1144
1145 return blk_index;
1146 }
1147
1148 static int
1149 replay_journal(journal *jnl)
1150 {
1151 int i, bad_blocks=0;
1152 unsigned int orig_checksum, checksum, check_block_checksums = 0;
1153 size_t ret;
1154 size_t max_bsize = 0; /* protected by block_ptr */
1155 block_list_header *blhdr;
1156 off_t offset, txn_start_offset=0, blhdr_offset, orig_jnl_start;
1157 char *buff, *block_ptr=NULL;
1158 struct bucket *co_buf;
1159 int num_buckets = STARTING_BUCKETS, num_full, check_past_jnl_end = 1, in_uncharted_territory=0;
1160 uint32_t last_sequence_num = 0;
1161 int replay_retry_count = 0;
1162
1163 // wrap the start ptr if it points to the very end of the journal
1164 if (jnl->jhdr->start == jnl->jhdr->size) {
1165 jnl->jhdr->start = jnl->jhdr->jhdr_size;
1166 }
1167 if (jnl->jhdr->end == jnl->jhdr->size) {
1168 jnl->jhdr->end = jnl->jhdr->jhdr_size;
1169 }
1170
1171 if (jnl->jhdr->start == jnl->jhdr->end) {
1172 return 0;
1173 }
1174
1175 orig_jnl_start = jnl->jhdr->start;
1176
1177 // allocate memory for the header_block. we'll read each blhdr into this
1178 buff = hfs_malloc(jnl->jhdr->blhdr_size);
1179
1180 // allocate memory for the coalesce buffer
1181 co_buf = hfs_malloc(num_buckets*sizeof(struct bucket));
1182
1183 restart_replay:
1184
1185 // initialize entries
1186 for(i = 0; i < num_buckets; i++) {
1187 co_buf[i].block_num = -1;
1188 }
1189 num_full = 0; // empty at first
1190
1191
1192 printf("jnl: %s: replay_journal: from: %lld to: %lld (joffset 0x%llx)\n",
1193 jnl->jdev_name, jnl->jhdr->start, jnl->jhdr->end, jnl->jdev_offset);
1194
1195 while (check_past_jnl_end || jnl->jhdr->start != jnl->jhdr->end) {
1196 offset = blhdr_offset = jnl->jhdr->start;
1197 ret = read_journal_data(jnl, &offset, buff, jnl->jhdr->blhdr_size);
1198 if (ret != (size_t)jnl->jhdr->blhdr_size) {
1199 printf("jnl: %s: replay_journal: Could not read block list header block @ 0x%llx!\n", jnl->jdev_name, offset);
1200 bad_blocks = 1;
1201 goto bad_txn_handling;
1202 }
1203
1204 blhdr = (block_list_header *)buff;
1205
1206 orig_checksum = blhdr->checksum;
1207 blhdr->checksum = 0;
1208 if (jnl->flags & JOURNAL_NEED_SWAP) {
1209 // calculate the checksum based on the unswapped data
1210 // because it is done byte-at-a-time.
1211 orig_checksum = (unsigned int)SWAP32(orig_checksum);
1212 checksum = calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE);
1213 swap_block_list_header(jnl, blhdr);
1214 } else {
1215 checksum = calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE);
1216 }
1217
1218
1219 //
1220 // XXXdbg - if these checks fail, we should replay as much
1221 // we can in the hopes that it will still leave the
1222 // drive in a better state than if we didn't replay
1223 // anything
1224 //
1225 if (checksum != orig_checksum) {
1226 if (check_past_jnl_end && in_uncharted_territory) {
1227
1228 if (blhdr_offset != jnl->jhdr->end) {
1229 printf("jnl: %s: Extra txn replay stopped @ %lld / 0x%llx\n", jnl->jdev_name, blhdr_offset, blhdr_offset);
1230 }
1231
1232 check_past_jnl_end = 0;
1233 jnl->jhdr->end = blhdr_offset;
1234 continue;
1235 }
1236
1237 printf("jnl: %s: replay_journal: bad block list header @ 0x%llx (checksum 0x%x != 0x%x)\n",
1238 jnl->jdev_name, blhdr_offset, orig_checksum, checksum);
1239
1240 if (blhdr_offset == orig_jnl_start) {
1241 // if there's nothing in the journal at all, just bail out altogether.
1242 goto bad_replay;
1243 }
1244
1245 bad_blocks = 1;
1246 goto bad_txn_handling;
1247 }
1248
1249 if ( (last_sequence_num != 0)
1250 && (blhdr->binfo[0].u.bi.b.sequence_num != 0)
1251 && (blhdr->binfo[0].u.bi.b.sequence_num != last_sequence_num)
1252 && (blhdr->binfo[0].u.bi.b.sequence_num != last_sequence_num+1)) {
1253
1254 txn_start_offset = jnl->jhdr->end = blhdr_offset;
1255
1256 if (check_past_jnl_end) {
1257 check_past_jnl_end = 0;
1258 printf("jnl: %s: 2: extra replay stopped @ %lld / 0x%llx (seq %d < %d)\n",
1259 jnl->jdev_name, blhdr_offset, blhdr_offset, blhdr->binfo[0].u.bi.b.sequence_num, last_sequence_num);
1260 continue;
1261 }
1262
1263 printf("jnl: %s: txn sequence numbers out of order in txn @ %lld / %llx! (%d < %d)\n",
1264 jnl->jdev_name, blhdr_offset, blhdr_offset, blhdr->binfo[0].u.bi.b.sequence_num, last_sequence_num);
1265 bad_blocks = 1;
1266 goto bad_txn_handling;
1267 }
1268 last_sequence_num = blhdr->binfo[0].u.bi.b.sequence_num;
1269
1270 if (blhdr_offset >= jnl->jhdr->end && jnl->jhdr->start <= jnl->jhdr->end) {
1271 if (last_sequence_num == 0) {
1272 check_past_jnl_end = 0;
1273 printf("jnl: %s: pre-sequence-num-enabled txn's - can not go further than end (%lld %lld).\n",
1274 jnl->jdev_name, jnl->jhdr->start, jnl->jhdr->end);
1275 if (jnl->jhdr->start != jnl->jhdr->end) {
1276 jnl->jhdr->start = jnl->jhdr->end;
1277 }
1278 continue;
1279 }
1280 printf("jnl: %s: examining extra transactions starting @ %lld / 0x%llx\n", jnl->jdev_name, blhdr_offset, blhdr_offset);
1281 }
1282
1283 if ( blhdr->max_blocks <= 0 || blhdr->max_blocks > (jnl->jhdr->size/jnl->jhdr->jhdr_size)
1284 || blhdr->num_blocks <= 0 || blhdr->num_blocks > blhdr->max_blocks) {
1285 printf("jnl: %s: replay_journal: bad looking journal entry: max: %d num: %d\n",
1286 jnl->jdev_name, blhdr->max_blocks, blhdr->num_blocks);
1287 bad_blocks = 1;
1288 goto bad_txn_handling;
1289 }
1290
1291 max_bsize = 0;
1292 for (i = 1; i < blhdr->num_blocks; i++) {
1293 if (blhdr->binfo[i].bnum < 0 && blhdr->binfo[i].bnum != (off_t)-1) {
1294 printf("jnl: %s: replay_journal: bogus block number 0x%llx\n", jnl->jdev_name, blhdr->binfo[i].bnum);
1295 bad_blocks = 1;
1296 goto bad_txn_handling;
1297 }
1298
1299 if ((size_t)blhdr->binfo[i].u.bi.bsize > max_bsize) {
1300 max_bsize = blhdr->binfo[i].u.bi.bsize;
1301 }
1302 }
1303
1304 if (blhdr->flags & BLHDR_CHECK_CHECKSUMS) {
1305 check_block_checksums = 1;
1306 block_ptr = hfs_malloc(max_bsize);
1307 } else {
1308 block_ptr = NULL;
1309 }
1310
1311 if (blhdr->flags & BLHDR_FIRST_HEADER) {
1312 txn_start_offset = blhdr_offset;
1313 }
1314
1315 //printf("jnl: replay_journal: adding %d blocks in journal entry @ 0x%llx to co_buf\n",
1316 // blhdr->num_blocks-1, jnl->jhdr->start);
1317 bad_blocks = 0;
1318 for (i = 1; i < blhdr->num_blocks; i++) {
1319 int size, ret_val;
1320 off_t number;
1321
1322 size = blhdr->binfo[i].u.bi.bsize;
1323 number = blhdr->binfo[i].bnum;
1324
1325 // don't add "killed" blocks
1326 if (number == (off_t)-1) {
1327 //printf("jnl: replay_journal: skipping killed fs block (index %d)\n", i);
1328 } else {
1329
1330 if (check_block_checksums) {
1331 int32_t disk_cksum;
1332 off_t block_offset;
1333
1334 block_offset = offset;
1335
1336 // read the block so we can check the checksum
1337 ret = read_journal_data(jnl, &block_offset, block_ptr, size);
1338 if (ret != (size_t)size) {
1339 printf("jnl: %s: replay_journal: Could not read journal entry data @ offset 0x%llx!\n", jnl->jdev_name, offset);
1340 bad_blocks = 1;
1341 goto bad_txn_handling;
1342 }
1343
1344 disk_cksum = calc_checksum(block_ptr, size);
1345
1346 // there is no need to swap the checksum from disk because
1347 // it got swapped when the blhdr was read in.
1348 if (blhdr->binfo[i].u.bi.b.cksum != 0 && disk_cksum != blhdr->binfo[i].u.bi.b.cksum) {
1349 printf("jnl: %s: txn starting at %lld (%lld) @ index %3d bnum %lld (%d) with disk cksum != blhdr cksum (0x%.8x 0x%.8x)\n",
1350 jnl->jdev_name, txn_start_offset, blhdr_offset, i, number, size, disk_cksum, blhdr->binfo[i].u.bi.b.cksum);
1351 printf("jnl: 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x\n",
1352 *(int *)&block_ptr[0*sizeof(int)], *(int *)&block_ptr[1*sizeof(int)], *(int *)&block_ptr[2*sizeof(int)], *(int *)&block_ptr[3*sizeof(int)],
1353 *(int *)&block_ptr[4*sizeof(int)], *(int *)&block_ptr[5*sizeof(int)], *(int *)&block_ptr[6*sizeof(int)], *(int *)&block_ptr[7*sizeof(int)]);
1354
1355 bad_blocks = 1;
1356 goto bad_txn_handling;
1357 }
1358 }
1359
1360
1361 // add this bucket to co_buf, coalescing where possible
1362 // printf("jnl: replay_journal: adding block 0x%llx\n", number);
1363 ret_val = add_block(jnl, &co_buf, number, size, (size_t) offset, blhdr->binfo[i].u.bi.b.cksum, &num_buckets, &num_full);
1364
1365 if (ret_val == -1) {
1366 printf("jnl: %s: replay_journal: trouble adding block to co_buf\n", jnl->jdev_name);
1367 goto bad_replay;
1368 } // else printf("jnl: replay_journal: added block 0x%llx at i=%d\n", number);
1369 }
1370
1371 // increment offset
1372 offset += size;
1373
1374 // check if the last block added puts us off the end of the jnl.
1375 // if so, we need to wrap to the beginning and take any remainder
1376 // into account
1377 //
1378 if (offset >= jnl->jhdr->size) {
1379 offset = jnl->jhdr->jhdr_size + (offset - jnl->jhdr->size);
1380 }
1381 }
1382
1383 if (block_ptr) {
1384 hfs_free(block_ptr, max_bsize);
1385 block_ptr = NULL;
1386 }
1387
1388 if (bad_blocks) {
1389 bad_txn_handling:
1390 /* Journal replay got error before it found any valid
1391 * transations, abort replay */
1392 if (txn_start_offset == 0) {
1393 printf("jnl: %s: no known good txn start offset! aborting journal replay.\n", jnl->jdev_name);
1394 goto bad_replay;
1395 }
1396
1397 /* Repeated error during journal replay, abort replay */
1398 if (replay_retry_count == 3) {
1399 printf("jnl: %s: repeated errors replaying journal! aborting journal replay.\n", jnl->jdev_name);
1400 goto bad_replay;
1401 }
1402 replay_retry_count++;
1403
1404 /* There was an error replaying the journal (possibly
1405 * EIO/ENXIO from the device). So retry replaying all
1406 * the good transactions that we found before getting
1407 * the error.
1408 */
1409 jnl->jhdr->start = orig_jnl_start;
1410 jnl->jhdr->end = txn_start_offset;
1411 check_past_jnl_end = 0;
1412 last_sequence_num = 0;
1413 printf("jnl: %s: restarting journal replay (%lld - %lld)!\n", jnl->jdev_name, jnl->jhdr->start, jnl->jhdr->end);
1414 goto restart_replay;
1415 }
1416
1417 jnl->jhdr->start += blhdr->bytes_used;
1418 if (jnl->jhdr->start >= jnl->jhdr->size) {
1419 // wrap around and skip the journal header block
1420 jnl->jhdr->start = (jnl->jhdr->start % jnl->jhdr->size) + jnl->jhdr->jhdr_size;
1421 }
1422
1423 if (jnl->jhdr->start == jnl->jhdr->end) {
1424 in_uncharted_territory = 1;
1425 }
1426 }
1427
1428 if (jnl->jhdr->start != jnl->jhdr->end) {
1429 printf("jnl: %s: start %lld != end %lld. resetting end.\n", jnl->jdev_name, jnl->jhdr->start, jnl->jhdr->end);
1430 jnl->jhdr->end = jnl->jhdr->start;
1431 }
1432
1433 //printf("jnl: replay_journal: replaying %d blocks\n", num_full);
1434
1435 /*
1436 * make sure it's at least one page in size, so
1437 * start max_bsize at PAGE_SIZE
1438 */
1439 for (i = 0, max_bsize = PAGE_SIZE; i < num_full; i++) {
1440
1441 if (co_buf[i].block_num == (off_t)-1)
1442 continue;
1443
1444 if (co_buf[i].block_size > max_bsize)
1445 max_bsize = co_buf[i].block_size;
1446 }
1447 /*
1448 * round max_bsize up to the nearest PAGE_SIZE multiple
1449 */
1450 if (max_bsize & (PAGE_SIZE - 1)) {
1451 max_bsize = (max_bsize + PAGE_SIZE) & ~(PAGE_SIZE - 1);
1452 }
1453
1454 block_ptr = hfs_malloc(max_bsize);
1455
1456 // Replay the coalesced entries in the co-buf
1457 for(i = 0; i < num_full; i++) {
1458 size_t size = co_buf[i].block_size;
1459 off_t jnl_offset = (off_t) co_buf[i].jnl_offset;
1460 off_t number = co_buf[i].block_num;
1461
1462
1463 // printf("replaying co_buf[%d]: block 0x%llx, size 0x%x, jnl_offset 0x%llx\n", i, co_buf[i].block_num,
1464 // co_buf[i].block_size, co_buf[i].jnl_offset);
1465
1466 if (number == (off_t)-1) {
1467 // printf("jnl: replay_journal: skipping killed fs block\n");
1468 } else {
1469
1470 // do journal read, and set the phys. block
1471 ret = read_journal_data(jnl, &jnl_offset, block_ptr, size);
1472 if (ret != size) {
1473 printf("jnl: %s: replay_journal: Could not read journal entry data @ offset 0x%llx!\n", jnl->jdev_name, jnl_offset);
1474 goto bad_replay;
1475 }
1476
1477 if (update_fs_block(jnl, block_ptr, number, size) != 0) {
1478 goto bad_replay;
1479 }
1480 }
1481 }
1482
1483
1484 // done replaying; update jnl header
1485 if (write_journal_header(jnl, 1, jnl->jhdr->sequence_num) != 0) {
1486 goto bad_replay;
1487 }
1488
1489 printf("jnl: %s: journal replay done.\n", jnl->jdev_name);
1490
1491 // free block_ptr
1492 if (block_ptr) {
1493 hfs_free(block_ptr, max_bsize);
1494 block_ptr = NULL;
1495 }
1496
1497 // free the coalesce buffer
1498 hfs_free(co_buf, num_buckets*sizeof(struct bucket));
1499 co_buf = NULL;
1500
1501 hfs_free(buff, jnl->jhdr->blhdr_size);
1502 return 0;
1503
1504 bad_replay:
1505 hfs_free(block_ptr, max_bsize);
1506 hfs_free(co_buf, num_buckets*sizeof(struct bucket));
1507 hfs_free(buff, jnl->jhdr->blhdr_size);
1508
1509 return -1;
1510 }
1511
1512
1513 #define DEFAULT_TRANSACTION_BUFFER_SIZE (128*1024)
1514 #define MAX_TRANSACTION_BUFFER_SIZE (3072*1024)
1515
1516 // XXXdbg - so I can change it in the debugger
1517 int def_tbuffer_size = 0;
1518
1519
1520 //
1521 // This function sets the size of the tbuffer and the
1522 // size of the blhdr. It assumes that jnl->jhdr->size
1523 // and jnl->jhdr->jhdr_size are already valid.
1524 //
1525 static void
1526 size_up_tbuffer(journal *jnl, int tbuffer_size, int phys_blksz)
1527 {
1528 //
1529 // one-time initialization based on how much memory
1530 // there is in the machine.
1531 //
1532 if (def_tbuffer_size == 0) {
1533 uint64_t memsize = 0;
1534 size_t l = sizeof(memsize);
1535 sysctlbyname("hw.memsize", &memsize, &l, NULL, 0);
1536
1537 if (memsize < (256*1024*1024)) {
1538 def_tbuffer_size = DEFAULT_TRANSACTION_BUFFER_SIZE;
1539 } else if (memsize < (512*1024*1024)) {
1540 def_tbuffer_size = DEFAULT_TRANSACTION_BUFFER_SIZE * 2;
1541 } else if (memsize < (1024*1024*1024)) {
1542 def_tbuffer_size = DEFAULT_TRANSACTION_BUFFER_SIZE * 3;
1543 } else {
1544 def_tbuffer_size = DEFAULT_TRANSACTION_BUFFER_SIZE * (memsize / (256*1024*1024));
1545 }
1546 }
1547
1548 // For analyzer
1549 hfs_assert(jnl->jhdr->jhdr_size > 0);
1550
1551 // size up the transaction buffer... can't be larger than the number
1552 // of blocks that can fit in a block_list_header block.
1553 if (tbuffer_size == 0) {
1554 jnl->tbuffer_size = def_tbuffer_size;
1555 } else {
1556 // make sure that the specified tbuffer_size isn't too small
1557 if (tbuffer_size < jnl->jhdr->blhdr_size * 2) {
1558 tbuffer_size = jnl->jhdr->blhdr_size * 2;
1559 }
1560 // and make sure it's an even multiple of the block size
1561 if ((tbuffer_size % jnl->jhdr->jhdr_size) != 0) {
1562 tbuffer_size -= (tbuffer_size % jnl->jhdr->jhdr_size);
1563 }
1564
1565 jnl->tbuffer_size = tbuffer_size;
1566 }
1567
1568 if (jnl->tbuffer_size > (jnl->jhdr->size / 2)) {
1569 jnl->tbuffer_size = (jnl->jhdr->size / 2);
1570 }
1571
1572 if (jnl->tbuffer_size > MAX_TRANSACTION_BUFFER_SIZE) {
1573 jnl->tbuffer_size = MAX_TRANSACTION_BUFFER_SIZE;
1574 }
1575
1576 jnl->jhdr->blhdr_size = (jnl->tbuffer_size / jnl->jhdr->jhdr_size) * sizeof(block_info);
1577 if (jnl->jhdr->blhdr_size < phys_blksz) {
1578 jnl->jhdr->blhdr_size = phys_blksz;
1579 } else if ((jnl->jhdr->blhdr_size % phys_blksz) != 0) {
1580 // have to round up so we're an even multiple of the physical block size
1581 jnl->jhdr->blhdr_size = (jnl->jhdr->blhdr_size + (phys_blksz - 1)) & ~(phys_blksz - 1);
1582 }
1583 }
1584
1585 static void
1586 get_io_info(struct vnode *devvp, size_t phys_blksz, journal *jnl, struct vfs_context *context)
1587 {
1588 off_t readblockcnt;
1589 off_t writeblockcnt;
1590 off_t readmaxcnt=0, tmp_readmaxcnt;
1591 off_t writemaxcnt=0, tmp_writemaxcnt;
1592 off_t readsegcnt, writesegcnt;
1593 int32_t features;
1594
1595 if (VNOP_IOCTL(devvp, DKIOCGETFEATURES, (caddr_t)&features, 0, context) == 0) {
1596 if (features & DK_FEATURE_FORCE_UNIT_ACCESS) {
1597 const char *name = vnode_getname_printable(devvp);
1598 jnl->flags |= JOURNAL_DO_FUA_WRITES;
1599 printf("jnl: %s: enabling FUA writes (features 0x%x)\n", name, features);
1600 vnode_putname_printable(name);
1601 }
1602 if (features & DK_FEATURE_UNMAP) {
1603 jnl->flags |= JOURNAL_USE_UNMAP;
1604 }
1605
1606 if (features & DK_FEATURE_BARRIER) {
1607 jnl->flags |= JOURNAL_FEATURE_BARRIER;
1608 }
1609 }
1610
1611 //
1612 // First check the max read size via several different mechanisms...
1613 //
1614 VNOP_IOCTL(devvp, DKIOCGETMAXBYTECOUNTREAD, (caddr_t)&readmaxcnt, 0, context);
1615
1616 if (VNOP_IOCTL(devvp, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t)&readblockcnt, 0, context) == 0) {
1617 tmp_readmaxcnt = readblockcnt * phys_blksz;
1618 if (readmaxcnt == 0 || (readblockcnt > 0 && tmp_readmaxcnt < readmaxcnt)) {
1619 readmaxcnt = tmp_readmaxcnt;
1620 }
1621 }
1622
1623 if (VNOP_IOCTL(devvp, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t)&readsegcnt, 0, context)) {
1624 readsegcnt = 0;
1625 }
1626
1627 if (readsegcnt > 0 && (readsegcnt * PAGE_SIZE) < readmaxcnt) {
1628 readmaxcnt = readsegcnt * PAGE_SIZE;
1629 }
1630
1631 if (readmaxcnt == 0) {
1632 readmaxcnt = 128 * 1024;
1633 } else if (readmaxcnt > UINT32_MAX) {
1634 readmaxcnt = UINT32_MAX;
1635 }
1636
1637
1638 //
1639 // Now check the max writes size via several different mechanisms...
1640 //
1641 VNOP_IOCTL(devvp, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t)&writemaxcnt, 0, context);
1642
1643 if (VNOP_IOCTL(devvp, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t)&writeblockcnt, 0, context) == 0) {
1644 tmp_writemaxcnt = writeblockcnt * phys_blksz;
1645 if (writemaxcnt == 0 || (writeblockcnt > 0 && tmp_writemaxcnt < writemaxcnt)) {
1646 writemaxcnt = tmp_writemaxcnt;
1647 }
1648 }
1649
1650 if (VNOP_IOCTL(devvp, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t)&writesegcnt, 0, context)) {
1651 writesegcnt = 0;
1652 }
1653
1654 if (writesegcnt > 0 && (writesegcnt * PAGE_SIZE) < writemaxcnt) {
1655 writemaxcnt = writesegcnt * PAGE_SIZE;
1656 }
1657
1658 if (writemaxcnt == 0) {
1659 writemaxcnt = 128 * 1024;
1660 } else if (writemaxcnt > UINT32_MAX) {
1661 writemaxcnt = UINT32_MAX;
1662 }
1663
1664 jnl->max_read_size = readmaxcnt;
1665 jnl->max_write_size = writemaxcnt;
1666 // printf("jnl: %s: max read/write: %lld k / %lld k\n",
1667 // jnl->jdev_name ? jnl->jdev_name : "unknown",
1668 // jnl->max_read_size/1024, jnl->max_write_size/1024);
1669 }
1670
1671
1672 journal *
1673 journal_create(struct vnode *jvp,
1674 off_t offset,
1675 off_t journal_size,
1676 struct vnode *fsvp,
1677 size_t min_fs_blksz,
1678 int32_t flags,
1679 int32_t tbuffer_size,
1680 void (*flush)(void *arg),
1681 void *arg,
1682 struct mount *fsmount)
1683 {
1684 journal *jnl;
1685 uint32_t phys_blksz, new_txn_base;
1686 u_int32_t min_size;
1687 const char *jdev_name;
1688 /*
1689 * Cap the journal max size to 2GB. On HFS, it will attempt to occupy
1690 * a full allocation block if the current size is smaller than the allocation
1691 * block on which it resides. Once we hit the exabyte filesystem range, then
1692 * it will use 2GB allocation blocks. As a result, make the cap 2GB.
1693 */
1694
1695 jdev_name = vnode_getname_printable(jvp);
1696
1697 /* Get the real physical block size. */
1698 if (VNOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, vfs_context_kernel())) {
1699 goto cleanup_jdev_name;
1700 }
1701
1702 if (journal_size < (256*1024) || journal_size > (MAX_JOURNAL_SIZE)) {
1703 printf("jnl: %s: create: journal size %lld looks bogus.\n", jdev_name, journal_size);
1704 goto cleanup_jdev_name;
1705 }
1706
1707 min_size = phys_blksz * (phys_blksz / sizeof(block_info));
1708 /* Reject journals that are too small given the sector size of the device */
1709 if (journal_size < min_size) {
1710 printf("jnl: %s: create: journal size (%lld) too small given sector size of (%u)\n",
1711 jdev_name, journal_size, phys_blksz);
1712 goto cleanup_jdev_name;
1713 }
1714
1715 if (phys_blksz > min_fs_blksz) {
1716 printf("jnl: %s: create: error: phys blksize %u bigger than min fs blksize %zd\n",
1717 jdev_name, phys_blksz, min_fs_blksz);
1718 goto cleanup_jdev_name;
1719 }
1720
1721 if ((journal_size % phys_blksz) != 0) {
1722 printf("jnl: %s: create: journal size 0x%llx is not an even multiple of block size 0x%ux\n",
1723 jdev_name, journal_size, phys_blksz);
1724 goto cleanup_jdev_name;
1725 }
1726
1727
1728 jnl = hfs_mallocz(sizeof(struct journal));
1729
1730 jnl->jdev = jvp;
1731 jnl->jdev_offset = offset;
1732 jnl->fsdev = fsvp;
1733 jnl->flush = flush;
1734 jnl->flush_arg = arg;
1735 jnl->flags = (flags & JOURNAL_OPTION_FLAGS_MASK);
1736 jnl->jdev_name = jdev_name;
1737 lck_mtx_init(&jnl->old_start_lock, jnl_mutex_group, jnl_lock_attr);
1738
1739 // Keep a point to the mount around for use in IO throttling.
1740 jnl->fsmount = fsmount;
1741
1742 get_io_info(jvp, phys_blksz, jnl, vfs_context_kernel());
1743
1744 jnl->header_buf = hfs_malloc(phys_blksz);
1745 jnl->header_buf_size = phys_blksz;
1746
1747 jnl->jhdr = (journal_header *)jnl->header_buf;
1748 memset(jnl->jhdr, 0, sizeof(journal_header));
1749
1750 // we have to set this up here so that do_journal_io() will work
1751 jnl->jhdr->jhdr_size = phys_blksz;
1752
1753 //
1754 // We try and read the journal header to see if there is already one
1755 // out there. If there is, it's possible that it has transactions
1756 // in it that we might replay if we happen to pick a sequence number
1757 // that is a little less than the old one, there is a crash and the
1758 // last txn written ends right at the start of a txn from the previous
1759 // incarnation of this file system. If all that happens we would
1760 // replay the transactions from the old file system and that would
1761 // destroy your disk. Although it is extremely unlikely for all those
1762 // conditions to happen, the probability is non-zero and the result is
1763 // severe - you lose your file system. Therefore if we find a valid
1764 // journal header and the sequence number is non-zero we write junk
1765 // over the entire journal so that there is no way we will encounter
1766 // any old transactions. This is slow but should be a rare event
1767 // since most tools erase the journal.
1768 //
1769 if ( read_journal_header(jnl, jnl->jhdr, phys_blksz) == phys_blksz
1770 && jnl->jhdr->magic == JOURNAL_HEADER_MAGIC
1771 && jnl->jhdr->sequence_num != 0) {
1772
1773 new_txn_base = (jnl->jhdr->sequence_num + (journal_size / phys_blksz) + (random() % 16384)) & 0x00ffffff;
1774 printf("jnl: %s: create: avoiding old sequence number 0x%x (0x%x)\n", jdev_name, jnl->jhdr->sequence_num, new_txn_base);
1775
1776 #if 0
1777 int i;
1778 off_t pos=0;
1779
1780 for(i = 1; i < journal_size / phys_blksz; i++) {
1781 pos = i*phys_blksz;
1782
1783 // we don't really care what data we write just so long
1784 // as it's not a valid transaction header. since we have
1785 // the header_buf sitting around we'll use that.
1786 write_journal_data(jnl, &pos, jnl->header_buf, phys_blksz);
1787 }
1788 printf("jnl: create: done clearing journal (i=%d)\n", i);
1789 #endif
1790 } else {
1791 new_txn_base = random() & 0x00ffffff;
1792 }
1793
1794 memset(jnl->header_buf, 0, phys_blksz);
1795
1796 jnl->jhdr->magic = JOURNAL_HEADER_MAGIC;
1797 jnl->jhdr->endian = ENDIAN_MAGIC;
1798 jnl->jhdr->start = phys_blksz; // start at block #1, block #0 is for the jhdr itself
1799 jnl->jhdr->end = phys_blksz;
1800 jnl->jhdr->size = journal_size;
1801 jnl->jhdr->jhdr_size = phys_blksz;
1802 size_up_tbuffer(jnl, tbuffer_size, phys_blksz);
1803
1804 jnl->active_start = jnl->jhdr->start;
1805
1806 // XXXdbg - for testing you can force the journal to wrap around
1807 // jnl->jhdr->start = jnl->jhdr->size - (phys_blksz*3);
1808 // jnl->jhdr->end = jnl->jhdr->size - (phys_blksz*3);
1809
1810 jnl->jhdr->sequence_num = new_txn_base;
1811
1812 lck_mtx_init(&jnl->jlock, jnl_mutex_group, jnl_lock_attr);
1813 lck_mtx_init(&jnl->flock, jnl_mutex_group, jnl_lock_attr);
1814 lck_rw_init(&jnl->trim_lock, jnl_mutex_group, jnl_lock_attr);
1815
1816
1817 jnl->flushing = FALSE;
1818 jnl->asyncIO = FALSE;
1819 jnl->flush_aborted = FALSE;
1820 jnl->writing_header = FALSE;
1821 jnl->async_trim = NULL;
1822 jnl->sequence_num = jnl->jhdr->sequence_num;
1823
1824 if (write_journal_header(jnl, 1, jnl->jhdr->sequence_num) != 0) {
1825 printf("jnl: %s: journal_create: failed to write journal header.\n", jdev_name);
1826 goto bad_write;
1827 }
1828
1829 goto journal_create_complete;
1830
1831
1832 bad_write:
1833 hfs_free(jnl->header_buf, phys_blksz);
1834 jnl->jhdr = NULL;
1835 hfs_free(jnl, sizeof(*jnl));
1836 cleanup_jdev_name:
1837 vnode_putname_printable(jdev_name);
1838 jnl = NULL;
1839 journal_create_complete:
1840 return jnl;
1841 }
1842
1843
1844 journal *
1845 journal_open(struct vnode *jvp,
1846 off_t offset,
1847 off_t journal_size,
1848 struct vnode *fsvp,
1849 size_t min_fs_blksz,
1850 int32_t flags,
1851 int32_t tbuffer_size,
1852 void (*flush)(void *arg),
1853 void *arg,
1854 struct mount *fsmount)
1855 {
1856 journal *jnl;
1857 uint32_t orig_blksz=0;
1858 uint32_t phys_blksz;
1859 u_int32_t min_size = 0;
1860 int orig_checksum, checksum;
1861 const char *jdev_name = vnode_getname_printable(jvp);
1862
1863 /* Get the real physical block size. */
1864 if (VNOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, vfs_context_kernel())) {
1865 goto cleanup_jdev_name;
1866 }
1867
1868 if (phys_blksz > min_fs_blksz) {
1869 printf("jnl: %s: open: error: phys blksize %u bigger than min fs blksize %zd\n",
1870 jdev_name, phys_blksz, min_fs_blksz);
1871 goto cleanup_jdev_name;
1872 }
1873
1874 if (journal_size < (256*1024) || journal_size > (1024*1024*1024)) {
1875 printf("jnl: %s: open: journal size %lld looks bogus.\n", jdev_name, journal_size);
1876 goto cleanup_jdev_name;
1877 }
1878
1879 min_size = phys_blksz * (phys_blksz / sizeof(block_info));
1880 /* Reject journals that are too small given the sector size of the device */
1881 if (journal_size < min_size) {
1882 printf("jnl: %s: open: journal size (%lld) too small given sector size of (%u)\n",
1883 jdev_name, journal_size, phys_blksz);
1884 goto cleanup_jdev_name;
1885 }
1886
1887 if ((journal_size % phys_blksz) != 0) {
1888 printf("jnl: %s: open: journal size 0x%llx is not an even multiple of block size 0x%x\n",
1889 jdev_name, journal_size, phys_blksz);
1890 goto cleanup_jdev_name;
1891 }
1892
1893 jnl = hfs_mallocz(sizeof(struct journal));
1894
1895 jnl->jdev = jvp;
1896 jnl->jdev_offset = offset;
1897 jnl->fsdev = fsvp;
1898 jnl->flush = flush;
1899 jnl->flush_arg = arg;
1900 jnl->flags = (flags & JOURNAL_OPTION_FLAGS_MASK);
1901 jnl->jdev_name = jdev_name;
1902 lck_mtx_init(&jnl->old_start_lock, jnl_mutex_group, jnl_lock_attr);
1903
1904 /* We hold the mount to later pass to the throttling code for IO
1905 * accounting.
1906 */
1907 jnl->fsmount = fsmount;
1908
1909 get_io_info(jvp, phys_blksz, jnl, vfs_context_kernel());
1910
1911 jnl->header_buf = hfs_malloc(phys_blksz);
1912 jnl->header_buf_size = phys_blksz;
1913
1914 jnl->jhdr = (journal_header *)jnl->header_buf;
1915 memset(jnl->jhdr, 0, sizeof(journal_header));
1916
1917 // we have to set this up here so that do_journal_io() will work
1918 jnl->jhdr->jhdr_size = phys_blksz;
1919
1920 if (read_journal_header(jnl, jnl->jhdr, phys_blksz) != phys_blksz) {
1921 printf("jnl: %s: open: could not read %u bytes for the journal header.\n",
1922 jdev_name, phys_blksz);
1923 goto bad_journal;
1924 }
1925
1926 orig_checksum = jnl->jhdr->checksum;
1927 jnl->jhdr->checksum = 0;
1928
1929 if (jnl->jhdr->magic == SWAP32(JOURNAL_HEADER_MAGIC)) {
1930 // do this before the swap since it's done byte-at-a-time
1931 orig_checksum = SWAP32(orig_checksum);
1932 checksum = calc_checksum((char *)jnl->jhdr, JOURNAL_HEADER_CKSUM_SIZE);
1933 swap_journal_header(jnl);
1934 jnl->flags |= JOURNAL_NEED_SWAP;
1935 } else {
1936 checksum = calc_checksum((char *)jnl->jhdr, JOURNAL_HEADER_CKSUM_SIZE);
1937 }
1938
1939 if (jnl->jhdr->magic != JOURNAL_HEADER_MAGIC && jnl->jhdr->magic != OLD_JOURNAL_HEADER_MAGIC) {
1940 printf("jnl: %s: open: journal magic is bad (0x%x != 0x%x)\n",
1941 jnl->jdev_name, jnl->jhdr->magic, JOURNAL_HEADER_MAGIC);
1942 goto bad_journal;
1943 }
1944
1945 // only check if we're the current journal header magic value
1946 if (jnl->jhdr->magic == JOURNAL_HEADER_MAGIC) {
1947
1948 if (orig_checksum != checksum) {
1949 printf("jnl: %s: open: journal checksum is bad (0x%x != 0x%x)\n",
1950 jdev_name, orig_checksum, checksum);
1951
1952 //goto bad_journal;
1953 }
1954 }
1955
1956 // XXXdbg - convert old style magic numbers to the new one
1957 if (jnl->jhdr->magic == OLD_JOURNAL_HEADER_MAGIC) {
1958 jnl->jhdr->magic = JOURNAL_HEADER_MAGIC;
1959 }
1960
1961 if (phys_blksz != (size_t)jnl->jhdr->jhdr_size && jnl->jhdr->jhdr_size != 0) {
1962 /*
1963 * The volume has probably been resized (such that we had to adjust the
1964 * logical sector size), or copied to media with a different logical
1965 * sector size.
1966 *
1967 * Temporarily change the device's logical block size to match the
1968 * journal's header size. This will allow us to replay the journal
1969 * safely. If the replay succeeds, we will update the journal's header
1970 * size (later in this function).
1971 */
1972 orig_blksz = phys_blksz;
1973 phys_blksz = jnl->jhdr->jhdr_size;
1974 VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&phys_blksz, FWRITE, vfs_context_kernel());
1975 printf("jnl: %s: open: temporarily switched block size from %u to %u\n",
1976 jdev_name, orig_blksz, phys_blksz);
1977 }
1978
1979 if ( jnl->jhdr->start <= 0
1980 || jnl->jhdr->start > jnl->jhdr->size
1981 || jnl->jhdr->start > 1024*1024*1024) {
1982 printf("jnl: %s: open: jhdr start looks bad (0x%llx max size 0x%llx)\n",
1983 jdev_name, jnl->jhdr->start, jnl->jhdr->size);
1984 goto bad_journal;
1985 }
1986
1987 if ( jnl->jhdr->end <= 0
1988 || jnl->jhdr->end > jnl->jhdr->size
1989 || jnl->jhdr->end > 1024*1024*1024) {
1990 printf("jnl: %s: open: jhdr end looks bad (0x%llx max size 0x%llx)\n",
1991 jdev_name, jnl->jhdr->end, jnl->jhdr->size);
1992 goto bad_journal;
1993 }
1994
1995 if (jnl->jhdr->size < (256*1024) || jnl->jhdr->size > 1024*1024*1024) {
1996 printf("jnl: %s: open: jhdr size looks bad (0x%llx)\n", jdev_name, jnl->jhdr->size);
1997 goto bad_journal;
1998 }
1999
2000 // XXXdbg - can't do these checks because hfs writes all kinds of
2001 // non-uniform sized blocks even on devices that have a block size
2002 // that is larger than 512 bytes (i.e. optical media w/2k blocks).
2003 // therefore these checks will fail and so we just have to punt and
2004 // do more relaxed checking...
2005 // XXXdbg if ((jnl->jhdr->start % jnl->jhdr->jhdr_size) != 0) {
2006 if ((jnl->jhdr->start % 512) != 0) {
2007 printf("jnl: %s: open: journal start (0x%llx) not a multiple of 512?\n",
2008 jdev_name, jnl->jhdr->start);
2009 goto bad_journal;
2010 }
2011
2012 //XXXdbg if ((jnl->jhdr->end % jnl->jhdr->jhdr_size) != 0) {
2013 if ((jnl->jhdr->end % 512) != 0) {
2014 printf("jnl: %s: open: journal end (0x%llx) not a multiple of block size (0x%x)?\n",
2015 jdev_name, jnl->jhdr->end, jnl->jhdr->jhdr_size);
2016 goto bad_journal;
2017 }
2018
2019 // take care of replaying the journal if necessary
2020 if (flags & JOURNAL_RESET) {
2021 printf("jnl: %s: journal start/end pointers reset! (s 0x%llx e 0x%llx)\n",
2022 jdev_name, jnl->jhdr->start, jnl->jhdr->end);
2023 jnl->jhdr->start = jnl->jhdr->end;
2024 } else if (replay_journal(jnl) != 0) {
2025 printf("jnl: %s: journal_open: Error replaying the journal!\n", jdev_name);
2026 goto bad_journal;
2027 }
2028
2029 /*
2030 * When we get here, we know that the journal is empty (jnl->jhdr->start ==
2031 * jnl->jhdr->end). If the device's logical block size was different from
2032 * the journal's header size, then we can now restore the device's logical
2033 * block size and update the journal's header size to match.
2034 *
2035 * Note that we also adjust the journal's start and end so that they will
2036 * be aligned on the new block size. We pick a new sequence number to
2037 * avoid any problems if a replay found previous transactions using the old
2038 * journal header size. (See the comments in journal_create(), above.)
2039 */
2040
2041 if (orig_blksz != 0) {
2042 VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, vfs_context_kernel());
2043 phys_blksz = orig_blksz;
2044
2045 orig_blksz = 0;
2046
2047 jnl->jhdr->jhdr_size = phys_blksz;
2048 jnl->jhdr->start = phys_blksz;
2049 jnl->jhdr->end = phys_blksz;
2050 jnl->jhdr->sequence_num = (jnl->jhdr->sequence_num +
2051 (journal_size / phys_blksz) +
2052 (random() % 16384)) & 0x00ffffff;
2053
2054 if (write_journal_header(jnl, 1, jnl->jhdr->sequence_num)) {
2055 printf("jnl: %s: open: failed to update journal header size\n", jdev_name);
2056 goto bad_journal;
2057 }
2058 }
2059
2060 // make sure this is in sync!
2061 jnl->active_start = jnl->jhdr->start;
2062 jnl->sequence_num = jnl->jhdr->sequence_num;
2063
2064 // set this now, after we've replayed the journal
2065 size_up_tbuffer(jnl, tbuffer_size, phys_blksz);
2066
2067 // TODO: Does this need to change if the device's logical block size changed?
2068 if ((off_t)(jnl->jhdr->blhdr_size/sizeof(block_info)-1) > (jnl->jhdr->size/jnl->jhdr->jhdr_size)) {
2069 printf("jnl: %s: open: jhdr size and blhdr size are not compatible (0x%llx, %d, %d)\n", jdev_name, jnl->jhdr->size,
2070 jnl->jhdr->blhdr_size, jnl->jhdr->jhdr_size);
2071 goto bad_journal;
2072 }
2073
2074 lck_mtx_init(&jnl->jlock, jnl_mutex_group, jnl_lock_attr);
2075 lck_mtx_init(&jnl->flock, jnl_mutex_group, jnl_lock_attr);
2076 lck_rw_init(&jnl->trim_lock, jnl_mutex_group, jnl_lock_attr);
2077
2078 goto journal_open_complete;
2079
2080 bad_journal:
2081 if (orig_blksz != 0) {
2082 phys_blksz = orig_blksz;
2083 VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, vfs_context_kernel());
2084 printf("jnl: %s: open: restored block size after error\n", jdev_name);
2085 }
2086 hfs_free(jnl->header_buf, jnl->header_buf_size);
2087 hfs_free(jnl, sizeof(*jnl));
2088 cleanup_jdev_name:
2089 vnode_putname_printable(jdev_name);
2090 jnl = NULL;
2091 journal_open_complete:
2092 return jnl;
2093 }
2094
2095
2096 int
2097 journal_is_clean(struct vnode *jvp,
2098 off_t offset,
2099 off_t journal_size,
2100 struct vnode *fsvp,
2101 size_t min_fs_block_size)
2102 {
2103 journal jnl;
2104 uint32_t phys_blksz;
2105 int ret;
2106 int orig_checksum, checksum;
2107 const char *jdev_name = vnode_getname_printable(jvp);
2108
2109 /* Get the real physical block size. */
2110 if (VNOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, vfs_context_kernel())) {
2111 printf("jnl: %s: is_clean: failed to get device block size.\n", jdev_name);
2112 ret = EINVAL;
2113 goto cleanup_jdev_name;
2114 }
2115
2116 if (phys_blksz > (uint32_t)min_fs_block_size) {
2117 printf("jnl: %s: is_clean: error: phys blksize %d bigger than min fs blksize %zd\n",
2118 jdev_name, phys_blksz, min_fs_block_size);
2119 ret = EINVAL;
2120 goto cleanup_jdev_name;
2121 }
2122
2123 if (journal_size < (256*1024) || journal_size > (MAX_JOURNAL_SIZE)) {
2124 printf("jnl: %s: is_clean: journal size %lld looks bogus.\n", jdev_name, journal_size);
2125 ret = EINVAL;
2126 goto cleanup_jdev_name;
2127 }
2128
2129 if ((journal_size % phys_blksz) != 0) {
2130 printf("jnl: %s: is_clean: journal size 0x%llx is not an even multiple of block size 0x%x\n",
2131 jdev_name, journal_size, phys_blksz);
2132 ret = EINVAL;
2133 goto cleanup_jdev_name;
2134 }
2135
2136 memset(&jnl, 0, sizeof(jnl));
2137
2138 jnl.header_buf = hfs_malloc(phys_blksz);
2139 jnl.header_buf_size = phys_blksz;
2140
2141 get_io_info(jvp, phys_blksz, &jnl, vfs_context_kernel());
2142
2143 jnl.jhdr = (journal_header *)jnl.header_buf;
2144 memset(jnl.jhdr, 0, sizeof(journal_header));
2145
2146 jnl.jdev = jvp;
2147 jnl.jdev_offset = offset;
2148 jnl.fsdev = fsvp;
2149
2150 // we have to set this up here so that do_journal_io() will work
2151 jnl.jhdr->jhdr_size = phys_blksz;
2152
2153 if (read_journal_header(&jnl, jnl.jhdr, phys_blksz) != (unsigned)phys_blksz) {
2154 printf("jnl: %s: is_clean: could not read %d bytes for the journal header.\n",
2155 jdev_name, phys_blksz);
2156 ret = EINVAL;
2157 goto get_out;
2158 }
2159
2160 orig_checksum = jnl.jhdr->checksum;
2161 jnl.jhdr->checksum = 0;
2162
2163 if (jnl.jhdr->magic == SWAP32(JOURNAL_HEADER_MAGIC)) {
2164 // do this before the swap since it's done byte-at-a-time
2165 orig_checksum = SWAP32(orig_checksum);
2166 checksum = calc_checksum((char *)jnl.jhdr, JOURNAL_HEADER_CKSUM_SIZE);
2167 swap_journal_header(&jnl);
2168 jnl.flags |= JOURNAL_NEED_SWAP;
2169 } else {
2170 checksum = calc_checksum((char *)jnl.jhdr, JOURNAL_HEADER_CKSUM_SIZE);
2171 }
2172
2173 if (jnl.jhdr->magic != JOURNAL_HEADER_MAGIC && jnl.jhdr->magic != OLD_JOURNAL_HEADER_MAGIC) {
2174 printf("jnl: %s: is_clean: journal magic is bad (0x%x != 0x%x)\n",
2175 jdev_name, jnl.jhdr->magic, JOURNAL_HEADER_MAGIC);
2176 ret = EINVAL;
2177 goto get_out;
2178 }
2179
2180 if (orig_checksum != checksum) {
2181 printf("jnl: %s: is_clean: journal checksum is bad (0x%x != 0x%x)\n", jdev_name, orig_checksum, checksum);
2182 ret = EINVAL;
2183 goto get_out;
2184 }
2185
2186 //
2187 // if the start and end are equal then the journal is clean.
2188 // otherwise it's not clean and therefore an error.
2189 //
2190 if (jnl.jhdr->start == jnl.jhdr->end) {
2191 ret = 0;
2192 } else {
2193 ret = EBUSY; // so the caller can differentiate an invalid journal from a "busy" one
2194 }
2195
2196 get_out:
2197 hfs_free(jnl.header_buf, jnl.header_buf_size);
2198 cleanup_jdev_name:
2199 vnode_putname_printable(jdev_name);
2200 return ret;
2201 }
2202
2203
2204 void
2205 journal_close(journal *jnl)
2206 {
2207 volatile off_t *start, *end;
2208 int counter=0;
2209
2210 CHECK_JOURNAL(jnl);
2211
2212 // set this before doing anything that would block so that
2213 // we start tearing things down properly.
2214 //
2215 jnl->flags |= JOURNAL_CLOSE_PENDING;
2216
2217 if (jnl->owner != current_thread()) {
2218 journal_lock(jnl);
2219 }
2220
2221 wait_condition(jnl, &jnl->flushing, "journal_close");
2222
2223 //
2224 // only write stuff to disk if the journal is still valid
2225 //
2226 if ((jnl->flags & JOURNAL_INVALID) == 0) {
2227
2228 if (jnl->active_tr) {
2229 /*
2230 * "journal_end_transaction" will fire the flush asynchronously
2231 */
2232 journal_end_transaction(jnl);
2233 }
2234
2235 // flush any buffered transactions
2236 if (jnl->cur_tr) {
2237 transaction *tr = jnl->cur_tr;
2238
2239 jnl->cur_tr = NULL;
2240 /*
2241 * "end_transaction" will wait for any in-progress flush to complete
2242 * before flushing "cur_tr" synchronously("must_wait" == TRUE)
2243 */
2244 end_transaction(tr, 1, NULL, NULL, FALSE, TRUE);
2245 }
2246 /*
2247 * if there was an "active_tr", make sure we wait for
2248 * it to flush if there was no "cur_tr" to process
2249 */
2250 wait_condition(jnl, &jnl->flushing, "journal_close");
2251
2252 //start = &jnl->jhdr->start;
2253 start = &jnl->active_start;
2254 end = &jnl->jhdr->end;
2255
2256 while (*start != *end && counter++ < 5000) {
2257 //printf("jnl: close: flushing the buffer cache (start 0x%llx end 0x%llx)\n", *start, *end);
2258 if (jnl->flush) {
2259 jnl->flush(jnl->flush_arg);
2260 }
2261 tsleep((caddr_t)jnl, PRIBIO, "jnl_close", 2);
2262 }
2263
2264 if (*start != *end) {
2265 printf("jnl: %s: close: buffer flushing didn't seem to flush out all the transactions! (0x%llx - 0x%llx)\n",
2266 jnl->jdev_name, *start, *end);
2267 }
2268
2269 // make sure this is in sync when we close the journal
2270 jnl->jhdr->start = jnl->active_start;
2271
2272 // if this fails there's not much we can do at this point...
2273 write_journal_header(jnl, 1, jnl->sequence_num);
2274 } else {
2275 // if we're here the journal isn't valid any more.
2276 // so make sure we don't leave any locked blocks lying around
2277 printf("jnl: %s: close: journal is invalid. aborting outstanding transactions\n", jnl->jdev_name);
2278 if (jnl->active_tr || jnl->cur_tr) {
2279 transaction *tr;
2280
2281 if (jnl->active_tr) {
2282 tr = jnl->active_tr;
2283 jnl->active_tr = NULL;
2284 } else {
2285 tr = jnl->cur_tr;
2286 jnl->cur_tr = NULL;
2287 }
2288 abort_transaction(jnl, tr);
2289
2290 if (jnl->active_tr || jnl->cur_tr) {
2291 panic("jnl: %s: close: jnl @ %p had both an active and cur tr\n", jnl->jdev_name, jnl);
2292 }
2293 }
2294 }
2295 wait_condition(jnl, &jnl->asyncIO, "journal_close");
2296
2297 free_old_stuff(jnl);
2298
2299 hfs_free(jnl->header_buf, jnl->header_buf_size);
2300 jnl->jhdr = (void *)0xbeefbabe;
2301
2302 vnode_putname_printable(jnl->jdev_name);
2303
2304 journal_unlock(jnl);
2305 lck_mtx_destroy(&jnl->old_start_lock, jnl_mutex_group);
2306 lck_mtx_destroy(&jnl->jlock, jnl_mutex_group);
2307 lck_mtx_destroy(&jnl->flock, jnl_mutex_group);
2308 hfs_free(jnl, sizeof(*jnl));
2309 }
2310
2311 static void
2312 dump_journal(journal *jnl)
2313 {
2314 transaction *ctr;
2315
2316 printf("journal for dev %s:", jnl->jdev_name);
2317 printf(" jdev_offset %.8llx\n", jnl->jdev_offset);
2318 printf(" magic: 0x%.8x\n", jnl->jhdr->magic);
2319 printf(" start: 0x%.8llx\n", jnl->jhdr->start);
2320 printf(" end: 0x%.8llx\n", jnl->jhdr->end);
2321 printf(" size: 0x%.8llx\n", jnl->jhdr->size);
2322 printf(" blhdr size: %d\n", jnl->jhdr->blhdr_size);
2323 printf(" jhdr size: %d\n", jnl->jhdr->jhdr_size);
2324 printf(" chksum: 0x%.8x\n", jnl->jhdr->checksum);
2325
2326 printf(" completed transactions:\n");
2327 for (ctr = jnl->completed_trs; ctr; ctr = ctr->next) {
2328 printf(" 0x%.8llx - 0x%.8llx\n", ctr->journal_start, ctr->journal_end);
2329 }
2330 }
2331
2332
2333
2334 static off_t
2335 free_space(journal *jnl)
2336 {
2337 off_t free_space_offset;
2338
2339 if (jnl->jhdr->start < jnl->jhdr->end) {
2340 free_space_offset = jnl->jhdr->size - (jnl->jhdr->end - jnl->jhdr->start) - jnl->jhdr->jhdr_size;
2341 } else if (jnl->jhdr->start > jnl->jhdr->end) {
2342 free_space_offset = jnl->jhdr->start - jnl->jhdr->end;
2343 } else {
2344 // journal is completely empty
2345 free_space_offset = jnl->jhdr->size - jnl->jhdr->jhdr_size;
2346 }
2347
2348 return free_space_offset;
2349 }
2350
2351
2352 //
2353 // The journal must be locked on entry to this function.
2354 // The "desired_size" is in bytes.
2355 //
2356 static int
2357 check_free_space(journal *jnl, int desired_size, boolean_t *delayed_header_write, uint32_t sequence_num)
2358 {
2359 size_t i;
2360 int counter=0;
2361
2362 //printf("jnl: check free space (desired 0x%x, avail 0x%Lx)\n",
2363 // desired_size, free_space(jnl));
2364
2365 if (delayed_header_write)
2366 *delayed_header_write = FALSE;
2367
2368 while (1) {
2369 int old_start_empty;
2370
2371 // make sure there's space in the journal to hold this transaction
2372 if (free_space(jnl) > desired_size && jnl->old_start[0] == 0) {
2373 break;
2374 }
2375 if (counter++ == 5000) {
2376 dump_journal(jnl);
2377 panic("jnl: check_free_space: buffer flushing isn't working "
2378 "(jnl @ %p s %lld e %lld f %lld [active start %lld]).\n", jnl,
2379 jnl->jhdr->start, jnl->jhdr->end, free_space(jnl), jnl->active_start);
2380 }
2381 if (counter > 7500) {
2382 printf("jnl: %s: check_free_space: giving up waiting for free space.\n", jnl->jdev_name);
2383 return ENOSPC;
2384 }
2385
2386 //
2387 // here's where we lazily bump up jnl->jhdr->start. we'll consume
2388 // entries until there is enough space for the next transaction.
2389 //
2390 old_start_empty = 1;
2391 lock_oldstart(jnl);
2392
2393 for (i = 0; i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0]); i++) {
2394 int lcl_counter;
2395
2396 lcl_counter = 0;
2397 while (jnl->old_start[i] & 0x8000000000000000LL) {
2398 if (lcl_counter++ > 10000) {
2399 panic("jnl: check_free_space: tr starting @ 0x%llx not flushing (jnl %p).\n",
2400 jnl->old_start[i], jnl);
2401 }
2402
2403 unlock_oldstart(jnl);
2404 if (jnl->flush) {
2405 jnl->flush(jnl->flush_arg);
2406 }
2407 tsleep((caddr_t)jnl, PRIBIO, "check_free_space1", 1);
2408 lock_oldstart(jnl);
2409 }
2410
2411 if (jnl->old_start[i] == 0) {
2412 continue;
2413 }
2414
2415 old_start_empty = 0;
2416 jnl->jhdr->start = jnl->old_start[i];
2417 jnl->old_start[i] = 0;
2418
2419 if (free_space(jnl) > desired_size) {
2420
2421 if (delayed_header_write)
2422 *delayed_header_write = TRUE;
2423 else {
2424 unlock_oldstart(jnl);
2425 write_journal_header(jnl, 1, sequence_num);
2426 lock_oldstart(jnl);
2427 }
2428 break;
2429 }
2430 }
2431 unlock_oldstart(jnl);
2432
2433 // if we bumped the start, loop and try again
2434 if (i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0])) {
2435 continue;
2436 } else if (old_start_empty) {
2437 //
2438 // if there is nothing in old_start anymore then we can
2439 // bump the jhdr->start to be the same as active_start
2440 // since it is possible there was only one very large
2441 // transaction in the old_start array. if we didn't do
2442 // this then jhdr->start would never get updated and we
2443 // would wind up looping until we hit the panic at the
2444 // start of the loop.
2445 //
2446 jnl->jhdr->start = jnl->active_start;
2447
2448 if (delayed_header_write)
2449 *delayed_header_write = TRUE;
2450 else
2451 write_journal_header(jnl, 1, sequence_num);
2452 continue;
2453 }
2454
2455
2456 // if the file system gave us a flush function, call it to so that
2457 // it can flush some blocks which hopefully will cause some transactions
2458 // to complete and thus free up space in the journal.
2459 if (jnl->flush) {
2460 jnl->flush(jnl->flush_arg);
2461 }
2462
2463 // wait for a while to avoid being cpu-bound (this will
2464 // put us to sleep for 10 milliseconds)
2465 tsleep((caddr_t)jnl, PRIBIO, "check_free_space2", 1);
2466 }
2467
2468 return 0;
2469 }
2470
2471 /*
2472 * Allocate a new active transaction.
2473 */
2474 static errno_t
2475 journal_allocate_transaction(journal *jnl)
2476 {
2477 transaction *tr;
2478 boolean_t was_vm_privileged = FALSE;
2479
2480 if (vfs_isswapmount(jnl->fsmount)) {
2481 /*
2482 * the disk driver can allocate memory on this path...
2483 * if we block waiting for memory, and there is enough pressure to
2484 * cause us to try and create a new swap file, we may end up deadlocking
2485 * due to waiting for the journal on the swap file creation path...
2486 * by making ourselves vm_privileged, we give ourselves the best chance
2487 * of not blocking
2488 */
2489 was_vm_privileged = set_vm_privilege(TRUE);
2490 }
2491 tr = hfs_mallocz(sizeof(transaction));
2492
2493 tr->tbuffer_size = jnl->tbuffer_size;
2494
2495 tr->tbuffer = hfs_malloc(tr->tbuffer_size);
2496
2497 if (vfs_isswapmount(jnl->fsmount) && (was_vm_privileged == FALSE))
2498 set_vm_privilege(FALSE);
2499
2500 // journal replay code checksum check depends on this.
2501 memset(tr->tbuffer, 0, BLHDR_CHECKSUM_SIZE);
2502 // Fill up the rest of the block with unimportant bytes (0x5a 'Z' chosen for visibility)
2503 memset(tr->tbuffer + BLHDR_CHECKSUM_SIZE, 0x5a, jnl->jhdr->blhdr_size - BLHDR_CHECKSUM_SIZE);
2504
2505 tr->blhdr = (block_list_header *)tr->tbuffer;
2506 tr->blhdr->max_blocks = (jnl->jhdr->blhdr_size / sizeof(block_info)) - 1;
2507 tr->blhdr->num_blocks = 1; // accounts for this header block
2508 tr->blhdr->bytes_used = jnl->jhdr->blhdr_size;
2509 tr->blhdr->flags = BLHDR_CHECK_CHECKSUMS | BLHDR_FIRST_HEADER;
2510
2511 tr->sequence_num = ++jnl->sequence_num;
2512 tr->num_blhdrs = 1;
2513 tr->total_bytes = jnl->jhdr->blhdr_size;
2514 tr->jnl = jnl;
2515
2516 jnl->active_tr = tr;
2517
2518 return 0;
2519 }
2520
2521 int
2522 journal_start_transaction(journal *jnl)
2523 {
2524 int ret;
2525
2526 CHECK_JOURNAL(jnl);
2527
2528 free_old_stuff(jnl);
2529
2530 if (jnl->flags & JOURNAL_INVALID) {
2531 return EINVAL;
2532 }
2533 if (jnl->owner == current_thread()) {
2534 if (jnl->active_tr == NULL) {
2535 panic("jnl: start_tr: active_tr is NULL (jnl @ %p, owner %p, current_thread %p\n",
2536 jnl, jnl->owner, current_thread());
2537 }
2538 jnl->nested_count++;
2539 return 0;
2540 }
2541
2542 journal_lock(jnl);
2543
2544 if (jnl->nested_count != 0 || jnl->active_tr != NULL) {
2545 panic("jnl: start_tr: owner %p, nested count %d, active_tr %p jnl @ %p\n",
2546 jnl->owner, jnl->nested_count, jnl->active_tr, jnl);
2547 }
2548
2549 jnl->nested_count = 1;
2550
2551 #if JOE
2552 // make sure there's room in the journal
2553 if (free_space(jnl) < jnl->tbuffer_size) {
2554
2555 KERNEL_DEBUG(0xbbbbc030 | DBG_FUNC_START, jnl, 0, 0, 0, 0);
2556
2557 // this is the call that really waits for space to free up
2558 // as well as updating jnl->jhdr->start
2559 if (check_free_space(jnl, jnl->tbuffer_size, NULL, jnl->sequence_num) != 0) {
2560 printf("jnl: %s: start transaction failed: no space\n", jnl->jdev_name);
2561 ret = ENOSPC;
2562 goto bad_start;
2563 }
2564 KERNEL_DEBUG(0xbbbbc030 | DBG_FUNC_END, jnl, 0, 0, 0, 0);
2565 }
2566 #endif
2567
2568 // if there's a buffered transaction, use it.
2569 if (jnl->cur_tr) {
2570 jnl->active_tr = jnl->cur_tr;
2571 jnl->cur_tr = NULL;
2572
2573 return 0;
2574 }
2575
2576 ret = journal_allocate_transaction(jnl);
2577 if (ret) {
2578 goto bad_start;
2579 }
2580
2581 // printf("jnl: start_tr: owner 0x%x new tr @ 0x%x\n", jnl->owner, jnl->active_tr);
2582
2583 return 0;
2584
2585 bad_start:
2586 jnl->nested_count = 0;
2587 journal_unlock(jnl);
2588
2589 return ret;
2590 }
2591
2592
2593 int
2594 journal_modify_block_start(journal *jnl, struct buf *bp)
2595 {
2596 transaction *tr;
2597 boolean_t was_vm_privileged = FALSE;
2598
2599 CHECK_JOURNAL(jnl);
2600
2601
2602 free_old_stuff(jnl);
2603
2604 if (jnl->flags & JOURNAL_INVALID) {
2605 return EINVAL;
2606 }
2607
2608 if (vfs_isswapmount(jnl->fsmount)) {
2609 /*
2610 * if we block waiting for memory, and there is enough pressure to
2611 * cause us to try and create a new swap file, we may end up deadlocking
2612 * due to waiting for the journal on the swap file creation path...
2613 * by making ourselves vm_privileged, we give ourselves the best chance
2614 * of not blocking
2615 */
2616 was_vm_privileged = set_vm_privilege(TRUE);
2617 }
2618
2619 // XXXdbg - for debugging I want this to be true. later it may
2620 // not be necessary.
2621 if ((buf_flags(bp) & B_META) == 0) {
2622 panic("jnl: modify_block_start: bp @ %p is not a meta-data block! (jnl %p)\n", bp, jnl);
2623 }
2624
2625 tr = jnl->active_tr;
2626 CHECK_TRANSACTION(tr);
2627
2628 if (jnl->owner != current_thread()) {
2629 panic("jnl: modify_block_start: called w/out a transaction! jnl %p, owner %p, curact %p\n",
2630 jnl, jnl->owner, current_thread());
2631 }
2632
2633 //printf("jnl: mod block start (bp 0x%x vp 0x%x l/blkno %qd/%qd bsz %d; total bytes %d)\n",
2634 // bp, buf_vnode(bp), buf_lblkno(bp), buf_blkno(bp), buf_size(bp), tr->total_bytes);
2635
2636 // can't allow blocks that aren't an even multiple of the
2637 // underlying block size.
2638 if ((buf_size(bp) % jnl->jhdr->jhdr_size) != 0) {
2639 uint32_t phys_blksz, bad=0;
2640
2641 if (VNOP_IOCTL(jnl->jdev, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, vfs_context_kernel())) {
2642 bad = 1;
2643 } else if (phys_blksz != (uint32_t)jnl->jhdr->jhdr_size) {
2644 if (phys_blksz < 512) {
2645 panic("jnl: mod block start: phys blksz %d is too small (%d, %d)\n",
2646 phys_blksz, buf_size(bp), jnl->jhdr->jhdr_size);
2647 }
2648
2649 if ((buf_size(bp) % phys_blksz) != 0) {
2650 bad = 1;
2651 } else if (phys_blksz < (uint32_t)jnl->jhdr->jhdr_size) {
2652 jnl->jhdr->jhdr_size = phys_blksz;
2653 } else {
2654 // the phys_blksz is now larger... need to realloc the jhdr
2655 char *new_header_buf;
2656
2657 printf("jnl: %s: phys blksz got bigger (was: %d/%d now %d)\n",
2658 jnl->jdev_name, jnl->header_buf_size, jnl->jhdr->jhdr_size, phys_blksz);
2659 new_header_buf = hfs_malloc(phys_blksz);
2660 memcpy(new_header_buf, jnl->header_buf, jnl->header_buf_size);
2661 memset(&new_header_buf[jnl->header_buf_size], 0x18, (phys_blksz - jnl->header_buf_size));
2662 hfs_free(jnl->header_buf, jnl->header_buf_size);
2663 jnl->header_buf = new_header_buf;
2664 jnl->header_buf_size = phys_blksz;
2665
2666 jnl->jhdr = (journal_header *)jnl->header_buf;
2667 jnl->jhdr->jhdr_size = phys_blksz;
2668 }
2669 } else {
2670 bad = 1;
2671 }
2672
2673 if (bad) {
2674 panic("jnl: mod block start: bufsize %d not a multiple of block size %d\n",
2675 buf_size(bp), jnl->jhdr->jhdr_size);
2676
2677 if (vfs_isswapmount(jnl->fsmount) && (was_vm_privileged == FALSE))
2678 set_vm_privilege(FALSE);
2679 return -1;
2680 }
2681 }
2682
2683 // make sure that this transaction isn't bigger than the whole journal
2684 if (tr->total_bytes+buf_size(bp) >= (jnl->jhdr->size - jnl->jhdr->jhdr_size)) {
2685 panic("jnl: transaction too big (%d >= %lld bytes, bufsize %d, tr %p bp %p)\n",
2686 tr->total_bytes, (tr->jnl->jhdr->size - jnl->jhdr->jhdr_size), buf_size(bp), tr, bp);
2687
2688 if (vfs_isswapmount(jnl->fsmount) && (was_vm_privileged == FALSE))
2689 set_vm_privilege(FALSE);
2690 return -1;
2691 }
2692
2693 #if DEBUG
2694 const int f = buf_flags(bp);
2695 hfs_assert(!ISSET(f, B_DELWRI) || ISSET(f, B_LOCKED));
2696 #endif
2697
2698 buf_setflags(bp, B_LOCKED);
2699
2700 if (vfs_isswapmount(jnl->fsmount) && (was_vm_privileged == FALSE))
2701 set_vm_privilege(FALSE);
2702
2703 return 0;
2704 }
2705
2706 int
2707 journal_modify_block_abort(journal *jnl, struct buf *bp)
2708 {
2709 transaction *tr;
2710 block_list_header *blhdr;
2711 int i;
2712
2713 CHECK_JOURNAL(jnl);
2714
2715 free_old_stuff(jnl);
2716
2717 tr = jnl->active_tr;
2718
2719 //
2720 // if there's no active transaction then we just want to
2721 // call buf_brelse() and return since this is just a block
2722 // that happened to be modified as part of another tr.
2723 //
2724 if (tr == NULL) {
2725 buf_brelse(bp);
2726 return 0;
2727 }
2728
2729 if (jnl->flags & JOURNAL_INVALID) {
2730 /* Still need to buf_brelse(). Callers assume we consume the bp. */
2731 buf_brelse(bp);
2732 return EINVAL;
2733 }
2734
2735 CHECK_TRANSACTION(tr);
2736
2737 if (jnl->owner != current_thread()) {
2738 panic("jnl: modify_block_abort: called w/out a transaction! jnl %p, owner %p, curact %p\n",
2739 jnl, jnl->owner, current_thread());
2740 }
2741
2742 // printf("jnl: modify_block_abort: tr 0x%x bp 0x%x\n", jnl->active_tr, bp);
2743
2744 // first check if it's already part of this transaction
2745 for (blhdr = tr->blhdr; blhdr; blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum)) {
2746 for (i = 1; i < blhdr->num_blocks; i++) {
2747 if (bp == blhdr->binfo[i].u.bp) {
2748 break;
2749 }
2750 }
2751
2752 if (i < blhdr->num_blocks) {
2753 break;
2754 }
2755 }
2756
2757 //
2758 // if blhdr is null, then this block has only had modify_block_start
2759 // called on it as part of the current transaction. that means that
2760 // it is ok to clear the LOCKED bit since it hasn't actually been
2761 // modified. if blhdr is non-null then modify_block_end was called
2762 // on it and so we need to keep it locked in memory.
2763 //
2764 if (blhdr == NULL) {
2765 buf_clearflags(bp, B_LOCKED);
2766 }
2767
2768 buf_brelse(bp);
2769 return 0;
2770 }
2771
2772
2773 int
2774 journal_modify_block_end(journal *jnl, struct buf *bp, void (*func)(buf_t bp, void *arg), void *arg)
2775 {
2776 int i = 1;
2777 int tbuffer_offset=0;
2778 block_list_header *blhdr, *prev=NULL;
2779 transaction *tr;
2780
2781 CHECK_JOURNAL(jnl);
2782
2783 free_old_stuff(jnl);
2784
2785 if (jnl->flags & JOURNAL_INVALID) {
2786 /* Still need to buf_brelse(). Callers assume we consume the bp. */
2787 buf_brelse(bp);
2788 return EINVAL;
2789 }
2790
2791 tr = jnl->active_tr;
2792 CHECK_TRANSACTION(tr);
2793
2794 if (jnl->owner != current_thread()) {
2795 panic("jnl: modify_block_end: called w/out a transaction! jnl %p, owner %p, curact %p\n",
2796 jnl, jnl->owner, current_thread());
2797 }
2798
2799 //printf("jnl: mod block end: (bp 0x%x vp 0x%x l/blkno %qd/%qd bsz %d, total bytes %d)\n",
2800 // bp, buf_vnode(bp), buf_lblkno(bp), buf_blkno(bp), buf_size(bp), tr->total_bytes);
2801
2802 if ((buf_flags(bp) & B_LOCKED) == 0) {
2803 panic("jnl: modify_block_end: bp %p not locked! jnl @ %p\n", bp, jnl);
2804 }
2805
2806 // first check if it's already part of this transaction
2807 for (blhdr = tr->blhdr; blhdr; prev = blhdr, blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum)) {
2808 tbuffer_offset = jnl->jhdr->blhdr_size;
2809
2810 for (i = 1; i < blhdr->num_blocks; i++) {
2811 if (bp == blhdr->binfo[i].u.bp) {
2812 break;
2813 }
2814 if (blhdr->binfo[i].bnum != (off_t)-1) {
2815 tbuffer_offset += buf_size(blhdr->binfo[i].u.bp);
2816 } else {
2817 tbuffer_offset += blhdr->binfo[i].u.bi.bsize;
2818 }
2819 }
2820
2821 if (i < blhdr->num_blocks) {
2822 break;
2823 }
2824 }
2825
2826 if (blhdr == NULL
2827 && prev
2828 && (prev->num_blocks+1) <= prev->max_blocks
2829 && (prev->bytes_used+buf_size(bp)) <= (uint32_t)tr->tbuffer_size) {
2830 blhdr = prev;
2831
2832 } else if (blhdr == NULL) {
2833 block_list_header *nblhdr;
2834 if (prev == NULL) {
2835 panic("jnl: modify block end: no way man, prev == NULL?!?, jnl %p, bp %p\n", jnl, bp);
2836 }
2837
2838 // we got to the end of the list, didn't find the block and there's
2839 // no room in the block_list_header pointed to by prev
2840
2841 // we allocate another tbuffer and link it in at the end of the list
2842 // through prev->binfo[0].bnum. that's a skanky way to do things but
2843 // avoids having yet another linked list of small data structures to manage.
2844
2845 nblhdr = hfs_malloc(tr->tbuffer_size);
2846
2847 // journal replay code checksum check depends on this.
2848 memset(nblhdr, 0, BLHDR_CHECKSUM_SIZE);
2849 // Fill up the rest of the block with unimportant bytes
2850 memset(nblhdr + BLHDR_CHECKSUM_SIZE, 0x5a, jnl->jhdr->blhdr_size - BLHDR_CHECKSUM_SIZE);
2851
2852 // initialize the new guy
2853 nblhdr->max_blocks = (jnl->jhdr->blhdr_size / sizeof(block_info)) - 1;
2854 nblhdr->num_blocks = 1; // accounts for this header block
2855 nblhdr->bytes_used = jnl->jhdr->blhdr_size;
2856 nblhdr->flags = BLHDR_CHECK_CHECKSUMS;
2857
2858 tr->num_blhdrs++;
2859 tr->total_bytes += jnl->jhdr->blhdr_size;
2860
2861 // then link him in at the end
2862 prev->binfo[0].bnum = (off_t)((long)nblhdr);
2863
2864 // and finally switch to using the new guy
2865 blhdr = nblhdr;
2866 tbuffer_offset = jnl->jhdr->blhdr_size;
2867 i = 1;
2868 }
2869
2870
2871 if ((i+1) > blhdr->max_blocks) {
2872 panic("jnl: modify_block_end: i = %d, max_blocks %d\n", i, blhdr->max_blocks);
2873 }
2874
2875 // if this is true then this is a new block we haven't seen
2876 if (i >= blhdr->num_blocks) {
2877 int bsize;
2878 vnode_t vp;
2879
2880 vp = buf_vnode(bp);
2881 if (vnode_ref(vp)) {
2882 // Nobody checks the return values, so...
2883 jnl->flags |= JOURNAL_INVALID;
2884
2885 buf_brelse(bp);
2886
2887 // We're probably here due to a force unmount, so EIO is appropriate
2888 return EIO;
2889 }
2890
2891 bsize = buf_size(bp);
2892
2893 blhdr->binfo[i].bnum = (off_t)(buf_blkno(bp));
2894 blhdr->binfo[i].u.bp = bp;
2895
2896 KERNEL_DEBUG_CONSTANT(0x3018004, kdebug_vnode(vp), blhdr->binfo[i].bnum, bsize, 0, 0);
2897 /*
2898 * Update the per-task logical counter for metadata write.
2899 * We use (2 * bsize) to account for the write to the journal and the
2900 * corresponding write to the btree.
2901 */
2902 task_update_logical_writes(current_task(), (2 * bsize), TASK_WRITE_METADATA, vp);
2903
2904 if (func) {
2905 void (*old_func)(buf_t, void *)=NULL, *old_arg=NULL;
2906
2907 buf_setfilter(bp, func, arg, &old_func, &old_arg);
2908 if (old_func != NULL && old_func != func) {
2909 panic("jnl: modify_block_end: old func %p / arg %p (func %p)", old_func, old_arg, func);
2910 }
2911 }
2912
2913 blhdr->bytes_used += bsize;
2914 tr->total_bytes += bsize;
2915
2916 blhdr->num_blocks++;
2917 }
2918 buf_bdwrite(bp);
2919
2920 return 0;
2921 }
2922
2923 int
2924 journal_kill_block(journal *jnl, struct buf *bp)
2925 {
2926 int i;
2927 int bflags;
2928 block_list_header *blhdr;
2929 transaction *tr;
2930
2931 CHECK_JOURNAL(jnl);
2932
2933 free_old_stuff(jnl);
2934
2935 if (jnl->flags & JOURNAL_INVALID) {
2936 buf_brelse(bp);
2937 return 0;
2938 }
2939
2940 tr = jnl->active_tr;
2941 CHECK_TRANSACTION(tr);
2942
2943 if (jnl->owner != current_thread()) {
2944 panic("jnl: modify_block_end: called w/out a transaction! jnl %p, owner %p, curact %p\n",
2945 jnl, jnl->owner, current_thread());
2946 }
2947
2948 bflags = buf_flags(bp);
2949
2950 if ( !(bflags & B_LOCKED))
2951 panic("jnl: modify_block_end: called with bp not B_LOCKED");
2952
2953 /*
2954 * bp must be BL_BUSY and B_LOCKED
2955 * first check if it's already part of this transaction
2956 */
2957 for (blhdr = tr->blhdr; blhdr; blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum)) {
2958
2959 for (i = 1; i < blhdr->num_blocks; i++) {
2960 if (bp == blhdr->binfo[i].u.bp) {
2961 vnode_t vp;
2962
2963 buf_clearflags(bp, B_LOCKED);
2964
2965 // this undoes the vnode_ref() in journal_modify_block_end()
2966 vp = buf_vnode(bp);
2967 vnode_rele_ext(vp, 0, 1);
2968
2969 // if the block has the DELWRI and FILTER bits sets, then
2970 // things are seriously weird. if it was part of another
2971 // transaction then journal_modify_block_start() should
2972 // have force it to be written.
2973 //
2974 //if ((bflags & B_DELWRI) && (bflags & B_FILTER)) {
2975 // panic("jnl: kill block: this defies all logic! bp 0x%x\n", bp);
2976 //} else {
2977 tr->num_killed += buf_size(bp);
2978 //}
2979 blhdr->binfo[i].bnum = (off_t)-1;
2980 blhdr->binfo[i].u.bp = NULL;
2981 blhdr->binfo[i].u.bi.bsize = buf_size(bp);
2982
2983 buf_markinvalid(bp);
2984 buf_brelse(bp);
2985
2986 return 0;
2987 }
2988 }
2989 }
2990
2991 /*
2992 * We did not find the block in any transaction buffer but we still
2993 * need to release it or else it will be left locked forever.
2994 */
2995 buf_brelse(bp);
2996
2997 return 0;
2998 }
2999
3000 /*
3001 ;________________________________________________________________________________
3002 ;
3003 ; Routine: journal_trim_set_callback
3004 ;
3005 ; Function: Provide the journal with a routine to be called back when a
3006 ; TRIM has (or would have) been issued to the device. That
3007 ; is, the transaction has been flushed to the device, and the
3008 ; blocks freed by the transaction are now safe for reuse.
3009 ;
3010 ; CAUTION: If the journal becomes invalid (eg., due to an I/O
3011 ; error when trying to write to the journal), this callback
3012 ; will stop getting called, even if extents got freed before
3013 ; the journal became invalid!
3014 ;
3015 ; Input Arguments:
3016 ; jnl - The journal structure for the filesystem.
3017 ; callback - The function to call when the TRIM is complete.
3018 ; arg - An argument to be passed to callback.
3019 ;________________________________________________________________________________
3020 */
3021 void
3022 journal_trim_set_callback(journal *jnl, jnl_trim_callback_t callback, void *arg)
3023 {
3024 jnl->trim_callback = callback;
3025 jnl->trim_callback_arg = arg;
3026 }
3027
3028
3029 /*
3030 ;________________________________________________________________________________
3031 ;
3032 ; Routine: journal_trim_realloc
3033 ;
3034 ; Function: Increase the amount of memory allocated for the list of extents
3035 ; to be unmapped (trimmed). This routine will be called when
3036 ; adding an extent to the list, and the list already occupies
3037 ; all of the space allocated to it. This routine returns ENOMEM
3038 ; if unable to allocate more space, or 0 if the extent list was
3039 ; grown successfully.
3040 ;
3041 ; Input Arguments:
3042 ; trim - The trim list to be resized.
3043 ;
3044 ; Output:
3045 ; (result) - ENOMEM or 0.
3046 ;
3047 ; Side effects:
3048 ; The allocated_count and extents fields of tr->trim are updated
3049 ; if the function returned 0.
3050 ;________________________________________________________________________________
3051 */
3052 static int
3053 trim_realloc(journal *jnl, struct jnl_trim_list *trim)
3054 {
3055 void *new_extents;
3056 uint32_t new_allocated_count;
3057 boolean_t was_vm_privileged = FALSE;
3058
3059 if (jnl_kdebug)
3060 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REALLOC | DBG_FUNC_START, obfuscate_addr(trim), 0, trim->allocated_count, trim->extent_count, 0);
3061
3062 new_allocated_count = trim->allocated_count + JOURNAL_DEFAULT_TRIM_EXTENTS;
3063
3064 if (vfs_isswapmount(jnl->fsmount)) {
3065 /*
3066 * if we block waiting for memory, and there is enough pressure to
3067 * cause us to try and create a new swap file, we may end up deadlocking
3068 * due to waiting for the journal on the swap file creation path...
3069 * by making ourselves vm_privileged, we give ourselves the best chance
3070 * of not blocking
3071 */
3072 was_vm_privileged = set_vm_privilege(TRUE);
3073 }
3074 new_extents = hfs_malloc(new_allocated_count * sizeof(dk_extent_t));
3075 if (vfs_isswapmount(jnl->fsmount) && (was_vm_privileged == FALSE))
3076 set_vm_privilege(FALSE);
3077
3078 if (new_extents == NULL) {
3079 printf("jnl: trim_realloc: unable to grow extent list!\n");
3080 /*
3081 * Since we could be called when allocating space previously marked
3082 * to be trimmed, we need to empty out the list to be safe.
3083 */
3084 trim->extent_count = 0;
3085 if (jnl_kdebug)
3086 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REALLOC | DBG_FUNC_END, ENOMEM, 0, trim->allocated_count, 0, 0);
3087 return ENOMEM;
3088 }
3089
3090 /* Copy the old extent list to the newly allocated list. */
3091 if (trim->extents != NULL) {
3092 memmove(new_extents,
3093 trim->extents,
3094 trim->allocated_count * sizeof(dk_extent_t));
3095 hfs_free(trim->extents, trim->allocated_count * sizeof(dk_extent_t));
3096 }
3097
3098 trim->allocated_count = new_allocated_count;
3099 trim->extents = new_extents;
3100
3101 if (jnl_kdebug)
3102 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REALLOC | DBG_FUNC_END, 0, 0, new_allocated_count, trim->extent_count, 0);
3103
3104 return 0;
3105 }
3106
3107 /*
3108 ;________________________________________________________________________________
3109 ;
3110 ; Routine: trim_search_extent
3111 ;
3112 ; Function: Search the given extent list to see if any of its extents
3113 ; overlap the given extent.
3114 ;
3115 ; Input Arguments:
3116 ; trim - The trim list to be searched.
3117 ; offset - The first byte of the range to be searched for.
3118 ; length - The number of bytes of the extent being searched for.
3119 ; overlap_start - start of the overlapping extent
3120 ; overlap_len - length of the overlapping extent
3121 ;
3122 ; Output:
3123 ; (result) - TRUE if one or more extents overlap, FALSE otherwise.
3124 ;________________________________________________________________________________
3125 */
3126 static int
3127 trim_search_extent(struct jnl_trim_list *trim, uint64_t offset,
3128 uint64_t length, uint64_t *overlap_start, uint64_t *overlap_len)
3129 {
3130 uint64_t end = offset + length;
3131 uint32_t lower = 0; /* Lowest index to search */
3132 uint32_t upper = trim->extent_count; /* Highest index to search + 1 */
3133 uint32_t middle;
3134
3135 /* A binary search over the extent list. */
3136 while (lower < upper) {
3137 middle = (lower + upper) / 2;
3138
3139 if (trim->extents[middle].offset >= end)
3140 upper = middle;
3141 else if (trim->extents[middle].offset + trim->extents[middle].length <= offset)
3142 lower = middle + 1;
3143 else {
3144 if (overlap_start) {
3145 *overlap_start = trim->extents[middle].offset;
3146 }
3147 if (overlap_len) {
3148 *overlap_len = trim->extents[middle].length;
3149 }
3150 return TRUE;
3151 }
3152 }
3153
3154 return FALSE;
3155 }
3156
3157
3158 /*
3159 ;________________________________________________________________________________
3160 ;
3161 ; Routine: journal_trim_add_extent
3162 ;
3163 ; Function: Keep track of extents that have been freed as part of this
3164 ; transaction. If the underlying device supports TRIM (UNMAP),
3165 ; then those extents will be trimmed/unmapped once the
3166 ; transaction has been written to the journal. (For example,
3167 ; SSDs can support trim/unmap and avoid having to recopy those
3168 ; blocks when doing wear leveling, and may reuse the same
3169 ; phsyical blocks for different logical blocks.)
3170 ;
3171 ; HFS also uses this, in combination with journal_trim_set_callback,
3172 ; to add recently freed extents to its free extent cache, but
3173 ; only after the transaction that freed them is committed to
3174 ; disk. (This reduces the chance of overwriting live data in
3175 ; a way that causes data loss if a transaction never gets
3176 ; written to the journal.)
3177 ;
3178 ; Input Arguments:
3179 ; jnl - The journal for the volume containing the byte range.
3180 ; offset - The first byte of the range to be trimmed.
3181 ; length - The number of bytes of the extent being trimmed.
3182 ;________________________________________________________________________________
3183 */
3184 int
3185 journal_trim_add_extent(journal *jnl, uint64_t offset, uint64_t length)
3186 {
3187 uint64_t end;
3188 transaction *tr;
3189 dk_extent_t *extent;
3190 uint32_t insert_index;
3191 uint32_t replace_count;
3192
3193 CHECK_JOURNAL(jnl);
3194
3195 /* TODO: Is it OK to manipulate the trim list even if JOURNAL_INVALID is set? I think so... */
3196 if (jnl->flags & JOURNAL_INVALID) {
3197 return EINVAL;
3198 }
3199
3200 tr = jnl->active_tr;
3201 CHECK_TRANSACTION(tr);
3202
3203 if (jnl_kdebug)
3204 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_START, obfuscate_addr(jnl), offset, length, tr->trim.extent_count, 0);
3205
3206 if (jnl->owner != current_thread()) {
3207 panic("jnl: trim_add_extent: called w/out a transaction! jnl %p, owner %p, curact %p\n",
3208 jnl, jnl->owner, current_thread());
3209 }
3210
3211 free_old_stuff(jnl);
3212
3213 end = offset + length;
3214
3215 /*
3216 * Find the range of existing extents that can be combined with the
3217 * input extent. We start by counting the number of extents that end
3218 * strictly before the input extent, then count the number of extents
3219 * that overlap or are contiguous with the input extent.
3220 */
3221 extent = tr->trim.extents;
3222 insert_index = 0;
3223 while (insert_index < tr->trim.extent_count && extent->offset + extent->length < offset) {
3224 ++insert_index;
3225 ++extent;
3226 }
3227 replace_count = 0;
3228 while (insert_index + replace_count < tr->trim.extent_count && extent->offset <= end) {
3229 ++replace_count;
3230 ++extent;
3231 }
3232
3233 /*
3234 * If none of the existing extents can be combined with the input extent,
3235 * then just insert it in the list (before item number insert_index).
3236 */
3237 if (replace_count == 0) {
3238 /* If the list was already full, we need to grow it. */
3239 if (tr->trim.extent_count == tr->trim.allocated_count) {
3240 if (trim_realloc(jnl, &tr->trim) != 0) {
3241 printf("jnl: trim_add_extent: out of memory!");
3242 if (jnl_kdebug)
3243 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, ENOMEM, 0, 0, tr->trim.extent_count, 0);
3244 return ENOMEM;
3245 }
3246 }
3247
3248 /* Shift any existing extents with larger offsets. */
3249 if (insert_index < tr->trim.extent_count) {
3250 memmove(&tr->trim.extents[insert_index+1],
3251 &tr->trim.extents[insert_index],
3252 (tr->trim.extent_count - insert_index) * sizeof(dk_extent_t));
3253 }
3254 tr->trim.extent_count++;
3255
3256 /* Store the new extent in the list. */
3257 tr->trim.extents[insert_index].offset = offset;
3258 tr->trim.extents[insert_index].length = length;
3259
3260 /* We're done. */
3261 if (jnl_kdebug)
3262 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, 0, 0, 0, tr->trim.extent_count, 0);
3263 return 0;
3264 }
3265
3266 /*
3267 * Update extent number insert_index to be the union of the input extent
3268 * and all of the replaced extents.
3269 */
3270 if (tr->trim.extents[insert_index].offset < offset)
3271 offset = tr->trim.extents[insert_index].offset;
3272 extent = &tr->trim.extents[insert_index + replace_count - 1];
3273 if (extent->offset + extent->length > end)
3274 end = extent->offset + extent->length;
3275 tr->trim.extents[insert_index].offset = offset;
3276 tr->trim.extents[insert_index].length = end - offset;
3277
3278 /*
3279 * If we were replacing more than one existing extent, then shift any
3280 * extents with larger offsets, and update the count of extents.
3281 *
3282 * We're going to leave extent #insert_index alone since it was just updated, above.
3283 * We need to move extents from index (insert_index + replace_count) through the end of
3284 * the list by (replace_count - 1) positions so that they overwrite extent #(insert_index + 1).
3285 */
3286 if (replace_count > 1 && (insert_index + replace_count) < tr->trim.extent_count) {
3287 memmove(&tr->trim.extents[insert_index + 1],
3288 &tr->trim.extents[insert_index + replace_count],
3289 (tr->trim.extent_count - insert_index - replace_count) * sizeof(dk_extent_t));
3290 }
3291 tr->trim.extent_count -= replace_count - 1;
3292
3293 if (jnl_kdebug)
3294 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_ADD | DBG_FUNC_END, 0, 0, 0, tr->trim.extent_count, 0);
3295 return 0;
3296 }
3297
3298 /*
3299 * journal_trim_extent_overlap
3300 *
3301 * Return 1 if there are any pending TRIMs that overlap with the given offset and length
3302 * Return 0 otherwise.
3303 */
3304
3305 int journal_trim_extent_overlap (journal *jnl, uint64_t offset, uint64_t length, uint64_t *end) {
3306 transaction *tr = NULL;
3307 int overlap = 0;
3308
3309 uint64_t overlap_start;
3310 uint64_t overlap_len;
3311 tr = jnl->active_tr;
3312 CHECK_TRANSACTION(tr);
3313
3314 /*
3315 * There are two lists that need to be examined for potential overlaps:
3316 *
3317 * The first is the current transaction. Since this function requires that
3318 * a transaction be active when this is called, this is the "active_tr"
3319 * pointer in the journal struct. This has a trimlist pointer which needs
3320 * to be searched.
3321 */
3322 overlap = trim_search_extent (&tr->trim, offset, length, &overlap_start, &overlap_len);
3323 if (overlap == 0) {
3324 /*
3325 * The second is the async trim list, which is only done if the current
3326 * transaction group (active transaction) did not overlap with our target
3327 * extent. This async trim list is the set of all previously
3328 * committed transaction groups whose I/Os are now in-flight. We need to hold the
3329 * trim lock in order to search this list. If we grab the list before the
3330 * TRIM has completed, then we will compare it. If it is grabbed AFTER the
3331 * TRIM has completed, then the pointer will be zeroed out and we won't have
3332 * to check anything.
3333 */
3334 lck_rw_lock_shared (&jnl->trim_lock);
3335 if (jnl->async_trim != NULL) {
3336 overlap = trim_search_extent(jnl->async_trim, offset, length, &overlap_start, &overlap_len);
3337 }
3338 lck_rw_unlock_shared (&jnl->trim_lock);
3339 }
3340
3341 if (overlap) {
3342 /* compute the end (min) of the overlapping range */
3343 if ( (overlap_start + overlap_len) < (offset + length)) {
3344 *end = (overlap_start + overlap_len);
3345 }
3346 else {
3347 *end = (offset + length);
3348 }
3349 }
3350
3351
3352 return overlap;
3353 }
3354
3355 /*
3356 * journal_request_immediate_flush
3357 *
3358 * FS requests that the journal flush immediately upon the
3359 * active transaction's completion.
3360 *
3361 * Returns 0 if operation succeeds
3362 * Returns EPERM if we failed to leave hint
3363 */
3364 int
3365 journal_request_immediate_flush (journal *jnl) {
3366
3367 transaction *tr = NULL;
3368 /*
3369 * Is a transaction still in process? You must do
3370 * this while there are txns open
3371 */
3372 tr = jnl->active_tr;
3373 if (tr != NULL) {
3374 CHECK_TRANSACTION(tr);
3375 tr->flush_on_completion = TRUE;
3376 }
3377 else {
3378 return EPERM;
3379 }
3380 return 0;
3381 }
3382
3383
3384
3385 /*
3386 ;________________________________________________________________________________
3387 ;
3388 ; Routine: trim_remove_extent
3389 ;
3390 ; Function: Indicate that a range of bytes, some of which may have previously
3391 ; been passed to journal_trim_add_extent, is now allocated.
3392 ; Any overlapping ranges currently in the journal's trim list will
3393 ; be removed. If the underlying device supports TRIM (UNMAP), then
3394 ; these extents will not be trimmed/unmapped when the transaction
3395 ; is written to the journal.
3396 ;
3397 ; HFS also uses this to prevent newly allocated space from being
3398 ; added to its free extent cache (if some portion of the newly
3399 ; allocated space was recently freed).
3400 ;
3401 ; Input Arguments:
3402 ; trim - The trim list to update.
3403 ; offset - The first byte of the range to be trimmed.
3404 ; length - The number of bytes of the extent being trimmed.
3405 ;________________________________________________________________________________
3406 */
3407 static int
3408 trim_remove_extent(journal *jnl, struct jnl_trim_list *trim, uint64_t offset, uint64_t length)
3409 {
3410 u_int64_t end;
3411 dk_extent_t *extent;
3412 u_int32_t keep_before;
3413 u_int32_t keep_after;
3414
3415 end = offset + length;
3416
3417 /*
3418 * Find any existing extents that start before or end after the input
3419 * extent. These extents will be modified if they overlap the input
3420 * extent. Other extents between them will be deleted.
3421 */
3422 extent = trim->extents;
3423 keep_before = 0;
3424 while (keep_before < trim->extent_count && extent->offset < offset) {
3425 ++keep_before;
3426 ++extent;
3427 }
3428 keep_after = keep_before;
3429 if (keep_after > 0) {
3430 /* See if previous extent extends beyond both ends of input extent. */
3431 --keep_after;
3432 --extent;
3433 }
3434 while (keep_after < trim->extent_count && (extent->offset + extent->length) <= end) {
3435 ++keep_after;
3436 ++extent;
3437 }
3438
3439 /*
3440 * When we get here, the first keep_before extents (0 .. keep_before-1)
3441 * start before the input extent, and extents (keep_after .. extent_count-1)
3442 * end after the input extent. We'll need to keep, all of those extents,
3443 * but possibly modify #(keep_before-1) and #keep_after to remove the portion
3444 * that overlaps with the input extent.
3445 */
3446
3447 /*
3448 * Does the input extent start after and end before the same existing
3449 * extent? If so, we have to "punch a hole" in that extent and convert
3450 * it to two separate extents.
3451 */
3452 if (keep_before > keep_after) {
3453 /* If the list was already full, we need to grow it. */
3454 if (trim->extent_count == trim->allocated_count) {
3455 if (trim_realloc(jnl, trim) != 0) {
3456 printf("jnl: trim_remove_extent: out of memory!");
3457 return ENOMEM;
3458 }
3459 }
3460
3461 /*
3462 * Make room for a new extent by shifting extents #keep_after and later
3463 * down by one extent. When we're done, extents #keep_before and
3464 * #keep_after will be identical, and we can fall through to removing
3465 * the portion that overlaps the input extent.
3466 */
3467 memmove(&trim->extents[keep_before],
3468 &trim->extents[keep_after],
3469 (trim->extent_count - keep_after) * sizeof(dk_extent_t));
3470 ++trim->extent_count;
3471 ++keep_after;
3472
3473 /*
3474 * Fall through. We now have the case where the length of extent
3475 * #(keep_before - 1) needs to be updated, and the start of extent
3476 * #(keep_after) needs to be updated.
3477 */
3478 }
3479
3480 /*
3481 * May need to truncate the end of extent #(keep_before - 1) if it overlaps
3482 * the input extent.
3483 */
3484 if (keep_before > 0) {
3485 extent = &trim->extents[keep_before - 1];
3486 if (extent->offset + extent->length > offset) {
3487 extent->length = offset - extent->offset;
3488 }
3489 }
3490
3491 /*
3492 * May need to update the start of extent #(keep_after) if it overlaps the
3493 * input extent.
3494 */
3495 if (keep_after < trim->extent_count) {
3496 extent = &trim->extents[keep_after];
3497 if (extent->offset < end) {
3498 extent->length = extent->offset + extent->length - end;
3499 extent->offset = end;
3500 }
3501 }
3502
3503 /*
3504 * If there were whole extents that overlapped the input extent, get rid
3505 * of them by shifting any following extents, and updating the count.
3506 */
3507 if (keep_after > keep_before && keep_after < trim->extent_count) {
3508 memmove(&trim->extents[keep_before],
3509 &trim->extents[keep_after],
3510 (trim->extent_count - keep_after) * sizeof(dk_extent_t));
3511 }
3512 trim->extent_count -= keep_after - keep_before;
3513
3514 return 0;
3515 }
3516
3517 /*
3518 ;________________________________________________________________________________
3519 ;
3520 ; Routine: journal_trim_remove_extent
3521 ;
3522 ; Function: Make note of a range of bytes, some of which may have previously
3523 ; been passed to journal_trim_add_extent, is now in use on the
3524 ; volume. The given bytes will be not be trimmed as part of
3525 ; this transaction, or a pending trim of a transaction being
3526 ; asynchronously flushed.
3527 ;
3528 ; Input Arguments:
3529 ; jnl - The journal for the volume containing the byte range.
3530 ; offset - The first byte of the range to be trimmed.
3531 ; length - The number of bytes of the extent being trimmed.
3532 ;________________________________________________________________________________
3533 */
3534 int
3535 journal_trim_remove_extent(journal *jnl, uint64_t offset, uint64_t length)
3536 {
3537 int error = 0;
3538 transaction *tr;
3539
3540 CHECK_JOURNAL(jnl);
3541
3542 /* TODO: Is it OK to manipulate the trim list even if JOURNAL_INVALID is set? I think so... */
3543 if (jnl->flags & JOURNAL_INVALID) {
3544 return EINVAL;
3545 }
3546
3547 tr = jnl->active_tr;
3548 CHECK_TRANSACTION(tr);
3549
3550 if (jnl_kdebug)
3551 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE | DBG_FUNC_START, obfuscate_addr(jnl), offset, length, tr->trim.extent_count, 0);
3552
3553 if (jnl->owner != current_thread()) {
3554 panic("jnl: trim_remove_extent: called w/out a transaction! jnl %p, owner %p, curact %p\n",
3555 jnl, jnl->owner, current_thread());
3556 }
3557
3558 free_old_stuff(jnl);
3559
3560 error = trim_remove_extent(jnl, &tr->trim, offset, length);
3561 if (error == 0) {
3562 int found = FALSE;
3563
3564 /*
3565 * See if a pending trim has any extents that overlap with the
3566 * one we were given.
3567 */
3568 lck_rw_lock_shared(&jnl->trim_lock);
3569 if (jnl->async_trim != NULL)
3570 found = trim_search_extent(jnl->async_trim, offset, length, NULL, NULL);
3571 lck_rw_unlock_shared(&jnl->trim_lock);
3572
3573 if (found) {
3574 /*
3575 * There was an overlap, so avoid trimming the extent we
3576 * just allocated. (Otherwise, it might get trimmed after
3577 * we've written to it, which will cause that data to be
3578 * corrupted.)
3579 */
3580 uint32_t async_extent_count = 0;
3581
3582 if (jnl_kdebug)
3583 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE_PENDING | DBG_FUNC_START, obfuscate_addr(jnl), offset, length, 0, 0);
3584 lck_rw_lock_exclusive(&jnl->trim_lock);
3585 if (jnl->async_trim != NULL) {
3586 error = trim_remove_extent(jnl, jnl->async_trim, offset, length);
3587 async_extent_count = jnl->async_trim->extent_count;
3588 }
3589 lck_rw_unlock_exclusive(&jnl->trim_lock);
3590 if (jnl_kdebug)
3591 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE_PENDING | DBG_FUNC_END, error, 0, 0, async_extent_count, 0);
3592 }
3593 }
3594
3595 if (jnl_kdebug)
3596 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_REMOVE | DBG_FUNC_END, error, 0, 0, tr->trim.extent_count, 0);
3597 return error;
3598 }
3599
3600
3601 static int
3602 journal_trim_flush(journal *jnl, transaction *tr)
3603 {
3604 int errno = 0;
3605 boolean_t was_vm_privileged = FALSE;
3606
3607 if (jnl_kdebug)
3608 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_FLUSH | DBG_FUNC_START, obfuscate_addr(jnl), tr, 0, tr->trim.extent_count, 0);
3609
3610 if (vfs_isswapmount(jnl->fsmount)) {
3611 /*
3612 * the disk driver can allocate memory on this path...
3613 * if we block waiting for memory, and there is enough pressure to
3614 * cause us to try and create a new swap file, we may end up deadlocking
3615 * due to waiting for the journal on the swap file creation path...
3616 * by making ourselves vm_privileged, we give ourselves the best chance
3617 * of not blocking
3618 */
3619 was_vm_privileged = set_vm_privilege(TRUE);
3620 }
3621 lck_rw_lock_shared(&jnl->trim_lock);
3622 if (tr->trim.extent_count > 0) {
3623 dk_unmap_t unmap;
3624
3625 bzero(&unmap, sizeof(unmap));
3626 if (jnl->flags & JOURNAL_USE_UNMAP) {
3627 unmap.extents = tr->trim.extents;
3628 unmap.extentsCount = tr->trim.extent_count;
3629 if (jnl_kdebug)
3630 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_UNMAP | DBG_FUNC_START, obfuscate_addr(jnl), tr, 0, tr->trim.extent_count, 0);
3631 errno = VNOP_IOCTL(jnl->fsdev, DKIOCUNMAP, (caddr_t)&unmap, FWRITE, vfs_context_kernel());
3632 if (jnl_kdebug)
3633 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_UNMAP | DBG_FUNC_END, errno, 0, 0, 0, 0);
3634 }
3635
3636 /*
3637 * Call back into the file system to tell them that we have
3638 * trimmed some extents and that they can now be reused.
3639 *
3640 * CAUTION: If the journal becomes invalid (eg., due to an I/O
3641 * error when trying to write to the journal), this callback
3642 * will stop getting called, even if extents got freed before
3643 * the journal became invalid!
3644 */
3645 if (jnl->trim_callback)
3646 jnl->trim_callback(jnl->trim_callback_arg, tr->trim.extent_count, tr->trim.extents);
3647 }
3648 lck_rw_unlock_shared(&jnl->trim_lock);
3649
3650 if (vfs_isswapmount(jnl->fsmount) && (was_vm_privileged == FALSE))
3651 set_vm_privilege(FALSE);
3652 /*
3653 * If the transaction we're flushing was the async transaction, then
3654 * tell the current transaction that there is no pending trim
3655 * any more.
3656 *
3657 * NOTE: Since we released the lock, another thread could have
3658 * removed one or more extents from our list. That's not a
3659 * problem since any writes to the re-allocated blocks
3660 * would get sent to the device after the DKIOCUNMAP.
3661 */
3662 lck_rw_lock_exclusive(&jnl->trim_lock);
3663 if (jnl->async_trim == &tr->trim)
3664 jnl->async_trim = NULL;
3665 lck_rw_unlock_exclusive(&jnl->trim_lock);
3666
3667 /*
3668 * By the time we get here, no other thread can discover the address
3669 * of "tr", so it is safe for us to manipulate tr->trim without
3670 * holding any locks.
3671 */
3672 if (tr->trim.extents) {
3673 hfs_free(tr->trim.extents, tr->trim.allocated_count * sizeof(dk_extent_t));
3674 tr->trim.allocated_count = 0;
3675 tr->trim.extent_count = 0;
3676 tr->trim.extents = NULL;
3677 }
3678
3679 if (jnl_kdebug)
3680 KERNEL_DEBUG_CONSTANT(DBG_JOURNAL_TRIM_FLUSH | DBG_FUNC_END, errno, 0, 0, 0, 0);
3681
3682 return errno;
3683 }
3684
3685 static int
3686 journal_binfo_cmp(const void *a, const void *b)
3687 {
3688 const block_info *bi_a = (const struct block_info *)a;
3689 const block_info *bi_b = (const struct block_info *)b;
3690 daddr64_t res;
3691
3692 if (bi_a->bnum == (off_t)-1) {
3693 return 1;
3694 }
3695 if (bi_b->bnum == (off_t)-1) {
3696 return -1;
3697 }
3698
3699 // don't have to worry about negative block
3700 // numbers so this is ok to do.
3701 //
3702 res = (buf_blkno(bi_a->u.bp) - buf_blkno(bi_b->u.bp));
3703
3704 return (int)res;
3705 }
3706
3707
3708 /*
3709 * End a transaction. If the transaction is small enough, and we're not forcing
3710 * a write to disk, the "active" transaction becomes the "current" transaction,
3711 * and will be reused for the next transaction that is started (group commit).
3712 *
3713 * If the transaction gets written to disk (because force_it is true, or no
3714 * group commit, or the transaction is sufficiently full), the blocks get
3715 * written into the journal first, then the are written asynchronously. When
3716 * those async writes complete, the transaction can be freed and removed from
3717 * the journal.
3718 *
3719 * An optional callback can be supplied. If given, it is called after the
3720 * the blocks have been written to the journal, but before the async writes
3721 * of those blocks to their normal on-disk locations. This is used by
3722 * journal_relocate so that the location of the journal can be changed and
3723 * flushed to disk before the blocks get written to their normal locations.
3724 * Note that the callback is only called if the transaction gets written to
3725 * the journal during this end_transaction call; you probably want to set the
3726 * force_it flag.
3727 *
3728 * Inputs:
3729 * tr Transaction to add to the journal
3730 * force_it If true, force this transaction to the on-disk journal immediately.
3731 * callback See description above. Pass NULL for no callback.
3732 * callback_arg Argument passed to callback routine.
3733 *
3734 * Result
3735 * 0 No errors
3736 * -1 An error occurred. The journal is marked invalid.
3737 */
3738 static int
3739 end_transaction(transaction *tr, int force_it, errno_t (*callback)(void*), void *callback_arg, boolean_t drop_lock, boolean_t must_wait)
3740 {
3741 block_list_header *blhdr=NULL, *next=NULL;
3742 int i, ret_val = 0;
3743 errno_t errno;
3744 journal *jnl = tr->jnl;
3745 struct buf *bp;
3746 size_t tbuffer_offset;
3747 boolean_t drop_lock_early;
3748
3749 if (jnl->cur_tr) {
3750 panic("jnl: jnl @ %p already has cur_tr %p, new tr: %p\n",
3751 jnl, jnl->cur_tr, tr);
3752 }
3753
3754 // if there weren't any modified blocks in the transaction
3755 // just save off the transaction pointer and return.
3756 if (tr->total_bytes == jnl->jhdr->blhdr_size) {
3757 jnl->cur_tr = tr;
3758 goto done;
3759 }
3760
3761 // if our transaction buffer isn't very full, just hang
3762 // on to it and don't actually flush anything. this is
3763 // what is known as "group commit". we will flush the
3764 // transaction buffer if it's full or if we have more than
3765 // one of them so we don't start hogging too much memory.
3766 //
3767 // We also check the device supports UNMAP/TRIM, and if so,
3768 // the number of extents waiting to be trimmed. If it is
3769 // small enough, then keep accumulating more (so we can
3770 // reduce the overhead of trimming). If there was a prior
3771 // trim error, then we stop issuing trims for this
3772 // volume, so we can also coalesce transactions.
3773 //
3774 if ( force_it == 0
3775 && (jnl->flags & JOURNAL_NO_GROUP_COMMIT) == 0
3776 && tr->num_blhdrs < 3
3777 && (tr->total_bytes <= ((tr->tbuffer_size*tr->num_blhdrs) - tr->tbuffer_size/8))
3778 && (!(jnl->flags & JOURNAL_USE_UNMAP) || (tr->trim.extent_count < jnl_trim_flush_limit))) {
3779
3780 jnl->cur_tr = tr;
3781 goto done;
3782 }
3783
3784 KERNEL_DEBUG(0xbbbbc018|DBG_FUNC_START, jnl, tr, drop_lock, must_wait, 0);
3785
3786 lock_condition(jnl, &jnl->flushing, "end_transaction");
3787
3788 /*
3789 * if the previous 'finish_end_transaction' was being run
3790 * asynchronously, it could have encountered a condition
3791 * that caused it to mark the journal invalid... if that
3792 * occurred while we were waiting for it to finish, we
3793 * need to notice and abort the current transaction
3794 */
3795 if ((jnl->flags & JOURNAL_INVALID) || jnl->flush_aborted == TRUE) {
3796 unlock_condition(jnl, &jnl->flushing);
3797
3798 abort_transaction(jnl, tr);
3799 ret_val = -1;
3800 KERNEL_DEBUG(0xbbbbc018|DBG_FUNC_END, jnl, tr, ret_val, 0, 0);
3801 goto done;
3802 }
3803
3804 /*
3805 * Store a pointer to this transaction's trim list so that
3806 * future transactions can find it.
3807 *
3808 * Note: if there are no extents in the trim list, then don't
3809 * bother saving the pointer since nothing can add new extents
3810 * to the list (and other threads/transactions only care if
3811 * there is a trim pending).
3812 */
3813 lck_rw_lock_exclusive(&jnl->trim_lock);
3814 if (jnl->async_trim != NULL)
3815 panic("jnl: end_transaction: async_trim already non-NULL!");
3816 if (tr->trim.extent_count > 0)
3817 jnl->async_trim = &tr->trim;
3818 lck_rw_unlock_exclusive(&jnl->trim_lock);
3819
3820 /*
3821 * snapshot the transaction sequence number while we are still behind
3822 * the journal lock since it will be bumped upon the start of the
3823 * next transaction group which may overlap the current journal flush...
3824 * we pass the snapshot into write_journal_header during the journal
3825 * flush so that it can write the correct version in the header...
3826 * because we hold the 'flushing' condition variable for the duration
3827 * of the journal flush, 'saved_sequence_num' remains stable
3828 */
3829 jnl->saved_sequence_num = jnl->sequence_num;
3830
3831 /*
3832 * if we're here we're going to flush the transaction buffer to disk.
3833 * 'check_free_space' will not return untl there is enough free
3834 * space for this transaction in the journal and jnl->old_start[0]
3835 * is avaiable for use
3836 */
3837 KERNEL_DEBUG(0xbbbbc030 | DBG_FUNC_START, jnl, 0, 0, 0, 0);
3838
3839 check_free_space(jnl, tr->total_bytes, &tr->delayed_header_write, jnl->saved_sequence_num);
3840
3841 KERNEL_DEBUG(0xbbbbc030 | DBG_FUNC_END, jnl, tr->delayed_header_write, 0, 0, 0);
3842
3843 // range check the end index
3844 if (jnl->jhdr->end <= 0 || jnl->jhdr->end > jnl->jhdr->size) {
3845 panic("jnl: end_transaction: end is bogus 0x%llx (sz 0x%llx)\n",
3846 jnl->jhdr->end, jnl->jhdr->size);
3847 }
3848 if (tr->delayed_header_write == TRUE) {
3849 thread_t thread = THREAD_NULL;
3850
3851 lock_condition(jnl, &jnl->writing_header, "end_transaction");
3852 /*
3853 * fire up a thread to write the journal header
3854 * asynchronously... when it finishes, it will call
3855 * unlock_condition... we can overlap the preparation of
3856 * the log and buffers during this time
3857 */
3858 kernel_thread_start((thread_continue_t)write_header_thread, jnl, &thread);
3859 } else
3860 jnl->write_header_failed = FALSE;
3861
3862
3863 // this transaction starts where the current journal ends
3864 tr->journal_start = jnl->jhdr->end;
3865
3866 lock_oldstart(jnl);
3867 /*
3868 * Because old_start is locked above, we can cast away the volatile qualifier before passing it to memcpy.
3869 * slide everyone else down and put our latest guy in the last
3870 * entry in the old_start array
3871 */
3872 memcpy(__CAST_AWAY_QUALIFIER(&jnl->old_start[0], volatile, void *), __CAST_AWAY_QUALIFIER(&jnl->old_start[1], volatile, void *), sizeof(jnl->old_start)-sizeof(jnl->old_start[0]));
3873 jnl->old_start[sizeof(jnl->old_start)/sizeof(jnl->old_start[0]) - 1] = tr->journal_start | 0x8000000000000000LL;
3874
3875 unlock_oldstart(jnl);
3876
3877
3878 for (blhdr = tr->blhdr; blhdr; blhdr = next) {
3879 char *blkptr;
3880 buf_t sbp;
3881 int32_t bsize;
3882
3883 tbuffer_offset = jnl->jhdr->blhdr_size;
3884
3885 for (i = 1; i < blhdr->num_blocks; i++) {
3886
3887 if (blhdr->binfo[i].bnum != (off_t)-1) {
3888 void (*func)(buf_t, void *);
3889 void *arg;
3890
3891 bp = blhdr->binfo[i].u.bp;
3892
3893 if (bp == NULL) {
3894 panic("jnl: inconsistent binfo (NULL bp w/bnum %lld; jnl @ %p, tr %p)\n",
3895 blhdr->binfo[i].bnum, jnl, tr);
3896 }
3897 /*
3898 * acquire the bp here so that we can safely
3899 * mess around with its data. buf_acquire()
3900 * will return EAGAIN if the buffer was busy,
3901 * so loop trying again.
3902 */
3903 do {
3904 errno = buf_acquire(bp, BAC_REMOVE, 0, 0);
3905 } while (errno == EAGAIN);
3906
3907 if (errno)
3908 panic("could not acquire bp %p (err %d)\n", bp, errno);
3909
3910 if ((buf_flags(bp) & (B_LOCKED|B_DELWRI)) != (B_LOCKED|B_DELWRI)) {
3911 if (jnl->flags & JOURNAL_CLOSE_PENDING) {
3912 buf_clearflags(bp, B_LOCKED);
3913 buf_brelse(bp);
3914
3915 /*
3916 * this is an odd case that appears to happen occasionally
3917 * make sure we mark this block as no longer valid
3918 * so that we don't process it in "finish_end_transaction" since
3919 * the bp that is recorded in our array no longer belongs
3920 * to us (normally we substitute a shadow bp to be processed
3921 * issuing a 'buf_bawrite' on a stale buf_t pointer leads
3922 * to all kinds of problems.
3923 */
3924 blhdr->binfo[i].bnum = (off_t)-1;
3925 continue;
3926 } else {
3927 panic("jnl: end_tr: !!!DANGER!!! bp %p flags (0x%x) not LOCKED & DELWRI\n", bp, buf_flags(bp));
3928 }
3929 }
3930 bsize = buf_size(bp);
3931
3932 buf_setfilter(bp, NULL, NULL, &func, &arg);
3933
3934 blkptr = (char *)&((char *)blhdr)[tbuffer_offset];
3935
3936 sbp = buf_create_shadow_priv(bp, FALSE, (uintptr_t)blkptr, 0, 0);
3937
3938 if (sbp == NULL)
3939 panic("jnl: buf_create_shadow returned NULL");
3940
3941 /*
3942 * copy the data into the transaction buffer...
3943 */
3944 memcpy(blkptr, (char *)buf_dataptr(bp), bsize);
3945
3946 buf_clearflags(bp, B_LOCKED);
3947 buf_markclean(bp);
3948 buf_drop(bp);
3949
3950 /*
3951 * adopt the shadow buffer for this block
3952 */
3953 if (func) {
3954 /*
3955 * transfer FS hook function to the
3956 * shadow buffer... it will get called
3957 * in finish_end_transaction
3958 */
3959 buf_setfilter(sbp, func, arg, NULL, NULL);
3960 }
3961 blhdr->binfo[i].u.bp = sbp;
3962
3963 } else {
3964 // bnum == -1, only true if a block was "killed"
3965 bsize = blhdr->binfo[i].u.bi.bsize;
3966 }
3967 tbuffer_offset += bsize;
3968 }
3969 next = (block_list_header *)((long)blhdr->binfo[0].bnum);
3970 }
3971 /*
3972 * if callback != NULL, we don't want to drop the journal
3973 * lock, or complete end_transaction asynchronously, since
3974 * the caller is expecting the callback to run in the calling
3975 * context
3976 *
3977 * if drop_lock == FALSE, we can't complete end_transaction
3978 * asynchronously
3979 */
3980 if (callback)
3981 drop_lock_early = FALSE;
3982 else
3983 drop_lock_early = drop_lock;
3984
3985 if (drop_lock_early == FALSE)
3986 must_wait = TRUE;
3987
3988 if (drop_lock_early == TRUE) {
3989 journal_unlock(jnl);
3990 drop_lock = FALSE;
3991 }
3992 if (must_wait == TRUE)
3993 ret_val = finish_end_transaction(tr, callback, callback_arg);
3994 else {
3995 thread_t thread = THREAD_NULL;
3996
3997 /*
3998 * fire up a thread to complete processing this transaction
3999 * asynchronously... when it finishes, it will call
4000 * unlock_condition
4001 */
4002 kernel_thread_start((thread_continue_t)finish_end_thread, tr, &thread);
4003 }
4004 KERNEL_DEBUG(0xbbbbc018|DBG_FUNC_END, jnl, tr, ret_val, 0, 0);
4005 done:
4006 if (drop_lock == TRUE) {
4007 journal_unlock(jnl);
4008 }
4009 return (ret_val);
4010 }
4011
4012
4013 static void
4014 finish_end_thread(transaction *tr)
4015 {
4016 throttle_set_thread_io_policy(IOPOL_PASSIVE);
4017
4018 finish_end_transaction(tr, NULL, NULL);
4019
4020 thread_deallocate(current_thread());
4021 thread_terminate(current_thread());
4022 }
4023
4024 static void
4025 write_header_thread(journal *jnl)
4026 {
4027 throttle_set_thread_io_policy(IOPOL_PASSIVE);
4028
4029 if (write_journal_header(jnl, 1, jnl->saved_sequence_num))
4030 jnl->write_header_failed = TRUE;
4031 else
4032 jnl->write_header_failed = FALSE;
4033 unlock_condition(jnl, &jnl->writing_header);
4034
4035 thread_deallocate(current_thread());
4036 thread_terminate(current_thread());
4037 }
4038
4039 static int
4040 finish_end_transaction(transaction *tr, errno_t (*callback)(void*), void *callback_arg)
4041 {
4042 int i, amt;
4043 int ret = 0;
4044 off_t end;
4045 journal *jnl = tr->jnl;
4046 buf_t bp, *bparray;
4047 vnode_t vp;
4048 block_list_header *blhdr=NULL, *next=NULL;
4049 size_t tbuffer_offset;
4050 int bufs_written = 0;
4051 int ret_val = 0;
4052 boolean_t was_vm_privileged = FALSE;
4053
4054 KERNEL_DEBUG(0xbbbbc028|DBG_FUNC_START, jnl, tr, 0, 0, 0);
4055
4056 if (vfs_isswapmount(jnl->fsmount)) {
4057 /*
4058 * if we block waiting for memory, and there is enough pressure to
4059 * cause us to try and create a new swap file, we may end up deadlocking
4060 * due to waiting for the journal on the swap file creation path...
4061 * by making ourselves vm_privileged, we give ourselves the best chance
4062 * of not blocking
4063 */
4064 was_vm_privileged = set_vm_privilege(TRUE);
4065 }
4066 end = jnl->jhdr->end;
4067
4068 for (blhdr = tr->blhdr; blhdr; blhdr = (block_list_header *)((long)blhdr->binfo[0].bnum)) {
4069
4070 amt = blhdr->bytes_used;
4071
4072 blhdr->binfo[0].u.bi.b.sequence_num = tr->sequence_num;
4073
4074 blhdr->checksum = 0;
4075 blhdr->checksum = calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE);
4076
4077 bparray = hfs_malloc(blhdr->num_blocks * sizeof(buf_t));
4078 tbuffer_offset = jnl->jhdr->blhdr_size;
4079
4080 for (i = 1; i < blhdr->num_blocks; i++) {
4081 void (*func)(buf_t, void *);
4082 void *arg;
4083 int32_t bsize;
4084
4085 /*
4086 * finish preparing the shadow buf_t before
4087 * calculating the individual block checksums
4088 */
4089 if (blhdr->binfo[i].bnum != (off_t)-1) {
4090 daddr64_t blkno;
4091 daddr64_t lblkno;
4092
4093 bp = blhdr->binfo[i].u.bp;
4094
4095 vp = buf_vnode(bp);
4096 blkno = buf_blkno(bp);
4097 lblkno = buf_lblkno(bp);
4098
4099 if (vp == NULL && lblkno == blkno) {
4100 printf("jnl: %s: end_tr: bad news! buffer w/null vp and l/blkno = %qd/%qd. aborting the transaction.\n",
4101 jnl->jdev_name, lblkno, blkno);
4102 ret_val = -1;
4103 goto bad_journal;
4104 }
4105
4106 // if the lblkno is the same as blkno and this bp isn't
4107 // associated with the underlying file system device then
4108 // we need to call bmap() to get the actual physical block.
4109 //
4110 if ((lblkno == blkno) && (vp != jnl->fsdev)) {
4111 off_t f_offset;
4112 size_t contig_bytes;
4113
4114 if (hfs_vnop_blktooff(&(struct vnop_blktooff_args){
4115 .a_vp = vp,
4116 .a_lblkno = lblkno,
4117 .a_offset = &f_offset
4118 })) {
4119 printf("jnl: %s: end_tr: vnop_blktooff failed\n", jnl->jdev_name);
4120 ret_val = -1;
4121 goto bad_journal;
4122 }
4123
4124 if (hfs_vnop_blockmap(&(struct vnop_blockmap_args) {
4125 .a_vp = vp,
4126 .a_foffset = f_offset,
4127 .a_size = buf_count(bp),
4128 .a_bpn = &blkno,
4129 .a_run = &contig_bytes
4130 })) {
4131 printf("jnl: %s: end_tr: can't blockmap the buffer\n", jnl->jdev_name);
4132 ret_val = -1;
4133 goto bad_journal;
4134 }
4135
4136 if ((uint32_t)contig_bytes < buf_count(bp)) {
4137 printf("jnl: %s: end_tr: blk not physically contiguous on disk\n", jnl->jdev_name);
4138 ret_val = -1;
4139 goto bad_journal;
4140 }
4141 buf_setblkno(bp, blkno);
4142 }
4143 // update this so we write out the correct physical block number!
4144 blhdr->binfo[i].bnum = (off_t)(blkno);
4145
4146 /*
4147 * pick up the FS hook function (if any) and prepare
4148 * to fire this buffer off in the next pass
4149 */
4150 buf_setfilter(bp, buffer_flushed_callback, tr, &func, &arg);
4151
4152 if (func) {
4153 /*
4154 * call the hook function supplied by the filesystem...
4155 * this needs to happen BEFORE cacl_checksum in case
4156 * the FS morphs the data in the buffer
4157 */
4158 func(bp, arg);
4159 }
4160 bparray[i] = bp;
4161 bsize = buf_size(bp);
4162 blhdr->binfo[i].u.bi.bsize = bsize;
4163 blhdr->binfo[i].u.bi.b.cksum = calc_checksum(&((char *)blhdr)[tbuffer_offset], bsize);
4164 } else {
4165 bparray[i] = NULL;
4166 bsize = blhdr->binfo[i].u.bi.bsize;
4167 blhdr->binfo[i].u.bi.b.cksum = 0;
4168 }
4169 tbuffer_offset += bsize;
4170 }
4171 /*
4172 * if we fired off the journal_write_header asynchronously in
4173 * 'end_transaction', we need to wait for its completion
4174 * before writing the actual journal data
4175 */
4176 wait_condition(jnl, &jnl->writing_header, "finish_end_transaction");
4177
4178 if (jnl->write_header_failed == FALSE)
4179 ret = write_journal_data(jnl, &end, blhdr, amt);
4180 else
4181 ret_val = -1;
4182 /*
4183 * put the bp pointers back so that we can
4184 * make the final pass on them
4185 */
4186 for (i = 1; i < blhdr->num_blocks; i++)
4187 blhdr->binfo[i].u.bp = bparray[i];
4188
4189 hfs_free(bparray, blhdr->num_blocks * sizeof(buf_t));
4190
4191 if (ret_val == -1)
4192 goto bad_journal;
4193
4194 if (ret != amt) {
4195 printf("jnl: %s: end_transaction: only wrote %d of %d bytes to the journal!\n",
4196 jnl->jdev_name, ret, amt);
4197
4198 ret_val = -1;
4199 goto bad_journal;
4200 }
4201 }
4202 jnl->jhdr->end = end; // update where the journal now ends
4203 tr->journal_end = end; // the transaction ends here too
4204
4205 if (tr->journal_start == 0 || tr->journal_end == 0) {
4206 panic("jnl: end_transaction: bad tr journal start/end: 0x%llx 0x%llx\n",
4207 tr->journal_start, tr->journal_end);
4208 }
4209
4210 if (write_journal_header(jnl, 0, jnl->saved_sequence_num) != 0) {
4211 ret_val = -1;
4212 goto bad_journal;
4213 }
4214 /*
4215 * If the caller supplied a callback, call it now that the blocks have been
4216 * written to the journal. This is used by journal_relocate so, for example,
4217 * the file system can change its pointer to the new journal.
4218 */
4219 if (callback != NULL && callback(callback_arg) != 0) {
4220 ret_val = -1;
4221 goto bad_journal;
4222 }
4223
4224 //
4225 // Send a DKIOCUNMAP for the extents trimmed by this transaction, and
4226 // free up the extent list.
4227 //
4228 journal_trim_flush(jnl, tr);
4229
4230 // the buffer_flushed_callback will only be called for the
4231 // real blocks that get flushed so we have to account for
4232 // the block_list_headers here.
4233 //
4234 tr->num_flushed = tr->num_blhdrs * jnl->jhdr->blhdr_size;
4235
4236 lock_condition(jnl, &jnl->asyncIO, "finish_end_transaction");
4237
4238 //
4239 // setup for looping through all the blhdr's.
4240 //
4241 for (blhdr = tr->blhdr; blhdr; blhdr = next) {
4242 uint16_t num_blocks;
4243
4244 /*
4245 * grab this info ahead of issuing the buf_bawrites...
4246 * once the last one goes out, its possible for blhdr
4247 * to be freed (especially if we get preempted) before
4248 * we do the last check of num_blocks or
4249 * grab the next blhdr pointer...
4250 */
4251 next = (block_list_header *)((long)blhdr->binfo[0].bnum);
4252 num_blocks = blhdr->num_blocks;
4253
4254 /*
4255 * we can re-order the buf ptrs because everything is written out already
4256 */
4257 kx_qsort(&blhdr->binfo[1], num_blocks-1, sizeof(block_info), journal_binfo_cmp);
4258
4259 /*
4260 * need to make sure that the loop issuing the buf_bawrite's
4261 * does not touch blhdr once the last buf_bawrite has been
4262 * issued... at that point, we no longer have a legitmate
4263 * reference on the associated storage since it will be
4264 * released upon the completion of that last buf_bawrite
4265 */
4266 for (i = num_blocks-1; i >= 1; i--) {
4267 if (blhdr->binfo[i].bnum != (off_t)-1)
4268 break;
4269 num_blocks--;
4270 }
4271 for (i = 1; i < num_blocks; i++) {
4272
4273 if ((bp = blhdr->binfo[i].u.bp)) {
4274 vp = buf_vnode(bp);
4275
4276 buf_bawrite(bp);
4277
4278 // this undoes the vnode_ref() in journal_modify_block_end()
4279 vnode_rele_ext(vp, 0, 1);
4280
4281 bufs_written++;
4282 }
4283 }
4284 }
4285 if (bufs_written == 0) {
4286 /*
4287 * since we didn't issue any buf_bawrite's, there is no
4288 * async trigger to cause the memory associated with this
4289 * transaction to be freed... so, move it to the garbage
4290 * list now
4291 */
4292 lock_oldstart(jnl);
4293
4294 tr->next = jnl->tr_freeme;
4295 jnl->tr_freeme = tr;
4296
4297 unlock_oldstart(jnl);
4298
4299 unlock_condition(jnl, &jnl->asyncIO);
4300 }
4301
4302 //printf("jnl: end_tr: tr @ 0x%x, jnl-blocks: 0x%llx - 0x%llx. exit!\n",
4303 // tr, tr->journal_start, tr->journal_end);
4304
4305 bad_journal:
4306 if (ret_val == -1) {
4307 abort_transaction(jnl, tr); // cleans up list of extents to be trimmed
4308
4309 /*
4310 * 'flush_aborted' is protected by the flushing condition... we need to
4311 * set it before dropping the condition so that it will be
4312 * noticed in 'end_transaction'... we add this additional
4313 * aborted condition so that we can drop the 'flushing' condition
4314 * before grabbing the journal lock... this avoids a deadlock
4315 * in 'end_transaction' which is holding the journal lock while
4316 * waiting for the 'flushing' condition to clear...
4317 * everyone else will notice the JOURNAL_INVALID flag
4318 */
4319 jnl->flush_aborted = TRUE;
4320
4321 unlock_condition(jnl, &jnl->flushing);
4322 journal_lock(jnl);
4323
4324 jnl->flags |= JOURNAL_INVALID;
4325 jnl->old_start[sizeof(jnl->old_start)/sizeof(jnl->old_start[0]) - 1] &= ~0x8000000000000000LL;
4326
4327 journal_unlock(jnl);
4328 } else
4329 unlock_condition(jnl, &jnl->flushing);
4330
4331 if (vfs_isswapmount(jnl->fsmount) && (was_vm_privileged == FALSE))
4332 set_vm_privilege(FALSE);
4333
4334 KERNEL_DEBUG(0xbbbbc028|DBG_FUNC_END, jnl, tr, bufs_written, ret_val, 0);
4335
4336 return (ret_val);
4337 }
4338
4339
4340 static void
4341 lock_condition(journal *jnl, boolean_t *condition, const char *condition_name)
4342 {
4343
4344 KERNEL_DEBUG(0xbbbbc020|DBG_FUNC_START, jnl, condition, 0, 0, 0);
4345
4346 lock_flush(jnl);
4347
4348 while (*condition == TRUE)
4349 msleep(condition, &jnl->flock, PRIBIO, condition_name, NULL);
4350
4351 *condition = TRUE;
4352 unlock_flush(jnl);
4353
4354 KERNEL_DEBUG(0xbbbbc020|DBG_FUNC_END, jnl, condition, 0, 0, 0);
4355 }
4356
4357 static void
4358 wait_condition(journal *jnl, boolean_t *condition, const char *condition_name)
4359 {
4360
4361 if (*condition == FALSE)
4362 return;
4363
4364 KERNEL_DEBUG(0xbbbbc02c|DBG_FUNC_START, jnl, condition, 0, 0, 0);
4365
4366 lock_flush(jnl);
4367
4368 while (*condition == TRUE)
4369 msleep(condition, &jnl->flock, PRIBIO, condition_name, NULL);
4370
4371 unlock_flush(jnl);
4372
4373 KERNEL_DEBUG(0xbbbbc02c|DBG_FUNC_END, jnl, condition, 0, 0, 0);
4374 }
4375
4376 static void
4377 unlock_condition(journal *jnl, boolean_t *condition)
4378 {
4379 lock_flush(jnl);
4380
4381 *condition = FALSE;
4382 wakeup(condition);
4383
4384 unlock_flush(jnl);
4385 }
4386
4387 static void
4388 abort_transaction(journal *jnl, transaction *tr)
4389 {
4390 block_list_header *blhdr, *next;
4391
4392 // for each block list header, iterate over the blocks then
4393 // free up the memory associated with the block list.
4394 //
4395 // find each of the primary blocks (i.e. the list could
4396 // contain a mix of shadowed and real buf_t's depending
4397 // on when the abort condition was detected) and mark them
4398 // clean and locked in the cache... this at least allows
4399 // the FS a consistent view between it's incore data structures
4400 // and the meta-data held in the cache
4401 //
4402 KERNEL_DEBUG(0xbbbbc034|DBG_FUNC_START, jnl, tr, 0, 0, 0);
4403
4404 for (blhdr = tr->blhdr; blhdr; blhdr = next) {
4405 int i;
4406
4407 for (i = 1; i < blhdr->num_blocks; i++) {
4408 buf_t bp, tbp, sbp;
4409 vnode_t bp_vp;
4410 errno_t errno;
4411
4412 if (blhdr->binfo[i].bnum == (off_t)-1)
4413 continue;
4414
4415 tbp = blhdr->binfo[i].u.bp;
4416
4417 bp_vp = buf_vnode(tbp);
4418
4419 if (buf_shadow(tbp)) {
4420 sbp = tbp;
4421 buf_setfilter(tbp, NULL, NULL, NULL, NULL);
4422 } else {
4423 hfs_assert(ISSET(buf_flags(tbp), B_LOCKED));
4424
4425 sbp = NULL;
4426
4427 do {
4428 errno = buf_acquire(tbp, BAC_REMOVE, 0, 0);
4429 } while (errno == EAGAIN);
4430
4431 if (!errno) {
4432 buf_setfilter(tbp, NULL, NULL, NULL, NULL);
4433 buf_brelse(tbp);
4434 }
4435 }
4436
4437 if (bp_vp) {
4438 errno = buf_meta_bread(bp_vp,
4439 buf_lblkno(tbp),
4440 buf_size(tbp),
4441 NOCRED,
4442 &bp);
4443 if (errno == 0) {
4444 if (sbp == NULL && bp != tbp && (buf_flags(tbp) & B_LOCKED)) {
4445 panic("jnl: abort_tr: got back a different bp! (bp %p should be %p, jnl %p\n",
4446 bp, tbp, jnl);
4447 }
4448 /*
4449 * once the journal has been marked INVALID and aborted,
4450 * NO meta data can be written back to the disk, so
4451 * mark the buf_t clean and make sure it's locked in the cache
4452 * note: if we found a shadow, the real buf_t needs to be relocked
4453 */
4454 buf_setflags(bp, B_LOCKED);
4455 buf_markclean(bp);
4456 buf_brelse(bp);
4457
4458 KERNEL_DEBUG(0xbbbbc034|DBG_FUNC_NONE, jnl, tr, bp, 0, 0);
4459
4460 /*
4461 * this undoes the vnode_ref() in journal_modify_block_end()
4462 */
4463 vnode_rele_ext(bp_vp, 0, 1);
4464 } else {
4465 printf("jnl: %s: abort_tr: could not find block %lld for vnode!\n",
4466 jnl->jdev_name, blhdr->binfo[i].bnum);
4467 if (bp) {
4468 buf_brelse(bp);
4469 }
4470 }
4471 }
4472 if (sbp)
4473 buf_brelse(sbp);
4474 }
4475 next = (block_list_header *)((long)blhdr->binfo[0].bnum);
4476
4477 // we can free blhdr here since we won't need it any more
4478 blhdr->binfo[0].bnum = 0xdeadc0de;
4479 hfs_free(blhdr, tr->tbuffer_size);
4480 }
4481
4482 /*
4483 * If the transaction we're aborting was the async transaction, then
4484 * tell the current transaction that there is no pending trim
4485 * any more.
4486 */
4487 lck_rw_lock_exclusive(&jnl->trim_lock);
4488 if (jnl->async_trim == &tr->trim)
4489 jnl->async_trim = NULL;
4490 lck_rw_unlock_exclusive(&jnl->trim_lock);
4491
4492
4493 if (tr->trim.extents) {
4494 hfs_free(tr->trim.extents, tr->trim.allocated_count * sizeof(dk_extent_t));
4495 }
4496 tr->trim.allocated_count = 0;
4497 tr->trim.extent_count = 0;
4498 tr->trim.extents = NULL;
4499 tr->tbuffer = NULL;
4500 tr->blhdr = NULL;
4501 tr->total_bytes = 0xdbadc0de;
4502 hfs_free(tr, sizeof(*tr));
4503
4504 KERNEL_DEBUG(0xbbbbc034|DBG_FUNC_END, jnl, tr, 0, 0, 0);
4505 }
4506
4507
4508 int
4509 journal_end_transaction(journal *jnl)
4510 {
4511 int ret;
4512 transaction *tr;
4513
4514 CHECK_JOURNAL(jnl);
4515
4516 free_old_stuff(jnl);
4517
4518 if ((jnl->flags & JOURNAL_INVALID) && jnl->owner == NULL) {
4519 return 0;
4520 }
4521
4522 if (jnl->owner != current_thread()) {
4523 panic("jnl: end_tr: I'm not the owner! jnl %p, owner %p, curact %p\n",
4524 jnl, jnl->owner, current_thread());
4525 }
4526 jnl->nested_count--;
4527
4528 if (jnl->nested_count > 0) {
4529 return 0;
4530 } else if (jnl->nested_count < 0) {
4531 panic("jnl: jnl @ %p has negative nested count (%d). bad boy.\n", jnl, jnl->nested_count);
4532 }
4533
4534 if (jnl->flags & JOURNAL_INVALID) {
4535 if (jnl->active_tr) {
4536 if (jnl->cur_tr != NULL) {
4537 panic("jnl: journal @ %p has active tr (%p) and cur tr (%p)\n",
4538 jnl, jnl->active_tr, jnl->cur_tr);
4539 }
4540 tr = jnl->active_tr;
4541 jnl->active_tr = NULL;
4542
4543 abort_transaction(jnl, tr);
4544 }
4545 journal_unlock(jnl);
4546
4547 return EINVAL;
4548 }
4549
4550 tr = jnl->active_tr;
4551 CHECK_TRANSACTION(tr);
4552
4553 // clear this out here so that when check_free_space() calls
4554 // the FS flush function, we don't panic in journal_flush()
4555 // if the FS were to call that. note: check_free_space() is
4556 // called from end_transaction().
4557 //
4558 jnl->active_tr = NULL;
4559
4560 /* Examine the force-journal-flush state in the active txn */
4561 if (tr->flush_on_completion == TRUE) {
4562 /*
4563 * If the FS requested it, disallow group commit and force the
4564 * transaction out to disk immediately.
4565 */
4566 ret = end_transaction(tr, 1, NULL, NULL, TRUE, TRUE);
4567 }
4568 else {
4569 /* in the common path we can simply use the double-buffered journal */
4570 ret = end_transaction(tr, 0, NULL, NULL, TRUE, FALSE);
4571 }
4572
4573 return ret;
4574 }
4575
4576
4577 /*
4578 * Flush the contents of the journal to the disk.
4579 *
4580 * Input:
4581 * wait_for_IO -
4582 * If TRUE, wait to write in-memory journal to the disk
4583 * consistently, and also wait to write all asynchronous
4584 * metadata blocks to its corresponding locations
4585 * consistently on the disk. This means that the journal
4586 * is empty at this point and does not contain any
4587 * transactions. This is overkill in normal scenarios
4588 * but is useful whenever the metadata blocks are required
4589 * to be consistent on-disk instead of just the journal
4590 * being consistent; like before live verification
4591 * and live volume resizing.
4592 *
4593 * If FALSE, only wait to write in-memory journal to the
4594 * disk consistently. This means that the journal still
4595 * contains uncommitted transactions and the file system
4596 * metadata blocks in the journal transactions might be
4597 * written asynchronously to the disk. But there is no
4598 * guarantee that they are written to the disk before
4599 * returning to the caller. Note that this option is
4600 * sufficient for file system data integrity as it
4601 * guarantees consistent journal content on the disk.
4602 */
4603 int
4604 journal_flush(journal *jnl, journal_flush_options_t options)
4605 {
4606 boolean_t drop_lock = FALSE;
4607 errno_t error = 0;
4608 uint32_t flush_count = 0;
4609
4610 CHECK_JOURNAL(jnl);
4611
4612 free_old_stuff(jnl);
4613
4614 if (jnl->flags & JOURNAL_INVALID) {
4615 return -1;
4616 }
4617
4618 KDBG(DBG_JOURNAL_FLUSH | DBG_FUNC_START, jnl);
4619
4620 if (jnl->owner != current_thread()) {
4621 journal_lock(jnl);
4622 drop_lock = TRUE;
4623 }
4624
4625 if (ISSET(options, JOURNAL_FLUSH_FULL))
4626 flush_count = jnl->flush_counter;
4627
4628 // if we're not active, flush any buffered transactions
4629 if (jnl->active_tr == NULL && jnl->cur_tr) {
4630 transaction *tr = jnl->cur_tr;
4631
4632 jnl->cur_tr = NULL;
4633
4634 if (ISSET(options, JOURNAL_WAIT_FOR_IO)) {
4635 wait_condition(jnl, &jnl->flushing, "journal_flush");
4636 wait_condition(jnl, &jnl->asyncIO, "journal_flush");
4637 }
4638 /*
4639 * "end_transction" will wait for any current async flush
4640 * to complete, before flushing "cur_tr"... because we've
4641 * specified the 'must_wait' arg as TRUE, it will then
4642 * synchronously flush the "cur_tr"
4643 */
4644 end_transaction(tr, 1, NULL, NULL, drop_lock, TRUE); // force it to get flushed
4645
4646 } else {
4647 if (drop_lock == TRUE) {
4648 journal_unlock(jnl);
4649 }
4650
4651 /* Because of pipelined journal, the journal transactions
4652 * might be in process of being flushed on another thread.
4653 * If there is nothing to flush currently, we should
4654 * synchronize ourselves with the pipelined journal thread
4655 * to ensure that all inflight transactions, if any, are
4656 * flushed before we return success to caller.
4657 */
4658 wait_condition(jnl, &jnl->flushing, "journal_flush");
4659 }
4660 if (ISSET(options, JOURNAL_WAIT_FOR_IO)) {
4661 wait_condition(jnl, &jnl->asyncIO, "journal_flush");
4662 }
4663
4664 if (ISSET(options, JOURNAL_FLUSH_FULL)) {
4665
4666 dk_synchronize_t sync_request = {
4667 .options = 0,
4668 };
4669
4670 // We need a full cache flush. If it has not been done, do it here.
4671 if (flush_count == jnl->flush_counter)
4672 error = VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZE, (caddr_t)&sync_request, FWRITE, vfs_context_kernel());
4673
4674 // If external journal partition is enabled, flush filesystem data partition.
4675 if (jnl->jdev != jnl->fsdev)
4676 error = VNOP_IOCTL(jnl->fsdev, DKIOCSYNCHRONIZE, (caddr_t)&sync_request, FWRITE, vfs_context_kernel());
4677
4678 }
4679
4680 KDBG(DBG_JOURNAL_FLUSH | DBG_FUNC_END, jnl);
4681
4682 return 0;
4683 }
4684
4685 int
4686 journal_active(journal *jnl)
4687 {
4688 if (jnl->flags & JOURNAL_INVALID) {
4689 return -1;
4690 }
4691
4692 return (jnl->active_tr == NULL) ? 0 : 1;
4693 }
4694
4695 void *
4696 journal_owner(journal *jnl)
4697 {
4698 return jnl->owner;
4699 }
4700
4701 int journal_uses_fua(journal *jnl)
4702 {
4703 if (jnl->flags & JOURNAL_DO_FUA_WRITES)
4704 return 1;
4705 return 0;
4706 }
4707
4708 /*
4709 * Relocate the journal.
4710 *
4711 * You provide the new starting offset and size for the journal. You may
4712 * optionally provide a new tbuffer_size; passing zero defaults to not
4713 * changing the tbuffer size except as needed to fit within the new journal
4714 * size.
4715 *
4716 * You must have already started a transaction. The transaction may contain
4717 * modified blocks (such as those needed to deallocate the old journal,
4718 * allocate the new journal, and update the location and size of the journal
4719 * in filesystem-private structures). Any transactions prior to the active
4720 * transaction will be flushed to the old journal. The new journal will be
4721 * initialized, and the blocks from the active transaction will be written to
4722 * the new journal.
4723 *
4724 * The caller will need to update the structures that identify the location
4725 * and size of the journal. These updates should be made in the supplied
4726 * callback routine. These updates must NOT go into a transaction. You should
4727 * force these updates to the media before returning from the callback. In the
4728 * even of a crash, either the old journal will be found, with an empty journal,
4729 * or the new journal will be found with the contents of the active transaction.
4730 *
4731 * Upon return from the callback, the blocks from the active transaction are
4732 * written to their normal locations on disk.
4733 *
4734 * (Remember that we have to ensure that blocks get committed to the journal
4735 * before being committed to their normal locations. But the blocks don't count
4736 * as committed until the new journal is pointed at.)
4737 *
4738 * Upon return, there is still an active transaction: newly allocated, and
4739 * with no modified blocks. Call journal_end_transaction as normal. You may
4740 * modifiy additional blocks before calling journal_end_transaction, and those
4741 * blocks will (eventually) go to the relocated journal.
4742 *
4743 * Inputs:
4744 * jnl The (opened) journal to relocate.
4745 * offset The new journal byte offset (from start of the journal device).
4746 * journal_size The size, in bytes, of the new journal.
4747 * tbuffer_size The new desired transaction buffer size. Pass zero to keep
4748 * the same size as the current journal. The size will be
4749 * modified as needed to fit the new journal.
4750 * callback Routine called after the new journal has been initialized,
4751 * and the active transaction written to the new journal, but
4752 * before the blocks are written to their normal locations.
4753 * Pass NULL for no callback.
4754 * callback_arg An argument passed to the callback routine.
4755 *
4756 * Result:
4757 * 0 No errors
4758 * EINVAL The offset is not block aligned
4759 * EINVAL The journal_size is not a multiple of the block size
4760 * EINVAL The journal is invalid
4761 * (any) An error returned by journal_flush.
4762 *
4763 */
4764 int journal_relocate(journal *jnl, off_t offset, off_t journal_size, int32_t tbuffer_size,
4765 errno_t (*callback)(void *), void *callback_arg)
4766 {
4767 int ret;
4768 transaction *tr;
4769 size_t i = 0;
4770
4771 /*
4772 * Sanity check inputs, and adjust the size of the transaction buffer.
4773 */
4774 if ((offset % jnl->jhdr->jhdr_size) != 0) {
4775 printf("jnl: %s: relocate: offset 0x%llx is not an even multiple of block size 0x%x\n",
4776 jnl->jdev_name, offset, jnl->jhdr->jhdr_size);
4777 return EINVAL;
4778 }
4779 if ((journal_size % jnl->jhdr->jhdr_size) != 0) {
4780 printf("jnl: %s: relocate: journal size 0x%llx is not an even multiple of block size 0x%x\n",
4781 jnl->jdev_name, journal_size, jnl->jhdr->jhdr_size);
4782 return EINVAL;
4783 }
4784
4785 CHECK_JOURNAL(jnl);
4786
4787 /* Guarantee we own the active transaction. */
4788 if (jnl->flags & JOURNAL_INVALID) {
4789 return EINVAL;
4790 }
4791 if (jnl->owner != current_thread()) {
4792 panic("jnl: relocate: Not the owner! jnl %p, owner %p, curact %p\n",
4793 jnl, jnl->owner, current_thread());
4794 }
4795
4796 if (tbuffer_size == 0)
4797 tbuffer_size = jnl->tbuffer_size;
4798 size_up_tbuffer(jnl, tbuffer_size, jnl->jhdr->jhdr_size);
4799
4800 /*
4801 * Flush any non-active transactions. We have to temporarily hide the
4802 * active transaction to make journal_flush flush out non-active but
4803 * current (unwritten) transactions.
4804 */
4805 tr = jnl->active_tr;
4806 CHECK_TRANSACTION(tr);
4807 jnl->active_tr = NULL;
4808 ret = journal_flush(jnl, JOURNAL_WAIT_FOR_IO);
4809 jnl->active_tr = tr;
4810
4811 if (ret) {
4812 return ret;
4813 }
4814 wait_condition(jnl, &jnl->flushing, "end_transaction");
4815
4816 /*
4817 * At this point, we have completely flushed the contents of the current
4818 * journal to disk (and have asynchronously written all of the txns to
4819 * their actual desired locations). As a result, we can (and must) clear
4820 * out the old_start array. If we do not, then if the last written transaction
4821 * started at the beginning of the journal (starting 1 block into the
4822 * journal file) it could confuse the buffer_flushed callback. This is
4823 * because we're about to reset the start/end pointers of the journal header
4824 * below.
4825 */
4826 lock_oldstart(jnl);
4827 for (i = 0; i < sizeof (jnl->old_start) / sizeof(jnl->old_start[0]); i++) {
4828 jnl->old_start[i] = 0;
4829 }
4830 unlock_oldstart(jnl);
4831
4832 /* Update the journal's offset and size in memory. */
4833 jnl->jdev_offset = offset;
4834 jnl->jhdr->start = jnl->jhdr->end = jnl->jhdr->jhdr_size;
4835 jnl->jhdr->size = journal_size;
4836 jnl->active_start = jnl->jhdr->start;
4837
4838 /*
4839 * Force the active transaction to be written to the new journal. Call the
4840 * supplied callback after the blocks have been written to the journal, but
4841 * before they get written to their normal on-disk locations.
4842 */
4843 jnl->active_tr = NULL;
4844 ret = end_transaction(tr, 1, callback, callback_arg, FALSE, TRUE);
4845 if (ret) {
4846 printf("jnl: %s: relocate: end_transaction failed (%d)\n", jnl->jdev_name, ret);
4847 goto bad_journal;
4848 }
4849
4850 /*
4851 * Create a new, empty transaction to be the active transaction. This way
4852 * our caller can use journal_end_transaction as usual.
4853 */
4854 ret = journal_allocate_transaction(jnl);
4855 if (ret) {
4856 printf("jnl: %s: relocate: could not allocate new transaction (%d)\n", jnl->jdev_name, ret);
4857 goto bad_journal;
4858 }
4859
4860 return 0;
4861
4862 bad_journal:
4863 jnl->flags |= JOURNAL_INVALID;
4864 abort_transaction(jnl, tr);
4865 return ret;
4866 }
4867
4868 uint32_t journal_current_txn(journal *jnl)
4869 {
4870 return jnl->sequence_num + (jnl->active_tr || jnl->cur_tr ? 0 : 1);
4871 }