livefiles_hfs_plugin/lf_hfs_journal.h

   1 //
   2 //  lf_hfs_journal.h
   3 //  livefiles_hfs
   4 //
   5 //  Created by Or Haimovich on 22/3/18.
   6 //
   7
   8 #ifndef lf_hfs_journal_h
   9 #define lf_hfs_journal_h
  10
  11 #include <sys/disk.h>
  12 #include "lf_hfs_generic_buf.h"
  13
  14 #define JOURNAL_DEBUG 0
  15
  16 typedef struct _blk_info {
  17     int32_t    bsize;
  18     union {
  19         int32_t    cksum;
  20         uint32_t   sequence_num;
  21     } b;
  22 } _blk_info;
  23
  24 typedef struct block_info {
  25     off_t       bnum;                // block # on the file system device
  26     union {
  27         _blk_info     bi;
  28         struct buf   *bp;
  29     } u;
  30 } __attribute__((__packed__)) block_info;
  31
  32 typedef struct block_list_header {
  33     u_int16_t   max_blocks;          // max number of blocks in this chunk
  34     u_int16_t   num_blocks;          // number of valid block numbers in block_nums
  35     int32_t     bytes_used;          // how many bytes of this tbuffer are used
  36     uint32_t    checksum;            // on-disk: checksum of this header and binfo[0]
  37     int32_t     flags;               // check-checksums, initial blhdr, etc
  38     block_info  binfo[1];            // so we can reference them by name
  39 } block_list_header;
  40
  41 #define BLHDR_CHECK_CHECKSUMS   0x0001
  42 #define BLHDR_FIRST_HEADER      0x0002
  43
  44
  45 struct journal;
  46
  47 struct jnl_trim_list {
  48     uint32_t    allocated_count;
  49     uint32_t    extent_count;
  50     dk_extent_t *extents;
  51 };
  52
  53 typedef void (*jnl_trim_callback_t)(void *arg, uint32_t extent_count, const dk_extent_t *extents);
  54
  55 typedef struct transaction {
  56     int                  tbuffer_size;  // in bytes
  57     char                *tbuffer;       // memory copy of the transaction
  58     block_list_header   *blhdr;         // points to the first byte of tbuffer
  59     int                  num_blhdrs;    // how many buffers we've allocated
  60     int                  total_bytes;   // total # of bytes in transaction
  61     int                  num_flushed;   // how many bytes have been flushed
  62     int                  num_killed;    // how many bytes were "killed"
  63     off_t                journal_start; // where in the journal this transaction starts
  64     off_t                journal_end;   // where in the journal this transaction ends
  65     struct journal      *jnl;           // ptr back to the journal structure
  66     struct transaction  *next;          // list of tr's (either completed or to be free'd)
  67     uint32_t             sequence_num;
  68     struct jnl_trim_list trim;
  69     boolean_t            delayed_header_write;
  70     boolean_t            flush_on_completion; //flush transaction immediately upon txn end.
  71 } transaction;
  72
  73
  74 /*
  75  * This is written to block zero of the journal and it
  76  * maintains overall state about the journal.
  77  */
  78 typedef struct journal_header {
  79     int32_t        magic;
  80     int32_t        endian;
  81     volatile off_t start;         // zero-based byte offset of the start of the first transaction
  82     volatile off_t end;           // zero-based byte offset of where free space begins
  83     off_t          size;          // size in bytes of the entire journal
  84     uint32_t       blhdr_size;    // size in bytes of each block_list_header in the journal
  85     uint32_t       checksum;
  86     int32_t        jhdr_size;     // block size (in bytes) of the journal header
  87     uint32_t       sequence_num;  // NEW FIELD: a monotonically increasing value assigned to all txn's
  88 } journal_header;
  89
  90 #define JOURNAL_HEADER_MAGIC  0x4a4e4c78   // 'JNLx'
  91 #define ENDIAN_MAGIC          0x12345678
  92
  93 //
  94 // we only checksum the original size of the journal_header to remain
  95 // backwards compatible.  the size of the original journal_heade is
  96 // everything up to the the sequence_num field, hence we use the
  97 // offsetof macro to calculate the size.
  98 //
  99 #define JOURNAL_HEADER_CKSUM_SIZE  (offsetof(struct journal_header, sequence_num))
 100
 101 #define OLD_JOURNAL_HEADER_MAGIC  0x4a484452   // 'JHDR'
 102
 103 typedef struct {
 104     pthread_cond_t sCond;
 105     uint32_t       uFlag;
 106 } ConditionalFlag_S;
 107
 108 /*
 109  * In memory structure about the journal.
 110  */
 111 typedef struct journal {
 112     pthread_mutex_t     jlock;             // protects the struct journal data
 113     pthread_mutex_t     flock;             // serializes flushing of journal
 114     pthread_rwlock_t    trim_lock;         // protects the async_trim field, below
 115
 116     struct vnode       *jdev;              // vnode of the device where the journal lives
 117     off_t               jdev_offset;       // byte offset to the start of the journal
 118     uint32_t            jdev_blknum;       // Physical block number of the journal
 119     //const char         *jdev_name;
 120
 121     struct vnode       *fsdev;             // vnode of the file system device
 122     struct mount       *fsmount;           // mount of the file system
 123
 124     void              (*flush)(void *arg); // fs callback to flush meta data blocks
 125     void               *flush_arg;         // arg that's passed to flush()
 126
 127     int32_t             flags;
 128     uint32_t            tbuffer_size;      // default transaction buffer size
 129     ConditionalFlag_S   flushing;
 130     ConditionalFlag_S   asyncIO;
 131     ConditionalFlag_S   writing_header;
 132     boolean_t           flush_aborted;
 133     boolean_t           write_header_failed;
 134
 135     struct jnl_trim_list *async_trim;      // extents to be trimmed by transaction being asynchronously flushed
 136     jnl_trim_callback_t    trim_callback;
 137     void                *trim_callback_arg;
 138
 139     char               *header_buf;        // in-memory copy of the journal header
 140     int32_t             header_buf_size;
 141     journal_header     *jhdr;              // points to the first byte of header_buf
 142
 143     uint32_t        saved_sequence_num;
 144     uint32_t        sequence_num;
 145
 146     off_t               max_read_size;
 147     off_t               max_write_size;
 148
 149     transaction        *cur_tr;            // for group-commit
 150     transaction        *completed_trs;     // out-of-order transactions that completed
 151     transaction        *active_tr;         // for nested transactions
 152     int32_t             nested_count;      // for nested transactions
 153     void               *owner;             // a ptr that's unique to the calling process
 154
 155     transaction        *tr_freeme;         // transaction structs that need to be free'd
 156
 157     volatile off_t      active_start;      // the active start that we only keep in memory
 158     pthread_mutex_t     old_start_lock;    // protects the old_start
 159     volatile off_t      old_start[16];     // this is how we do lazy start update
 160
 161     int                 last_flush_err;    // last error from flushing the cache
 162     uint32_t            flush_counter;     // a monotonically increasing value assigned on track cache flush
 163 } journal;
 164
 165 /* internal-only journal flags (top 16 bits) */
 166 #define JOURNAL_CLOSE_PENDING     0x00010000
 167 #define JOURNAL_INVALID           0x00020000
 168 #define JOURNAL_FLUSHCACHE_ERR    0x00040000   // means we already printed this err
 169 #define JOURNAL_NEED_SWAP         0x00080000   // swap any data read from disk
 170 #define JOURNAL_DO_FUA_WRITES     0x00100000   // do force-unit-access writes
 171 #define JOURNAL_USE_UNMAP         0x00200000   // device supports UNMAP (TRIM)
 172 #define JOURNAL_FEATURE_BARRIER   0x00400000   // device supports barrier-only flush
 173
 174
 175 /* journal_open/create options are always in the low-16 bits */
 176 #define JOURNAL_OPTION_FLAGS_MASK 0x0000ffff
 177
 178 __BEGIN_DECLS
 179 /*
 180  * Prototypes.
 181  */
 182
 183 /*
 184  * Call journal_init() to initialize the journaling code (sets up lock attributes)
 185  */
 186 void      journal_init(void);
 187
 188 /*
 189  * Call journal_open() when mounting an existing file system
 190  * that has a previously created journal.  It will take care
 191  * of validating the journal and replaying it if necessary.
 192  *
 193  * The "jvp" argument is the vnode where the journal is written.
 194  * The journal starts at "offset" and is "journal_size" bytes long.
 195  *
 196  * The "fsvp" argument is the vnode of your file system.  It may be
 197  * the same as "jvp".
 198  *
 199  * The "min_fs_block_size" argument is the minimum block size
 200  * (in bytes) that the file system will ever write.  Typically
 201  * this is the block size of the file system (1k, 4k, etc) but
 202  * on HFS+ it is the minimum block size of the underlying device.
 203  *
 204  * The flags argument lets you disable group commit if you
 205  * want tighter guarantees on transactions (in exchange for
 206  * lower performance).
 207  *
 208  * The tbuffer_size is the size of the transaction buffer
 209  * used by the journal. If you specify zero, the journal code
 210  * will use a reasonable defaults.  The tbuffer_size should
 211  * be an integer multiple of the min_fs_block_size.
 212  *
 213  * Returns a valid journal pointer of NULL if it runs into
 214  * trouble reading/playing back the journal.
 215  */
 216 journal  *journal_open(struct vnode *jvp,
 217                        off_t         offset,
 218                        off_t         journal_size,
 219                        struct vnode *fsvp,
 220                        size_t        min_fs_block_size,
 221                        int32_t       flags,
 222                        int32_t       tbuffer_size,
 223                        void        (*flush)(void *arg),
 224                        void         *arg,
 225                        struct mount *fsmount);
 226 /*
 227  * Call journal_create() to create a new journal.  You only
 228  * call this once, typically at file system creation time.
 229  *
 230  * The "jvp" argument is the vnode where the journal is written.
 231  * The journal starts at "offset" and is "journal_size" bytes long.
 232  *
 233  * The "fsvp" argument is the vnode of your file system.  It may be
 234  * the same as "jvp".
 235  *
 236  * The "min_fs_block_size" argument is the minimum block size
 237  * (in bytes) that the file system will ever write.  Typically
 238  * this is the block size of the file system (1k, 4k, etc) but
 239  * on HFS+ it is the minimum block size of the underlying device.
 240  *
 241  * The flags argument lets you disable group commit if you
 242  * want tighter guarantees on transactions (in exchange for
 243  * lower performance).
 244  *
 245  * The tbuffer_size is the size of the transaction buffer
 246  * used by the journal. If you specify zero, the journal code
 247  * will use a reasonable defaults.  The tbuffer_size should
 248  * be an integer multiple of the min_fs_block_size.
 249  *
 250  * Returns a valid journal pointer or NULL if one could not
 251  * be created.
 252  */
 253 journal *journal_create(struct vnode *jvp,
 254                         off_t         offset,
 255                         off_t         journal_size,
 256                         struct vnode *fsvp,
 257                         size_t        min_fs_block_size,
 258                         int32_t       flags,
 259                         int32_t       tbuffer_size,
 260                         void        (*flush)(void *arg),
 261                         void         *arg,
 262                         struct mount *fsmount);
 263
 264 /*
 265  * Test whether the journal is clean or not.  This is intended
 266  * to be used when you're mounting read-only.  If the journal
 267  * is not clean for some reason then you should not mount the
 268  * volume as your data structures may be in an unknown state.
 269  */
 270 int journal_is_clean(struct vnode *jvp,
 271                      off_t         offset,
 272                      off_t         journal_size,
 273                      struct vnode *fsvp,
 274                      size_t        min_fs_block_size,
 275                      struct mount  *fsmount);
 276
 277
 278
 279 /*
 280  * Call journal_release() to release all buffers held by the journal.
 281  * This is used incase of live-files unmount, since the media is no longer
 282  * available at this time.
 283  */
 284 void journal_release(journal *jnl);
 285
 286 /*
 287  * Call journal_close() just before your file system is unmounted.
 288  * It flushes any outstanding transactions and makes sure the
 289  * journal is in a consistent state.
 290  */
 291 void      journal_close(journal *journalp);
 292
 293 /*
 294  * flags for journal_create/open.  only can use
 295  * the low 16 bits for flags because internal
 296  * bits go in the high 16.
 297  */
 298 #define JOURNAL_NO_GROUP_COMMIT   0x00000001
 299 #define JOURNAL_RESET             0x00000002
 300
 301 /*
 302  * Transaction related functions.
 303  *
 304  * Before you start modifying file system meta data, you
 305  * should call journal_start_transaction().  Then before
 306  * you modify each block, call journal_modify_block_start()
 307  * and when you're done, journal_modify_block_end().  When
 308  * you've modified the last block as part of a transaction,
 309  * call journal_end_transaction() to commit the changes.
 310  *
 311  * If you decide to abort the modifications to a block you
 312  * should call journal_modify_block_abort().
 313  *
 314  * If as part of a transaction you need want to throw out
 315  * any previous copies of a block (because it got deleted)
 316  * then call journal_kill_block().  This will mark it so
 317  * that the journal does not play it back (effectively
 318  * dropping it).
 319  *
 320  * journal_trim_add_extent() marks a range of bytes on the device which should
 321  * be trimmed (invalidated, unmapped).  journal_trim_remove_extent() marks a
 322  * range of bytes which should no longer be trimmed.  Accumulated extents
 323  * will be trimmed when the transaction is flushed to the on-disk journal.
 324  */
 325 int   journal_start_transaction(journal *jnl);
 326 int   journal_modify_block_start(journal *jnl, GenericLFBuf *psGenBuf);
 327 int   journal_modify_block_abort(journal *jnl, struct buf *bp);
 328 int   journal_modify_block_end(journal *jnl, GenericLFBuf *psGenBuf, void (*func)(GenericLFBuf *bp, void *arg), void *arg);
 329 int   journal_kill_block(journal *jnl, GenericLFBuf *bp);
 330 int   journal_trim_add_extent(journal *jnl, uint64_t offset, uint64_t length);
 331 int   journal_trim_remove_extent(journal *jnl, uint64_t offset, uint64_t length);
 332 void  journal_trim_set_callback(journal *jnl, jnl_trim_callback_t callback, void *arg);
 333 int   journal_trim_extent_overlap (journal *jnl, uint64_t offset, uint64_t length, uint64_t *end);
 334 /* Mark state in the journal that requests an immediate journal flush upon txn completion */
 335 int   journal_request_immediate_flush (journal *jnl);
 336 int   journal_end_transaction(journal *jnl);
 337
 338 int   journal_active(journal *jnl);
 339
 340 typedef enum journal_flush_options {
 341     JOURNAL_WAIT_FOR_IO       = 0x01,   // Flush journal and metadata blocks, wait for async IO to complete.
 342     JOURNAL_FLUSH_FULL        = 0x02,   // Flush track cache to media
 343 } journal_flush_options_t;
 344
 345 int   journal_flush(journal *jnl, journal_flush_options_t options);
 346 void *journal_owner(journal *jnl);    // compare against current_thread()
 347 int   journal_uses_fua(journal *jnl);
 348 void  journal_lock(journal *jnl);
 349 void  journal_unlock(journal *jnl);
 350 uint32_t journal_current_txn(journal *jnl);
 351
 352
 353 /*
 354  * Relocate the journal.
 355  *
 356  * You provide the new starting offset and size for the journal. You may
 357  * optionally provide a new tbuffer_size; passing zero defaults to not
 358  * changing the tbuffer size except as needed to fit within the new journal
 359  * size.
 360  *
 361  * You must have already started a transaction. The transaction may contain
 362  * modified blocks (such as those needed to deallocate the old journal,
 363  * allocate the new journal, and update the location and size of the journal
 364  * in filesystem-private structures). Any transactions prior to the active
 365  * transaction will be flushed to the old journal. The new journal will be
 366  * initialized, and the blocks from the active transaction will be written to
 367  * the new journal. The caller will need to update the structures that
 368  * identify the location and size of the journal from the callback routine.
 369  */
 370 int journal_relocate(journal *jnl, off_t offset, off_t journal_size, int32_t tbuffer_size,
 371                      errno_t (*callback)(void *), void *callback_arg);
 372
 373 uint32_t journal_current_txn(journal *jnl);
 374 _Bool hfs_is_journal_file(struct hfsmount *hfsmp, struct cnode *cp);
 375 bool  is_journaled(UVFSFileNode *psRootNode);
 376
 377 __END_DECLS
 378
 379 #endif /* lf_hfs_journal_h */