2 * Copyright (c) 2008-2015 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
30 /* We need these symbols even though compression is turned off */
32 #define UNUSED_SYMBOL(x) asm(".global _" #x "\n.set _" #x ", 0\n");
34 UNUSED_SYMBOL(register_decmpfs_decompressor
)
35 UNUSED_SYMBOL(unregister_decmpfs_decompressor
)
36 UNUSED_SYMBOL(decmpfs_init
)
37 UNUSED_SYMBOL(decmpfs_read_compressed
)
38 UNUSED_SYMBOL(decmpfs_cnode_cmp_type
)
39 UNUSED_SYMBOL(decmpfs_cnode_get_vnode_state
)
40 UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_size
)
41 UNUSED_SYMBOL(decmpfs_lock_compressed_data
)
42 UNUSED_SYMBOL(decmpfs_cnode_free
)
43 UNUSED_SYMBOL(decmpfs_cnode_alloc
)
44 UNUSED_SYMBOL(decmpfs_cnode_destroy
)
45 UNUSED_SYMBOL(decmpfs_decompress_file
)
46 UNUSED_SYMBOL(decmpfs_unlock_compressed_data
)
47 UNUSED_SYMBOL(decmpfs_cnode_init
)
48 UNUSED_SYMBOL(decmpfs_cnode_set_vnode_state
)
49 UNUSED_SYMBOL(decmpfs_hides_xattr
)
50 UNUSED_SYMBOL(decmpfs_ctx
)
51 UNUSED_SYMBOL(decmpfs_file_is_compressed
)
52 UNUSED_SYMBOL(decmpfs_update_attributes
)
53 UNUSED_SYMBOL(decmpfs_hides_rsrc
)
54 UNUSED_SYMBOL(decmpfs_pagein_compressed
)
55 UNUSED_SYMBOL(decmpfs_validate_compressed_file
)
57 #else /* FS_COMPRESSION */
58 #include <sys/kernel.h>
59 #include <sys/vnode_internal.h>
60 #include <sys/file_internal.h>
62 #include <sys/fcntl.h>
63 #include <sys/xattr.h>
64 #include <sys/namei.h>
66 #include <sys/mount_internal.h>
68 #include <sys/decmpfs.h>
69 #include <sys/uio_internal.h>
70 #include <libkern/OSByteOrder.h>
72 #pragma mark --- debugging ---
74 #define COMPRESSION_DEBUG 0
75 #define COMPRESSION_DEBUG_VERBOSE 0
76 #define MALLOC_DEBUG 0
79 baseName(const char *path
)
83 const char *ret
= path
;
85 for (i
= 0; path
[i
] != 0; i
++) {
93 vnpath(vnode_t vp
, char *path
, int len
)
97 vn_getpath(vp
, path
, &len
);
98 path
[origlen
- 1] = 0;
102 #define ErrorLog(x, args...) printf("%s:%d:%s: " x, baseName(__FILE__), __LINE__, __FUNCTION__, ## args)
103 #define ErrorLogWithPath(x, args...) do { char *path; MALLOC(path, char *, PATH_MAX, M_TEMP, M_WAITOK); printf("%s:%d:%s: %s: " x, baseName(__FILE__), __LINE__, __FUNCTION__, vnpath(vp, path, PATH_MAX), ## args); FREE(path, M_TEMP); } while(0)
105 #if COMPRESSION_DEBUG
106 #define DebugLog ErrorLog
107 #define DebugLogWithPath ErrorLogWithPath
109 #define DebugLog(x...) do { } while(0)
110 #define DebugLogWithPath(x...) do { } while(0)
113 #if COMPRESSION_DEBUG_VERBOSE
114 #define VerboseLog ErrorLog
115 #define VerboseLogWithPath ErrorLogWithPath
117 #define VerboseLog(x...) do { } while(0)
118 #define VerboseLogWithPath(x...) do { } while(0)
123 static SInt32 totalAlloc
;
133 _malloc(uint32_t sz
, __unused
int type
, __unused
int flags
, const char *file
, int line
)
135 uint32_t allocSz
= sz
+ 2 * sizeof(allocated
);
137 allocated
*alloc
= NULL
;
138 MALLOC(alloc
, allocated
*, allocSz
, type
, flags
);
140 ErrorLog("malloc failed\n");
144 char *ret
= (char*)&alloc
[1];
145 allocated
*alloc2
= (allocated
*)(ret
+ sz
);
147 alloc
->allocSz
= allocSz
;
148 alloc
->magic
= 0xdadadada;
154 int s
= OSAddAtomic(sz
, &totalAlloc
);
155 ErrorLog("malloc(%d) -> %p, total allocations %d\n", sz
, ret
, s
+ sz
);
161 _free(char *ret
, __unused
int type
, const char *file
, int line
)
164 ErrorLog("freeing null\n");
167 allocated
*alloc
= (allocated
*)ret
;
169 uint32_t sz
= alloc
->allocSz
- 2 * sizeof(allocated
);
170 allocated
*alloc2
= (allocated
*)(ret
+ sz
);
172 if (alloc
->magic
!= 0xdadadada) {
173 panic("freeing bad pointer");
176 if (memcmp(alloc
, alloc2
, sizeof(*alloc
)) != 0) {
177 panic("clobbered data");
180 memset(ret
, 0xce, sz
);
184 int s
= OSAddAtomic(-sz
, &totalAlloc
);
185 ErrorLog("free(%p,%d) -> total allocations %d\n", ret
, sz
, s
- sz
);
190 #define MALLOC(space, cast, size, type, flags) (space) = (cast)_malloc(size, type, flags, __FILE__, __LINE__)
191 #define FREE(addr, type) _free((void *)addr, type, __FILE__, __LINE__)
193 #endif /* MALLOC_DEBUG */
195 #pragma mark --- globals ---
197 static lck_grp_t
*decmpfs_lockgrp
;
199 static decmpfs_registration
* decompressors
[CMP_MAX
]; /* the registered compressors */
200 static lck_rw_t
* decompressorsLock
;
201 static int decompress_channel
; /* channel used by decompress_file to wake up waiters */
202 static lck_mtx_t
*decompress_channel_mtx
;
204 vfs_context_t decmpfs_ctx
;
206 #pragma mark --- decmp_get_func ---
208 #define offsetof_func(func) ((uintptr_t)(&(((decmpfs_registration*)NULL)->func)))
211 _func_from_offset(uint32_t type
, uintptr_t offset
)
213 /* get the function at the given offset in the registration for the given type */
214 decmpfs_registration
*reg
= decompressors
[type
];
215 char *regChar
= (char*)reg
;
216 char *func
= ®Char
[offset
];
217 void **funcPtr
= (void**)func
;
219 switch (reg
->decmpfs_registration
) {
220 case DECMPFS_REGISTRATION_VERSION_V1
:
221 if (offset
> offsetof_func(free_data
))
224 case DECMPFS_REGISTRATION_VERSION_V3
:
225 if (offset
> offsetof_func(get_flags
))
235 extern void IOServicePublishResource( const char * property
, boolean_t value
);
236 extern boolean_t
IOServiceWaitForMatchingResource( const char * property
, uint64_t timeout
);
237 extern boolean_t
IOCatalogueMatchingDriversPresent( const char * property
);
240 _decmp_get_func(vnode_t vp
, uint32_t type
, uintptr_t offset
)
243 this function should be called while holding a shared lock to decompressorsLock,
244 and will return with the lock held
250 if (decompressors
[type
] != NULL
) {
251 // the compressor has already registered but the function might be null
252 return _func_from_offset(type
, offset
);
255 // does IOKit know about a kext that is supposed to provide this type?
256 char providesName
[80];
257 snprintf(providesName
, sizeof(providesName
), "com.apple.AppleFSCompression.providesType%u", type
);
258 if (IOCatalogueMatchingDriversPresent(providesName
)) {
259 // there is a kext that says it will register for this type, so let's wait for it
260 char resourceName
[80];
261 uint64_t delay
= 10000000ULL; // 10 milliseconds.
262 snprintf(resourceName
, sizeof(resourceName
), "com.apple.AppleFSCompression.Type%u", type
);
263 ErrorLogWithPath("waiting for %s\n", resourceName
);
264 while(decompressors
[type
] == NULL
) {
265 lck_rw_unlock_shared(decompressorsLock
); // we have to unlock to allow the kext to register
266 if (IOServiceWaitForMatchingResource(resourceName
, delay
)) {
267 lck_rw_lock_shared(decompressorsLock
);
270 if (!IOCatalogueMatchingDriversPresent(providesName
)) {
272 ErrorLogWithPath("the kext with %s is no longer present\n", providesName
);
273 lck_rw_lock_shared(decompressorsLock
);
276 ErrorLogWithPath("still waiting for %s\n", resourceName
);
278 lck_rw_lock_shared(decompressorsLock
);
280 // IOKit says the kext is loaded, so it should be registered too!
281 if (decompressors
[type
] == NULL
) {
282 ErrorLogWithPath("we found %s, but the type still isn't registered\n", providesName
);
285 // it's now registered, so let's return the function
286 return _func_from_offset(type
, offset
);
289 // the compressor hasn't registered, so it never will unless someone manually kextloads it
290 ErrorLogWithPath("tried to access a compressed file of unregistered type %d\n", type
);
294 #define decmp_get_func(vp, type, func) ((typeof(((decmpfs_registration*)NULL)->func))_decmp_get_func(vp, type, offsetof_func(func)))
296 #pragma mark --- utilities ---
298 #if COMPRESSION_DEBUG
300 vnsize(vnode_t vp
, uint64_t *size
)
302 struct vnode_attr va
;
304 VATTR_WANTED(&va
, va_data_size
);
305 int error
= vnode_getattr(vp
, &va
, decmpfs_ctx
);
307 ErrorLogWithPath("vnode_getattr err %d\n", error
);
310 *size
= va
.va_data_size
;
313 #endif /* COMPRESSION_DEBUG */
315 #pragma mark --- cnode routines ---
317 decmpfs_cnode
*decmpfs_cnode_alloc(void)
320 MALLOC_ZONE(dp
, decmpfs_cnode
*, sizeof(decmpfs_cnode
), M_DECMPFS_CNODE
, M_WAITOK
);
324 void decmpfs_cnode_free(decmpfs_cnode
*dp
)
326 FREE_ZONE(dp
, sizeof(*dp
), M_DECMPFS_CNODE
);
330 decmpfs_cnode_init(decmpfs_cnode
*cp
)
332 memset(cp
, 0, sizeof(*cp
));
333 lck_rw_init(&cp
->compressed_data_lock
, decmpfs_lockgrp
, NULL
);
337 decmpfs_cnode_destroy(decmpfs_cnode
*cp
)
339 lck_rw_destroy(&cp
->compressed_data_lock
, decmpfs_lockgrp
);
343 decmpfs_trylock_compressed_data(decmpfs_cnode
*cp
, int exclusive
)
345 void *thread
= current_thread();
348 if (cp
->lockowner
== thread
) {
349 /* this thread is already holding an exclusive lock, so bump the count */
352 } else if (exclusive
) {
353 if ((retval
= lck_rw_try_lock_exclusive(&cp
->compressed_data_lock
))) {
354 cp
->lockowner
= thread
;
358 if ((retval
= lck_rw_try_lock_shared(&cp
->compressed_data_lock
))) {
359 cp
->lockowner
= (void *)-1;
366 decmpfs_lock_compressed_data(decmpfs_cnode
*cp
, int exclusive
)
368 void *thread
= current_thread();
370 if (cp
->lockowner
== thread
) {
371 /* this thread is already holding an exclusive lock, so bump the count */
373 } else if (exclusive
) {
374 lck_rw_lock_exclusive(&cp
->compressed_data_lock
);
375 cp
->lockowner
= thread
;
378 lck_rw_lock_shared(&cp
->compressed_data_lock
);
379 cp
->lockowner
= (void *)-1;
384 decmpfs_unlock_compressed_data(decmpfs_cnode
*cp
, __unused
int exclusive
)
386 void *thread
= current_thread();
388 if (cp
->lockowner
== thread
) {
389 /* this thread is holding an exclusive lock, so decrement the count */
390 if ((--cp
->lockcount
) > 0) {
391 /* the caller still has outstanding locks, so we're done */
394 cp
->lockowner
= NULL
;
397 lck_rw_done(&cp
->compressed_data_lock
);
401 decmpfs_cnode_get_vnode_state(decmpfs_cnode
*cp
)
403 return cp
->cmp_state
;
407 decmpfs_cnode_set_vnode_state(decmpfs_cnode
*cp
, uint32_t state
, int skiplock
)
409 if (!skiplock
) decmpfs_lock_compressed_data(cp
, 1);
410 cp
->cmp_state
= state
;
411 if (state
== FILE_TYPE_UNKNOWN
) {
412 /* clear out the compression type too */
415 if (!skiplock
) decmpfs_unlock_compressed_data(cp
, 1);
419 decmpfs_cnode_set_vnode_cmp_type(decmpfs_cnode
*cp
, uint32_t cmp_type
, int skiplock
)
421 if (!skiplock
) decmpfs_lock_compressed_data(cp
, 1);
422 cp
->cmp_type
= cmp_type
;
423 if (!skiplock
) decmpfs_unlock_compressed_data(cp
, 1);
427 decmpfs_cnode_set_vnode_minimal_xattr(decmpfs_cnode
*cp
, int minimal_xattr
, int skiplock
)
429 if (!skiplock
) decmpfs_lock_compressed_data(cp
, 1);
430 cp
->cmp_minimal_xattr
= minimal_xattr
;
431 if (!skiplock
) decmpfs_unlock_compressed_data(cp
, 1);
435 decmpfs_cnode_get_vnode_cached_size(decmpfs_cnode
*cp
)
437 return cp
->uncompressed_size
;
441 decmpfs_cnode_set_vnode_cached_size(decmpfs_cnode
*cp
, uint64_t size
)
444 uint64_t old
= cp
->uncompressed_size
;
445 if (OSCompareAndSwap64(old
, size
, (UInt64
*)&cp
->uncompressed_size
)) {
448 /* failed to write our value, so loop */
454 decmpfs_cnode_get_decompression_flags(decmpfs_cnode
*cp
)
456 return cp
->decompression_flags
;
460 decmpfs_cnode_set_decompression_flags(decmpfs_cnode
*cp
, uint64_t flags
)
463 uint64_t old
= cp
->decompression_flags
;
464 if (OSCompareAndSwap64(old
, flags
, (UInt64
*)&cp
->decompression_flags
)) {
467 /* failed to write our value, so loop */
472 uint32_t decmpfs_cnode_cmp_type(decmpfs_cnode
*cp
)
477 #pragma mark --- decmpfs state routines ---
480 decmpfs_fetch_compressed_header(vnode_t vp
, decmpfs_cnode
*cp
, decmpfs_header
**hdrOut
, int returnInvalid
)
483 fetches vp's compression xattr, converting it into a decmpfs_header; returns 0 or errno
484 if returnInvalid == 1, returns the header even if the type was invalid (out of range),
485 and return ERANGE in that case
488 size_t read_size
= 0;
489 size_t attr_size
= 0;
490 uio_t attr_uio
= NULL
;
493 decmpfs_header
*hdr
= NULL
;
494 char uio_buf
[ UIO_SIZEOF(1) ];
497 (cp
->cmp_type
!= 0) &&
498 (cp
->cmp_minimal_xattr
!= 0)) {
499 /* this file's xattr didn't have any extra data when we fetched it, so we can synthesize a header from the data in the cnode */
501 MALLOC(data
, char *, sizeof(decmpfs_header
), M_TEMP
, M_WAITOK
);
506 hdr
= (decmpfs_header
*)data
;
507 hdr
->attr_size
= sizeof(decmpfs_disk_header
);
508 hdr
->compression_magic
= DECMPFS_MAGIC
;
509 hdr
->compression_type
= cp
->cmp_type
;
510 hdr
->uncompressed_size
= decmpfs_cnode_get_vnode_cached_size(cp
);
512 /* figure out how big the xattr is on disk */
513 err
= vn_getxattr(vp
, DECMPFS_XATTR_NAME
, NULL
, &attr_size
, XATTR_NOSECURITY
, decmpfs_ctx
);
517 if (attr_size
< sizeof(decmpfs_disk_header
) || attr_size
> MAX_DECMPFS_XATTR_SIZE
) {
522 /* allocation includes space for the extra attr_size field of a compressed_header */
523 MALLOC(data
, char *, attr_size
+ sizeof(hdr
->attr_size
), M_TEMP
, M_WAITOK
);
529 /* read the xattr into our buffer, skipping over the attr_size field at the beginning */
530 attr_uio
= uio_createwithbuffer(1, 0, UIO_SYSSPACE
, UIO_READ
, &uio_buf
[0], sizeof(uio_buf
));
531 uio_addiov(attr_uio
, CAST_USER_ADDR_T(data
+ sizeof(hdr
->attr_size
)), attr_size
);
533 err
= vn_getxattr(vp
, DECMPFS_XATTR_NAME
, attr_uio
, &read_size
, XATTR_NOSECURITY
, decmpfs_ctx
);
536 if (read_size
!= attr_size
) {
540 hdr
= (decmpfs_header
*)data
;
541 hdr
->attr_size
= attr_size
;
542 /* swap the fields to native endian */
543 hdr
->compression_magic
= OSSwapLittleToHostInt32(hdr
->compression_magic
);
544 hdr
->compression_type
= OSSwapLittleToHostInt32(hdr
->compression_type
);
545 hdr
->uncompressed_size
= OSSwapLittleToHostInt64(hdr
->uncompressed_size
);
548 if (hdr
->compression_magic
!= DECMPFS_MAGIC
) {
549 ErrorLogWithPath("invalid compression_magic 0x%08x, should be 0x%08x\n", hdr
->compression_magic
, DECMPFS_MAGIC
);
554 if (hdr
->compression_type
>= CMP_MAX
) {
556 /* return the header even though the type is out of range */
559 ErrorLogWithPath("compression_type %d out of range\n", hdr
->compression_type
);
566 if (err
&& (err
!= ERANGE
)) {
567 DebugLogWithPath("err %d\n", err
);
568 if (data
) FREE(data
, M_TEMP
);
577 decmpfs_fast_get_state(decmpfs_cnode
*cp
)
580 return the cached state
581 this should *only* be called when we know that decmpfs_file_is_compressed has already been called,
582 because this implies that the cached state is valid
584 int cmp_state
= decmpfs_cnode_get_vnode_state(cp
);
587 case FILE_IS_NOT_COMPRESSED
:
588 case FILE_IS_COMPRESSED
:
589 case FILE_IS_CONVERTING
:
591 case FILE_TYPE_UNKNOWN
:
593 we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode,
594 which should not be possible
596 ErrorLog("decmpfs_fast_get_state called on unknown file\n");
597 return FILE_IS_NOT_COMPRESSED
;
600 ErrorLog("unknown cmp_state %d\n", cmp_state
);
601 return FILE_IS_NOT_COMPRESSED
;
606 decmpfs_fast_file_is_compressed(decmpfs_cnode
*cp
)
608 int cmp_state
= decmpfs_cnode_get_vnode_state(cp
);
611 case FILE_IS_NOT_COMPRESSED
:
613 case FILE_IS_COMPRESSED
:
614 case FILE_IS_CONVERTING
:
616 case FILE_TYPE_UNKNOWN
:
618 we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode,
619 which should not be possible
621 ErrorLog("decmpfs_fast_get_state called on unknown file\n");
625 ErrorLog("unknown cmp_state %d\n", cmp_state
);
631 decmpfs_validate_compressed_file(vnode_t vp
, decmpfs_cnode
*cp
)
633 /* give a compressor a chance to indicate that a compressed file is invalid */
635 decmpfs_header
*hdr
= NULL
;
636 errno_t err
= decmpfs_fetch_compressed_header(vp
, cp
, &hdr
, 0);
638 /* we couldn't get the header */
639 if (decmpfs_fast_get_state(cp
) == FILE_IS_NOT_COMPRESSED
) {
640 /* the file is no longer compressed, so return success */
646 lck_rw_lock_shared(decompressorsLock
);
647 decmpfs_validate_compressed_file_func validate
= decmp_get_func(vp
, hdr
->compression_type
, validate
);
648 if (validate
) { /* make sure this validation function is valid */
649 /* is the data okay? */
650 err
= validate(vp
, decmpfs_ctx
, hdr
);
651 } else if (decmp_get_func(vp
, hdr
->compression_type
, fetch
) == NULL
) {
652 /* the type isn't registered */
655 /* no validate registered, so nothing to do */
658 lck_rw_unlock_shared(decompressorsLock
);
660 if (hdr
) FREE(hdr
, M_TEMP
);
661 #if COMPRESSION_DEBUG
663 DebugLogWithPath("decmpfs_validate_compressed_file ret %d, vp->v_flag %d\n", err
, vp
->v_flag
);
670 decmpfs_file_is_compressed(vnode_t vp
, decmpfs_cnode
*cp
)
673 determines whether vp points to a compressed file
675 to speed up this operation, we cache the result in the cnode, and do as little as possible
676 in the case where the cnode already has a valid cached state
683 struct vnode_attr va_fetch
;
684 decmpfs_header
*hdr
= NULL
;
686 int cnode_locked
= 0;
687 int saveInvalid
= 0; // save the header data even though the type was out of range
688 uint64_t decompression_flags
= 0;
690 if (vnode_isnamedstream(vp
)) {
692 named streams can't be compressed
693 since named streams of the same file share the same cnode,
694 we don't want to get/set the state in the cnode, just return 0
699 /* examine the cached a state in this cnode */
700 cmp_state
= decmpfs_cnode_get_vnode_state(cp
);
702 case FILE_IS_NOT_COMPRESSED
:
704 case FILE_IS_COMPRESSED
:
706 case FILE_IS_CONVERTING
:
707 /* treat the file as compressed, because this gives us a way to block future reads until decompression is done */
709 case FILE_TYPE_UNKNOWN
:
710 /* the first time we encountered this vnode, so we need to check it out */
713 /* unknown state, assume file is not compressed */
714 ErrorLogWithPath("unknown cmp_state %d\n", cmp_state
);
718 if (!vnode_isreg(vp
)) {
719 /* only regular files can be compressed */
720 ret
= FILE_IS_NOT_COMPRESSED
;
724 mp
= vnode_mount(vp
);
727 this should only be true before we mount the root filesystem
728 we short-cut this return to avoid the call to getattr below, which
729 will fail before root is mounted
731 ret
= FILE_IS_NOT_COMPRESSED
;
734 if ((mp
->mnt_flag
& MNT_LOCAL
) == 0) {
735 /* compression only supported on local filesystems */
736 ret
= FILE_IS_NOT_COMPRESSED
;
740 /* lock our cnode data so that another caller doesn't change the state under us */
741 decmpfs_lock_compressed_data(cp
, 1);
744 VATTR_INIT(&va_fetch
);
745 VATTR_WANTED(&va_fetch
, va_flags
);
746 error
= vnode_getattr(vp
, &va_fetch
, decmpfs_ctx
);
748 /* failed to get the bsd flags so the file is not compressed */
749 ret
= FILE_IS_NOT_COMPRESSED
;
752 if (va_fetch
.va_flags
& UF_COMPRESSED
) {
753 /* UF_COMPRESSED is on, make sure the file has the DECMPFS_XATTR_NAME xattr */
754 error
= decmpfs_fetch_compressed_header(vp
, cp
, &hdr
, 1);
755 if ((hdr
!= NULL
) && (error
== ERANGE
)) {
759 /* failed to get the xattr so the file is not compressed */
760 ret
= FILE_IS_NOT_COMPRESSED
;
763 /* we got the xattr, so the file is compressed */
764 ret
= FILE_IS_COMPRESSED
;
767 /* UF_COMPRESSED isn't on, so the file isn't compressed */
768 ret
= FILE_IS_NOT_COMPRESSED
;
771 if (((ret
== FILE_IS_COMPRESSED
) || saveInvalid
) && hdr
) {
773 cache the uncompressed size away in the cnode
778 we should never get here since the only place ret is set to FILE_IS_COMPRESSED
779 is after the call to decmpfs_lock_compressed_data above
781 decmpfs_lock_compressed_data(cp
, 1);
785 decmpfs_cnode_set_vnode_cached_size(cp
, hdr
->uncompressed_size
);
786 decmpfs_cnode_set_vnode_state(cp
, ret
, 1);
787 decmpfs_cnode_set_vnode_cmp_type(cp
, hdr
->compression_type
, 1);
788 /* remember if the xattr's size was equal to the minimal xattr */
789 if (hdr
->attr_size
== sizeof(decmpfs_disk_header
)) {
790 decmpfs_cnode_set_vnode_minimal_xattr(cp
, 1, 1);
792 if (ret
== FILE_IS_COMPRESSED
) {
793 /* update the ubc's size for this file */
794 ubc_setsize(vp
, hdr
->uncompressed_size
);
796 /* update the decompression flags in the decmpfs cnode */
797 lck_rw_lock_shared(decompressorsLock
);
798 decmpfs_get_decompression_flags_func get_flags
= decmp_get_func(vp
, hdr
->compression_type
, get_flags
);
800 decompression_flags
= get_flags(vp
, decmpfs_ctx
, hdr
);
802 lck_rw_unlock_shared(decompressorsLock
);
803 decmpfs_cnode_set_decompression_flags(cp
, decompression_flags
);
806 /* we might have already taken the lock above; if so, skip taking it again by passing cnode_locked as the skiplock parameter */
807 decmpfs_cnode_set_vnode_state(cp
, ret
, cnode_locked
);
810 if (cnode_locked
) decmpfs_unlock_compressed_data(cp
, 1);
812 if (hdr
) FREE(hdr
, M_TEMP
);
815 case FILE_IS_NOT_COMPRESSED
:
817 case FILE_IS_COMPRESSED
:
818 case FILE_IS_CONVERTING
:
821 /* unknown state, assume file is not compressed */
822 ErrorLogWithPath("unknown ret %d\n", ret
);
828 decmpfs_update_attributes(vnode_t vp
, struct vnode_attr
*vap
)
832 if (VATTR_IS_ACTIVE(vap
, va_flags
)) {
833 /* the BSD flags are being updated */
834 if (vap
->va_flags
& UF_COMPRESSED
) {
835 /* the compressed bit is being set, did it change? */
836 struct vnode_attr va_fetch
;
838 VATTR_INIT(&va_fetch
);
839 VATTR_WANTED(&va_fetch
, va_flags
);
840 error
= vnode_getattr(vp
, &va_fetch
, decmpfs_ctx
);
844 old_flags
= va_fetch
.va_flags
;
846 if (!(old_flags
& UF_COMPRESSED
)) {
848 * Compression bit was turned on, make sure the file has the DECMPFS_XATTR_NAME attribute.
849 * This precludes anyone from using the UF_COMPRESSED bit for anything else, and it enforces
850 * an order of operation -- you must first do the setxattr and then the chflags.
853 if (VATTR_IS_ACTIVE(vap
, va_data_size
)) {
855 * don't allow the caller to set the BSD flag and the size in the same call
856 * since this doesn't really make sense
858 vap
->va_flags
&= ~UF_COMPRESSED
;
862 decmpfs_header
*hdr
= NULL
;
863 error
= decmpfs_fetch_compressed_header(vp
, NULL
, &hdr
, 1);
866 allow the flag to be set since the decmpfs attribute is present
867 in that case, we also want to truncate the data fork of the file
869 VATTR_SET_ACTIVE(vap
, va_data_size
);
870 vap
->va_data_size
= 0;
871 } else if (error
== ERANGE
) {
872 /* the file had a decmpfs attribute but the type was out of range, so don't muck with the file's data size */
874 /* no DECMPFS_XATTR_NAME attribute, so deny the update */
875 vap
->va_flags
&= ~UF_COMPRESSED
;
877 if (hdr
) FREE(hdr
, M_TEMP
);
886 wait_for_decompress(decmpfs_cnode
*cp
)
889 lck_mtx_lock(decompress_channel_mtx
);
891 state
= decmpfs_fast_get_state(cp
);
892 if (state
!= FILE_IS_CONVERTING
) {
893 /* file is not decompressing */
894 lck_mtx_unlock(decompress_channel_mtx
);
897 msleep((caddr_t
)&decompress_channel
, decompress_channel_mtx
, PINOD
, "wait_for_decompress", NULL
);
901 #pragma mark --- decmpfs hide query routines ---
904 decmpfs_hides_rsrc(vfs_context_t ctx
, decmpfs_cnode
*cp
)
908 callers may (and do) pass NULL for ctx, so we should only use it
909 for this equality comparison
911 This routine should only be called after a file has already been through decmpfs_file_is_compressed
914 if (ctx
== decmpfs_ctx
)
917 if (!decmpfs_fast_file_is_compressed(cp
))
920 /* all compressed files hide their resource fork */
925 decmpfs_hides_xattr(vfs_context_t ctx
, decmpfs_cnode
*cp
, const char *xattr
)
929 callers may (and do) pass NULL for ctx, so we should only use it
930 for this equality comparison
932 This routine should only be called after a file has already been through decmpfs_file_is_compressed
935 if (ctx
== decmpfs_ctx
)
937 if (strncmp(xattr
, XATTR_RESOURCEFORK_NAME
, sizeof(XATTR_RESOURCEFORK_NAME
) - 1) == 0)
938 return decmpfs_hides_rsrc(ctx
, cp
);
939 if (!decmpfs_fast_file_is_compressed(cp
))
940 /* file is not compressed, so don't hide this xattr */
942 if (strncmp(xattr
, DECMPFS_XATTR_NAME
, sizeof(DECMPFS_XATTR_NAME
) - 1) == 0)
943 /* it's our xattr, so hide it */
945 /* don't hide this xattr */
949 #pragma mark --- registration/validation routines ---
951 static inline int registration_valid(decmpfs_registration
*registration
)
953 return registration
&& ((registration
->decmpfs_registration
== DECMPFS_REGISTRATION_VERSION_V1
) || (registration
->decmpfs_registration
== DECMPFS_REGISTRATION_VERSION_V3
));
957 register_decmpfs_decompressor(uint32_t compression_type
, decmpfs_registration
*registration
)
959 /* called by kexts to register decompressors */
963 char resourceName
[80];
965 if ((compression_type
>= CMP_MAX
) || !registration_valid(registration
)) {
970 lck_rw_lock_exclusive(decompressorsLock
); locked
= 1;
972 /* make sure the registration for this type is zero */
973 if (decompressors
[compression_type
] != NULL
) {
977 decompressors
[compression_type
] = registration
;
978 snprintf(resourceName
, sizeof(resourceName
), "com.apple.AppleFSCompression.Type%u", compression_type
);
979 IOServicePublishResource(resourceName
, TRUE
);
982 if (locked
) lck_rw_unlock_exclusive(decompressorsLock
);
987 unregister_decmpfs_decompressor(uint32_t compression_type
, decmpfs_registration
*registration
)
989 /* called by kexts to unregister decompressors */
993 char resourceName
[80];
995 if ((compression_type
>= CMP_MAX
) || !registration_valid(registration
)) {
1000 lck_rw_lock_exclusive(decompressorsLock
); locked
= 1;
1001 if (decompressors
[compression_type
] != registration
) {
1005 decompressors
[compression_type
] = NULL
;
1006 snprintf(resourceName
, sizeof(resourceName
), "com.apple.AppleFSCompression.Type%u", compression_type
);
1007 IOServicePublishResource(resourceName
, FALSE
);
1010 if (locked
) lck_rw_unlock_exclusive(decompressorsLock
);
1015 compression_type_valid(vnode_t vp
, decmpfs_header
*hdr
)
1017 /* fast pre-check to determine if the given compressor has checked in */
1020 /* every compressor must have at least a fetch function */
1021 lck_rw_lock_shared(decompressorsLock
);
1022 if (decmp_get_func(vp
, hdr
->compression_type
, fetch
) != NULL
) {
1025 lck_rw_unlock_shared(decompressorsLock
);
1030 #pragma mark --- compression/decompression routines ---
1033 decmpfs_fetch_uncompressed_data(vnode_t vp
, decmpfs_cnode
*cp
, decmpfs_header
*hdr
, off_t offset
, user_ssize_t size
, int nvec
, decmpfs_vector
*vec
, uint64_t *bytes_read
)
1035 /* get the uncompressed bytes for the specified region of vp by calling out to the registered compressor */
1041 if ((uint64_t)offset
>= hdr
->uncompressed_size
) {
1042 /* reading past end of file; nothing to do */
1047 /* tried to read from before start of file */
1051 if ((uint64_t)(offset
+ size
) > hdr
->uncompressed_size
) {
1052 /* adjust size so we don't read past the end of the file */
1053 size
= hdr
->uncompressed_size
- offset
;
1056 /* nothing to read */
1061 lck_rw_lock_shared(decompressorsLock
);
1062 decmpfs_fetch_uncompressed_data_func fetch
= decmp_get_func(vp
, hdr
->compression_type
, fetch
);
1064 err
= fetch(vp
, decmpfs_ctx
, hdr
, offset
, size
, nvec
, vec
, bytes_read
);
1065 lck_rw_unlock_shared(decompressorsLock
);
1067 uint64_t decompression_flags
= decmpfs_cnode_get_decompression_flags(cp
);
1068 if (decompression_flags
& DECMPFS_FLAGS_FORCE_FLUSH_ON_DECOMPRESS
) {
1069 #if !defined(__i386__) && !defined(__x86_64__)
1071 for (i
= 0; i
< nvec
; i
++) {
1072 flush_dcache64((addr64_t
)(uintptr_t)vec
[i
].buf
, vec
[i
].size
, FALSE
);
1079 lck_rw_unlock_shared(decompressorsLock
);
1086 static kern_return_t
1087 commit_upl(upl_t upl
, upl_offset_t pl_offset
, size_t uplSize
, int flags
, int abort
)
1089 kern_return_t kr
= 0;
1092 upl_unmark_decmp(upl
);
1093 #endif /* CONFIG_IOSCHED */
1095 /* commit the upl pages */
1097 VerboseLog("aborting upl, flags 0x%08x\n", flags
);
1098 kr
= ubc_upl_abort_range(upl
, pl_offset
, uplSize
, flags
);
1099 if (kr
!= KERN_SUCCESS
)
1100 ErrorLog("ubc_upl_abort_range error %d\n", (int)kr
);
1102 VerboseLog("committing upl, flags 0x%08x\n", flags
| UPL_COMMIT_CLEAR_DIRTY
);
1103 kr
= ubc_upl_commit_range(upl
, pl_offset
, uplSize
, flags
| UPL_COMMIT_CLEAR_DIRTY
| UPL_COMMIT_WRITTEN_BY_KERNEL
);
1104 if (kr
!= KERN_SUCCESS
)
1105 ErrorLog("ubc_upl_commit_range error %d\n", (int)kr
);
1112 decmpfs_pagein_compressed(struct vnop_pagein_args
*ap
, int *is_compressed
, decmpfs_cnode
*cp
)
1114 /* handles a page-in request from vfs for a compressed file */
1117 vnode_t vp
= ap
->a_vp
;
1118 upl_t pl
= ap
->a_pl
;
1119 upl_offset_t pl_offset
= ap
->a_pl_offset
;
1120 off_t f_offset
= ap
->a_f_offset
;
1121 size_t size
= ap
->a_size
;
1122 int flags
= ap
->a_flags
;
1124 user_ssize_t uplSize
= 0;
1126 decmpfs_header
*hdr
= NULL
;
1127 uint64_t cachedSize
= 0;
1128 int cmpdata_locked
= 0;
1130 if(!decmpfs_trylock_compressed_data(cp
, 0)) {
1136 if (flags
& ~(UPL_IOSYNC
| UPL_NOCOMMIT
| UPL_NORDAHEAD
)) {
1137 DebugLogWithPath("pagein: unknown flags 0x%08x\n", (flags
& ~(UPL_IOSYNC
| UPL_NOCOMMIT
| UPL_NORDAHEAD
)));
1140 err
= decmpfs_fetch_compressed_header(vp
, cp
, &hdr
, 0);
1145 cachedSize
= hdr
->uncompressed_size
;
1147 if (!compression_type_valid(vp
, hdr
)) {
1148 /* compressor not registered */
1154 /* Mark the UPL as the requesting UPL for decompression */
1156 #endif /* CONFIG_IOSCHED */
1158 /* map the upl so we can fetch into it */
1159 kern_return_t kr
= ubc_upl_map(pl
, (vm_offset_t
*)&data
);
1160 if ((kr
!= KERN_SUCCESS
) || (data
== NULL
)) {
1164 upl_unmark_decmp(pl
);
1165 #endif /* CONFIG_IOSCHED */
1172 /* clip the size to the size of the file */
1173 if ((uint64_t)uplPos
+ uplSize
> cachedSize
) {
1174 /* truncate the read to the size of the file */
1175 uplSize
= cachedSize
- uplPos
;
1182 /* the mapped data pointer points to the first page of the page list, so we want to start filling in at an offset of pl_offset */
1183 vec
.buf
= (char*)data
+ pl_offset
;
1186 uint64_t did_read
= 0;
1187 if (decmpfs_fast_get_state(cp
) == FILE_IS_CONVERTING
) {
1188 ErrorLogWithPath("unexpected pagein during decompress\n");
1190 if the file is converting, this must be a recursive call to pagein from underneath a call to decmpfs_decompress_file;
1191 pretend that it succeeded but don't do anything since we're just going to write over the pages anyway
1196 err
= decmpfs_fetch_uncompressed_data(vp
, cp
, hdr
, uplPos
, uplSize
, 1, &vec
, &did_read
);
1199 DebugLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err
);
1200 int cmp_state
= decmpfs_fast_get_state(cp
);
1201 if (cmp_state
== FILE_IS_CONVERTING
) {
1202 DebugLogWithPath("cmp_state == FILE_IS_CONVERTING\n");
1203 cmp_state
= wait_for_decompress(cp
);
1204 if (cmp_state
== FILE_IS_COMPRESSED
) {
1205 DebugLogWithPath("cmp_state == FILE_IS_COMPRESSED\n");
1206 /* a decompress was attempted but it failed, let's try calling fetch again */
1210 if (cmp_state
== FILE_IS_NOT_COMPRESSED
) {
1211 DebugLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n");
1212 /* the file was decompressed after we started reading it */
1213 *is_compressed
= 0; /* instruct caller to fall back to its normal path */
1217 /* zero out whatever we didn't read, and zero out the end of the last page(s) */
1218 uint64_t total_size
= (size
+ (PAGE_SIZE
- 1)) & ~(PAGE_SIZE
- 1);
1219 if (did_read
< total_size
) {
1220 memset((char*)vec
.buf
+ did_read
, 0, total_size
- did_read
);
1224 upl_unmark_decmp(pl
);
1225 #endif /* CONFIG_IOSCHED */
1227 kr
= ubc_upl_unmap(pl
); data
= NULL
; /* make sure to set data to NULL so we don't try to unmap again below */
1228 if (kr
!= KERN_SUCCESS
)
1229 ErrorLogWithPath("ubc_upl_unmap error %d\n", (int)kr
);
1232 /* commit our pages */
1233 kr
= commit_upl(pl
, pl_offset
, total_size
, UPL_COMMIT_FREE_ON_EMPTY
, 0);
1238 if (data
) ubc_upl_unmap(pl
);
1239 if (hdr
) FREE(hdr
, M_TEMP
);
1240 if (cmpdata_locked
) decmpfs_unlock_compressed_data(cp
, 0);
1243 if (err
!= ENXIO
&& err
!= ENOSPC
) {
1245 MALLOC(path
, char *, PATH_MAX
, M_TEMP
, M_WAITOK
);
1246 panic("%s: decmpfs_pagein_compressed: err %d", vnpath(vp
, path
, PATH_MAX
), err
);
1250 ErrorLogWithPath("err %d\n", err
);
1256 decmpfs_read_compressed(struct vnop_read_args
*ap
, int *is_compressed
, decmpfs_cnode
*cp
)
1258 /* handles a read request from vfs for a compressed file */
1260 uio_t uio
= ap
->a_uio
;
1261 vnode_t vp
= ap
->a_vp
;
1265 user_ssize_t uplSize
= 0;
1266 user_ssize_t uplRemaining
= 0;
1267 off_t curUplPos
= 0;
1268 user_ssize_t curUplSize
= 0;
1269 kern_return_t kr
= KERN_SUCCESS
;
1272 uint64_t did_read
= 0;
1274 upl_page_info_t
*pli
= NULL
;
1275 decmpfs_header
*hdr
= NULL
;
1276 uint64_t cachedSize
= 0;
1278 user_ssize_t uioRemaining
= 0;
1279 int cmpdata_locked
= 0;
1281 decmpfs_lock_compressed_data(cp
, 0); cmpdata_locked
= 1;
1283 uplPos
= uio_offset(uio
);
1284 uplSize
= uio_resid(uio
);
1285 VerboseLogWithPath("uplPos %lld uplSize %lld\n", uplPos
, uplSize
);
1287 cachedSize
= decmpfs_cnode_get_vnode_cached_size(cp
);
1289 if ((uint64_t)uplPos
+ uplSize
> cachedSize
) {
1290 /* truncate the read to the size of the file */
1291 uplSize
= cachedSize
- uplPos
;
1294 /* give the cluster layer a chance to fill in whatever it already has */
1295 countInt
= (uplSize
> INT_MAX
) ? INT_MAX
: uplSize
;
1296 err
= cluster_copy_ubc_data(vp
, uio
, &countInt
, 0);
1300 /* figure out what's left */
1301 uioPos
= uio_offset(uio
);
1302 uioRemaining
= uio_resid(uio
);
1303 if ((uint64_t)uioPos
+ uioRemaining
> cachedSize
) {
1304 /* truncate the read to the size of the file */
1305 uioRemaining
= cachedSize
- uioPos
;
1308 if (uioRemaining
<= 0) {
1313 err
= decmpfs_fetch_compressed_header(vp
, cp
, &hdr
, 0);
1317 if (!compression_type_valid(vp
, hdr
)) {
1323 uplSize
= uioRemaining
;
1324 #if COMPRESSION_DEBUG
1325 DebugLogWithPath("uplPos %lld uplSize %lld\n", (uint64_t)uplPos
, (uint64_t)uplSize
);
1328 lck_rw_lock_shared(decompressorsLock
);
1329 decmpfs_adjust_fetch_region_func adjust_fetch
= decmp_get_func(vp
, hdr
->compression_type
, adjust_fetch
);
1331 /* give the compressor a chance to adjust the portion of the file that we read */
1332 adjust_fetch(vp
, decmpfs_ctx
, hdr
, &uplPos
, &uplSize
);
1333 VerboseLogWithPath("adjusted uplPos %lld uplSize %lld\n", (uint64_t)uplPos
, (uint64_t)uplSize
);
1335 lck_rw_unlock_shared(decompressorsLock
);
1337 /* clip the adjusted size to the size of the file */
1338 if ((uint64_t)uplPos
+ uplSize
> cachedSize
) {
1339 /* truncate the read to the size of the file */
1340 uplSize
= cachedSize
- uplPos
;
1349 since we're going to create a upl for the given region of the file,
1350 make sure we're on page boundaries
1353 if (uplPos
& (PAGE_SIZE
- 1)) {
1354 /* round position down to page boundary */
1355 uplSize
+= (uplPos
& (PAGE_SIZE
- 1));
1356 uplPos
&= ~(PAGE_SIZE
- 1);
1358 /* round size up to page multiple */
1359 uplSize
= (uplSize
+ (PAGE_SIZE
- 1)) & ~(PAGE_SIZE
- 1);
1361 VerboseLogWithPath("new uplPos %lld uplSize %lld\n", (uint64_t)uplPos
, (uint64_t)uplSize
);
1363 uplRemaining
= uplSize
;
1367 while(uplRemaining
> 0) {
1368 /* start after the last upl */
1369 curUplPos
+= curUplSize
;
1371 /* clip to max upl size */
1372 curUplSize
= uplRemaining
;
1373 if (curUplSize
> MAX_UPL_SIZE_BYTES
) {
1374 curUplSize
= MAX_UPL_SIZE_BYTES
;
1377 /* create the upl */
1378 kr
= ubc_create_upl(vp
, curUplPos
, curUplSize
, &upl
, &pli
, UPL_SET_LITE
);
1379 if (kr
!= KERN_SUCCESS
) {
1380 ErrorLogWithPath("ubc_create_upl error %d\n", (int)kr
);
1384 VerboseLogWithPath("curUplPos %lld curUplSize %lld\n", (uint64_t)curUplPos
, (uint64_t)curUplSize
);
1387 /* Mark the UPL as the requesting UPL for decompression */
1388 upl_mark_decmp(upl
);
1389 #endif /* CONFIG_IOSCHED */
1392 kr
= ubc_upl_map(upl
, (vm_offset_t
*)&data
);
1393 if (kr
!= KERN_SUCCESS
) {
1395 commit_upl(upl
, 0, curUplSize
, UPL_ABORT_FREE_ON_EMPTY
, 1);
1398 MALLOC(path
, char *, PATH_MAX
, M_TEMP
, M_WAITOK
);
1399 panic("%s: decmpfs_read_compressed: ubc_upl_map error %d", vnpath(vp
, path
, PATH_MAX
), (int)kr
);
1402 ErrorLogWithPath("ubc_upl_map kr=0x%x\n", (int)kr
);
1408 /* make sure the map succeeded */
1411 commit_upl(upl
, 0, curUplSize
, UPL_ABORT_FREE_ON_EMPTY
, 1);
1413 ErrorLogWithPath("ubc_upl_map mapped null\n");
1418 /* fetch uncompressed data into the mapped upl */
1421 vec
= (decmpfs_vector
){ .buf
= data
, .size
= curUplSize
};
1422 err
= decmpfs_fetch_uncompressed_data(vp
, cp
, hdr
, curUplPos
, curUplSize
, 1, &vec
, &did_read
);
1424 ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err
);
1426 /* maybe the file is converting to decompressed */
1427 int cmp_state
= decmpfs_fast_get_state(cp
);
1428 if (cmp_state
== FILE_IS_CONVERTING
) {
1429 ErrorLogWithPath("cmp_state == FILE_IS_CONVERTING\n");
1430 cmp_state
= wait_for_decompress(cp
);
1431 if (cmp_state
== FILE_IS_COMPRESSED
) {
1432 ErrorLogWithPath("cmp_state == FILE_IS_COMPRESSED\n");
1433 /* a decompress was attempted but it failed, let's try fetching again */
1437 if (cmp_state
== FILE_IS_NOT_COMPRESSED
) {
1438 ErrorLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n");
1439 /* the file was decompressed after we started reading it */
1440 abort_read
= 1; /* we're not going to commit our data */
1441 *is_compressed
= 0; /* instruct caller to fall back to its normal path */
1446 /* zero out the remainder of the last page */
1447 memset((char*)data
+ did_read
, 0, curUplSize
- did_read
);
1448 kr
= ubc_upl_unmap(upl
);
1449 if (kr
== KERN_SUCCESS
) {
1451 kr
= commit_upl(upl
, 0, curUplSize
, UPL_ABORT_FREE_ON_EMPTY
, 1);
1453 VerboseLogWithPath("uioPos %lld uioRemaining %lld\n", (uint64_t)uioPos
, (uint64_t)uioRemaining
);
1455 off_t uplOff
= uioPos
- curUplPos
;
1457 ErrorLogWithPath("uplOff %lld should never be negative\n", (int64_t)uplOff
);
1460 off_t count
= curUplPos
+ curUplSize
- uioPos
;
1462 /* this upl is entirely before the uio */
1464 if (count
> uioRemaining
)
1465 count
= uioRemaining
;
1466 int io_resid
= count
;
1467 err
= cluster_copy_upl_data(uio
, upl
, uplOff
, &io_resid
);
1468 int copied
= count
- io_resid
;
1469 VerboseLogWithPath("uplOff %lld count %lld copied %lld\n", (uint64_t)uplOff
, (uint64_t)count
, (uint64_t)copied
);
1471 ErrorLogWithPath("cluster_copy_upl_data err %d\n", err
);
1474 uioRemaining
-= copied
;
1478 kr
= commit_upl(upl
, 0, curUplSize
, UPL_COMMIT_FREE_ON_EMPTY
| UPL_COMMIT_INACTIVATE
, 0);
1484 ErrorLogWithPath("ubc_upl_unmap error %d\n", (int)kr
);
1487 uplRemaining
-= curUplSize
;
1492 if (hdr
) FREE(hdr
, M_TEMP
);
1493 if (cmpdata_locked
) decmpfs_unlock_compressed_data(cp
, 0);
1494 if (err
) {/* something went wrong */
1495 ErrorLogWithPath("err %d\n", err
);
1499 #if COMPRESSION_DEBUG
1500 uplSize
= uio_resid(uio
);
1502 VerboseLogWithPath("still %lld bytes to copy\n", uplSize
);
1508 decmpfs_free_compressed_data(vnode_t vp
, decmpfs_cnode
*cp
)
1511 call out to the decompressor to free remove any data associated with this compressed file
1512 then delete the file's compression xattr
1515 decmpfs_header
*hdr
= NULL
;
1516 int err
= decmpfs_fetch_compressed_header(vp
, cp
, &hdr
, 0);
1518 ErrorLogWithPath("decmpfs_fetch_compressed_header err %d\n", err
);
1520 lck_rw_lock_shared(decompressorsLock
);
1521 decmpfs_free_compressed_data_func free_data
= decmp_get_func(vp
, hdr
->compression_type
, free_data
);
1523 err
= free_data(vp
, decmpfs_ctx
, hdr
);
1525 /* nothing to do, so no error */
1528 lck_rw_unlock_shared(decompressorsLock
);
1531 ErrorLogWithPath("decompressor err %d\n", err
);
1535 /* delete the xattr */
1536 err
= vn_removexattr(vp
, DECMPFS_XATTR_NAME
, 0, decmpfs_ctx
);
1542 if (hdr
) FREE(hdr
, M_TEMP
);
1546 #pragma mark --- file conversion routines ---
1549 unset_compressed_flag(vnode_t vp
)
1552 struct vnode_attr va
;
1553 int new_bsdflags
= 0;
1556 VATTR_WANTED(&va
, va_flags
);
1557 err
= vnode_getattr(vp
, &va
, decmpfs_ctx
);
1560 ErrorLogWithPath("vnode_getattr err %d\n", err
);
1562 new_bsdflags
= va
.va_flags
& ~UF_COMPRESSED
;
1565 VATTR_SET(&va
, va_flags
, new_bsdflags
);
1566 err
= vnode_setattr(vp
, &va
, decmpfs_ctx
);
1568 ErrorLogWithPath("vnode_setattr err %d\n", err
);
1575 decmpfs_decompress_file(vnode_t vp
, decmpfs_cnode
*cp
, off_t toSize
, int truncate_okay
, int skiplock
)
1577 /* convert a compressed file to an uncompressed file */
1583 uint32_t old_state
= 0;
1584 uint32_t new_state
= 0;
1585 int update_file_state
= 0;
1587 decmpfs_header
*hdr
= NULL
;
1588 int cmpdata_locked
= 0;
1589 off_t remaining
= 0;
1590 uint64_t uncompressed_size
= 0;
1593 decmpfs_lock_compressed_data(cp
, 1); cmpdata_locked
= 1;
1597 old_state
= decmpfs_fast_get_state(cp
);
1600 case FILE_IS_NOT_COMPRESSED
:
1602 /* someone else decompressed the file */
1607 case FILE_TYPE_UNKNOWN
:
1609 /* the file is in an unknown state, so update the state and retry */
1610 (void)decmpfs_file_is_compressed(vp
, cp
);
1616 case FILE_IS_COMPRESSED
:
1618 /* the file is compressed, so decompress it */
1625 this shouldn't happen since multiple calls to decmpfs_decompress_file lock each other out,
1626 and when decmpfs_decompress_file returns, the state should be always be set back to
1627 FILE_IS_NOT_COMPRESSED or FILE_IS_UNKNOWN
1634 err
= decmpfs_fetch_compressed_header(vp
, cp
, &hdr
, 0);
1639 uncompressed_size
= hdr
->uncompressed_size
;
1641 toSize
= hdr
->uncompressed_size
;
1644 /* special case truncating the file to zero bytes */
1646 } else if ((uint64_t)toSize
> hdr
->uncompressed_size
) {
1647 /* the caller is trying to grow the file, so we should decompress all the data */
1648 toSize
= hdr
->uncompressed_size
;
1651 allocSize
= MIN(64*1024, toSize
);
1652 MALLOC(data
, char *, allocSize
, M_TEMP
, M_WAITOK
);
1658 uio_w
= uio_create(1, 0LL, UIO_SYSSPACE
, UIO_WRITE
);
1663 uio_w
->uio_flags
|= UIO_FLAGS_IS_COMPRESSED_FILE
;
1667 /* tell the buffer cache that this is an empty file */
1670 /* if we got here, we need to decompress the file */
1671 decmpfs_cnode_set_vnode_state(cp
, FILE_IS_CONVERTING
, 1);
1673 while(remaining
> 0) {
1674 /* loop decompressing data from the file and writing it into the data fork */
1676 uint64_t bytes_read
= 0;
1677 decmpfs_vector vec
= { .buf
= data
, .size
= MIN(allocSize
, remaining
) };
1678 err
= decmpfs_fetch_uncompressed_data(vp
, cp
, hdr
, offset
, vec
.size
, 1, &vec
, &bytes_read
);
1680 ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err
);
1684 if (bytes_read
== 0) {
1685 /* we're done reading data */
1689 uio_reset(uio_w
, offset
, UIO_SYSSPACE
, UIO_WRITE
);
1690 err
= uio_addiov(uio_w
, CAST_USER_ADDR_T(data
), bytes_read
);
1692 ErrorLogWithPath("uio_addiov err %d\n", err
);
1697 err
= VNOP_WRITE(vp
, uio_w
, 0, decmpfs_ctx
);
1699 /* if the write failed, truncate the file to zero bytes */
1700 ErrorLogWithPath("VNOP_WRITE err %d\n", err
);
1703 offset
+= bytes_read
;
1704 remaining
-= bytes_read
;
1708 if (offset
!= toSize
) {
1709 ErrorLogWithPath("file decompressed to %lld instead of %lld\n", offset
, toSize
);
1716 /* sync the data and metadata */
1717 err
= VNOP_FSYNC(vp
, MNT_WAIT
, decmpfs_ctx
);
1719 ErrorLogWithPath("VNOP_FSYNC err %d\n", err
);
1725 /* write, setattr, or fsync failed */
1726 ErrorLogWithPath("aborting decompress, err %d\n", err
);
1727 if (truncate_okay
) {
1728 /* truncate anything we might have written */
1729 int error
= vnode_setsize(vp
, 0, 0, decmpfs_ctx
);
1730 ErrorLogWithPath("vnode_setsize err %d\n", error
);
1736 /* if we're truncating the file to zero bytes, we'll skip ahead to here */
1738 /* unset the compressed flag */
1739 unset_compressed_flag(vp
);
1741 /* free the compressed data associated with this file */
1742 err
= decmpfs_free_compressed_data(vp
, cp
);
1744 ErrorLogWithPath("decmpfs_free_compressed_data err %d\n", err
);
1748 even if free_compressed_data or vnode_getattr/vnode_setattr failed, return success
1749 since we succeeded in writing all of the file data to the data fork
1753 /* if we got this far, the file was successfully decompressed */
1754 update_file_state
= 1;
1755 new_state
= FILE_IS_NOT_COMPRESSED
;
1757 #if COMPRESSION_DEBUG
1759 uint64_t filesize
= 0;
1760 vnsize(vp
, &filesize
);
1761 DebugLogWithPath("new file size %lld\n", filesize
);
1766 if (hdr
) FREE(hdr
, M_TEMP
);
1767 if (data
) FREE(data
, M_TEMP
);
1768 if (uio_w
) uio_free(uio_w
);
1771 /* if there was a failure, reset compression flags to unknown and clear the buffer cache data */
1772 update_file_state
= 1;
1773 new_state
= FILE_TYPE_UNKNOWN
;
1774 if (uncompressed_size
) {
1776 ubc_setsize(vp
, uncompressed_size
);
1780 if (update_file_state
) {
1781 lck_mtx_lock(decompress_channel_mtx
);
1782 decmpfs_cnode_set_vnode_state(cp
, new_state
, 1);
1783 wakeup((caddr_t
)&decompress_channel
); /* wake up anyone who might have been waiting for decompression */
1784 lck_mtx_unlock(decompress_channel_mtx
);
1787 if (cmpdata_locked
) decmpfs_unlock_compressed_data(cp
, 1);
1792 #pragma mark --- Type1 compressor ---
1795 The "Type1" compressor stores the data fork directly in the compression xattr
1799 decmpfs_validate_compressed_file_Type1(__unused vnode_t vp
, __unused vfs_context_t ctx
, decmpfs_header
*hdr
)
1803 if (hdr
->uncompressed_size
+ sizeof(decmpfs_disk_header
) != (uint64_t)hdr
->attr_size
) {
1812 decmpfs_fetch_uncompressed_data_Type1(__unused vnode_t vp
, __unused vfs_context_t ctx
, decmpfs_header
*hdr
, off_t offset
, user_ssize_t size
, int nvec
, decmpfs_vector
*vec
, uint64_t *bytes_read
)
1816 user_ssize_t remaining
;
1818 if (hdr
->uncompressed_size
+ sizeof(decmpfs_disk_header
) != (uint64_t)hdr
->attr_size
) {
1823 #if COMPRESSION_DEBUG
1824 static int dummy
= 0; // prevent syslog from coalescing printfs
1825 DebugLogWithPath("%d memcpy %lld at %lld\n", dummy
++, size
, (uint64_t)offset
);
1829 for (i
= 0; (i
< nvec
) && (remaining
> 0); i
++) {
1830 user_ssize_t curCopy
= vec
[i
].size
;
1831 if (curCopy
> remaining
)
1832 curCopy
= remaining
;
1833 memcpy(vec
[i
].buf
, hdr
->attr_bytes
+ offset
, curCopy
);
1835 remaining
-= curCopy
;
1838 if ((bytes_read
) && (err
== 0))
1839 *bytes_read
= (size
- remaining
);
1845 static decmpfs_registration Type1Reg
=
1847 .decmpfs_registration
= DECMPFS_REGISTRATION_VERSION
,
1848 .validate
= decmpfs_validate_compressed_file_Type1
,
1849 .adjust_fetch
= NULL
, /* no adjust necessary */
1850 .fetch
= decmpfs_fetch_uncompressed_data_Type1
,
1851 .free_data
= NULL
, /* no free necessary */
1852 .get_flags
= NULL
/* no flags */
1855 #pragma mark --- decmpfs initialization ---
1859 static int done
= 0;
1862 decmpfs_ctx
= vfs_context_create(vfs_context_kernel());
1864 lck_grp_attr_t
*attr
= lck_grp_attr_alloc_init();
1865 decmpfs_lockgrp
= lck_grp_alloc_init("VFSCOMP", attr
);
1866 lck_grp_attr_free(attr
);
1867 decompressorsLock
= lck_rw_alloc_init(decmpfs_lockgrp
, NULL
);
1868 decompress_channel_mtx
= lck_mtx_alloc_init(decmpfs_lockgrp
, NULL
);
1870 register_decmpfs_decompressor(CMP_Type1
, &Type1Reg
);
1874 #endif /* FS_COMPRESSION */