]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/decmpfs.c
xnu-3789.70.16.tar.gz
[apple/xnu.git] / bsd / kern / decmpfs.c
1 /*
2 * Copyright (c) 2008-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #if !FS_COMPRESSION
29
30 /* We need these symbols even though compression is turned off */
31
32 #define UNUSED_SYMBOL(x) asm(".global _" #x "\n.set _" #x ", 0\n");
33
34 UNUSED_SYMBOL(register_decmpfs_decompressor)
35 UNUSED_SYMBOL(unregister_decmpfs_decompressor)
36 UNUSED_SYMBOL(decmpfs_init)
37 UNUSED_SYMBOL(decmpfs_read_compressed)
38 UNUSED_SYMBOL(decmpfs_cnode_cmp_type)
39 UNUSED_SYMBOL(decmpfs_cnode_get_vnode_state)
40 UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_size)
41 UNUSED_SYMBOL(decmpfs_lock_compressed_data)
42 UNUSED_SYMBOL(decmpfs_cnode_free)
43 UNUSED_SYMBOL(decmpfs_cnode_alloc)
44 UNUSED_SYMBOL(decmpfs_cnode_destroy)
45 UNUSED_SYMBOL(decmpfs_decompress_file)
46 UNUSED_SYMBOL(decmpfs_unlock_compressed_data)
47 UNUSED_SYMBOL(decmpfs_cnode_init)
48 UNUSED_SYMBOL(decmpfs_cnode_set_vnode_state)
49 UNUSED_SYMBOL(decmpfs_hides_xattr)
50 UNUSED_SYMBOL(decmpfs_ctx)
51 UNUSED_SYMBOL(decmpfs_file_is_compressed)
52 UNUSED_SYMBOL(decmpfs_update_attributes)
53 UNUSED_SYMBOL(decmpfs_hides_rsrc)
54 UNUSED_SYMBOL(decmpfs_pagein_compressed)
55 UNUSED_SYMBOL(decmpfs_validate_compressed_file)
56
57 #else /* FS_COMPRESSION */
58 #include <sys/kernel.h>
59 #include <sys/vnode_internal.h>
60 #include <sys/file_internal.h>
61 #include <sys/stat.h>
62 #include <sys/fcntl.h>
63 #include <sys/xattr.h>
64 #include <sys/namei.h>
65 #include <sys/user.h>
66 #include <sys/mount_internal.h>
67 #include <sys/ubc.h>
68 #include <sys/decmpfs.h>
69 #include <sys/uio_internal.h>
70 #include <libkern/OSByteOrder.h>
71
72 #pragma mark --- debugging ---
73
74 #define COMPRESSION_DEBUG 0
75 #define COMPRESSION_DEBUG_VERBOSE 0
76 #define MALLOC_DEBUG 0
77
78 static const char *
79 baseName(const char *path)
80 {
81 if (!path)
82 return NULL;
83 const char *ret = path;
84 int i;
85 for (i = 0; path[i] != 0; i++) {
86 if (path[i] == '/')
87 ret = &path[i + 1];
88 }
89 return ret;
90 }
91
92 static char*
93 vnpath(vnode_t vp, char *path, int len)
94 {
95 int origlen = len;
96 path[0] = 0;
97 vn_getpath(vp, path, &len);
98 path[origlen - 1] = 0;
99 return path;
100 }
101
102 #define ErrorLog(x, args...) printf("%s:%d:%s: " x, baseName(__FILE__), __LINE__, __FUNCTION__, ## args)
103 #define ErrorLogWithPath(x, args...) do { char *path; MALLOC(path, char *, PATH_MAX, M_TEMP, M_WAITOK); printf("%s:%d:%s: %s: " x, baseName(__FILE__), __LINE__, __FUNCTION__, vnpath(vp, path, PATH_MAX), ## args); FREE(path, M_TEMP); } while(0)
104
105 #if COMPRESSION_DEBUG
106 #define DebugLog ErrorLog
107 #define DebugLogWithPath ErrorLogWithPath
108 #else
109 #define DebugLog(x...) do { } while(0)
110 #define DebugLogWithPath(x...) do { } while(0)
111 #endif
112
113 #if COMPRESSION_DEBUG_VERBOSE
114 #define VerboseLog ErrorLog
115 #define VerboseLogWithPath ErrorLogWithPath
116 #else
117 #define VerboseLog(x...) do { } while(0)
118 #define VerboseLogWithPath(x...) do { } while(0)
119 #endif
120
121 #if MALLOC_DEBUG
122
123 static SInt32 totalAlloc;
124
125 typedef struct {
126 uint32_t allocSz;
127 uint32_t magic;
128 const char *file;
129 int line;
130 } allocated;
131
132 static void *
133 _malloc(uint32_t sz, __unused int type, __unused int flags, const char *file, int line)
134 {
135 uint32_t allocSz = sz + 2 * sizeof(allocated);
136
137 allocated *alloc = NULL;
138 MALLOC(alloc, allocated *, allocSz, type, flags);
139 if (!alloc) {
140 ErrorLog("malloc failed\n");
141 return NULL;
142 }
143
144 char *ret = (char*)&alloc[1];
145 allocated *alloc2 = (allocated*)(ret + sz);
146
147 alloc->allocSz = allocSz;
148 alloc->magic = 0xdadadada;
149 alloc->file = file;
150 alloc->line = line;
151
152 *alloc2 = *alloc;
153
154 int s = OSAddAtomic(sz, &totalAlloc);
155 ErrorLog("malloc(%d) -> %p, total allocations %d\n", sz, ret, s + sz);
156
157 return ret;
158 }
159
160 static void
161 _free(char *ret, __unused int type, const char *file, int line)
162 {
163 if (!ret) {
164 ErrorLog("freeing null\n");
165 return;
166 }
167 allocated *alloc = (allocated*)ret;
168 alloc--;
169 uint32_t sz = alloc->allocSz - 2 * sizeof(allocated);
170 allocated *alloc2 = (allocated*)(ret + sz);
171
172 if (alloc->magic != 0xdadadada) {
173 panic("freeing bad pointer");
174 }
175
176 if (memcmp(alloc, alloc2, sizeof(*alloc)) != 0) {
177 panic("clobbered data");
178 }
179
180 memset(ret, 0xce, sz);
181 alloc2->file = file;
182 alloc2->line = line;
183 FREE(alloc, type);
184 int s = OSAddAtomic(-sz, &totalAlloc);
185 ErrorLog("free(%p,%d) -> total allocations %d\n", ret, sz, s - sz);
186 }
187
188 #undef MALLOC
189 #undef FREE
190 #define MALLOC(space, cast, size, type, flags) (space) = (cast)_malloc(size, type, flags, __FILE__, __LINE__)
191 #define FREE(addr, type) _free((void *)addr, type, __FILE__, __LINE__)
192
193 #endif /* MALLOC_DEBUG */
194
195 #pragma mark --- globals ---
196
197 static lck_grp_t *decmpfs_lockgrp;
198
199 static decmpfs_registration * decompressors[CMP_MAX]; /* the registered compressors */
200 static lck_rw_t * decompressorsLock;
201 static int decompress_channel; /* channel used by decompress_file to wake up waiters */
202 static lck_mtx_t *decompress_channel_mtx;
203
204 vfs_context_t decmpfs_ctx;
205
206 #pragma mark --- decmp_get_func ---
207
208 #define offsetof_func(func) ((uintptr_t)(&(((decmpfs_registration*)NULL)->func)))
209
210 static void *
211 _func_from_offset(uint32_t type, uintptr_t offset)
212 {
213 /* get the function at the given offset in the registration for the given type */
214 decmpfs_registration *reg = decompressors[type];
215 char *regChar = (char*)reg;
216 char *func = &regChar[offset];
217 void **funcPtr = (void**)func;
218
219 switch (reg->decmpfs_registration) {
220 case DECMPFS_REGISTRATION_VERSION_V1:
221 if (offset > offsetof_func(free_data))
222 return NULL;
223 break;
224 case DECMPFS_REGISTRATION_VERSION_V3:
225 if (offset > offsetof_func(get_flags))
226 return NULL;
227 break;
228 default:
229 return NULL;
230 }
231
232 return funcPtr[0];
233 }
234
235 extern void IOServicePublishResource( const char * property, boolean_t value );
236 extern boolean_t IOServiceWaitForMatchingResource( const char * property, uint64_t timeout );
237 extern boolean_t IOCatalogueMatchingDriversPresent( const char * property );
238
239 static void *
240 _decmp_get_func(vnode_t vp, uint32_t type, uintptr_t offset)
241 {
242 /*
243 this function should be called while holding a shared lock to decompressorsLock,
244 and will return with the lock held
245 */
246
247 if (type >= CMP_MAX)
248 return NULL;
249
250 if (decompressors[type] != NULL) {
251 // the compressor has already registered but the function might be null
252 return _func_from_offset(type, offset);
253 }
254
255 // does IOKit know about a kext that is supposed to provide this type?
256 char providesName[80];
257 snprintf(providesName, sizeof(providesName), "com.apple.AppleFSCompression.providesType%u", type);
258 if (IOCatalogueMatchingDriversPresent(providesName)) {
259 // there is a kext that says it will register for this type, so let's wait for it
260 char resourceName[80];
261 uint64_t delay = 10000000ULL; // 10 milliseconds.
262 snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", type);
263 ErrorLogWithPath("waiting for %s\n", resourceName);
264 while(decompressors[type] == NULL) {
265 lck_rw_unlock_shared(decompressorsLock); // we have to unlock to allow the kext to register
266 if (IOServiceWaitForMatchingResource(resourceName, delay)) {
267 lck_rw_lock_shared(decompressorsLock);
268 break;
269 }
270 if (!IOCatalogueMatchingDriversPresent(providesName)) {
271 //
272 ErrorLogWithPath("the kext with %s is no longer present\n", providesName);
273 lck_rw_lock_shared(decompressorsLock);
274 break;
275 }
276 ErrorLogWithPath("still waiting for %s\n", resourceName);
277 delay *= 2;
278 lck_rw_lock_shared(decompressorsLock);
279 }
280 // IOKit says the kext is loaded, so it should be registered too!
281 if (decompressors[type] == NULL) {
282 ErrorLogWithPath("we found %s, but the type still isn't registered\n", providesName);
283 return NULL;
284 }
285 // it's now registered, so let's return the function
286 return _func_from_offset(type, offset);
287 }
288
289 // the compressor hasn't registered, so it never will unless someone manually kextloads it
290 ErrorLogWithPath("tried to access a compressed file of unregistered type %d\n", type);
291 return NULL;
292 }
293
294 #define decmp_get_func(vp, type, func) ((typeof(((decmpfs_registration*)NULL)->func))_decmp_get_func(vp, type, offsetof_func(func)))
295
296 #pragma mark --- utilities ---
297
298 #if COMPRESSION_DEBUG
299 static int
300 vnsize(vnode_t vp, uint64_t *size)
301 {
302 struct vnode_attr va;
303 VATTR_INIT(&va);
304 VATTR_WANTED(&va, va_data_size);
305 int error = vnode_getattr(vp, &va, decmpfs_ctx);
306 if (error != 0) {
307 ErrorLogWithPath("vnode_getattr err %d\n", error);
308 return error;
309 }
310 *size = va.va_data_size;
311 return 0;
312 }
313 #endif /* COMPRESSION_DEBUG */
314
315 #pragma mark --- cnode routines ---
316
317 decmpfs_cnode *decmpfs_cnode_alloc(void)
318 {
319 decmpfs_cnode *dp;
320 MALLOC_ZONE(dp, decmpfs_cnode *, sizeof(decmpfs_cnode), M_DECMPFS_CNODE, M_WAITOK);
321 return dp;
322 }
323
324 void decmpfs_cnode_free(decmpfs_cnode *dp)
325 {
326 FREE_ZONE(dp, sizeof(*dp), M_DECMPFS_CNODE);
327 }
328
329 void
330 decmpfs_cnode_init(decmpfs_cnode *cp)
331 {
332 memset(cp, 0, sizeof(*cp));
333 lck_rw_init(&cp->compressed_data_lock, decmpfs_lockgrp, NULL);
334 }
335
336 void
337 decmpfs_cnode_destroy(decmpfs_cnode *cp)
338 {
339 lck_rw_destroy(&cp->compressed_data_lock, decmpfs_lockgrp);
340 }
341
342 bool
343 decmpfs_trylock_compressed_data(decmpfs_cnode *cp, int exclusive)
344 {
345 void *thread = current_thread();
346 bool retval = false;
347
348 if (cp->lockowner == thread) {
349 /* this thread is already holding an exclusive lock, so bump the count */
350 cp->lockcount++;
351 retval = true;
352 } else if (exclusive) {
353 if ((retval = lck_rw_try_lock_exclusive(&cp->compressed_data_lock))) {
354 cp->lockowner = thread;
355 cp->lockcount = 1;
356 }
357 } else {
358 if ((retval = lck_rw_try_lock_shared(&cp->compressed_data_lock))) {
359 cp->lockowner = (void *)-1;
360 }
361 }
362 return retval;
363 }
364
365 void
366 decmpfs_lock_compressed_data(decmpfs_cnode *cp, int exclusive)
367 {
368 void *thread = current_thread();
369
370 if (cp->lockowner == thread) {
371 /* this thread is already holding an exclusive lock, so bump the count */
372 cp->lockcount++;
373 } else if (exclusive) {
374 lck_rw_lock_exclusive(&cp->compressed_data_lock);
375 cp->lockowner = thread;
376 cp->lockcount = 1;
377 } else {
378 lck_rw_lock_shared(&cp->compressed_data_lock);
379 cp->lockowner = (void *)-1;
380 }
381 }
382
383 void
384 decmpfs_unlock_compressed_data(decmpfs_cnode *cp, __unused int exclusive)
385 {
386 void *thread = current_thread();
387
388 if (cp->lockowner == thread) {
389 /* this thread is holding an exclusive lock, so decrement the count */
390 if ((--cp->lockcount) > 0) {
391 /* the caller still has outstanding locks, so we're done */
392 return;
393 }
394 cp->lockowner = NULL;
395 }
396
397 lck_rw_done(&cp->compressed_data_lock);
398 }
399
400 uint32_t
401 decmpfs_cnode_get_vnode_state(decmpfs_cnode *cp)
402 {
403 return cp->cmp_state;
404 }
405
406 void
407 decmpfs_cnode_set_vnode_state(decmpfs_cnode *cp, uint32_t state, int skiplock)
408 {
409 if (!skiplock) decmpfs_lock_compressed_data(cp, 1);
410 cp->cmp_state = state;
411 if (state == FILE_TYPE_UNKNOWN) {
412 /* clear out the compression type too */
413 cp->cmp_type = 0;
414 }
415 if (!skiplock) decmpfs_unlock_compressed_data(cp, 1);
416 }
417
418 static void
419 decmpfs_cnode_set_vnode_cmp_type(decmpfs_cnode *cp, uint32_t cmp_type, int skiplock)
420 {
421 if (!skiplock) decmpfs_lock_compressed_data(cp, 1);
422 cp->cmp_type = cmp_type;
423 if (!skiplock) decmpfs_unlock_compressed_data(cp, 1);
424 }
425
426 static void
427 decmpfs_cnode_set_vnode_minimal_xattr(decmpfs_cnode *cp, int minimal_xattr, int skiplock)
428 {
429 if (!skiplock) decmpfs_lock_compressed_data(cp, 1);
430 cp->cmp_minimal_xattr = minimal_xattr;
431 if (!skiplock) decmpfs_unlock_compressed_data(cp, 1);
432 }
433
434 uint64_t
435 decmpfs_cnode_get_vnode_cached_size(decmpfs_cnode *cp)
436 {
437 return cp->uncompressed_size;
438 }
439
440 static void
441 decmpfs_cnode_set_vnode_cached_size(decmpfs_cnode *cp, uint64_t size)
442 {
443 while(1) {
444 uint64_t old = cp->uncompressed_size;
445 if (OSCompareAndSwap64(old, size, (UInt64*)&cp->uncompressed_size)) {
446 return;
447 } else {
448 /* failed to write our value, so loop */
449 }
450 }
451 }
452
453 static uint64_t
454 decmpfs_cnode_get_decompression_flags(decmpfs_cnode *cp)
455 {
456 return cp->decompression_flags;
457 }
458
459 static void
460 decmpfs_cnode_set_decompression_flags(decmpfs_cnode *cp, uint64_t flags)
461 {
462 while(1) {
463 uint64_t old = cp->decompression_flags;
464 if (OSCompareAndSwap64(old, flags, (UInt64*)&cp->decompression_flags)) {
465 return;
466 } else {
467 /* failed to write our value, so loop */
468 }
469 }
470 }
471
472 uint32_t decmpfs_cnode_cmp_type(decmpfs_cnode *cp)
473 {
474 return cp->cmp_type;
475 }
476
477 #pragma mark --- decmpfs state routines ---
478
479 static int
480 decmpfs_fetch_compressed_header(vnode_t vp, decmpfs_cnode *cp, decmpfs_header **hdrOut, int returnInvalid)
481 {
482 /*
483 fetches vp's compression xattr, converting it into a decmpfs_header; returns 0 or errno
484 if returnInvalid == 1, returns the header even if the type was invalid (out of range),
485 and return ERANGE in that case
486 */
487
488 size_t read_size = 0;
489 size_t attr_size = 0;
490 uio_t attr_uio = NULL;
491 int err = 0;
492 char *data = NULL;
493 decmpfs_header *hdr = NULL;
494 char uio_buf[ UIO_SIZEOF(1) ];
495
496 if ((cp != NULL) &&
497 (cp->cmp_type != 0) &&
498 (cp->cmp_minimal_xattr != 0)) {
499 /* this file's xattr didn't have any extra data when we fetched it, so we can synthesize a header from the data in the cnode */
500
501 MALLOC(data, char *, sizeof(decmpfs_header), M_TEMP, M_WAITOK);
502 if (!data) {
503 err = ENOMEM;
504 goto out;
505 }
506 hdr = (decmpfs_header*)data;
507 hdr->attr_size = sizeof(decmpfs_disk_header);
508 hdr->compression_magic = DECMPFS_MAGIC;
509 hdr->compression_type = cp->cmp_type;
510 hdr->uncompressed_size = decmpfs_cnode_get_vnode_cached_size(cp);
511 } else {
512 /* figure out how big the xattr is on disk */
513 err = vn_getxattr(vp, DECMPFS_XATTR_NAME, NULL, &attr_size, XATTR_NOSECURITY, decmpfs_ctx);
514 if (err != 0)
515 goto out;
516
517 if (attr_size < sizeof(decmpfs_disk_header) || attr_size > MAX_DECMPFS_XATTR_SIZE) {
518 err = EINVAL;
519 goto out;
520 }
521
522 /* allocation includes space for the extra attr_size field of a compressed_header */
523 MALLOC(data, char *, attr_size + sizeof(hdr->attr_size), M_TEMP, M_WAITOK);
524 if (!data) {
525 err = ENOMEM;
526 goto out;
527 }
528
529 /* read the xattr into our buffer, skipping over the attr_size field at the beginning */
530 attr_uio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, &uio_buf[0], sizeof(uio_buf));
531 uio_addiov(attr_uio, CAST_USER_ADDR_T(data + sizeof(hdr->attr_size)), attr_size);
532
533 err = vn_getxattr(vp, DECMPFS_XATTR_NAME, attr_uio, &read_size, XATTR_NOSECURITY, decmpfs_ctx);
534 if (err != 0)
535 goto out;
536 if (read_size != attr_size) {
537 err = EINVAL;
538 goto out;
539 }
540 hdr = (decmpfs_header*)data;
541 hdr->attr_size = attr_size;
542 /* swap the fields to native endian */
543 hdr->compression_magic = OSSwapLittleToHostInt32(hdr->compression_magic);
544 hdr->compression_type = OSSwapLittleToHostInt32(hdr->compression_type);
545 hdr->uncompressed_size = OSSwapLittleToHostInt64(hdr->uncompressed_size);
546 }
547
548 if (hdr->compression_magic != DECMPFS_MAGIC) {
549 ErrorLogWithPath("invalid compression_magic 0x%08x, should be 0x%08x\n", hdr->compression_magic, DECMPFS_MAGIC);
550 err = EINVAL;
551 goto out;
552 }
553
554 if (hdr->compression_type >= CMP_MAX) {
555 if (returnInvalid) {
556 /* return the header even though the type is out of range */
557 err = ERANGE;
558 } else {
559 ErrorLogWithPath("compression_type %d out of range\n", hdr->compression_type);
560 err = EINVAL;
561 }
562 goto out;
563 }
564
565 out:
566 if (err && (err != ERANGE)) {
567 DebugLogWithPath("err %d\n", err);
568 if (data) FREE(data, M_TEMP);
569 *hdrOut = NULL;
570 } else {
571 *hdrOut = hdr;
572 }
573 return err;
574 }
575
576 static int
577 decmpfs_fast_get_state(decmpfs_cnode *cp)
578 {
579 /*
580 return the cached state
581 this should *only* be called when we know that decmpfs_file_is_compressed has already been called,
582 because this implies that the cached state is valid
583 */
584 int cmp_state = decmpfs_cnode_get_vnode_state(cp);
585
586 switch(cmp_state) {
587 case FILE_IS_NOT_COMPRESSED:
588 case FILE_IS_COMPRESSED:
589 case FILE_IS_CONVERTING:
590 return cmp_state;
591 case FILE_TYPE_UNKNOWN:
592 /*
593 we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode,
594 which should not be possible
595 */
596 ErrorLog("decmpfs_fast_get_state called on unknown file\n");
597 return FILE_IS_NOT_COMPRESSED;
598 default:
599 /* */
600 ErrorLog("unknown cmp_state %d\n", cmp_state);
601 return FILE_IS_NOT_COMPRESSED;
602 }
603 }
604
605 static int
606 decmpfs_fast_file_is_compressed(decmpfs_cnode *cp)
607 {
608 int cmp_state = decmpfs_cnode_get_vnode_state(cp);
609
610 switch(cmp_state) {
611 case FILE_IS_NOT_COMPRESSED:
612 return 0;
613 case FILE_IS_COMPRESSED:
614 case FILE_IS_CONVERTING:
615 return 1;
616 case FILE_TYPE_UNKNOWN:
617 /*
618 we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode,
619 which should not be possible
620 */
621 ErrorLog("decmpfs_fast_get_state called on unknown file\n");
622 return 0;
623 default:
624 /* */
625 ErrorLog("unknown cmp_state %d\n", cmp_state);
626 return 0;
627 }
628 }
629
630 errno_t
631 decmpfs_validate_compressed_file(vnode_t vp, decmpfs_cnode *cp)
632 {
633 /* give a compressor a chance to indicate that a compressed file is invalid */
634
635 decmpfs_header *hdr = NULL;
636 errno_t err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
637 if (err) {
638 /* we couldn't get the header */
639 if (decmpfs_fast_get_state(cp) == FILE_IS_NOT_COMPRESSED) {
640 /* the file is no longer compressed, so return success */
641 err = 0;
642 }
643 goto out;
644 }
645
646 lck_rw_lock_shared(decompressorsLock);
647 decmpfs_validate_compressed_file_func validate = decmp_get_func(vp, hdr->compression_type, validate);
648 if (validate) { /* make sure this validation function is valid */
649 /* is the data okay? */
650 err = validate(vp, decmpfs_ctx, hdr);
651 } else if (decmp_get_func(vp, hdr->compression_type, fetch) == NULL) {
652 /* the type isn't registered */
653 err = EIO;
654 } else {
655 /* no validate registered, so nothing to do */
656 err = 0;
657 }
658 lck_rw_unlock_shared(decompressorsLock);
659 out:
660 if (hdr) FREE(hdr, M_TEMP);
661 #if COMPRESSION_DEBUG
662 if (err) {
663 DebugLogWithPath("decmpfs_validate_compressed_file ret %d, vp->v_flag %d\n", err, vp->v_flag);
664 }
665 #endif
666 return err;
667 }
668
669 int
670 decmpfs_file_is_compressed(vnode_t vp, decmpfs_cnode *cp)
671 {
672 /*
673 determines whether vp points to a compressed file
674
675 to speed up this operation, we cache the result in the cnode, and do as little as possible
676 in the case where the cnode already has a valid cached state
677
678 */
679
680 int ret = 0;
681 int error = 0;
682 uint32_t cmp_state;
683 struct vnode_attr va_fetch;
684 decmpfs_header *hdr = NULL;
685 mount_t mp = NULL;
686 int cnode_locked = 0;
687 int saveInvalid = 0; // save the header data even though the type was out of range
688 uint64_t decompression_flags = 0;
689
690 if (vnode_isnamedstream(vp)) {
691 /*
692 named streams can't be compressed
693 since named streams of the same file share the same cnode,
694 we don't want to get/set the state in the cnode, just return 0
695 */
696 return 0;
697 }
698
699 /* examine the cached a state in this cnode */
700 cmp_state = decmpfs_cnode_get_vnode_state(cp);
701 switch(cmp_state) {
702 case FILE_IS_NOT_COMPRESSED:
703 return 0;
704 case FILE_IS_COMPRESSED:
705 return 1;
706 case FILE_IS_CONVERTING:
707 /* treat the file as compressed, because this gives us a way to block future reads until decompression is done */
708 return 1;
709 case FILE_TYPE_UNKNOWN:
710 /* the first time we encountered this vnode, so we need to check it out */
711 break;
712 default:
713 /* unknown state, assume file is not compressed */
714 ErrorLogWithPath("unknown cmp_state %d\n", cmp_state);
715 return 0;
716 }
717
718 if (!vnode_isreg(vp)) {
719 /* only regular files can be compressed */
720 ret = FILE_IS_NOT_COMPRESSED;
721 goto done;
722 }
723
724 mp = vnode_mount(vp);
725 if (mp == NULL) {
726 /*
727 this should only be true before we mount the root filesystem
728 we short-cut this return to avoid the call to getattr below, which
729 will fail before root is mounted
730 */
731 ret = FILE_IS_NOT_COMPRESSED;
732 goto done;
733 }
734 if ((mp->mnt_flag & MNT_LOCAL) == 0) {
735 /* compression only supported on local filesystems */
736 ret = FILE_IS_NOT_COMPRESSED;
737 goto done;
738 }
739
740 /* lock our cnode data so that another caller doesn't change the state under us */
741 decmpfs_lock_compressed_data(cp, 1);
742 cnode_locked = 1;
743
744 VATTR_INIT(&va_fetch);
745 VATTR_WANTED(&va_fetch, va_flags);
746 error = vnode_getattr(vp, &va_fetch, decmpfs_ctx);
747 if (error) {
748 /* failed to get the bsd flags so the file is not compressed */
749 ret = FILE_IS_NOT_COMPRESSED;
750 goto done;
751 }
752 if (va_fetch.va_flags & UF_COMPRESSED) {
753 /* UF_COMPRESSED is on, make sure the file has the DECMPFS_XATTR_NAME xattr */
754 error = decmpfs_fetch_compressed_header(vp, cp, &hdr, 1);
755 if ((hdr != NULL) && (error == ERANGE)) {
756 saveInvalid = 1;
757 }
758 if (error) {
759 /* failed to get the xattr so the file is not compressed */
760 ret = FILE_IS_NOT_COMPRESSED;
761 goto done;
762 }
763 /* we got the xattr, so the file is compressed */
764 ret = FILE_IS_COMPRESSED;
765 goto done;
766 }
767 /* UF_COMPRESSED isn't on, so the file isn't compressed */
768 ret = FILE_IS_NOT_COMPRESSED;
769
770 done:
771 if (((ret == FILE_IS_COMPRESSED) || saveInvalid) && hdr) {
772 /*
773 cache the uncompressed size away in the cnode
774 */
775
776 if (!cnode_locked) {
777 /*
778 we should never get here since the only place ret is set to FILE_IS_COMPRESSED
779 is after the call to decmpfs_lock_compressed_data above
780 */
781 decmpfs_lock_compressed_data(cp, 1);
782 cnode_locked = 1;
783 }
784
785 decmpfs_cnode_set_vnode_cached_size(cp, hdr->uncompressed_size);
786 decmpfs_cnode_set_vnode_state(cp, ret, 1);
787 decmpfs_cnode_set_vnode_cmp_type(cp, hdr->compression_type, 1);
788 /* remember if the xattr's size was equal to the minimal xattr */
789 if (hdr->attr_size == sizeof(decmpfs_disk_header)) {
790 decmpfs_cnode_set_vnode_minimal_xattr(cp, 1, 1);
791 }
792 if (ret == FILE_IS_COMPRESSED) {
793 /* update the ubc's size for this file */
794 ubc_setsize(vp, hdr->uncompressed_size);
795
796 /* update the decompression flags in the decmpfs cnode */
797 lck_rw_lock_shared(decompressorsLock);
798 decmpfs_get_decompression_flags_func get_flags = decmp_get_func(vp, hdr->compression_type, get_flags);
799 if (get_flags) {
800 decompression_flags = get_flags(vp, decmpfs_ctx, hdr);
801 }
802 lck_rw_unlock_shared(decompressorsLock);
803 decmpfs_cnode_set_decompression_flags(cp, decompression_flags);
804 }
805 } else {
806 /* we might have already taken the lock above; if so, skip taking it again by passing cnode_locked as the skiplock parameter */
807 decmpfs_cnode_set_vnode_state(cp, ret, cnode_locked);
808 }
809
810 if (cnode_locked) decmpfs_unlock_compressed_data(cp, 1);
811
812 if (hdr) FREE(hdr, M_TEMP);
813
814 switch(ret) {
815 case FILE_IS_NOT_COMPRESSED:
816 return 0;
817 case FILE_IS_COMPRESSED:
818 case FILE_IS_CONVERTING:
819 return 1;
820 default:
821 /* unknown state, assume file is not compressed */
822 ErrorLogWithPath("unknown ret %d\n", ret);
823 return 0;
824 }
825 }
826
827 int
828 decmpfs_update_attributes(vnode_t vp, struct vnode_attr *vap)
829 {
830 int error = 0;
831
832 if (VATTR_IS_ACTIVE(vap, va_flags)) {
833 /* the BSD flags are being updated */
834 if (vap->va_flags & UF_COMPRESSED) {
835 /* the compressed bit is being set, did it change? */
836 struct vnode_attr va_fetch;
837 int old_flags = 0;
838 VATTR_INIT(&va_fetch);
839 VATTR_WANTED(&va_fetch, va_flags);
840 error = vnode_getattr(vp, &va_fetch, decmpfs_ctx);
841 if (error)
842 return error;
843
844 old_flags = va_fetch.va_flags;
845
846 if (!(old_flags & UF_COMPRESSED)) {
847 /*
848 * Compression bit was turned on, make sure the file has the DECMPFS_XATTR_NAME attribute.
849 * This precludes anyone from using the UF_COMPRESSED bit for anything else, and it enforces
850 * an order of operation -- you must first do the setxattr and then the chflags.
851 */
852
853 if (VATTR_IS_ACTIVE(vap, va_data_size)) {
854 /*
855 * don't allow the caller to set the BSD flag and the size in the same call
856 * since this doesn't really make sense
857 */
858 vap->va_flags &= ~UF_COMPRESSED;
859 return 0;
860 }
861
862 decmpfs_header *hdr = NULL;
863 error = decmpfs_fetch_compressed_header(vp, NULL, &hdr, 1);
864 if (error == 0) {
865 /*
866 allow the flag to be set since the decmpfs attribute is present
867 in that case, we also want to truncate the data fork of the file
868 */
869 VATTR_SET_ACTIVE(vap, va_data_size);
870 vap->va_data_size = 0;
871 } else if (error == ERANGE) {
872 /* the file had a decmpfs attribute but the type was out of range, so don't muck with the file's data size */
873 } else {
874 /* no DECMPFS_XATTR_NAME attribute, so deny the update */
875 vap->va_flags &= ~UF_COMPRESSED;
876 }
877 if (hdr) FREE(hdr, M_TEMP);
878 }
879 }
880 }
881
882 return 0;
883 }
884
885 static int
886 wait_for_decompress(decmpfs_cnode *cp)
887 {
888 int state;
889 lck_mtx_lock(decompress_channel_mtx);
890 do {
891 state = decmpfs_fast_get_state(cp);
892 if (state != FILE_IS_CONVERTING) {
893 /* file is not decompressing */
894 lck_mtx_unlock(decompress_channel_mtx);
895 return state;
896 }
897 msleep((caddr_t)&decompress_channel, decompress_channel_mtx, PINOD, "wait_for_decompress", NULL);
898 } while(1);
899 }
900
901 #pragma mark --- decmpfs hide query routines ---
902
903 int
904 decmpfs_hides_rsrc(vfs_context_t ctx, decmpfs_cnode *cp)
905 {
906 /*
907 WARNING!!!
908 callers may (and do) pass NULL for ctx, so we should only use it
909 for this equality comparison
910
911 This routine should only be called after a file has already been through decmpfs_file_is_compressed
912 */
913
914 if (ctx == decmpfs_ctx)
915 return 0;
916
917 if (!decmpfs_fast_file_is_compressed(cp))
918 return 0;
919
920 /* all compressed files hide their resource fork */
921 return 1;
922 }
923
924 int
925 decmpfs_hides_xattr(vfs_context_t ctx, decmpfs_cnode *cp, const char *xattr)
926 {
927 /*
928 WARNING!!!
929 callers may (and do) pass NULL for ctx, so we should only use it
930 for this equality comparison
931
932 This routine should only be called after a file has already been through decmpfs_file_is_compressed
933 */
934
935 if (ctx == decmpfs_ctx)
936 return 0;
937 if (strncmp(xattr, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME) - 1) == 0)
938 return decmpfs_hides_rsrc(ctx, cp);
939 if (!decmpfs_fast_file_is_compressed(cp))
940 /* file is not compressed, so don't hide this xattr */
941 return 0;
942 if (strncmp(xattr, DECMPFS_XATTR_NAME, sizeof(DECMPFS_XATTR_NAME) - 1) == 0)
943 /* it's our xattr, so hide it */
944 return 1;
945 /* don't hide this xattr */
946 return 0;
947 }
948
949 #pragma mark --- registration/validation routines ---
950
951 static inline int registration_valid(decmpfs_registration *registration)
952 {
953 return registration && ((registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V1) || (registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V3));
954 }
955
956 errno_t
957 register_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration)
958 {
959 /* called by kexts to register decompressors */
960
961 errno_t ret = 0;
962 int locked = 0;
963 char resourceName[80];
964
965 if ((compression_type >= CMP_MAX) || !registration_valid(registration)) {
966 ret = EINVAL;
967 goto out;
968 }
969
970 lck_rw_lock_exclusive(decompressorsLock); locked = 1;
971
972 /* make sure the registration for this type is zero */
973 if (decompressors[compression_type] != NULL) {
974 ret = EEXIST;
975 goto out;
976 }
977 decompressors[compression_type] = registration;
978 snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type);
979 IOServicePublishResource(resourceName, TRUE);
980
981 out:
982 if (locked) lck_rw_unlock_exclusive(decompressorsLock);
983 return ret;
984 }
985
986 errno_t
987 unregister_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration)
988 {
989 /* called by kexts to unregister decompressors */
990
991 errno_t ret = 0;
992 int locked = 0;
993 char resourceName[80];
994
995 if ((compression_type >= CMP_MAX) || !registration_valid(registration)) {
996 ret = EINVAL;
997 goto out;
998 }
999
1000 lck_rw_lock_exclusive(decompressorsLock); locked = 1;
1001 if (decompressors[compression_type] != registration) {
1002 ret = EEXIST;
1003 goto out;
1004 }
1005 decompressors[compression_type] = NULL;
1006 snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type);
1007 IOServicePublishResource(resourceName, FALSE);
1008
1009 out:
1010 if (locked) lck_rw_unlock_exclusive(decompressorsLock);
1011 return ret;
1012 }
1013
1014 static int
1015 compression_type_valid(vnode_t vp, decmpfs_header *hdr)
1016 {
1017 /* fast pre-check to determine if the given compressor has checked in */
1018 int ret = 0;
1019
1020 /* every compressor must have at least a fetch function */
1021 lck_rw_lock_shared(decompressorsLock);
1022 if (decmp_get_func(vp, hdr->compression_type, fetch) != NULL) {
1023 ret = 1;
1024 }
1025 lck_rw_unlock_shared(decompressorsLock);
1026
1027 return ret;
1028 }
1029
1030 #pragma mark --- compression/decompression routines ---
1031
1032 static int
1033 decmpfs_fetch_uncompressed_data(vnode_t vp, decmpfs_cnode *cp, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read)
1034 {
1035 /* get the uncompressed bytes for the specified region of vp by calling out to the registered compressor */
1036
1037 int err = 0;
1038
1039 *bytes_read = 0;
1040
1041 if ((uint64_t)offset >= hdr->uncompressed_size) {
1042 /* reading past end of file; nothing to do */
1043 err = 0;
1044 goto out;
1045 }
1046 if (offset < 0) {
1047 /* tried to read from before start of file */
1048 err = EINVAL;
1049 goto out;
1050 }
1051 if ((uint64_t)(offset + size) > hdr->uncompressed_size) {
1052 /* adjust size so we don't read past the end of the file */
1053 size = hdr->uncompressed_size - offset;
1054 }
1055 if (size == 0) {
1056 /* nothing to read */
1057 err = 0;
1058 goto out;
1059 }
1060
1061 lck_rw_lock_shared(decompressorsLock);
1062 decmpfs_fetch_uncompressed_data_func fetch = decmp_get_func(vp, hdr->compression_type, fetch);
1063 if (fetch) {
1064 err = fetch(vp, decmpfs_ctx, hdr, offset, size, nvec, vec, bytes_read);
1065 lck_rw_unlock_shared(decompressorsLock);
1066 if (err == 0) {
1067 uint64_t decompression_flags = decmpfs_cnode_get_decompression_flags(cp);
1068 if (decompression_flags & DECMPFS_FLAGS_FORCE_FLUSH_ON_DECOMPRESS) {
1069 #if !defined(__i386__) && !defined(__x86_64__)
1070 int i;
1071 for (i = 0; i < nvec; i++) {
1072 flush_dcache64((addr64_t)(uintptr_t)vec[i].buf, vec[i].size, FALSE);
1073 }
1074 #endif
1075 }
1076 }
1077 } else {
1078 err = ENOTSUP;
1079 lck_rw_unlock_shared(decompressorsLock);
1080 }
1081
1082 out:
1083 return err;
1084 }
1085
1086 static kern_return_t
1087 commit_upl(upl_t upl, upl_offset_t pl_offset, size_t uplSize, int flags, int abort)
1088 {
1089 kern_return_t kr = 0;
1090
1091 #if CONFIG_IOSCHED
1092 upl_unmark_decmp(upl);
1093 #endif /* CONFIG_IOSCHED */
1094
1095 /* commit the upl pages */
1096 if (abort) {
1097 VerboseLog("aborting upl, flags 0x%08x\n", flags);
1098 kr = ubc_upl_abort_range(upl, pl_offset, uplSize, flags);
1099 if (kr != KERN_SUCCESS)
1100 ErrorLog("ubc_upl_abort_range error %d\n", (int)kr);
1101 } else {
1102 VerboseLog("committing upl, flags 0x%08x\n", flags | UPL_COMMIT_CLEAR_DIRTY);
1103 kr = ubc_upl_commit_range(upl, pl_offset, uplSize, flags | UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_WRITTEN_BY_KERNEL);
1104 if (kr != KERN_SUCCESS)
1105 ErrorLog("ubc_upl_commit_range error %d\n", (int)kr);
1106 }
1107 return kr;
1108 }
1109
1110
1111 errno_t
1112 decmpfs_pagein_compressed(struct vnop_pagein_args *ap, int *is_compressed, decmpfs_cnode *cp)
1113 {
1114 /* handles a page-in request from vfs for a compressed file */
1115
1116 int err = 0;
1117 vnode_t vp = ap->a_vp;
1118 upl_t pl = ap->a_pl;
1119 upl_offset_t pl_offset = ap->a_pl_offset;
1120 off_t f_offset = ap->a_f_offset;
1121 size_t size = ap->a_size;
1122 int flags = ap->a_flags;
1123 off_t uplPos = 0;
1124 user_ssize_t uplSize = 0;
1125 void *data = NULL;
1126 decmpfs_header *hdr = NULL;
1127 uint64_t cachedSize = 0;
1128 int cmpdata_locked = 0;
1129
1130 if(!decmpfs_trylock_compressed_data(cp, 0)) {
1131 return EAGAIN;
1132 }
1133 cmpdata_locked = 1;
1134
1135
1136 if (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)) {
1137 DebugLogWithPath("pagein: unknown flags 0x%08x\n", (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)));
1138 }
1139
1140 err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
1141 if (err != 0) {
1142 goto out;
1143 }
1144
1145 cachedSize = hdr->uncompressed_size;
1146
1147 if (!compression_type_valid(vp, hdr)) {
1148 /* compressor not registered */
1149 err = ENOTSUP;
1150 goto out;
1151 }
1152
1153 #if CONFIG_IOSCHED
1154 /* Mark the UPL as the requesting UPL for decompression */
1155 upl_mark_decmp(pl);
1156 #endif /* CONFIG_IOSCHED */
1157
1158 /* map the upl so we can fetch into it */
1159 kern_return_t kr = ubc_upl_map(pl, (vm_offset_t*)&data);
1160 if ((kr != KERN_SUCCESS) || (data == NULL)) {
1161 err = ENOSPC;
1162 data = NULL;
1163 #if CONFIG_IOSCHED
1164 upl_unmark_decmp(pl);
1165 #endif /* CONFIG_IOSCHED */
1166 goto out;
1167 }
1168
1169 uplPos = f_offset;
1170 uplSize = size;
1171
1172 /* clip the size to the size of the file */
1173 if ((uint64_t)uplPos + uplSize > cachedSize) {
1174 /* truncate the read to the size of the file */
1175 uplSize = cachedSize - uplPos;
1176 }
1177
1178 /* do the fetch */
1179 decmpfs_vector vec;
1180
1181 decompress:
1182 /* the mapped data pointer points to the first page of the page list, so we want to start filling in at an offset of pl_offset */
1183 vec.buf = (char*)data + pl_offset;
1184 vec.size = size;
1185
1186 uint64_t did_read = 0;
1187 if (decmpfs_fast_get_state(cp) == FILE_IS_CONVERTING) {
1188 ErrorLogWithPath("unexpected pagein during decompress\n");
1189 /*
1190 if the file is converting, this must be a recursive call to pagein from underneath a call to decmpfs_decompress_file;
1191 pretend that it succeeded but don't do anything since we're just going to write over the pages anyway
1192 */
1193 err = 0;
1194 did_read = 0;
1195 } else {
1196 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, uplPos, uplSize, 1, &vec, &did_read);
1197 }
1198 if (err) {
1199 DebugLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err);
1200 int cmp_state = decmpfs_fast_get_state(cp);
1201 if (cmp_state == FILE_IS_CONVERTING) {
1202 DebugLogWithPath("cmp_state == FILE_IS_CONVERTING\n");
1203 cmp_state = wait_for_decompress(cp);
1204 if (cmp_state == FILE_IS_COMPRESSED) {
1205 DebugLogWithPath("cmp_state == FILE_IS_COMPRESSED\n");
1206 /* a decompress was attempted but it failed, let's try calling fetch again */
1207 goto decompress;
1208 }
1209 }
1210 if (cmp_state == FILE_IS_NOT_COMPRESSED) {
1211 DebugLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n");
1212 /* the file was decompressed after we started reading it */
1213 *is_compressed = 0; /* instruct caller to fall back to its normal path */
1214 }
1215 }
1216
1217 /* zero out whatever we didn't read, and zero out the end of the last page(s) */
1218 uint64_t total_size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
1219 if (did_read < total_size) {
1220 memset((char*)vec.buf + did_read, 0, total_size - did_read);
1221 }
1222
1223 #if CONFIG_IOSCHED
1224 upl_unmark_decmp(pl);
1225 #endif /* CONFIG_IOSCHED */
1226
1227 kr = ubc_upl_unmap(pl); data = NULL; /* make sure to set data to NULL so we don't try to unmap again below */
1228 if (kr != KERN_SUCCESS)
1229 ErrorLogWithPath("ubc_upl_unmap error %d\n", (int)kr);
1230 else {
1231 if (!err) {
1232 /* commit our pages */
1233 kr = commit_upl(pl, pl_offset, total_size, UPL_COMMIT_FREE_ON_EMPTY, 0);
1234 }
1235 }
1236
1237 out:
1238 if (data) ubc_upl_unmap(pl);
1239 if (hdr) FREE(hdr, M_TEMP);
1240 if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 0);
1241 if (err) {
1242 #if 0
1243 if (err != ENXIO && err != ENOSPC) {
1244 char *path;
1245 MALLOC(path, char *, PATH_MAX, M_TEMP, M_WAITOK);
1246 panic("%s: decmpfs_pagein_compressed: err %d", vnpath(vp, path, PATH_MAX), err);
1247 FREE(path, M_TEMP);
1248 }
1249 #endif /* 0 */
1250 ErrorLogWithPath("err %d\n", err);
1251 }
1252 return err;
1253 }
1254
1255 errno_t
1256 decmpfs_read_compressed(struct vnop_read_args *ap, int *is_compressed, decmpfs_cnode *cp)
1257 {
1258 /* handles a read request from vfs for a compressed file */
1259
1260 uio_t uio = ap->a_uio;
1261 vnode_t vp = ap->a_vp;
1262 int err = 0;
1263 int countInt = 0;
1264 off_t uplPos = 0;
1265 user_ssize_t uplSize = 0;
1266 user_ssize_t uplRemaining = 0;
1267 off_t curUplPos = 0;
1268 user_ssize_t curUplSize = 0;
1269 kern_return_t kr = KERN_SUCCESS;
1270 int abort_read = 0;
1271 void *data = NULL;
1272 uint64_t did_read = 0;
1273 upl_t upl = NULL;
1274 upl_page_info_t *pli = NULL;
1275 decmpfs_header *hdr = NULL;
1276 uint64_t cachedSize = 0;
1277 off_t uioPos = 0;
1278 user_ssize_t uioRemaining = 0;
1279 int cmpdata_locked = 0;
1280
1281 decmpfs_lock_compressed_data(cp, 0); cmpdata_locked = 1;
1282
1283 uplPos = uio_offset(uio);
1284 uplSize = uio_resid(uio);
1285 VerboseLogWithPath("uplPos %lld uplSize %lld\n", uplPos, uplSize);
1286
1287 cachedSize = decmpfs_cnode_get_vnode_cached_size(cp);
1288
1289 if ((uint64_t)uplPos + uplSize > cachedSize) {
1290 /* truncate the read to the size of the file */
1291 uplSize = cachedSize - uplPos;
1292 }
1293
1294 /* give the cluster layer a chance to fill in whatever it already has */
1295 countInt = (uplSize > INT_MAX) ? INT_MAX : uplSize;
1296 err = cluster_copy_ubc_data(vp, uio, &countInt, 0);
1297 if (err != 0)
1298 goto out;
1299
1300 /* figure out what's left */
1301 uioPos = uio_offset(uio);
1302 uioRemaining = uio_resid(uio);
1303 if ((uint64_t)uioPos + uioRemaining > cachedSize) {
1304 /* truncate the read to the size of the file */
1305 uioRemaining = cachedSize - uioPos;
1306 }
1307
1308 if (uioRemaining <= 0) {
1309 /* nothing left */
1310 goto out;
1311 }
1312
1313 err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
1314 if (err != 0) {
1315 goto out;
1316 }
1317 if (!compression_type_valid(vp, hdr)) {
1318 err = ENOTSUP;
1319 goto out;
1320 }
1321
1322 uplPos = uioPos;
1323 uplSize = uioRemaining;
1324 #if COMPRESSION_DEBUG
1325 DebugLogWithPath("uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize);
1326 #endif
1327
1328 lck_rw_lock_shared(decompressorsLock);
1329 decmpfs_adjust_fetch_region_func adjust_fetch = decmp_get_func(vp, hdr->compression_type, adjust_fetch);
1330 if (adjust_fetch) {
1331 /* give the compressor a chance to adjust the portion of the file that we read */
1332 adjust_fetch(vp, decmpfs_ctx, hdr, &uplPos, &uplSize);
1333 VerboseLogWithPath("adjusted uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize);
1334 }
1335 lck_rw_unlock_shared(decompressorsLock);
1336
1337 /* clip the adjusted size to the size of the file */
1338 if ((uint64_t)uplPos + uplSize > cachedSize) {
1339 /* truncate the read to the size of the file */
1340 uplSize = cachedSize - uplPos;
1341 }
1342
1343 if (uplSize <= 0) {
1344 /* nothing left */
1345 goto out;
1346 }
1347
1348 /*
1349 since we're going to create a upl for the given region of the file,
1350 make sure we're on page boundaries
1351 */
1352
1353 if (uplPos & (PAGE_SIZE - 1)) {
1354 /* round position down to page boundary */
1355 uplSize += (uplPos & (PAGE_SIZE - 1));
1356 uplPos &= ~(PAGE_SIZE - 1);
1357 }
1358 /* round size up to page multiple */
1359 uplSize = (uplSize + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
1360
1361 VerboseLogWithPath("new uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize);
1362
1363 uplRemaining = uplSize;
1364 curUplPos = uplPos;
1365 curUplSize = 0;
1366
1367 while(uplRemaining > 0) {
1368 /* start after the last upl */
1369 curUplPos += curUplSize;
1370
1371 /* clip to max upl size */
1372 curUplSize = uplRemaining;
1373 if (curUplSize > MAX_UPL_SIZE_BYTES) {
1374 curUplSize = MAX_UPL_SIZE_BYTES;
1375 }
1376
1377 /* create the upl */
1378 kr = ubc_create_upl(vp, curUplPos, curUplSize, &upl, &pli, UPL_SET_LITE);
1379 if (kr != KERN_SUCCESS) {
1380 ErrorLogWithPath("ubc_create_upl error %d\n", (int)kr);
1381 err = EINVAL;
1382 goto out;
1383 }
1384 VerboseLogWithPath("curUplPos %lld curUplSize %lld\n", (uint64_t)curUplPos, (uint64_t)curUplSize);
1385
1386 #if CONFIG_IOSCHED
1387 /* Mark the UPL as the requesting UPL for decompression */
1388 upl_mark_decmp(upl);
1389 #endif /* CONFIG_IOSCHED */
1390
1391 /* map the upl */
1392 kr = ubc_upl_map(upl, (vm_offset_t*)&data);
1393 if (kr != KERN_SUCCESS) {
1394
1395 commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1);
1396 #if 0
1397 char *path;
1398 MALLOC(path, char *, PATH_MAX, M_TEMP, M_WAITOK);
1399 panic("%s: decmpfs_read_compressed: ubc_upl_map error %d", vnpath(vp, path, PATH_MAX), (int)kr);
1400 FREE(path, M_TEMP);
1401 #else /* 0 */
1402 ErrorLogWithPath("ubc_upl_map kr=0x%x\n", (int)kr);
1403 #endif /* 0 */
1404 err = EINVAL;
1405 goto out;
1406 }
1407
1408 /* make sure the map succeeded */
1409 if (!data) {
1410
1411 commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1);
1412
1413 ErrorLogWithPath("ubc_upl_map mapped null\n");
1414 err = EINVAL;
1415 goto out;
1416 }
1417
1418 /* fetch uncompressed data into the mapped upl */
1419 decmpfs_vector vec;
1420 decompress:
1421 vec = (decmpfs_vector){ .buf = data, .size = curUplSize };
1422 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, curUplPos, curUplSize, 1, &vec, &did_read);
1423 if (err) {
1424 ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err);
1425
1426 /* maybe the file is converting to decompressed */
1427 int cmp_state = decmpfs_fast_get_state(cp);
1428 if (cmp_state == FILE_IS_CONVERTING) {
1429 ErrorLogWithPath("cmp_state == FILE_IS_CONVERTING\n");
1430 cmp_state = wait_for_decompress(cp);
1431 if (cmp_state == FILE_IS_COMPRESSED) {
1432 ErrorLogWithPath("cmp_state == FILE_IS_COMPRESSED\n");
1433 /* a decompress was attempted but it failed, let's try fetching again */
1434 goto decompress;
1435 }
1436 }
1437 if (cmp_state == FILE_IS_NOT_COMPRESSED) {
1438 ErrorLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n");
1439 /* the file was decompressed after we started reading it */
1440 abort_read = 1; /* we're not going to commit our data */
1441 *is_compressed = 0; /* instruct caller to fall back to its normal path */
1442 }
1443 kr = KERN_FAILURE;
1444 did_read = 0;
1445 }
1446 /* zero out the remainder of the last page */
1447 memset((char*)data + did_read, 0, curUplSize - did_read);
1448 kr = ubc_upl_unmap(upl);
1449 if (kr == KERN_SUCCESS) {
1450 if (abort_read) {
1451 kr = commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1);
1452 } else {
1453 VerboseLogWithPath("uioPos %lld uioRemaining %lld\n", (uint64_t)uioPos, (uint64_t)uioRemaining);
1454 if (uioRemaining) {
1455 off_t uplOff = uioPos - curUplPos;
1456 if (uplOff < 0) {
1457 ErrorLogWithPath("uplOff %lld should never be negative\n", (int64_t)uplOff);
1458 err = EINVAL;
1459 } else {
1460 off_t count = curUplPos + curUplSize - uioPos;
1461 if (count < 0) {
1462 /* this upl is entirely before the uio */
1463 } else {
1464 if (count > uioRemaining)
1465 count = uioRemaining;
1466 int io_resid = count;
1467 err = cluster_copy_upl_data(uio, upl, uplOff, &io_resid);
1468 int copied = count - io_resid;
1469 VerboseLogWithPath("uplOff %lld count %lld copied %lld\n", (uint64_t)uplOff, (uint64_t)count, (uint64_t)copied);
1470 if (err) {
1471 ErrorLogWithPath("cluster_copy_upl_data err %d\n", err);
1472 }
1473 uioPos += copied;
1474 uioRemaining -= copied;
1475 }
1476 }
1477 }
1478 kr = commit_upl(upl, 0, curUplSize, UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE, 0);
1479 if (err) {
1480 goto out;
1481 }
1482 }
1483 } else {
1484 ErrorLogWithPath("ubc_upl_unmap error %d\n", (int)kr);
1485 }
1486
1487 uplRemaining -= curUplSize;
1488 }
1489
1490 out:
1491
1492 if (hdr) FREE(hdr, M_TEMP);
1493 if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 0);
1494 if (err) {/* something went wrong */
1495 ErrorLogWithPath("err %d\n", err);
1496 return err;
1497 }
1498
1499 #if COMPRESSION_DEBUG
1500 uplSize = uio_resid(uio);
1501 if (uplSize)
1502 VerboseLogWithPath("still %lld bytes to copy\n", uplSize);
1503 #endif
1504 return 0;
1505 }
1506
1507 int
1508 decmpfs_free_compressed_data(vnode_t vp, decmpfs_cnode *cp)
1509 {
1510 /*
1511 call out to the decompressor to free remove any data associated with this compressed file
1512 then delete the file's compression xattr
1513 */
1514
1515 decmpfs_header *hdr = NULL;
1516 int err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
1517 if (err) {
1518 ErrorLogWithPath("decmpfs_fetch_compressed_header err %d\n", err);
1519 } else {
1520 lck_rw_lock_shared(decompressorsLock);
1521 decmpfs_free_compressed_data_func free_data = decmp_get_func(vp, hdr->compression_type, free_data);
1522 if (free_data) {
1523 err = free_data(vp, decmpfs_ctx, hdr);
1524 } else {
1525 /* nothing to do, so no error */
1526 err = 0;
1527 }
1528 lck_rw_unlock_shared(decompressorsLock);
1529
1530 if (err != 0) {
1531 ErrorLogWithPath("decompressor err %d\n", err);
1532 }
1533 }
1534
1535 /* delete the xattr */
1536 err = vn_removexattr(vp, DECMPFS_XATTR_NAME, 0, decmpfs_ctx);
1537 if (err != 0) {
1538 goto out;
1539 }
1540
1541 out:
1542 if (hdr) FREE(hdr, M_TEMP);
1543 return err;
1544 }
1545
1546 #pragma mark --- file conversion routines ---
1547
1548 static int
1549 unset_compressed_flag(vnode_t vp)
1550 {
1551 int err = 0;
1552 struct vnode_attr va;
1553 int new_bsdflags = 0;
1554
1555 VATTR_INIT(&va);
1556 VATTR_WANTED(&va, va_flags);
1557 err = vnode_getattr(vp, &va, decmpfs_ctx);
1558
1559 if (err != 0) {
1560 ErrorLogWithPath("vnode_getattr err %d\n", err);
1561 } else {
1562 new_bsdflags = va.va_flags & ~UF_COMPRESSED;
1563
1564 VATTR_INIT(&va);
1565 VATTR_SET(&va, va_flags, new_bsdflags);
1566 err = vnode_setattr(vp, &va, decmpfs_ctx);
1567 if (err != 0) {
1568 ErrorLogWithPath("vnode_setattr err %d\n", err);
1569 }
1570 }
1571 return err;
1572 }
1573
1574 int
1575 decmpfs_decompress_file(vnode_t vp, decmpfs_cnode *cp, off_t toSize, int truncate_okay, int skiplock)
1576 {
1577 /* convert a compressed file to an uncompressed file */
1578
1579 int err = 0;
1580 char *data = NULL;
1581 uio_t uio_w = 0;
1582 off_t offset = 0;
1583 uint32_t old_state = 0;
1584 uint32_t new_state = 0;
1585 int update_file_state = 0;
1586 int allocSize = 0;
1587 decmpfs_header *hdr = NULL;
1588 int cmpdata_locked = 0;
1589 off_t remaining = 0;
1590 uint64_t uncompressed_size = 0;
1591
1592 if (!skiplock) {
1593 decmpfs_lock_compressed_data(cp, 1); cmpdata_locked = 1;
1594 }
1595
1596 decompress:
1597 old_state = decmpfs_fast_get_state(cp);
1598
1599 switch(old_state) {
1600 case FILE_IS_NOT_COMPRESSED:
1601 {
1602 /* someone else decompressed the file */
1603 err = 0;
1604 goto out;
1605 }
1606
1607 case FILE_TYPE_UNKNOWN:
1608 {
1609 /* the file is in an unknown state, so update the state and retry */
1610 (void)decmpfs_file_is_compressed(vp, cp);
1611
1612 /* try again */
1613 goto decompress;
1614 }
1615
1616 case FILE_IS_COMPRESSED:
1617 {
1618 /* the file is compressed, so decompress it */
1619 break;
1620 }
1621
1622 default:
1623 {
1624 /*
1625 this shouldn't happen since multiple calls to decmpfs_decompress_file lock each other out,
1626 and when decmpfs_decompress_file returns, the state should be always be set back to
1627 FILE_IS_NOT_COMPRESSED or FILE_IS_UNKNOWN
1628 */
1629 err = EINVAL;
1630 goto out;
1631 }
1632 }
1633
1634 err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0);
1635 if (err != 0) {
1636 goto out;
1637 }
1638
1639 uncompressed_size = hdr->uncompressed_size;
1640 if (toSize == -1)
1641 toSize = hdr->uncompressed_size;
1642
1643 if (toSize == 0) {
1644 /* special case truncating the file to zero bytes */
1645 goto nodecmp;
1646 } else if ((uint64_t)toSize > hdr->uncompressed_size) {
1647 /* the caller is trying to grow the file, so we should decompress all the data */
1648 toSize = hdr->uncompressed_size;
1649 }
1650
1651 allocSize = MIN(64*1024, toSize);
1652 MALLOC(data, char *, allocSize, M_TEMP, M_WAITOK);
1653 if (!data) {
1654 err = ENOMEM;
1655 goto out;
1656 }
1657
1658 uio_w = uio_create(1, 0LL, UIO_SYSSPACE, UIO_WRITE);
1659 if (!uio_w) {
1660 err = ENOMEM;
1661 goto out;
1662 }
1663 uio_w->uio_flags |= UIO_FLAGS_IS_COMPRESSED_FILE;
1664
1665 remaining = toSize;
1666
1667 /* tell the buffer cache that this is an empty file */
1668 ubc_setsize(vp, 0);
1669
1670 /* if we got here, we need to decompress the file */
1671 decmpfs_cnode_set_vnode_state(cp, FILE_IS_CONVERTING, 1);
1672
1673 while(remaining > 0) {
1674 /* loop decompressing data from the file and writing it into the data fork */
1675
1676 uint64_t bytes_read = 0;
1677 decmpfs_vector vec = { .buf = data, .size = MIN(allocSize, remaining) };
1678 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset, vec.size, 1, &vec, &bytes_read);
1679 if (err != 0) {
1680 ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err);
1681 goto out;
1682 }
1683
1684 if (bytes_read == 0) {
1685 /* we're done reading data */
1686 break;
1687 }
1688
1689 uio_reset(uio_w, offset, UIO_SYSSPACE, UIO_WRITE);
1690 err = uio_addiov(uio_w, CAST_USER_ADDR_T(data), bytes_read);
1691 if (err != 0) {
1692 ErrorLogWithPath("uio_addiov err %d\n", err);
1693 err = ENOMEM;
1694 goto out;
1695 }
1696
1697 err = VNOP_WRITE(vp, uio_w, 0, decmpfs_ctx);
1698 if (err != 0) {
1699 /* if the write failed, truncate the file to zero bytes */
1700 ErrorLogWithPath("VNOP_WRITE err %d\n", err);
1701 break;
1702 }
1703 offset += bytes_read;
1704 remaining -= bytes_read;
1705 }
1706
1707 if (err == 0) {
1708 if (offset != toSize) {
1709 ErrorLogWithPath("file decompressed to %lld instead of %lld\n", offset, toSize);
1710 err = EINVAL;
1711 goto out;
1712 }
1713 }
1714
1715 if (err == 0) {
1716 /* sync the data and metadata */
1717 err = VNOP_FSYNC(vp, MNT_WAIT, decmpfs_ctx);
1718 if (err != 0) {
1719 ErrorLogWithPath("VNOP_FSYNC err %d\n", err);
1720 goto out;
1721 }
1722 }
1723
1724 if (err != 0) {
1725 /* write, setattr, or fsync failed */
1726 ErrorLogWithPath("aborting decompress, err %d\n", err);
1727 if (truncate_okay) {
1728 /* truncate anything we might have written */
1729 int error = vnode_setsize(vp, 0, 0, decmpfs_ctx);
1730 ErrorLogWithPath("vnode_setsize err %d\n", error);
1731 }
1732 goto out;
1733 }
1734
1735 nodecmp:
1736 /* if we're truncating the file to zero bytes, we'll skip ahead to here */
1737
1738 /* unset the compressed flag */
1739 unset_compressed_flag(vp);
1740
1741 /* free the compressed data associated with this file */
1742 err = decmpfs_free_compressed_data(vp, cp);
1743 if (err != 0) {
1744 ErrorLogWithPath("decmpfs_free_compressed_data err %d\n", err);
1745 }
1746
1747 /*
1748 even if free_compressed_data or vnode_getattr/vnode_setattr failed, return success
1749 since we succeeded in writing all of the file data to the data fork
1750 */
1751 err = 0;
1752
1753 /* if we got this far, the file was successfully decompressed */
1754 update_file_state = 1;
1755 new_state = FILE_IS_NOT_COMPRESSED;
1756
1757 #if COMPRESSION_DEBUG
1758 {
1759 uint64_t filesize = 0;
1760 vnsize(vp, &filesize);
1761 DebugLogWithPath("new file size %lld\n", filesize);
1762 }
1763 #endif
1764
1765 out:
1766 if (hdr) FREE(hdr, M_TEMP);
1767 if (data) FREE(data, M_TEMP);
1768 if (uio_w) uio_free(uio_w);
1769
1770 if (err != 0) {
1771 /* if there was a failure, reset compression flags to unknown and clear the buffer cache data */
1772 update_file_state = 1;
1773 new_state = FILE_TYPE_UNKNOWN;
1774 if (uncompressed_size) {
1775 ubc_setsize(vp, 0);
1776 ubc_setsize(vp, uncompressed_size);
1777 }
1778 }
1779
1780 if (update_file_state) {
1781 lck_mtx_lock(decompress_channel_mtx);
1782 decmpfs_cnode_set_vnode_state(cp, new_state, 1);
1783 wakeup((caddr_t)&decompress_channel); /* wake up anyone who might have been waiting for decompression */
1784 lck_mtx_unlock(decompress_channel_mtx);
1785 }
1786
1787 if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 1);
1788
1789 return err;
1790 }
1791
1792 #pragma mark --- Type1 compressor ---
1793
1794 /*
1795 The "Type1" compressor stores the data fork directly in the compression xattr
1796 */
1797
1798 static int
1799 decmpfs_validate_compressed_file_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr)
1800 {
1801 int err = 0;
1802
1803 if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) {
1804 err = EINVAL;
1805 goto out;
1806 }
1807 out:
1808 return err;
1809 }
1810
1811 static int
1812 decmpfs_fetch_uncompressed_data_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read)
1813 {
1814 int err = 0;
1815 int i;
1816 user_ssize_t remaining;
1817
1818 if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) {
1819 err = EINVAL;
1820 goto out;
1821 }
1822
1823 #if COMPRESSION_DEBUG
1824 static int dummy = 0; // prevent syslog from coalescing printfs
1825 DebugLogWithPath("%d memcpy %lld at %lld\n", dummy++, size, (uint64_t)offset);
1826 #endif
1827
1828 remaining = size;
1829 for (i = 0; (i < nvec) && (remaining > 0); i++) {
1830 user_ssize_t curCopy = vec[i].size;
1831 if (curCopy > remaining)
1832 curCopy = remaining;
1833 memcpy(vec[i].buf, hdr->attr_bytes + offset, curCopy);
1834 offset += curCopy;
1835 remaining -= curCopy;
1836 }
1837
1838 if ((bytes_read) && (err == 0))
1839 *bytes_read = (size - remaining);
1840
1841 out:
1842 return err;
1843 }
1844
1845 static decmpfs_registration Type1Reg =
1846 {
1847 .decmpfs_registration = DECMPFS_REGISTRATION_VERSION,
1848 .validate = decmpfs_validate_compressed_file_Type1,
1849 .adjust_fetch = NULL, /* no adjust necessary */
1850 .fetch = decmpfs_fetch_uncompressed_data_Type1,
1851 .free_data = NULL, /* no free necessary */
1852 .get_flags = NULL /* no flags */
1853 };
1854
1855 #pragma mark --- decmpfs initialization ---
1856
1857 void decmpfs_init()
1858 {
1859 static int done = 0;
1860 if (done) return;
1861
1862 decmpfs_ctx = vfs_context_create(vfs_context_kernel());
1863
1864 lck_grp_attr_t *attr = lck_grp_attr_alloc_init();
1865 decmpfs_lockgrp = lck_grp_alloc_init("VFSCOMP", attr);
1866 lck_grp_attr_free(attr);
1867 decompressorsLock = lck_rw_alloc_init(decmpfs_lockgrp, NULL);
1868 decompress_channel_mtx = lck_mtx_alloc_init(decmpfs_lockgrp, NULL);
1869
1870 register_decmpfs_decompressor(CMP_Type1, &Type1Reg);
1871
1872 done = 1;
1873 }
1874 #endif /* FS_COMPRESSION */