| 1 | /* |
| 2 | * Copyright (c) 2008-2015 Apple Inc. All rights reserved. |
| 3 | * |
| 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
| 5 | * |
| 6 | * This file contains Original Code and/or Modifications of Original Code |
| 7 | * as defined in and that are subject to the Apple Public Source License |
| 8 | * Version 2.0 (the 'License'). You may not use this file except in |
| 9 | * compliance with the License. The rights granted to you under the License |
| 10 | * may not be used to create, or enable the creation or redistribution of, |
| 11 | * unlawful or unlicensed copies of an Apple operating system, or to |
| 12 | * circumvent, violate, or enable the circumvention or violation of, any |
| 13 | * terms of an Apple operating system software license agreement. |
| 14 | * |
| 15 | * Please obtain a copy of the License at |
| 16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
| 17 | * |
| 18 | * The Original Code and all software distributed under the License are |
| 19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
| 20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
| 21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
| 22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
| 23 | * Please see the License for the specific language governing rights and |
| 24 | * limitations under the License. |
| 25 | * |
| 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
| 27 | */ |
| 28 | #if !FS_COMPRESSION |
| 29 | |
| 30 | /* We need these symbols even though compression is turned off */ |
| 31 | |
| 32 | #define UNUSED_SYMBOL(x) asm(".global _" #x "\n.set _" #x ", 0\n"); |
| 33 | |
| 34 | UNUSED_SYMBOL(register_decmpfs_decompressor) |
| 35 | UNUSED_SYMBOL(unregister_decmpfs_decompressor) |
| 36 | UNUSED_SYMBOL(decmpfs_init) |
| 37 | UNUSED_SYMBOL(decmpfs_read_compressed) |
| 38 | UNUSED_SYMBOL(decmpfs_cnode_cmp_type) |
| 39 | UNUSED_SYMBOL(decmpfs_cnode_get_vnode_state) |
| 40 | UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_size) |
| 41 | UNUSED_SYMBOL(decmpfs_lock_compressed_data) |
| 42 | UNUSED_SYMBOL(decmpfs_cnode_free) |
| 43 | UNUSED_SYMBOL(decmpfs_cnode_alloc) |
| 44 | UNUSED_SYMBOL(decmpfs_cnode_destroy) |
| 45 | UNUSED_SYMBOL(decmpfs_decompress_file) |
| 46 | UNUSED_SYMBOL(decmpfs_unlock_compressed_data) |
| 47 | UNUSED_SYMBOL(decmpfs_cnode_init) |
| 48 | UNUSED_SYMBOL(decmpfs_cnode_set_vnode_state) |
| 49 | UNUSED_SYMBOL(decmpfs_hides_xattr) |
| 50 | UNUSED_SYMBOL(decmpfs_ctx) |
| 51 | UNUSED_SYMBOL(decmpfs_file_is_compressed) |
| 52 | UNUSED_SYMBOL(decmpfs_update_attributes) |
| 53 | UNUSED_SYMBOL(decmpfs_hides_rsrc) |
| 54 | UNUSED_SYMBOL(decmpfs_pagein_compressed) |
| 55 | UNUSED_SYMBOL(decmpfs_validate_compressed_file) |
| 56 | |
| 57 | #else /* FS_COMPRESSION */ |
| 58 | #include <sys/kernel.h> |
| 59 | #include <sys/vnode_internal.h> |
| 60 | #include <sys/file_internal.h> |
| 61 | #include <sys/stat.h> |
| 62 | #include <sys/fcntl.h> |
| 63 | #include <sys/xattr.h> |
| 64 | #include <sys/namei.h> |
| 65 | #include <sys/user.h> |
| 66 | #include <sys/mount_internal.h> |
| 67 | #include <sys/ubc.h> |
| 68 | #include <sys/decmpfs.h> |
| 69 | #include <sys/uio_internal.h> |
| 70 | #include <libkern/OSByteOrder.h> |
| 71 | |
| 72 | #pragma mark --- debugging --- |
| 73 | |
| 74 | #define COMPRESSION_DEBUG 0 |
| 75 | #define COMPRESSION_DEBUG_VERBOSE 0 |
| 76 | #define MALLOC_DEBUG 0 |
| 77 | |
| 78 | static const char * |
| 79 | baseName(const char *path) |
| 80 | { |
| 81 | if (!path) |
| 82 | return NULL; |
| 83 | const char *ret = path; |
| 84 | int i; |
| 85 | for (i = 0; path[i] != 0; i++) { |
| 86 | if (path[i] == '/') |
| 87 | ret = &path[i + 1]; |
| 88 | } |
| 89 | return ret; |
| 90 | } |
| 91 | |
| 92 | static char* |
| 93 | vnpath(vnode_t vp, char *path, int len) |
| 94 | { |
| 95 | int origlen = len; |
| 96 | path[0] = 0; |
| 97 | vn_getpath(vp, path, &len); |
| 98 | path[origlen - 1] = 0; |
| 99 | return path; |
| 100 | } |
| 101 | |
| 102 | #define ErrorLog(x, args...) printf("%s:%d:%s: " x, baseName(__FILE__), __LINE__, __FUNCTION__, ## args) |
| 103 | #define ErrorLogWithPath(x, args...) do { char *path; MALLOC(path, char *, PATH_MAX, M_TEMP, M_WAITOK); printf("%s:%d:%s: %s: " x, baseName(__FILE__), __LINE__, __FUNCTION__, vnpath(vp, path, PATH_MAX), ## args); FREE(path, M_TEMP); } while(0) |
| 104 | |
| 105 | #if COMPRESSION_DEBUG |
| 106 | #define DebugLog ErrorLog |
| 107 | #define DebugLogWithPath ErrorLogWithPath |
| 108 | #else |
| 109 | #define DebugLog(x...) do { } while(0) |
| 110 | #define DebugLogWithPath(x...) do { } while(0) |
| 111 | #endif |
| 112 | |
| 113 | #if COMPRESSION_DEBUG_VERBOSE |
| 114 | #define VerboseLog ErrorLog |
| 115 | #define VerboseLogWithPath ErrorLogWithPath |
| 116 | #else |
| 117 | #define VerboseLog(x...) do { } while(0) |
| 118 | #define VerboseLogWithPath(x...) do { } while(0) |
| 119 | #endif |
| 120 | |
| 121 | #if MALLOC_DEBUG |
| 122 | |
| 123 | static SInt32 totalAlloc; |
| 124 | |
| 125 | typedef struct { |
| 126 | uint32_t allocSz; |
| 127 | uint32_t magic; |
| 128 | const char *file; |
| 129 | int line; |
| 130 | } allocated; |
| 131 | |
| 132 | static void * |
| 133 | _malloc(uint32_t sz, __unused int type, __unused int flags, const char *file, int line) |
| 134 | { |
| 135 | uint32_t allocSz = sz + 2 * sizeof(allocated); |
| 136 | |
| 137 | allocated *alloc = NULL; |
| 138 | MALLOC(alloc, allocated *, allocSz, type, flags); |
| 139 | if (!alloc) { |
| 140 | ErrorLog("malloc failed\n"); |
| 141 | return NULL; |
| 142 | } |
| 143 | |
| 144 | char *ret = (char*)&alloc[1]; |
| 145 | allocated *alloc2 = (allocated*)(ret + sz); |
| 146 | |
| 147 | alloc->allocSz = allocSz; |
| 148 | alloc->magic = 0xdadadada; |
| 149 | alloc->file = file; |
| 150 | alloc->line = line; |
| 151 | |
| 152 | *alloc2 = *alloc; |
| 153 | |
| 154 | int s = OSAddAtomic(sz, &totalAlloc); |
| 155 | ErrorLog("malloc(%d) -> %p, total allocations %d\n", sz, ret, s + sz); |
| 156 | |
| 157 | return ret; |
| 158 | } |
| 159 | |
| 160 | static void |
| 161 | _free(char *ret, __unused int type, const char *file, int line) |
| 162 | { |
| 163 | if (!ret) { |
| 164 | ErrorLog("freeing null\n"); |
| 165 | return; |
| 166 | } |
| 167 | allocated *alloc = (allocated*)ret; |
| 168 | alloc--; |
| 169 | uint32_t sz = alloc->allocSz - 2 * sizeof(allocated); |
| 170 | allocated *alloc2 = (allocated*)(ret + sz); |
| 171 | |
| 172 | if (alloc->magic != 0xdadadada) { |
| 173 | panic("freeing bad pointer"); |
| 174 | } |
| 175 | |
| 176 | if (memcmp(alloc, alloc2, sizeof(*alloc)) != 0) { |
| 177 | panic("clobbered data"); |
| 178 | } |
| 179 | |
| 180 | memset(ret, 0xce, sz); |
| 181 | alloc2->file = file; |
| 182 | alloc2->line = line; |
| 183 | FREE(alloc, type); |
| 184 | int s = OSAddAtomic(-sz, &totalAlloc); |
| 185 | ErrorLog("free(%p,%d) -> total allocations %d\n", ret, sz, s - sz); |
| 186 | } |
| 187 | |
| 188 | #undef MALLOC |
| 189 | #undef FREE |
| 190 | #define MALLOC(space, cast, size, type, flags) (space) = (cast)_malloc(size, type, flags, __FILE__, __LINE__) |
| 191 | #define FREE(addr, type) _free((void *)addr, type, __FILE__, __LINE__) |
| 192 | |
| 193 | #endif /* MALLOC_DEBUG */ |
| 194 | |
| 195 | #pragma mark --- globals --- |
| 196 | |
| 197 | static lck_grp_t *decmpfs_lockgrp; |
| 198 | |
| 199 | static decmpfs_registration * decompressors[CMP_MAX]; /* the registered compressors */ |
| 200 | static lck_rw_t * decompressorsLock; |
| 201 | static int decompress_channel; /* channel used by decompress_file to wake up waiters */ |
| 202 | static lck_mtx_t *decompress_channel_mtx; |
| 203 | |
| 204 | vfs_context_t decmpfs_ctx; |
| 205 | |
| 206 | #pragma mark --- decmp_get_func --- |
| 207 | |
| 208 | #define offsetof_func(func) ((uintptr_t)(&(((decmpfs_registration*)NULL)->func))) |
| 209 | |
| 210 | static void * |
| 211 | _func_from_offset(uint32_t type, uintptr_t offset) |
| 212 | { |
| 213 | /* get the function at the given offset in the registration for the given type */ |
| 214 | decmpfs_registration *reg = decompressors[type]; |
| 215 | char *regChar = (char*)reg; |
| 216 | char *func = ®Char[offset]; |
| 217 | void **funcPtr = (void**)func; |
| 218 | |
| 219 | switch (reg->decmpfs_registration) { |
| 220 | case DECMPFS_REGISTRATION_VERSION_V1: |
| 221 | if (offset > offsetof_func(free_data)) |
| 222 | return NULL; |
| 223 | break; |
| 224 | case DECMPFS_REGISTRATION_VERSION_V3: |
| 225 | if (offset > offsetof_func(get_flags)) |
| 226 | return NULL; |
| 227 | break; |
| 228 | default: |
| 229 | return NULL; |
| 230 | } |
| 231 | |
| 232 | return funcPtr[0]; |
| 233 | } |
| 234 | |
| 235 | extern void IOServicePublishResource( const char * property, boolean_t value ); |
| 236 | extern boolean_t IOServiceWaitForMatchingResource( const char * property, uint64_t timeout ); |
| 237 | extern boolean_t IOCatalogueMatchingDriversPresent( const char * property ); |
| 238 | |
| 239 | static void * |
| 240 | _decmp_get_func(vnode_t vp, uint32_t type, uintptr_t offset) |
| 241 | { |
| 242 | /* |
| 243 | this function should be called while holding a shared lock to decompressorsLock, |
| 244 | and will return with the lock held |
| 245 | */ |
| 246 | |
| 247 | if (type >= CMP_MAX) |
| 248 | return NULL; |
| 249 | |
| 250 | if (decompressors[type] != NULL) { |
| 251 | // the compressor has already registered but the function might be null |
| 252 | return _func_from_offset(type, offset); |
| 253 | } |
| 254 | |
| 255 | // does IOKit know about a kext that is supposed to provide this type? |
| 256 | char providesName[80]; |
| 257 | snprintf(providesName, sizeof(providesName), "com.apple.AppleFSCompression.providesType%u", type); |
| 258 | if (IOCatalogueMatchingDriversPresent(providesName)) { |
| 259 | // there is a kext that says it will register for this type, so let's wait for it |
| 260 | char resourceName[80]; |
| 261 | uint64_t delay = 10000000ULL; // 10 milliseconds. |
| 262 | snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", type); |
| 263 | ErrorLogWithPath("waiting for %s\n", resourceName); |
| 264 | while(decompressors[type] == NULL) { |
| 265 | lck_rw_unlock_shared(decompressorsLock); // we have to unlock to allow the kext to register |
| 266 | if (IOServiceWaitForMatchingResource(resourceName, delay)) { |
| 267 | lck_rw_lock_shared(decompressorsLock); |
| 268 | break; |
| 269 | } |
| 270 | if (!IOCatalogueMatchingDriversPresent(providesName)) { |
| 271 | // |
| 272 | ErrorLogWithPath("the kext with %s is no longer present\n", providesName); |
| 273 | lck_rw_lock_shared(decompressorsLock); |
| 274 | break; |
| 275 | } |
| 276 | ErrorLogWithPath("still waiting for %s\n", resourceName); |
| 277 | delay *= 2; |
| 278 | lck_rw_lock_shared(decompressorsLock); |
| 279 | } |
| 280 | // IOKit says the kext is loaded, so it should be registered too! |
| 281 | if (decompressors[type] == NULL) { |
| 282 | ErrorLogWithPath("we found %s, but the type still isn't registered\n", providesName); |
| 283 | return NULL; |
| 284 | } |
| 285 | // it's now registered, so let's return the function |
| 286 | return _func_from_offset(type, offset); |
| 287 | } |
| 288 | |
| 289 | // the compressor hasn't registered, so it never will unless someone manually kextloads it |
| 290 | ErrorLogWithPath("tried to access a compressed file of unregistered type %d\n", type); |
| 291 | return NULL; |
| 292 | } |
| 293 | |
| 294 | #define decmp_get_func(vp, type, func) ((typeof(((decmpfs_registration*)NULL)->func))_decmp_get_func(vp, type, offsetof_func(func))) |
| 295 | |
| 296 | #pragma mark --- utilities --- |
| 297 | |
| 298 | #if COMPRESSION_DEBUG |
| 299 | static int |
| 300 | vnsize(vnode_t vp, uint64_t *size) |
| 301 | { |
| 302 | struct vnode_attr va; |
| 303 | VATTR_INIT(&va); |
| 304 | VATTR_WANTED(&va, va_data_size); |
| 305 | int error = vnode_getattr(vp, &va, decmpfs_ctx); |
| 306 | if (error != 0) { |
| 307 | ErrorLogWithPath("vnode_getattr err %d\n", error); |
| 308 | return error; |
| 309 | } |
| 310 | *size = va.va_data_size; |
| 311 | return 0; |
| 312 | } |
| 313 | #endif /* COMPRESSION_DEBUG */ |
| 314 | |
| 315 | #pragma mark --- cnode routines --- |
| 316 | |
| 317 | decmpfs_cnode *decmpfs_cnode_alloc(void) |
| 318 | { |
| 319 | decmpfs_cnode *dp; |
| 320 | MALLOC_ZONE(dp, decmpfs_cnode *, sizeof(decmpfs_cnode), M_DECMPFS_CNODE, M_WAITOK); |
| 321 | return dp; |
| 322 | } |
| 323 | |
| 324 | void decmpfs_cnode_free(decmpfs_cnode *dp) |
| 325 | { |
| 326 | FREE_ZONE(dp, sizeof(*dp), M_DECMPFS_CNODE); |
| 327 | } |
| 328 | |
| 329 | void |
| 330 | decmpfs_cnode_init(decmpfs_cnode *cp) |
| 331 | { |
| 332 | memset(cp, 0, sizeof(*cp)); |
| 333 | lck_rw_init(&cp->compressed_data_lock, decmpfs_lockgrp, NULL); |
| 334 | } |
| 335 | |
| 336 | void |
| 337 | decmpfs_cnode_destroy(decmpfs_cnode *cp) |
| 338 | { |
| 339 | lck_rw_destroy(&cp->compressed_data_lock, decmpfs_lockgrp); |
| 340 | } |
| 341 | |
| 342 | bool |
| 343 | decmpfs_trylock_compressed_data(decmpfs_cnode *cp, int exclusive) |
| 344 | { |
| 345 | void *thread = current_thread(); |
| 346 | bool retval = false; |
| 347 | |
| 348 | if (cp->lockowner == thread) { |
| 349 | /* this thread is already holding an exclusive lock, so bump the count */ |
| 350 | cp->lockcount++; |
| 351 | retval = true; |
| 352 | } else if (exclusive) { |
| 353 | if ((retval = lck_rw_try_lock_exclusive(&cp->compressed_data_lock))) { |
| 354 | cp->lockowner = thread; |
| 355 | cp->lockcount = 1; |
| 356 | } |
| 357 | } else { |
| 358 | if ((retval = lck_rw_try_lock_shared(&cp->compressed_data_lock))) { |
| 359 | cp->lockowner = (void *)-1; |
| 360 | } |
| 361 | } |
| 362 | return retval; |
| 363 | } |
| 364 | |
| 365 | void |
| 366 | decmpfs_lock_compressed_data(decmpfs_cnode *cp, int exclusive) |
| 367 | { |
| 368 | void *thread = current_thread(); |
| 369 | |
| 370 | if (cp->lockowner == thread) { |
| 371 | /* this thread is already holding an exclusive lock, so bump the count */ |
| 372 | cp->lockcount++; |
| 373 | } else if (exclusive) { |
| 374 | lck_rw_lock_exclusive(&cp->compressed_data_lock); |
| 375 | cp->lockowner = thread; |
| 376 | cp->lockcount = 1; |
| 377 | } else { |
| 378 | lck_rw_lock_shared(&cp->compressed_data_lock); |
| 379 | cp->lockowner = (void *)-1; |
| 380 | } |
| 381 | } |
| 382 | |
| 383 | void |
| 384 | decmpfs_unlock_compressed_data(decmpfs_cnode *cp, __unused int exclusive) |
| 385 | { |
| 386 | void *thread = current_thread(); |
| 387 | |
| 388 | if (cp->lockowner == thread) { |
| 389 | /* this thread is holding an exclusive lock, so decrement the count */ |
| 390 | if ((--cp->lockcount) > 0) { |
| 391 | /* the caller still has outstanding locks, so we're done */ |
| 392 | return; |
| 393 | } |
| 394 | cp->lockowner = NULL; |
| 395 | } |
| 396 | |
| 397 | lck_rw_done(&cp->compressed_data_lock); |
| 398 | } |
| 399 | |
| 400 | uint32_t |
| 401 | decmpfs_cnode_get_vnode_state(decmpfs_cnode *cp) |
| 402 | { |
| 403 | return cp->cmp_state; |
| 404 | } |
| 405 | |
| 406 | void |
| 407 | decmpfs_cnode_set_vnode_state(decmpfs_cnode *cp, uint32_t state, int skiplock) |
| 408 | { |
| 409 | if (!skiplock) decmpfs_lock_compressed_data(cp, 1); |
| 410 | cp->cmp_state = state; |
| 411 | if (state == FILE_TYPE_UNKNOWN) { |
| 412 | /* clear out the compression type too */ |
| 413 | cp->cmp_type = 0; |
| 414 | } |
| 415 | if (!skiplock) decmpfs_unlock_compressed_data(cp, 1); |
| 416 | } |
| 417 | |
| 418 | static void |
| 419 | decmpfs_cnode_set_vnode_cmp_type(decmpfs_cnode *cp, uint32_t cmp_type, int skiplock) |
| 420 | { |
| 421 | if (!skiplock) decmpfs_lock_compressed_data(cp, 1); |
| 422 | cp->cmp_type = cmp_type; |
| 423 | if (!skiplock) decmpfs_unlock_compressed_data(cp, 1); |
| 424 | } |
| 425 | |
| 426 | static void |
| 427 | decmpfs_cnode_set_vnode_minimal_xattr(decmpfs_cnode *cp, int minimal_xattr, int skiplock) |
| 428 | { |
| 429 | if (!skiplock) decmpfs_lock_compressed_data(cp, 1); |
| 430 | cp->cmp_minimal_xattr = minimal_xattr; |
| 431 | if (!skiplock) decmpfs_unlock_compressed_data(cp, 1); |
| 432 | } |
| 433 | |
| 434 | uint64_t |
| 435 | decmpfs_cnode_get_vnode_cached_size(decmpfs_cnode *cp) |
| 436 | { |
| 437 | return cp->uncompressed_size; |
| 438 | } |
| 439 | |
| 440 | static void |
| 441 | decmpfs_cnode_set_vnode_cached_size(decmpfs_cnode *cp, uint64_t size) |
| 442 | { |
| 443 | while(1) { |
| 444 | uint64_t old = cp->uncompressed_size; |
| 445 | if (OSCompareAndSwap64(old, size, (UInt64*)&cp->uncompressed_size)) { |
| 446 | return; |
| 447 | } else { |
| 448 | /* failed to write our value, so loop */ |
| 449 | } |
| 450 | } |
| 451 | } |
| 452 | |
| 453 | static uint64_t |
| 454 | decmpfs_cnode_get_decompression_flags(decmpfs_cnode *cp) |
| 455 | { |
| 456 | return cp->decompression_flags; |
| 457 | } |
| 458 | |
| 459 | static void |
| 460 | decmpfs_cnode_set_decompression_flags(decmpfs_cnode *cp, uint64_t flags) |
| 461 | { |
| 462 | while(1) { |
| 463 | uint64_t old = cp->decompression_flags; |
| 464 | if (OSCompareAndSwap64(old, flags, (UInt64*)&cp->decompression_flags)) { |
| 465 | return; |
| 466 | } else { |
| 467 | /* failed to write our value, so loop */ |
| 468 | } |
| 469 | } |
| 470 | } |
| 471 | |
| 472 | uint32_t decmpfs_cnode_cmp_type(decmpfs_cnode *cp) |
| 473 | { |
| 474 | return cp->cmp_type; |
| 475 | } |
| 476 | |
| 477 | #pragma mark --- decmpfs state routines --- |
| 478 | |
| 479 | static int |
| 480 | decmpfs_fetch_compressed_header(vnode_t vp, decmpfs_cnode *cp, decmpfs_header **hdrOut, int returnInvalid) |
| 481 | { |
| 482 | /* |
| 483 | fetches vp's compression xattr, converting it into a decmpfs_header; returns 0 or errno |
| 484 | if returnInvalid == 1, returns the header even if the type was invalid (out of range), |
| 485 | and return ERANGE in that case |
| 486 | */ |
| 487 | |
| 488 | size_t read_size = 0; |
| 489 | size_t attr_size = 0; |
| 490 | uio_t attr_uio = NULL; |
| 491 | int err = 0; |
| 492 | char *data = NULL; |
| 493 | decmpfs_header *hdr = NULL; |
| 494 | char uio_buf[ UIO_SIZEOF(1) ]; |
| 495 | |
| 496 | if ((cp != NULL) && |
| 497 | (cp->cmp_type != 0) && |
| 498 | (cp->cmp_minimal_xattr != 0)) { |
| 499 | /* this file's xattr didn't have any extra data when we fetched it, so we can synthesize a header from the data in the cnode */ |
| 500 | |
| 501 | MALLOC(data, char *, sizeof(decmpfs_header), M_TEMP, M_WAITOK); |
| 502 | if (!data) { |
| 503 | err = ENOMEM; |
| 504 | goto out; |
| 505 | } |
| 506 | hdr = (decmpfs_header*)data; |
| 507 | hdr->attr_size = sizeof(decmpfs_disk_header); |
| 508 | hdr->compression_magic = DECMPFS_MAGIC; |
| 509 | hdr->compression_type = cp->cmp_type; |
| 510 | hdr->uncompressed_size = decmpfs_cnode_get_vnode_cached_size(cp); |
| 511 | } else { |
| 512 | /* figure out how big the xattr is on disk */ |
| 513 | err = vn_getxattr(vp, DECMPFS_XATTR_NAME, NULL, &attr_size, XATTR_NOSECURITY, decmpfs_ctx); |
| 514 | if (err != 0) |
| 515 | goto out; |
| 516 | |
| 517 | if (attr_size < sizeof(decmpfs_disk_header) || attr_size > MAX_DECMPFS_XATTR_SIZE) { |
| 518 | err = EINVAL; |
| 519 | goto out; |
| 520 | } |
| 521 | |
| 522 | /* allocation includes space for the extra attr_size field of a compressed_header */ |
| 523 | MALLOC(data, char *, attr_size + sizeof(hdr->attr_size), M_TEMP, M_WAITOK); |
| 524 | if (!data) { |
| 525 | err = ENOMEM; |
| 526 | goto out; |
| 527 | } |
| 528 | |
| 529 | /* read the xattr into our buffer, skipping over the attr_size field at the beginning */ |
| 530 | attr_uio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, &uio_buf[0], sizeof(uio_buf)); |
| 531 | uio_addiov(attr_uio, CAST_USER_ADDR_T(data + sizeof(hdr->attr_size)), attr_size); |
| 532 | |
| 533 | err = vn_getxattr(vp, DECMPFS_XATTR_NAME, attr_uio, &read_size, XATTR_NOSECURITY, decmpfs_ctx); |
| 534 | if (err != 0) |
| 535 | goto out; |
| 536 | if (read_size != attr_size) { |
| 537 | err = EINVAL; |
| 538 | goto out; |
| 539 | } |
| 540 | hdr = (decmpfs_header*)data; |
| 541 | hdr->attr_size = attr_size; |
| 542 | /* swap the fields to native endian */ |
| 543 | hdr->compression_magic = OSSwapLittleToHostInt32(hdr->compression_magic); |
| 544 | hdr->compression_type = OSSwapLittleToHostInt32(hdr->compression_type); |
| 545 | hdr->uncompressed_size = OSSwapLittleToHostInt64(hdr->uncompressed_size); |
| 546 | } |
| 547 | |
| 548 | if (hdr->compression_magic != DECMPFS_MAGIC) { |
| 549 | ErrorLogWithPath("invalid compression_magic 0x%08x, should be 0x%08x\n", hdr->compression_magic, DECMPFS_MAGIC); |
| 550 | err = EINVAL; |
| 551 | goto out; |
| 552 | } |
| 553 | |
| 554 | if (hdr->compression_type >= CMP_MAX) { |
| 555 | if (returnInvalid) { |
| 556 | /* return the header even though the type is out of range */ |
| 557 | err = ERANGE; |
| 558 | } else { |
| 559 | ErrorLogWithPath("compression_type %d out of range\n", hdr->compression_type); |
| 560 | err = EINVAL; |
| 561 | } |
| 562 | goto out; |
| 563 | } |
| 564 | |
| 565 | out: |
| 566 | if (err && (err != ERANGE)) { |
| 567 | DebugLogWithPath("err %d\n", err); |
| 568 | if (data) FREE(data, M_TEMP); |
| 569 | *hdrOut = NULL; |
| 570 | } else { |
| 571 | *hdrOut = hdr; |
| 572 | } |
| 573 | return err; |
| 574 | } |
| 575 | |
| 576 | static int |
| 577 | decmpfs_fast_get_state(decmpfs_cnode *cp) |
| 578 | { |
| 579 | /* |
| 580 | return the cached state |
| 581 | this should *only* be called when we know that decmpfs_file_is_compressed has already been called, |
| 582 | because this implies that the cached state is valid |
| 583 | */ |
| 584 | int cmp_state = decmpfs_cnode_get_vnode_state(cp); |
| 585 | |
| 586 | switch(cmp_state) { |
| 587 | case FILE_IS_NOT_COMPRESSED: |
| 588 | case FILE_IS_COMPRESSED: |
| 589 | case FILE_IS_CONVERTING: |
| 590 | return cmp_state; |
| 591 | case FILE_TYPE_UNKNOWN: |
| 592 | /* |
| 593 | we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode, |
| 594 | which should not be possible |
| 595 | */ |
| 596 | ErrorLog("decmpfs_fast_get_state called on unknown file\n"); |
| 597 | return FILE_IS_NOT_COMPRESSED; |
| 598 | default: |
| 599 | /* */ |
| 600 | ErrorLog("unknown cmp_state %d\n", cmp_state); |
| 601 | return FILE_IS_NOT_COMPRESSED; |
| 602 | } |
| 603 | } |
| 604 | |
| 605 | static int |
| 606 | decmpfs_fast_file_is_compressed(decmpfs_cnode *cp) |
| 607 | { |
| 608 | int cmp_state = decmpfs_cnode_get_vnode_state(cp); |
| 609 | |
| 610 | switch(cmp_state) { |
| 611 | case FILE_IS_NOT_COMPRESSED: |
| 612 | return 0; |
| 613 | case FILE_IS_COMPRESSED: |
| 614 | case FILE_IS_CONVERTING: |
| 615 | return 1; |
| 616 | case FILE_TYPE_UNKNOWN: |
| 617 | /* |
| 618 | we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode, |
| 619 | which should not be possible |
| 620 | */ |
| 621 | ErrorLog("decmpfs_fast_get_state called on unknown file\n"); |
| 622 | return 0; |
| 623 | default: |
| 624 | /* */ |
| 625 | ErrorLog("unknown cmp_state %d\n", cmp_state); |
| 626 | return 0; |
| 627 | } |
| 628 | } |
| 629 | |
| 630 | errno_t |
| 631 | decmpfs_validate_compressed_file(vnode_t vp, decmpfs_cnode *cp) |
| 632 | { |
| 633 | /* give a compressor a chance to indicate that a compressed file is invalid */ |
| 634 | |
| 635 | decmpfs_header *hdr = NULL; |
| 636 | errno_t err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); |
| 637 | if (err) { |
| 638 | /* we couldn't get the header */ |
| 639 | if (decmpfs_fast_get_state(cp) == FILE_IS_NOT_COMPRESSED) { |
| 640 | /* the file is no longer compressed, so return success */ |
| 641 | err = 0; |
| 642 | } |
| 643 | goto out; |
| 644 | } |
| 645 | |
| 646 | lck_rw_lock_shared(decompressorsLock); |
| 647 | decmpfs_validate_compressed_file_func validate = decmp_get_func(vp, hdr->compression_type, validate); |
| 648 | if (validate) { /* make sure this validation function is valid */ |
| 649 | /* is the data okay? */ |
| 650 | err = validate(vp, decmpfs_ctx, hdr); |
| 651 | } else if (decmp_get_func(vp, hdr->compression_type, fetch) == NULL) { |
| 652 | /* the type isn't registered */ |
| 653 | err = EIO; |
| 654 | } else { |
| 655 | /* no validate registered, so nothing to do */ |
| 656 | err = 0; |
| 657 | } |
| 658 | lck_rw_unlock_shared(decompressorsLock); |
| 659 | out: |
| 660 | if (hdr) FREE(hdr, M_TEMP); |
| 661 | #if COMPRESSION_DEBUG |
| 662 | if (err) { |
| 663 | DebugLogWithPath("decmpfs_validate_compressed_file ret %d, vp->v_flag %d\n", err, vp->v_flag); |
| 664 | } |
| 665 | #endif |
| 666 | return err; |
| 667 | } |
| 668 | |
| 669 | int |
| 670 | decmpfs_file_is_compressed(vnode_t vp, decmpfs_cnode *cp) |
| 671 | { |
| 672 | /* |
| 673 | determines whether vp points to a compressed file |
| 674 | |
| 675 | to speed up this operation, we cache the result in the cnode, and do as little as possible |
| 676 | in the case where the cnode already has a valid cached state |
| 677 | |
| 678 | */ |
| 679 | |
| 680 | int ret = 0; |
| 681 | int error = 0; |
| 682 | uint32_t cmp_state; |
| 683 | struct vnode_attr va_fetch; |
| 684 | decmpfs_header *hdr = NULL; |
| 685 | mount_t mp = NULL; |
| 686 | int cnode_locked = 0; |
| 687 | int saveInvalid = 0; // save the header data even though the type was out of range |
| 688 | uint64_t decompression_flags = 0; |
| 689 | |
| 690 | if (vnode_isnamedstream(vp)) { |
| 691 | /* |
| 692 | named streams can't be compressed |
| 693 | since named streams of the same file share the same cnode, |
| 694 | we don't want to get/set the state in the cnode, just return 0 |
| 695 | */ |
| 696 | return 0; |
| 697 | } |
| 698 | |
| 699 | /* examine the cached a state in this cnode */ |
| 700 | cmp_state = decmpfs_cnode_get_vnode_state(cp); |
| 701 | switch(cmp_state) { |
| 702 | case FILE_IS_NOT_COMPRESSED: |
| 703 | return 0; |
| 704 | case FILE_IS_COMPRESSED: |
| 705 | return 1; |
| 706 | case FILE_IS_CONVERTING: |
| 707 | /* treat the file as compressed, because this gives us a way to block future reads until decompression is done */ |
| 708 | return 1; |
| 709 | case FILE_TYPE_UNKNOWN: |
| 710 | /* the first time we encountered this vnode, so we need to check it out */ |
| 711 | break; |
| 712 | default: |
| 713 | /* unknown state, assume file is not compressed */ |
| 714 | ErrorLogWithPath("unknown cmp_state %d\n", cmp_state); |
| 715 | return 0; |
| 716 | } |
| 717 | |
| 718 | if (!vnode_isreg(vp)) { |
| 719 | /* only regular files can be compressed */ |
| 720 | ret = FILE_IS_NOT_COMPRESSED; |
| 721 | goto done; |
| 722 | } |
| 723 | |
| 724 | mp = vnode_mount(vp); |
| 725 | if (mp == NULL) { |
| 726 | /* |
| 727 | this should only be true before we mount the root filesystem |
| 728 | we short-cut this return to avoid the call to getattr below, which |
| 729 | will fail before root is mounted |
| 730 | */ |
| 731 | ret = FILE_IS_NOT_COMPRESSED; |
| 732 | goto done; |
| 733 | } |
| 734 | if ((mp->mnt_flag & MNT_LOCAL) == 0) { |
| 735 | /* compression only supported on local filesystems */ |
| 736 | ret = FILE_IS_NOT_COMPRESSED; |
| 737 | goto done; |
| 738 | } |
| 739 | |
| 740 | /* lock our cnode data so that another caller doesn't change the state under us */ |
| 741 | decmpfs_lock_compressed_data(cp, 1); |
| 742 | cnode_locked = 1; |
| 743 | |
| 744 | VATTR_INIT(&va_fetch); |
| 745 | VATTR_WANTED(&va_fetch, va_flags); |
| 746 | error = vnode_getattr(vp, &va_fetch, decmpfs_ctx); |
| 747 | if (error) { |
| 748 | /* failed to get the bsd flags so the file is not compressed */ |
| 749 | ret = FILE_IS_NOT_COMPRESSED; |
| 750 | goto done; |
| 751 | } |
| 752 | if (va_fetch.va_flags & UF_COMPRESSED) { |
| 753 | /* UF_COMPRESSED is on, make sure the file has the DECMPFS_XATTR_NAME xattr */ |
| 754 | error = decmpfs_fetch_compressed_header(vp, cp, &hdr, 1); |
| 755 | if ((hdr != NULL) && (error == ERANGE)) { |
| 756 | saveInvalid = 1; |
| 757 | } |
| 758 | if (error) { |
| 759 | /* failed to get the xattr so the file is not compressed */ |
| 760 | ret = FILE_IS_NOT_COMPRESSED; |
| 761 | goto done; |
| 762 | } |
| 763 | /* we got the xattr, so the file is compressed */ |
| 764 | ret = FILE_IS_COMPRESSED; |
| 765 | goto done; |
| 766 | } |
| 767 | /* UF_COMPRESSED isn't on, so the file isn't compressed */ |
| 768 | ret = FILE_IS_NOT_COMPRESSED; |
| 769 | |
| 770 | done: |
| 771 | if (((ret == FILE_IS_COMPRESSED) || saveInvalid) && hdr) { |
| 772 | /* |
| 773 | cache the uncompressed size away in the cnode |
| 774 | */ |
| 775 | |
| 776 | if (!cnode_locked) { |
| 777 | /* |
| 778 | we should never get here since the only place ret is set to FILE_IS_COMPRESSED |
| 779 | is after the call to decmpfs_lock_compressed_data above |
| 780 | */ |
| 781 | decmpfs_lock_compressed_data(cp, 1); |
| 782 | cnode_locked = 1; |
| 783 | } |
| 784 | |
| 785 | decmpfs_cnode_set_vnode_cached_size(cp, hdr->uncompressed_size); |
| 786 | decmpfs_cnode_set_vnode_state(cp, ret, 1); |
| 787 | decmpfs_cnode_set_vnode_cmp_type(cp, hdr->compression_type, 1); |
| 788 | /* remember if the xattr's size was equal to the minimal xattr */ |
| 789 | if (hdr->attr_size == sizeof(decmpfs_disk_header)) { |
| 790 | decmpfs_cnode_set_vnode_minimal_xattr(cp, 1, 1); |
| 791 | } |
| 792 | if (ret == FILE_IS_COMPRESSED) { |
| 793 | /* update the ubc's size for this file */ |
| 794 | ubc_setsize(vp, hdr->uncompressed_size); |
| 795 | |
| 796 | /* update the decompression flags in the decmpfs cnode */ |
| 797 | lck_rw_lock_shared(decompressorsLock); |
| 798 | decmpfs_get_decompression_flags_func get_flags = decmp_get_func(vp, hdr->compression_type, get_flags); |
| 799 | if (get_flags) { |
| 800 | decompression_flags = get_flags(vp, decmpfs_ctx, hdr); |
| 801 | } |
| 802 | lck_rw_unlock_shared(decompressorsLock); |
| 803 | decmpfs_cnode_set_decompression_flags(cp, decompression_flags); |
| 804 | } |
| 805 | } else { |
| 806 | /* we might have already taken the lock above; if so, skip taking it again by passing cnode_locked as the skiplock parameter */ |
| 807 | decmpfs_cnode_set_vnode_state(cp, ret, cnode_locked); |
| 808 | } |
| 809 | |
| 810 | if (cnode_locked) decmpfs_unlock_compressed_data(cp, 1); |
| 811 | |
| 812 | if (hdr) FREE(hdr, M_TEMP); |
| 813 | |
| 814 | switch(ret) { |
| 815 | case FILE_IS_NOT_COMPRESSED: |
| 816 | return 0; |
| 817 | case FILE_IS_COMPRESSED: |
| 818 | case FILE_IS_CONVERTING: |
| 819 | return 1; |
| 820 | default: |
| 821 | /* unknown state, assume file is not compressed */ |
| 822 | ErrorLogWithPath("unknown ret %d\n", ret); |
| 823 | return 0; |
| 824 | } |
| 825 | } |
| 826 | |
| 827 | int |
| 828 | decmpfs_update_attributes(vnode_t vp, struct vnode_attr *vap) |
| 829 | { |
| 830 | int error = 0; |
| 831 | |
| 832 | if (VATTR_IS_ACTIVE(vap, va_flags)) { |
| 833 | /* the BSD flags are being updated */ |
| 834 | if (vap->va_flags & UF_COMPRESSED) { |
| 835 | /* the compressed bit is being set, did it change? */ |
| 836 | struct vnode_attr va_fetch; |
| 837 | int old_flags = 0; |
| 838 | VATTR_INIT(&va_fetch); |
| 839 | VATTR_WANTED(&va_fetch, va_flags); |
| 840 | error = vnode_getattr(vp, &va_fetch, decmpfs_ctx); |
| 841 | if (error) |
| 842 | return error; |
| 843 | |
| 844 | old_flags = va_fetch.va_flags; |
| 845 | |
| 846 | if (!(old_flags & UF_COMPRESSED)) { |
| 847 | /* |
| 848 | * Compression bit was turned on, make sure the file has the DECMPFS_XATTR_NAME attribute. |
| 849 | * This precludes anyone from using the UF_COMPRESSED bit for anything else, and it enforces |
| 850 | * an order of operation -- you must first do the setxattr and then the chflags. |
| 851 | */ |
| 852 | |
| 853 | if (VATTR_IS_ACTIVE(vap, va_data_size)) { |
| 854 | /* |
| 855 | * don't allow the caller to set the BSD flag and the size in the same call |
| 856 | * since this doesn't really make sense |
| 857 | */ |
| 858 | vap->va_flags &= ~UF_COMPRESSED; |
| 859 | return 0; |
| 860 | } |
| 861 | |
| 862 | decmpfs_header *hdr = NULL; |
| 863 | error = decmpfs_fetch_compressed_header(vp, NULL, &hdr, 1); |
| 864 | if (error == 0) { |
| 865 | /* |
| 866 | allow the flag to be set since the decmpfs attribute is present |
| 867 | in that case, we also want to truncate the data fork of the file |
| 868 | */ |
| 869 | VATTR_SET_ACTIVE(vap, va_data_size); |
| 870 | vap->va_data_size = 0; |
| 871 | } else if (error == ERANGE) { |
| 872 | /* the file had a decmpfs attribute but the type was out of range, so don't muck with the file's data size */ |
| 873 | } else { |
| 874 | /* no DECMPFS_XATTR_NAME attribute, so deny the update */ |
| 875 | vap->va_flags &= ~UF_COMPRESSED; |
| 876 | } |
| 877 | if (hdr) FREE(hdr, M_TEMP); |
| 878 | } |
| 879 | } |
| 880 | } |
| 881 | |
| 882 | return 0; |
| 883 | } |
| 884 | |
| 885 | static int |
| 886 | wait_for_decompress(decmpfs_cnode *cp) |
| 887 | { |
| 888 | int state; |
| 889 | lck_mtx_lock(decompress_channel_mtx); |
| 890 | do { |
| 891 | state = decmpfs_fast_get_state(cp); |
| 892 | if (state != FILE_IS_CONVERTING) { |
| 893 | /* file is not decompressing */ |
| 894 | lck_mtx_unlock(decompress_channel_mtx); |
| 895 | return state; |
| 896 | } |
| 897 | msleep((caddr_t)&decompress_channel, decompress_channel_mtx, PINOD, "wait_for_decompress", NULL); |
| 898 | } while(1); |
| 899 | } |
| 900 | |
| 901 | #pragma mark --- decmpfs hide query routines --- |
| 902 | |
| 903 | int |
| 904 | decmpfs_hides_rsrc(vfs_context_t ctx, decmpfs_cnode *cp) |
| 905 | { |
| 906 | /* |
| 907 | WARNING!!! |
| 908 | callers may (and do) pass NULL for ctx, so we should only use it |
| 909 | for this equality comparison |
| 910 | |
| 911 | This routine should only be called after a file has already been through decmpfs_file_is_compressed |
| 912 | */ |
| 913 | |
| 914 | if (ctx == decmpfs_ctx) |
| 915 | return 0; |
| 916 | |
| 917 | if (!decmpfs_fast_file_is_compressed(cp)) |
| 918 | return 0; |
| 919 | |
| 920 | /* all compressed files hide their resource fork */ |
| 921 | return 1; |
| 922 | } |
| 923 | |
| 924 | int |
| 925 | decmpfs_hides_xattr(vfs_context_t ctx, decmpfs_cnode *cp, const char *xattr) |
| 926 | { |
| 927 | /* |
| 928 | WARNING!!! |
| 929 | callers may (and do) pass NULL for ctx, so we should only use it |
| 930 | for this equality comparison |
| 931 | |
| 932 | This routine should only be called after a file has already been through decmpfs_file_is_compressed |
| 933 | */ |
| 934 | |
| 935 | if (ctx == decmpfs_ctx) |
| 936 | return 0; |
| 937 | if (strncmp(xattr, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME) - 1) == 0) |
| 938 | return decmpfs_hides_rsrc(ctx, cp); |
| 939 | if (!decmpfs_fast_file_is_compressed(cp)) |
| 940 | /* file is not compressed, so don't hide this xattr */ |
| 941 | return 0; |
| 942 | if (strncmp(xattr, DECMPFS_XATTR_NAME, sizeof(DECMPFS_XATTR_NAME) - 1) == 0) |
| 943 | /* it's our xattr, so hide it */ |
| 944 | return 1; |
| 945 | /* don't hide this xattr */ |
| 946 | return 0; |
| 947 | } |
| 948 | |
| 949 | #pragma mark --- registration/validation routines --- |
| 950 | |
| 951 | static inline int registration_valid(decmpfs_registration *registration) |
| 952 | { |
| 953 | return registration && ((registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V1) || (registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V3)); |
| 954 | } |
| 955 | |
| 956 | errno_t |
| 957 | register_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration) |
| 958 | { |
| 959 | /* called by kexts to register decompressors */ |
| 960 | |
| 961 | errno_t ret = 0; |
| 962 | int locked = 0; |
| 963 | char resourceName[80]; |
| 964 | |
| 965 | if ((compression_type >= CMP_MAX) || !registration_valid(registration)) { |
| 966 | ret = EINVAL; |
| 967 | goto out; |
| 968 | } |
| 969 | |
| 970 | lck_rw_lock_exclusive(decompressorsLock); locked = 1; |
| 971 | |
| 972 | /* make sure the registration for this type is zero */ |
| 973 | if (decompressors[compression_type] != NULL) { |
| 974 | ret = EEXIST; |
| 975 | goto out; |
| 976 | } |
| 977 | decompressors[compression_type] = registration; |
| 978 | snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type); |
| 979 | IOServicePublishResource(resourceName, TRUE); |
| 980 | |
| 981 | out: |
| 982 | if (locked) lck_rw_unlock_exclusive(decompressorsLock); |
| 983 | return ret; |
| 984 | } |
| 985 | |
| 986 | errno_t |
| 987 | unregister_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration) |
| 988 | { |
| 989 | /* called by kexts to unregister decompressors */ |
| 990 | |
| 991 | errno_t ret = 0; |
| 992 | int locked = 0; |
| 993 | char resourceName[80]; |
| 994 | |
| 995 | if ((compression_type >= CMP_MAX) || !registration_valid(registration)) { |
| 996 | ret = EINVAL; |
| 997 | goto out; |
| 998 | } |
| 999 | |
| 1000 | lck_rw_lock_exclusive(decompressorsLock); locked = 1; |
| 1001 | if (decompressors[compression_type] != registration) { |
| 1002 | ret = EEXIST; |
| 1003 | goto out; |
| 1004 | } |
| 1005 | decompressors[compression_type] = NULL; |
| 1006 | snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type); |
| 1007 | IOServicePublishResource(resourceName, FALSE); |
| 1008 | |
| 1009 | out: |
| 1010 | if (locked) lck_rw_unlock_exclusive(decompressorsLock); |
| 1011 | return ret; |
| 1012 | } |
| 1013 | |
| 1014 | static int |
| 1015 | compression_type_valid(vnode_t vp, decmpfs_header *hdr) |
| 1016 | { |
| 1017 | /* fast pre-check to determine if the given compressor has checked in */ |
| 1018 | int ret = 0; |
| 1019 | |
| 1020 | /* every compressor must have at least a fetch function */ |
| 1021 | lck_rw_lock_shared(decompressorsLock); |
| 1022 | if (decmp_get_func(vp, hdr->compression_type, fetch) != NULL) { |
| 1023 | ret = 1; |
| 1024 | } |
| 1025 | lck_rw_unlock_shared(decompressorsLock); |
| 1026 | |
| 1027 | return ret; |
| 1028 | } |
| 1029 | |
| 1030 | #pragma mark --- compression/decompression routines --- |
| 1031 | |
| 1032 | static int |
| 1033 | decmpfs_fetch_uncompressed_data(vnode_t vp, decmpfs_cnode *cp, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read) |
| 1034 | { |
| 1035 | /* get the uncompressed bytes for the specified region of vp by calling out to the registered compressor */ |
| 1036 | |
| 1037 | int err = 0; |
| 1038 | |
| 1039 | *bytes_read = 0; |
| 1040 | |
| 1041 | if ((uint64_t)offset >= hdr->uncompressed_size) { |
| 1042 | /* reading past end of file; nothing to do */ |
| 1043 | err = 0; |
| 1044 | goto out; |
| 1045 | } |
| 1046 | if (offset < 0) { |
| 1047 | /* tried to read from before start of file */ |
| 1048 | err = EINVAL; |
| 1049 | goto out; |
| 1050 | } |
| 1051 | if ((uint64_t)(offset + size) > hdr->uncompressed_size) { |
| 1052 | /* adjust size so we don't read past the end of the file */ |
| 1053 | size = hdr->uncompressed_size - offset; |
| 1054 | } |
| 1055 | if (size == 0) { |
| 1056 | /* nothing to read */ |
| 1057 | err = 0; |
| 1058 | goto out; |
| 1059 | } |
| 1060 | |
| 1061 | lck_rw_lock_shared(decompressorsLock); |
| 1062 | decmpfs_fetch_uncompressed_data_func fetch = decmp_get_func(vp, hdr->compression_type, fetch); |
| 1063 | if (fetch) { |
| 1064 | err = fetch(vp, decmpfs_ctx, hdr, offset, size, nvec, vec, bytes_read); |
| 1065 | lck_rw_unlock_shared(decompressorsLock); |
| 1066 | if (err == 0) { |
| 1067 | uint64_t decompression_flags = decmpfs_cnode_get_decompression_flags(cp); |
| 1068 | if (decompression_flags & DECMPFS_FLAGS_FORCE_FLUSH_ON_DECOMPRESS) { |
| 1069 | #if !defined(__i386__) && !defined(__x86_64__) |
| 1070 | int i; |
| 1071 | for (i = 0; i < nvec; i++) { |
| 1072 | flush_dcache64((addr64_t)(uintptr_t)vec[i].buf, vec[i].size, FALSE); |
| 1073 | } |
| 1074 | #endif |
| 1075 | } |
| 1076 | } |
| 1077 | } else { |
| 1078 | err = ENOTSUP; |
| 1079 | lck_rw_unlock_shared(decompressorsLock); |
| 1080 | } |
| 1081 | |
| 1082 | out: |
| 1083 | return err; |
| 1084 | } |
| 1085 | |
| 1086 | static kern_return_t |
| 1087 | commit_upl(upl_t upl, upl_offset_t pl_offset, size_t uplSize, int flags, int abort) |
| 1088 | { |
| 1089 | kern_return_t kr = 0; |
| 1090 | |
| 1091 | #if CONFIG_IOSCHED |
| 1092 | upl_unmark_decmp(upl); |
| 1093 | #endif /* CONFIG_IOSCHED */ |
| 1094 | |
| 1095 | /* commit the upl pages */ |
| 1096 | if (abort) { |
| 1097 | VerboseLog("aborting upl, flags 0x%08x\n", flags); |
| 1098 | kr = ubc_upl_abort_range(upl, pl_offset, uplSize, flags); |
| 1099 | if (kr != KERN_SUCCESS) |
| 1100 | ErrorLog("ubc_upl_abort_range error %d\n", (int)kr); |
| 1101 | } else { |
| 1102 | VerboseLog("committing upl, flags 0x%08x\n", flags | UPL_COMMIT_CLEAR_DIRTY); |
| 1103 | kr = ubc_upl_commit_range(upl, pl_offset, uplSize, flags | UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_WRITTEN_BY_KERNEL); |
| 1104 | if (kr != KERN_SUCCESS) |
| 1105 | ErrorLog("ubc_upl_commit_range error %d\n", (int)kr); |
| 1106 | } |
| 1107 | return kr; |
| 1108 | } |
| 1109 | |
| 1110 | |
| 1111 | errno_t |
| 1112 | decmpfs_pagein_compressed(struct vnop_pagein_args *ap, int *is_compressed, decmpfs_cnode *cp) |
| 1113 | { |
| 1114 | /* handles a page-in request from vfs for a compressed file */ |
| 1115 | |
| 1116 | int err = 0; |
| 1117 | vnode_t vp = ap->a_vp; |
| 1118 | upl_t pl = ap->a_pl; |
| 1119 | upl_offset_t pl_offset = ap->a_pl_offset; |
| 1120 | off_t f_offset = ap->a_f_offset; |
| 1121 | size_t size = ap->a_size; |
| 1122 | int flags = ap->a_flags; |
| 1123 | off_t uplPos = 0; |
| 1124 | user_ssize_t uplSize = 0; |
| 1125 | void *data = NULL; |
| 1126 | decmpfs_header *hdr = NULL; |
| 1127 | uint64_t cachedSize = 0; |
| 1128 | int cmpdata_locked = 0; |
| 1129 | |
| 1130 | if(!decmpfs_trylock_compressed_data(cp, 0)) { |
| 1131 | return EAGAIN; |
| 1132 | } |
| 1133 | cmpdata_locked = 1; |
| 1134 | |
| 1135 | |
| 1136 | if (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)) { |
| 1137 | DebugLogWithPath("pagein: unknown flags 0x%08x\n", (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD))); |
| 1138 | } |
| 1139 | |
| 1140 | err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); |
| 1141 | if (err != 0) { |
| 1142 | goto out; |
| 1143 | } |
| 1144 | |
| 1145 | cachedSize = hdr->uncompressed_size; |
| 1146 | |
| 1147 | if (!compression_type_valid(vp, hdr)) { |
| 1148 | /* compressor not registered */ |
| 1149 | err = ENOTSUP; |
| 1150 | goto out; |
| 1151 | } |
| 1152 | |
| 1153 | #if CONFIG_IOSCHED |
| 1154 | /* Mark the UPL as the requesting UPL for decompression */ |
| 1155 | upl_mark_decmp(pl); |
| 1156 | #endif /* CONFIG_IOSCHED */ |
| 1157 | |
| 1158 | /* map the upl so we can fetch into it */ |
| 1159 | kern_return_t kr = ubc_upl_map(pl, (vm_offset_t*)&data); |
| 1160 | if ((kr != KERN_SUCCESS) || (data == NULL)) { |
| 1161 | err = ENOSPC; |
| 1162 | data = NULL; |
| 1163 | #if CONFIG_IOSCHED |
| 1164 | upl_unmark_decmp(pl); |
| 1165 | #endif /* CONFIG_IOSCHED */ |
| 1166 | goto out; |
| 1167 | } |
| 1168 | |
| 1169 | uplPos = f_offset; |
| 1170 | uplSize = size; |
| 1171 | |
| 1172 | /* clip the size to the size of the file */ |
| 1173 | if ((uint64_t)uplPos + uplSize > cachedSize) { |
| 1174 | /* truncate the read to the size of the file */ |
| 1175 | uplSize = cachedSize - uplPos; |
| 1176 | } |
| 1177 | |
| 1178 | /* do the fetch */ |
| 1179 | decmpfs_vector vec; |
| 1180 | |
| 1181 | decompress: |
| 1182 | /* the mapped data pointer points to the first page of the page list, so we want to start filling in at an offset of pl_offset */ |
| 1183 | vec.buf = (char*)data + pl_offset; |
| 1184 | vec.size = size; |
| 1185 | |
| 1186 | uint64_t did_read = 0; |
| 1187 | if (decmpfs_fast_get_state(cp) == FILE_IS_CONVERTING) { |
| 1188 | ErrorLogWithPath("unexpected pagein during decompress\n"); |
| 1189 | /* |
| 1190 | if the file is converting, this must be a recursive call to pagein from underneath a call to decmpfs_decompress_file; |
| 1191 | pretend that it succeeded but don't do anything since we're just going to write over the pages anyway |
| 1192 | */ |
| 1193 | err = 0; |
| 1194 | did_read = 0; |
| 1195 | } else { |
| 1196 | err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, uplPos, uplSize, 1, &vec, &did_read); |
| 1197 | } |
| 1198 | if (err) { |
| 1199 | DebugLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err); |
| 1200 | int cmp_state = decmpfs_fast_get_state(cp); |
| 1201 | if (cmp_state == FILE_IS_CONVERTING) { |
| 1202 | DebugLogWithPath("cmp_state == FILE_IS_CONVERTING\n"); |
| 1203 | cmp_state = wait_for_decompress(cp); |
| 1204 | if (cmp_state == FILE_IS_COMPRESSED) { |
| 1205 | DebugLogWithPath("cmp_state == FILE_IS_COMPRESSED\n"); |
| 1206 | /* a decompress was attempted but it failed, let's try calling fetch again */ |
| 1207 | goto decompress; |
| 1208 | } |
| 1209 | } |
| 1210 | if (cmp_state == FILE_IS_NOT_COMPRESSED) { |
| 1211 | DebugLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n"); |
| 1212 | /* the file was decompressed after we started reading it */ |
| 1213 | *is_compressed = 0; /* instruct caller to fall back to its normal path */ |
| 1214 | } |
| 1215 | } |
| 1216 | |
| 1217 | /* zero out whatever we didn't read, and zero out the end of the last page(s) */ |
| 1218 | uint64_t total_size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); |
| 1219 | if (did_read < total_size) { |
| 1220 | memset((char*)vec.buf + did_read, 0, total_size - did_read); |
| 1221 | } |
| 1222 | |
| 1223 | #if CONFIG_IOSCHED |
| 1224 | upl_unmark_decmp(pl); |
| 1225 | #endif /* CONFIG_IOSCHED */ |
| 1226 | |
| 1227 | kr = ubc_upl_unmap(pl); data = NULL; /* make sure to set data to NULL so we don't try to unmap again below */ |
| 1228 | if (kr != KERN_SUCCESS) |
| 1229 | ErrorLogWithPath("ubc_upl_unmap error %d\n", (int)kr); |
| 1230 | else { |
| 1231 | if (!err) { |
| 1232 | /* commit our pages */ |
| 1233 | kr = commit_upl(pl, pl_offset, total_size, UPL_COMMIT_FREE_ON_EMPTY, 0); |
| 1234 | } |
| 1235 | } |
| 1236 | |
| 1237 | out: |
| 1238 | if (data) ubc_upl_unmap(pl); |
| 1239 | if (hdr) FREE(hdr, M_TEMP); |
| 1240 | if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 0); |
| 1241 | if (err) { |
| 1242 | #if 0 |
| 1243 | if (err != ENXIO && err != ENOSPC) { |
| 1244 | char *path; |
| 1245 | MALLOC(path, char *, PATH_MAX, M_TEMP, M_WAITOK); |
| 1246 | panic("%s: decmpfs_pagein_compressed: err %d", vnpath(vp, path, PATH_MAX), err); |
| 1247 | FREE(path, M_TEMP); |
| 1248 | } |
| 1249 | #endif /* 0 */ |
| 1250 | ErrorLogWithPath("err %d\n", err); |
| 1251 | } |
| 1252 | return err; |
| 1253 | } |
| 1254 | |
| 1255 | errno_t |
| 1256 | decmpfs_read_compressed(struct vnop_read_args *ap, int *is_compressed, decmpfs_cnode *cp) |
| 1257 | { |
| 1258 | /* handles a read request from vfs for a compressed file */ |
| 1259 | |
| 1260 | uio_t uio = ap->a_uio; |
| 1261 | vnode_t vp = ap->a_vp; |
| 1262 | int err = 0; |
| 1263 | int countInt = 0; |
| 1264 | off_t uplPos = 0; |
| 1265 | user_ssize_t uplSize = 0; |
| 1266 | user_ssize_t uplRemaining = 0; |
| 1267 | off_t curUplPos = 0; |
| 1268 | user_ssize_t curUplSize = 0; |
| 1269 | kern_return_t kr = KERN_SUCCESS; |
| 1270 | int abort_read = 0; |
| 1271 | void *data = NULL; |
| 1272 | uint64_t did_read = 0; |
| 1273 | upl_t upl = NULL; |
| 1274 | upl_page_info_t *pli = NULL; |
| 1275 | decmpfs_header *hdr = NULL; |
| 1276 | uint64_t cachedSize = 0; |
| 1277 | off_t uioPos = 0; |
| 1278 | user_ssize_t uioRemaining = 0; |
| 1279 | int cmpdata_locked = 0; |
| 1280 | |
| 1281 | decmpfs_lock_compressed_data(cp, 0); cmpdata_locked = 1; |
| 1282 | |
| 1283 | uplPos = uio_offset(uio); |
| 1284 | uplSize = uio_resid(uio); |
| 1285 | VerboseLogWithPath("uplPos %lld uplSize %lld\n", uplPos, uplSize); |
| 1286 | |
| 1287 | cachedSize = decmpfs_cnode_get_vnode_cached_size(cp); |
| 1288 | |
| 1289 | if ((uint64_t)uplPos + uplSize > cachedSize) { |
| 1290 | /* truncate the read to the size of the file */ |
| 1291 | uplSize = cachedSize - uplPos; |
| 1292 | } |
| 1293 | |
| 1294 | /* give the cluster layer a chance to fill in whatever it already has */ |
| 1295 | countInt = (uplSize > INT_MAX) ? INT_MAX : uplSize; |
| 1296 | err = cluster_copy_ubc_data(vp, uio, &countInt, 0); |
| 1297 | if (err != 0) |
| 1298 | goto out; |
| 1299 | |
| 1300 | /* figure out what's left */ |
| 1301 | uioPos = uio_offset(uio); |
| 1302 | uioRemaining = uio_resid(uio); |
| 1303 | if ((uint64_t)uioPos + uioRemaining > cachedSize) { |
| 1304 | /* truncate the read to the size of the file */ |
| 1305 | uioRemaining = cachedSize - uioPos; |
| 1306 | } |
| 1307 | |
| 1308 | if (uioRemaining <= 0) { |
| 1309 | /* nothing left */ |
| 1310 | goto out; |
| 1311 | } |
| 1312 | |
| 1313 | err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); |
| 1314 | if (err != 0) { |
| 1315 | goto out; |
| 1316 | } |
| 1317 | if (!compression_type_valid(vp, hdr)) { |
| 1318 | err = ENOTSUP; |
| 1319 | goto out; |
| 1320 | } |
| 1321 | |
| 1322 | uplPos = uioPos; |
| 1323 | uplSize = uioRemaining; |
| 1324 | #if COMPRESSION_DEBUG |
| 1325 | DebugLogWithPath("uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize); |
| 1326 | #endif |
| 1327 | |
| 1328 | lck_rw_lock_shared(decompressorsLock); |
| 1329 | decmpfs_adjust_fetch_region_func adjust_fetch = decmp_get_func(vp, hdr->compression_type, adjust_fetch); |
| 1330 | if (adjust_fetch) { |
| 1331 | /* give the compressor a chance to adjust the portion of the file that we read */ |
| 1332 | adjust_fetch(vp, decmpfs_ctx, hdr, &uplPos, &uplSize); |
| 1333 | VerboseLogWithPath("adjusted uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize); |
| 1334 | } |
| 1335 | lck_rw_unlock_shared(decompressorsLock); |
| 1336 | |
| 1337 | /* clip the adjusted size to the size of the file */ |
| 1338 | if ((uint64_t)uplPos + uplSize > cachedSize) { |
| 1339 | /* truncate the read to the size of the file */ |
| 1340 | uplSize = cachedSize - uplPos; |
| 1341 | } |
| 1342 | |
| 1343 | if (uplSize <= 0) { |
| 1344 | /* nothing left */ |
| 1345 | goto out; |
| 1346 | } |
| 1347 | |
| 1348 | /* |
| 1349 | since we're going to create a upl for the given region of the file, |
| 1350 | make sure we're on page boundaries |
| 1351 | */ |
| 1352 | |
| 1353 | if (uplPos & (PAGE_SIZE - 1)) { |
| 1354 | /* round position down to page boundary */ |
| 1355 | uplSize += (uplPos & (PAGE_SIZE - 1)); |
| 1356 | uplPos &= ~(PAGE_SIZE - 1); |
| 1357 | } |
| 1358 | /* round size up to page multiple */ |
| 1359 | uplSize = (uplSize + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); |
| 1360 | |
| 1361 | VerboseLogWithPath("new uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize); |
| 1362 | |
| 1363 | uplRemaining = uplSize; |
| 1364 | curUplPos = uplPos; |
| 1365 | curUplSize = 0; |
| 1366 | |
| 1367 | while(uplRemaining > 0) { |
| 1368 | /* start after the last upl */ |
| 1369 | curUplPos += curUplSize; |
| 1370 | |
| 1371 | /* clip to max upl size */ |
| 1372 | curUplSize = uplRemaining; |
| 1373 | if (curUplSize > MAX_UPL_SIZE_BYTES) { |
| 1374 | curUplSize = MAX_UPL_SIZE_BYTES; |
| 1375 | } |
| 1376 | |
| 1377 | /* create the upl */ |
| 1378 | kr = ubc_create_upl(vp, curUplPos, curUplSize, &upl, &pli, UPL_SET_LITE); |
| 1379 | if (kr != KERN_SUCCESS) { |
| 1380 | ErrorLogWithPath("ubc_create_upl error %d\n", (int)kr); |
| 1381 | err = EINVAL; |
| 1382 | goto out; |
| 1383 | } |
| 1384 | VerboseLogWithPath("curUplPos %lld curUplSize %lld\n", (uint64_t)curUplPos, (uint64_t)curUplSize); |
| 1385 | |
| 1386 | #if CONFIG_IOSCHED |
| 1387 | /* Mark the UPL as the requesting UPL for decompression */ |
| 1388 | upl_mark_decmp(upl); |
| 1389 | #endif /* CONFIG_IOSCHED */ |
| 1390 | |
| 1391 | /* map the upl */ |
| 1392 | kr = ubc_upl_map(upl, (vm_offset_t*)&data); |
| 1393 | if (kr != KERN_SUCCESS) { |
| 1394 | |
| 1395 | commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1); |
| 1396 | #if 0 |
| 1397 | char *path; |
| 1398 | MALLOC(path, char *, PATH_MAX, M_TEMP, M_WAITOK); |
| 1399 | panic("%s: decmpfs_read_compressed: ubc_upl_map error %d", vnpath(vp, path, PATH_MAX), (int)kr); |
| 1400 | FREE(path, M_TEMP); |
| 1401 | #else /* 0 */ |
| 1402 | ErrorLogWithPath("ubc_upl_map kr=0x%x\n", (int)kr); |
| 1403 | #endif /* 0 */ |
| 1404 | err = EINVAL; |
| 1405 | goto out; |
| 1406 | } |
| 1407 | |
| 1408 | /* make sure the map succeeded */ |
| 1409 | if (!data) { |
| 1410 | |
| 1411 | commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1); |
| 1412 | |
| 1413 | ErrorLogWithPath("ubc_upl_map mapped null\n"); |
| 1414 | err = EINVAL; |
| 1415 | goto out; |
| 1416 | } |
| 1417 | |
| 1418 | /* fetch uncompressed data into the mapped upl */ |
| 1419 | decmpfs_vector vec; |
| 1420 | decompress: |
| 1421 | vec = (decmpfs_vector){ .buf = data, .size = curUplSize }; |
| 1422 | err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, curUplPos, curUplSize, 1, &vec, &did_read); |
| 1423 | if (err) { |
| 1424 | ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err); |
| 1425 | |
| 1426 | /* maybe the file is converting to decompressed */ |
| 1427 | int cmp_state = decmpfs_fast_get_state(cp); |
| 1428 | if (cmp_state == FILE_IS_CONVERTING) { |
| 1429 | ErrorLogWithPath("cmp_state == FILE_IS_CONVERTING\n"); |
| 1430 | cmp_state = wait_for_decompress(cp); |
| 1431 | if (cmp_state == FILE_IS_COMPRESSED) { |
| 1432 | ErrorLogWithPath("cmp_state == FILE_IS_COMPRESSED\n"); |
| 1433 | /* a decompress was attempted but it failed, let's try fetching again */ |
| 1434 | goto decompress; |
| 1435 | } |
| 1436 | } |
| 1437 | if (cmp_state == FILE_IS_NOT_COMPRESSED) { |
| 1438 | ErrorLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n"); |
| 1439 | /* the file was decompressed after we started reading it */ |
| 1440 | abort_read = 1; /* we're not going to commit our data */ |
| 1441 | *is_compressed = 0; /* instruct caller to fall back to its normal path */ |
| 1442 | } |
| 1443 | kr = KERN_FAILURE; |
| 1444 | did_read = 0; |
| 1445 | } |
| 1446 | /* zero out the remainder of the last page */ |
| 1447 | memset((char*)data + did_read, 0, curUplSize - did_read); |
| 1448 | kr = ubc_upl_unmap(upl); |
| 1449 | if (kr == KERN_SUCCESS) { |
| 1450 | if (abort_read) { |
| 1451 | kr = commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1); |
| 1452 | } else { |
| 1453 | VerboseLogWithPath("uioPos %lld uioRemaining %lld\n", (uint64_t)uioPos, (uint64_t)uioRemaining); |
| 1454 | if (uioRemaining) { |
| 1455 | off_t uplOff = uioPos - curUplPos; |
| 1456 | if (uplOff < 0) { |
| 1457 | ErrorLogWithPath("uplOff %lld should never be negative\n", (int64_t)uplOff); |
| 1458 | err = EINVAL; |
| 1459 | } else { |
| 1460 | off_t count = curUplPos + curUplSize - uioPos; |
| 1461 | if (count < 0) { |
| 1462 | /* this upl is entirely before the uio */ |
| 1463 | } else { |
| 1464 | if (count > uioRemaining) |
| 1465 | count = uioRemaining; |
| 1466 | int io_resid = count; |
| 1467 | err = cluster_copy_upl_data(uio, upl, uplOff, &io_resid); |
| 1468 | int copied = count - io_resid; |
| 1469 | VerboseLogWithPath("uplOff %lld count %lld copied %lld\n", (uint64_t)uplOff, (uint64_t)count, (uint64_t)copied); |
| 1470 | if (err) { |
| 1471 | ErrorLogWithPath("cluster_copy_upl_data err %d\n", err); |
| 1472 | } |
| 1473 | uioPos += copied; |
| 1474 | uioRemaining -= copied; |
| 1475 | } |
| 1476 | } |
| 1477 | } |
| 1478 | kr = commit_upl(upl, 0, curUplSize, UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE, 0); |
| 1479 | if (err) { |
| 1480 | goto out; |
| 1481 | } |
| 1482 | } |
| 1483 | } else { |
| 1484 | ErrorLogWithPath("ubc_upl_unmap error %d\n", (int)kr); |
| 1485 | } |
| 1486 | |
| 1487 | uplRemaining -= curUplSize; |
| 1488 | } |
| 1489 | |
| 1490 | out: |
| 1491 | |
| 1492 | if (hdr) FREE(hdr, M_TEMP); |
| 1493 | if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 0); |
| 1494 | if (err) {/* something went wrong */ |
| 1495 | ErrorLogWithPath("err %d\n", err); |
| 1496 | return err; |
| 1497 | } |
| 1498 | |
| 1499 | #if COMPRESSION_DEBUG |
| 1500 | uplSize = uio_resid(uio); |
| 1501 | if (uplSize) |
| 1502 | VerboseLogWithPath("still %lld bytes to copy\n", uplSize); |
| 1503 | #endif |
| 1504 | return 0; |
| 1505 | } |
| 1506 | |
| 1507 | int |
| 1508 | decmpfs_free_compressed_data(vnode_t vp, decmpfs_cnode *cp) |
| 1509 | { |
| 1510 | /* |
| 1511 | call out to the decompressor to free remove any data associated with this compressed file |
| 1512 | then delete the file's compression xattr |
| 1513 | */ |
| 1514 | |
| 1515 | decmpfs_header *hdr = NULL; |
| 1516 | int err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); |
| 1517 | if (err) { |
| 1518 | ErrorLogWithPath("decmpfs_fetch_compressed_header err %d\n", err); |
| 1519 | } else { |
| 1520 | lck_rw_lock_shared(decompressorsLock); |
| 1521 | decmpfs_free_compressed_data_func free_data = decmp_get_func(vp, hdr->compression_type, free_data); |
| 1522 | if (free_data) { |
| 1523 | err = free_data(vp, decmpfs_ctx, hdr); |
| 1524 | } else { |
| 1525 | /* nothing to do, so no error */ |
| 1526 | err = 0; |
| 1527 | } |
| 1528 | lck_rw_unlock_shared(decompressorsLock); |
| 1529 | |
| 1530 | if (err != 0) { |
| 1531 | ErrorLogWithPath("decompressor err %d\n", err); |
| 1532 | } |
| 1533 | } |
| 1534 | |
| 1535 | /* delete the xattr */ |
| 1536 | err = vn_removexattr(vp, DECMPFS_XATTR_NAME, 0, decmpfs_ctx); |
| 1537 | if (err != 0) { |
| 1538 | goto out; |
| 1539 | } |
| 1540 | |
| 1541 | out: |
| 1542 | if (hdr) FREE(hdr, M_TEMP); |
| 1543 | return err; |
| 1544 | } |
| 1545 | |
| 1546 | #pragma mark --- file conversion routines --- |
| 1547 | |
| 1548 | static int |
| 1549 | unset_compressed_flag(vnode_t vp) |
| 1550 | { |
| 1551 | int err = 0; |
| 1552 | struct vnode_attr va; |
| 1553 | int new_bsdflags = 0; |
| 1554 | |
| 1555 | VATTR_INIT(&va); |
| 1556 | VATTR_WANTED(&va, va_flags); |
| 1557 | err = vnode_getattr(vp, &va, decmpfs_ctx); |
| 1558 | |
| 1559 | if (err != 0) { |
| 1560 | ErrorLogWithPath("vnode_getattr err %d\n", err); |
| 1561 | } else { |
| 1562 | new_bsdflags = va.va_flags & ~UF_COMPRESSED; |
| 1563 | |
| 1564 | VATTR_INIT(&va); |
| 1565 | VATTR_SET(&va, va_flags, new_bsdflags); |
| 1566 | err = vnode_setattr(vp, &va, decmpfs_ctx); |
| 1567 | if (err != 0) { |
| 1568 | ErrorLogWithPath("vnode_setattr err %d\n", err); |
| 1569 | } |
| 1570 | } |
| 1571 | return err; |
| 1572 | } |
| 1573 | |
| 1574 | int |
| 1575 | decmpfs_decompress_file(vnode_t vp, decmpfs_cnode *cp, off_t toSize, int truncate_okay, int skiplock) |
| 1576 | { |
| 1577 | /* convert a compressed file to an uncompressed file */ |
| 1578 | |
| 1579 | int err = 0; |
| 1580 | char *data = NULL; |
| 1581 | uio_t uio_w = 0; |
| 1582 | off_t offset = 0; |
| 1583 | uint32_t old_state = 0; |
| 1584 | uint32_t new_state = 0; |
| 1585 | int update_file_state = 0; |
| 1586 | int allocSize = 0; |
| 1587 | decmpfs_header *hdr = NULL; |
| 1588 | int cmpdata_locked = 0; |
| 1589 | off_t remaining = 0; |
| 1590 | uint64_t uncompressed_size = 0; |
| 1591 | |
| 1592 | if (!skiplock) { |
| 1593 | decmpfs_lock_compressed_data(cp, 1); cmpdata_locked = 1; |
| 1594 | } |
| 1595 | |
| 1596 | decompress: |
| 1597 | old_state = decmpfs_fast_get_state(cp); |
| 1598 | |
| 1599 | switch(old_state) { |
| 1600 | case FILE_IS_NOT_COMPRESSED: |
| 1601 | { |
| 1602 | /* someone else decompressed the file */ |
| 1603 | err = 0; |
| 1604 | goto out; |
| 1605 | } |
| 1606 | |
| 1607 | case FILE_TYPE_UNKNOWN: |
| 1608 | { |
| 1609 | /* the file is in an unknown state, so update the state and retry */ |
| 1610 | (void)decmpfs_file_is_compressed(vp, cp); |
| 1611 | |
| 1612 | /* try again */ |
| 1613 | goto decompress; |
| 1614 | } |
| 1615 | |
| 1616 | case FILE_IS_COMPRESSED: |
| 1617 | { |
| 1618 | /* the file is compressed, so decompress it */ |
| 1619 | break; |
| 1620 | } |
| 1621 | |
| 1622 | default: |
| 1623 | { |
| 1624 | /* |
| 1625 | this shouldn't happen since multiple calls to decmpfs_decompress_file lock each other out, |
| 1626 | and when decmpfs_decompress_file returns, the state should be always be set back to |
| 1627 | FILE_IS_NOT_COMPRESSED or FILE_IS_UNKNOWN |
| 1628 | */ |
| 1629 | err = EINVAL; |
| 1630 | goto out; |
| 1631 | } |
| 1632 | } |
| 1633 | |
| 1634 | err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); |
| 1635 | if (err != 0) { |
| 1636 | goto out; |
| 1637 | } |
| 1638 | |
| 1639 | uncompressed_size = hdr->uncompressed_size; |
| 1640 | if (toSize == -1) |
| 1641 | toSize = hdr->uncompressed_size; |
| 1642 | |
| 1643 | if (toSize == 0) { |
| 1644 | /* special case truncating the file to zero bytes */ |
| 1645 | goto nodecmp; |
| 1646 | } else if ((uint64_t)toSize > hdr->uncompressed_size) { |
| 1647 | /* the caller is trying to grow the file, so we should decompress all the data */ |
| 1648 | toSize = hdr->uncompressed_size; |
| 1649 | } |
| 1650 | |
| 1651 | allocSize = MIN(64*1024, toSize); |
| 1652 | MALLOC(data, char *, allocSize, M_TEMP, M_WAITOK); |
| 1653 | if (!data) { |
| 1654 | err = ENOMEM; |
| 1655 | goto out; |
| 1656 | } |
| 1657 | |
| 1658 | uio_w = uio_create(1, 0LL, UIO_SYSSPACE, UIO_WRITE); |
| 1659 | if (!uio_w) { |
| 1660 | err = ENOMEM; |
| 1661 | goto out; |
| 1662 | } |
| 1663 | uio_w->uio_flags |= UIO_FLAGS_IS_COMPRESSED_FILE; |
| 1664 | |
| 1665 | remaining = toSize; |
| 1666 | |
| 1667 | /* tell the buffer cache that this is an empty file */ |
| 1668 | ubc_setsize(vp, 0); |
| 1669 | |
| 1670 | /* if we got here, we need to decompress the file */ |
| 1671 | decmpfs_cnode_set_vnode_state(cp, FILE_IS_CONVERTING, 1); |
| 1672 | |
| 1673 | while(remaining > 0) { |
| 1674 | /* loop decompressing data from the file and writing it into the data fork */ |
| 1675 | |
| 1676 | uint64_t bytes_read = 0; |
| 1677 | decmpfs_vector vec = { .buf = data, .size = MIN(allocSize, remaining) }; |
| 1678 | err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset, vec.size, 1, &vec, &bytes_read); |
| 1679 | if (err != 0) { |
| 1680 | ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err); |
| 1681 | goto out; |
| 1682 | } |
| 1683 | |
| 1684 | if (bytes_read == 0) { |
| 1685 | /* we're done reading data */ |
| 1686 | break; |
| 1687 | } |
| 1688 | |
| 1689 | uio_reset(uio_w, offset, UIO_SYSSPACE, UIO_WRITE); |
| 1690 | err = uio_addiov(uio_w, CAST_USER_ADDR_T(data), bytes_read); |
| 1691 | if (err != 0) { |
| 1692 | ErrorLogWithPath("uio_addiov err %d\n", err); |
| 1693 | err = ENOMEM; |
| 1694 | goto out; |
| 1695 | } |
| 1696 | |
| 1697 | err = VNOP_WRITE(vp, uio_w, 0, decmpfs_ctx); |
| 1698 | if (err != 0) { |
| 1699 | /* if the write failed, truncate the file to zero bytes */ |
| 1700 | ErrorLogWithPath("VNOP_WRITE err %d\n", err); |
| 1701 | break; |
| 1702 | } |
| 1703 | offset += bytes_read; |
| 1704 | remaining -= bytes_read; |
| 1705 | } |
| 1706 | |
| 1707 | if (err == 0) { |
| 1708 | if (offset != toSize) { |
| 1709 | ErrorLogWithPath("file decompressed to %lld instead of %lld\n", offset, toSize); |
| 1710 | err = EINVAL; |
| 1711 | goto out; |
| 1712 | } |
| 1713 | } |
| 1714 | |
| 1715 | if (err == 0) { |
| 1716 | /* sync the data and metadata */ |
| 1717 | err = VNOP_FSYNC(vp, MNT_WAIT, decmpfs_ctx); |
| 1718 | if (err != 0) { |
| 1719 | ErrorLogWithPath("VNOP_FSYNC err %d\n", err); |
| 1720 | goto out; |
| 1721 | } |
| 1722 | } |
| 1723 | |
| 1724 | if (err != 0) { |
| 1725 | /* write, setattr, or fsync failed */ |
| 1726 | ErrorLogWithPath("aborting decompress, err %d\n", err); |
| 1727 | if (truncate_okay) { |
| 1728 | /* truncate anything we might have written */ |
| 1729 | int error = vnode_setsize(vp, 0, 0, decmpfs_ctx); |
| 1730 | ErrorLogWithPath("vnode_setsize err %d\n", error); |
| 1731 | } |
| 1732 | goto out; |
| 1733 | } |
| 1734 | |
| 1735 | nodecmp: |
| 1736 | /* if we're truncating the file to zero bytes, we'll skip ahead to here */ |
| 1737 | |
| 1738 | /* unset the compressed flag */ |
| 1739 | unset_compressed_flag(vp); |
| 1740 | |
| 1741 | /* free the compressed data associated with this file */ |
| 1742 | err = decmpfs_free_compressed_data(vp, cp); |
| 1743 | if (err != 0) { |
| 1744 | ErrorLogWithPath("decmpfs_free_compressed_data err %d\n", err); |
| 1745 | } |
| 1746 | |
| 1747 | /* |
| 1748 | even if free_compressed_data or vnode_getattr/vnode_setattr failed, return success |
| 1749 | since we succeeded in writing all of the file data to the data fork |
| 1750 | */ |
| 1751 | err = 0; |
| 1752 | |
| 1753 | /* if we got this far, the file was successfully decompressed */ |
| 1754 | update_file_state = 1; |
| 1755 | new_state = FILE_IS_NOT_COMPRESSED; |
| 1756 | |
| 1757 | #if COMPRESSION_DEBUG |
| 1758 | { |
| 1759 | uint64_t filesize = 0; |
| 1760 | vnsize(vp, &filesize); |
| 1761 | DebugLogWithPath("new file size %lld\n", filesize); |
| 1762 | } |
| 1763 | #endif |
| 1764 | |
| 1765 | out: |
| 1766 | if (hdr) FREE(hdr, M_TEMP); |
| 1767 | if (data) FREE(data, M_TEMP); |
| 1768 | if (uio_w) uio_free(uio_w); |
| 1769 | |
| 1770 | if (err != 0) { |
| 1771 | /* if there was a failure, reset compression flags to unknown and clear the buffer cache data */ |
| 1772 | update_file_state = 1; |
| 1773 | new_state = FILE_TYPE_UNKNOWN; |
| 1774 | if (uncompressed_size) { |
| 1775 | ubc_setsize(vp, 0); |
| 1776 | ubc_setsize(vp, uncompressed_size); |
| 1777 | } |
| 1778 | } |
| 1779 | |
| 1780 | if (update_file_state) { |
| 1781 | lck_mtx_lock(decompress_channel_mtx); |
| 1782 | decmpfs_cnode_set_vnode_state(cp, new_state, 1); |
| 1783 | wakeup((caddr_t)&decompress_channel); /* wake up anyone who might have been waiting for decompression */ |
| 1784 | lck_mtx_unlock(decompress_channel_mtx); |
| 1785 | } |
| 1786 | |
| 1787 | if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 1); |
| 1788 | |
| 1789 | return err; |
| 1790 | } |
| 1791 | |
| 1792 | #pragma mark --- Type1 compressor --- |
| 1793 | |
| 1794 | /* |
| 1795 | The "Type1" compressor stores the data fork directly in the compression xattr |
| 1796 | */ |
| 1797 | |
| 1798 | static int |
| 1799 | decmpfs_validate_compressed_file_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr) |
| 1800 | { |
| 1801 | int err = 0; |
| 1802 | |
| 1803 | if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) { |
| 1804 | err = EINVAL; |
| 1805 | goto out; |
| 1806 | } |
| 1807 | out: |
| 1808 | return err; |
| 1809 | } |
| 1810 | |
| 1811 | static int |
| 1812 | decmpfs_fetch_uncompressed_data_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read) |
| 1813 | { |
| 1814 | int err = 0; |
| 1815 | int i; |
| 1816 | user_ssize_t remaining; |
| 1817 | |
| 1818 | if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) { |
| 1819 | err = EINVAL; |
| 1820 | goto out; |
| 1821 | } |
| 1822 | |
| 1823 | #if COMPRESSION_DEBUG |
| 1824 | static int dummy = 0; // prevent syslog from coalescing printfs |
| 1825 | DebugLogWithPath("%d memcpy %lld at %lld\n", dummy++, size, (uint64_t)offset); |
| 1826 | #endif |
| 1827 | |
| 1828 | remaining = size; |
| 1829 | for (i = 0; (i < nvec) && (remaining > 0); i++) { |
| 1830 | user_ssize_t curCopy = vec[i].size; |
| 1831 | if (curCopy > remaining) |
| 1832 | curCopy = remaining; |
| 1833 | memcpy(vec[i].buf, hdr->attr_bytes + offset, curCopy); |
| 1834 | offset += curCopy; |
| 1835 | remaining -= curCopy; |
| 1836 | } |
| 1837 | |
| 1838 | if ((bytes_read) && (err == 0)) |
| 1839 | *bytes_read = (size - remaining); |
| 1840 | |
| 1841 | out: |
| 1842 | return err; |
| 1843 | } |
| 1844 | |
| 1845 | static decmpfs_registration Type1Reg = |
| 1846 | { |
| 1847 | .decmpfs_registration = DECMPFS_REGISTRATION_VERSION, |
| 1848 | .validate = decmpfs_validate_compressed_file_Type1, |
| 1849 | .adjust_fetch = NULL, /* no adjust necessary */ |
| 1850 | .fetch = decmpfs_fetch_uncompressed_data_Type1, |
| 1851 | .free_data = NULL, /* no free necessary */ |
| 1852 | .get_flags = NULL /* no flags */ |
| 1853 | }; |
| 1854 | |
| 1855 | #pragma mark --- decmpfs initialization --- |
| 1856 | |
| 1857 | void decmpfs_init() |
| 1858 | { |
| 1859 | static int done = 0; |
| 1860 | if (done) return; |
| 1861 | |
| 1862 | decmpfs_ctx = vfs_context_create(vfs_context_kernel()); |
| 1863 | |
| 1864 | lck_grp_attr_t *attr = lck_grp_attr_alloc_init(); |
| 1865 | decmpfs_lockgrp = lck_grp_alloc_init("VFSCOMP", attr); |
| 1866 | lck_grp_attr_free(attr); |
| 1867 | decompressorsLock = lck_rw_alloc_init(decmpfs_lockgrp, NULL); |
| 1868 | decompress_channel_mtx = lck_mtx_alloc_init(decmpfs_lockgrp, NULL); |
| 1869 | |
| 1870 | register_decmpfs_decompressor(CMP_Type1, &Type1Reg); |
| 1871 | |
| 1872 | done = 1; |
| 1873 | } |
| 1874 | #endif /* FS_COMPRESSION */ |