]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOPolledInterface.cpp
projects / apple / xnu.git / blob
? search:


9ff9f8043c7631244f9bca0f29e6d333701a357e
[apple/xnu.git] / iokit / Kernel / IOPolledInterface.cpp
1 /*
2 * Copyright (c) 2006-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/uio.h>
30 #include <sys/conf.h>
31
32 #include <IOKit/IOLib.h>
33 #include <IOKit/IOBSD.h>
34 #include <IOKit/IOService.h>
35 #include <IOKit/IOPlatformExpert.h>
36 #include <IOKit/IOPolledInterface.h>
37 #include <IOKit/IOHibernatePrivate.h>
38 #include <IOKit/IOBufferMemoryDescriptor.h>
39 #include <IOKit/AppleKeyStoreInterface.h>
40 #include <libkern/c++/OSSharedPtr.h>
41 #include "IOKitKernelInternal.h"
42
43 #if defined(__arm64__)
44 #include <pexpert/arm64/board_config.h>
45 #if XNU_MONITOR_PPL_HIB
46 #include <IOKit/SEPHibernator.h>
47 #endif /* XNU_MONITOR_PPL_HIB */
48 #endif /* defined(__arm64__) */
49
50 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
51
52 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
53
54 OSMetaClassDefineReservedUsedX86(IOPolledInterface, 0);
55 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
56 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
57 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
58 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
59 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
60 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
61 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
62 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
63 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
64 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
65 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
66 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
67 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
68 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
69 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
70
71 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
72
73 #ifndef kIOMediaPreferredBlockSizeKey
74 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
75 #endif
76
77 enum { kDefaultIOSize = 128 * 1024 };
78
79 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
80
81 class IOPolledFilePollers : public OSObject
82 {
83 OSDeclareDefaultStructors(IOPolledFilePollers);
84
85 public:
86 IOService * media;
87 OSArray * pollers;
88 IOBufferMemoryDescriptor * ioBuffer;
89 bool abortable;
90 bool io;
91 IOReturn ioStatus;
92 uint32_t openCount;
93
94 static IOPolledFilePollers * copyPollers(IOService * media);
95 };
96
97 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
98
99 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
100
101 IOPolledFilePollers *
102 IOPolledFilePollers::copyPollers(IOService * media)
103 {
104 IOPolledFilePollers * vars;
105 IOReturn err;
106 IOService * service;
107 OSObject * obj;
108 IORegistryEntry * next;
109 IORegistryEntry * child;
110
111 if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) {
112 return OSDynamicCast(IOPolledFilePollers, obj);
113 }
114
115 do{
116 vars = OSTypeAlloc(IOPolledFilePollers);
117 vars->init();
118
119 vars->pollers = OSArray::withCapacity(4);
120 if (!vars->pollers) {
121 err = kIOReturnNoMemory;
122 break;
123 }
124
125 next = vars->media = media;
126 do{
127 IOPolledInterface * poller;
128 OSObject * obj;
129
130 obj = next->getProperty(kIOPolledInterfaceSupportKey);
131 if (kOSBooleanFalse == obj) {
132 vars->pollers->flushCollection();
133 break;
134 } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) {
135 vars->pollers->setObject(poller);
136 }
137
138 if ((service = OSDynamicCast(IOService, next))
139 && service->getDeviceMemory()
140 && !vars->pollers->getCount()) {
141 break;
142 }
143
144 child = next;
145 }while ((next = child->getParentEntry(gIOServicePlane))
146 && child->isParent(next, gIOServicePlane, true));
147
148 if (!vars->pollers->getCount()) {
149 err = kIOReturnUnsupported;
150 break;
151 }
152 }while (false);
153
154 media->setProperty(kIOPolledInterfaceStackKey, vars);
155
156 return vars;
157 }
158
159 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
160
161 static IOReturn
162 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
163
164 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
165
166 static IOReturn
167 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
168 {
169 IOReturn err = kIOReturnError;
170 int32_t idx;
171 IOPolledInterface * poller;
172
173 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
174 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
175 err = poller->probe(vars->media);
176 if (err) {
177 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
178 break;
179 }
180 }
181
182 return err;
183 }
184
185 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
186
187 IOReturn
188 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
189 {
190 IOPolledFilePollers * vars = filevars->pollers;
191 IOBufferMemoryDescriptor * ioBuffer;
192 IOPolledInterface * poller;
193 IOService * next;
194 IOReturn err = kIOReturnError;
195 int32_t idx;
196
197 vars->abortable = abortable;
198 ioBuffer = NULL;
199
200 if (kIOPolledAfterSleepState == state) {
201 vars->ioStatus = 0;
202 vars->io = false;
203 }
204 (void) IOPolledFilePollersIODone(vars, false);
205
206 if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) {
207 ioBuffer = vars->ioBuffer;
208 if (!ioBuffer) {
209 vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
210 2 * kDefaultIOSize, page_size);
211 if (!ioBuffer) {
212 return kIOReturnNoMemory;
213 }
214 }
215 }
216
217 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
218 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
219 err = poller->open(state, ioBuffer);
220 if (kIOReturnSuccess != err) {
221 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
222 break;
223 }
224 }
225 if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) {
226 next = vars->media;
227 while (next) {
228 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
229 next = next->getProvider();
230 }
231 }
232
233 return err;
234 }
235
236 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
237
238 IOReturn
239 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
240 {
241 IOPolledFilePollers * vars = filevars->pollers;
242 IOPolledInterface * poller;
243 IORegistryEntry * next;
244 IOReturn err;
245 int32_t idx;
246
247 (void) IOPolledFilePollersIODone(vars, false);
248
249 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
250 vars->openCount--;
251 }
252
253 for (idx = 0, err = kIOReturnSuccess;
254 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
255 idx++) {
256 err = poller->close(state);
257 if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) {
258 err = poller->close(kIOPolledBeforeSleepState);
259 }
260 if (err) {
261 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
262 }
263 }
264
265 if (kIOPolledPostflightState == state) {
266 next = vars->media;
267 while (next) {
268 next->removeProperty(kIOPolledInterfaceActiveKey);
269 next = next->getParentEntry(gIOServicePlane);
270 }
271 }
272
273 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
274 do{
275 if (vars->openCount) {
276 break;
277 }
278 if (vars->ioBuffer) {
279 vars->ioBuffer->release();
280 vars->ioBuffer = NULL;
281 }
282 }while (false);
283 }
284
285 return err;
286 }
287
288 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
289
290 IOReturn
291 IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
292 {
293 return kIOReturnUnsupported;
294 }
295
296 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
297
298 IOReturn
299 IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
300 const uint8_t * key, size_t keySize)
301 {
302 IOReturn ret = kIOReturnUnsupported;
303 IOReturn err;
304 int32_t idx;
305 IOPolledFilePollers * vars = filevars->pollers;
306 IOPolledInterface * poller;
307
308 for (idx = 0;
309 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
310 idx++) {
311 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
312 err = poller->setEncryptionKey(key, keySize);
313 if (kIOReturnSuccess == err) {
314 ret = err;
315 }
316 }
317
318 return ret;
319 }
320
321 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
322
323 IOMemoryDescriptor *
324 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
325 {
326 return vars->pollers->ioBuffer;
327 }
328
329 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
330
331 static void
332 IOPolledIOComplete(void * target,
333 void * parameter,
334 IOReturn status,
335 UInt64 actualByteCount)
336 {
337 IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
338
339 vars->ioStatus = status;
340 }
341
342 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
343
344 static IOReturn
345 IOStartPolledIO(IOPolledFilePollers * vars,
346 uint32_t operation, uint32_t bufferOffset,
347 uint64_t deviceOffset, uint64_t length)
348 {
349 IOReturn err;
350 IOPolledInterface * poller;
351 IOPolledCompletion completion;
352
353 err = vars->ioStatus;
354 if (kIOReturnSuccess != err) {
355 return err;
356 }
357
358 completion.target = NULL;
359 completion.action = &IOPolledIOComplete;
360 completion.parameter = vars;
361
362 vars->ioStatus = -1;
363
364 poller = (IOPolledInterface *) vars->pollers->getObject(0);
365 err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
366 if (err) {
367 if (kernel_debugger_entry_count) {
368 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
369 } else {
370 HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
371 vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
372 }
373 }
374 return err;
375 }
376
377 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
378
379 static IOReturn
380 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
381 {
382 IOReturn err = kIOReturnSuccess;
383 int32_t idx = 0;
384 IOPolledInterface * poller;
385 AbsoluteTime deadline;
386
387 if (!vars->io) {
388 return kIOReturnSuccess;
389 }
390
391 abortable &= vars->abortable;
392
393 clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
394
395 while (-1 == vars->ioStatus) {
396 for (idx = 0;
397 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
398 idx++) {
399 IOReturn newErr;
400 newErr = poller->checkForWork();
401 if ((newErr == kIOReturnAborted) && !abortable) {
402 newErr = kIOReturnSuccess;
403 }
404 if (kIOReturnSuccess == err) {
405 err = newErr;
406 }
407 }
408 if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) {
409 HIBLOG("IOPolledInterface::forced timeout\n");
410 vars->ioStatus = kIOReturnTimeout;
411 }
412 }
413 vars->io = false;
414
415 #if HIBERNATION
416 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) {
417 err = kIOReturnAborted;
418 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
419 }
420 #endif
421
422 if (err) {
423 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
424 } else {
425 err = vars->ioStatus;
426 if (kIOReturnSuccess != err) {
427 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
428 }
429 }
430
431 return err;
432 }
433
434 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
435 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
436
437 struct _OpenFileContext {
438 OSData * extents;
439 uint64_t size;
440 };
441
442 static void
443 file_extent_callback(void * ref, uint64_t start, uint64_t length)
444 {
445 _OpenFileContext * ctx = (_OpenFileContext *) ref;
446 IOPolledFileExtent extent;
447
448 extent.start = start;
449 extent.length = length;
450 ctx->extents->appendBytes(&extent, sizeof(extent));
451 ctx->size += length;
452 }
453
454 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
455
456 static IOService *
457 IOCopyMediaForDev(dev_t device)
458 {
459 OSDictionary * matching;
460 OSNumber * num;
461 OSIterator * iter;
462 IOService * result = NULL;
463
464 matching = IOService::serviceMatching("IOMedia");
465 if (!matching) {
466 return NULL;
467 }
468 do{
469 num = OSNumber::withNumber(major(device), 32);
470 if (!num) {
471 break;
472 }
473 matching->setObject(kIOBSDMajorKey, num);
474 num->release();
475 num = OSNumber::withNumber(minor(device), 32);
476 if (!num) {
477 break;
478 }
479 matching->setObject(kIOBSDMinorKey, num);
480 num->release();
481 if (!num) {
482 break;
483 }
484 iter = IOService::getMatchingServices(matching);
485 if (iter) {
486 result = (IOService *) iter->getNextObject();
487 result->retain();
488 iter->release();
489 }
490 }while (false);
491 matching->release();
492
493 return result;
494 }
495
496 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
497
498 #if defined(__i386__) || defined(__x86_64__)
499 #define APFSMEDIA_GETHIBERKEY "getHiberKey"
500
501 static IOReturn
502 IOGetVolumeCryptKey(dev_t block_dev,
503 LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,
504 uint8_t * volumeCryptKey,
505 size_t * keySize)
506 {
507 IOReturn err;
508 IOService * part;
509 OSString * keyUUID = NULL;
510 OSString * keyStoreUUID = NULL;
511 uuid_t volumeKeyUUID;
512 aks_volume_key_t vek;
513 size_t callerKeySize;
514
515 static IOService * sKeyStore;
516
517 part = IOCopyMediaForDev(block_dev);
518 if (!part) {
519 return kIOReturnNotFound;
520 }
521
522 callerKeySize = *keySize;
523 // Try APFS first
524 {
525 uuid_t volUuid = {0};
526 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
527 if (kIOReturnBadArgument == err) {
528 // apfs fails on buffer size >32
529 *keySize = 32;
530 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
531 }
532 if (err != kIOReturnSuccess) {
533 *keySize = 0;
534 } else {
535 // No need to create uuid string if it's not requested
536 if (pKeyUUID) {
537 uuid_string_t volUuidStr;
538 uuid_unparse(volUuid, volUuidStr);
539 *pKeyUUID = OSString::withCString(volUuidStr);
540 }
541
542 part->release();
543 return kIOReturnSuccess;
544 }
545 }
546
547 // Then old CS path
548 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
549 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
550 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) {
551 // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
552
553 if (!sKeyStore) {
554 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
555 }
556 if (sKeyStore) {
557 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
558 } else {
559 err = kIOReturnNoResources;
560 }
561 if (kIOReturnSuccess == err) {
562 err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
563 }
564 if (kIOReturnSuccess != err) {
565 IOLog("volume key err 0x%x\n", err);
566 } else {
567 if (vek.key.keybytecount <= callerKeySize) {
568 *keySize = vek.key.keybytecount;
569 }
570 bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
571 }
572 bzero(&vek, sizeof(vek));
573
574 if (pKeyUUID) {
575 // Create a copy because the caller would release it
576 *pKeyUUID = OSString::withString(keyUUID);
577 }
578 }
579
580 part->release();
581 return err;
582 }
583 #endif /* defined(__i386__) || defined(__x86_64__) */
584
585 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
586
587 #if defined(__arm64__)
588 static IOReturn
589 IOGetHibernationCryptKey(uint8_t * hibernationKey,
590 size_t * keySize,
591 uint32_t *swSeed
592 )
593 {
594 return kIOReturnNotFound;
595 }
596 #endif /* defined(__arm64__) */
597
598 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
599
600 IOReturn
601 IOPolledFileOpen(const char * filename,
602 uint32_t flags,
603 uint64_t setFileSize, uint64_t fsFreeSize,
604 void * write_file_addr, size_t write_file_len,
605 IOPolledFileIOVars ** fileVars,
606 OSData ** imagePath,
607 uint8_t * volumeCryptKey, size_t * keySize)
608 {
609 IOReturn err = kIOReturnSuccess;
610 IOPolledFileIOVars * vars;
611 _OpenFileContext ctx;
612 OSData * extentsData = NULL;
613 OSNumber * num;
614 IOService * part = NULL;
615 dev_t block_dev;
616 dev_t image_dev;
617 AbsoluteTime startTime, endTime;
618 uint64_t nsec;
619
620 vars = IONew(IOPolledFileIOVars, 1);
621 if (!vars) {
622 return kIOReturnNoMemory;
623 }
624 bzero(vars, sizeof(*vars));
625 vars->allocated = true;
626
627 do{
628 extentsData = OSData::withCapacity(32);
629 ctx.extents = extentsData;
630 ctx.size = 0;
631 clock_get_uptime(&startTime);
632
633 vars->fileRef = kern_open_file_for_direct_io(filename,
634 flags,
635 &file_extent_callback, &ctx,
636 setFileSize,
637 fsFreeSize,
638 // write file:
639 0, write_file_addr, write_file_len,
640 // results
641 &block_dev,
642 &image_dev,
643 &vars->block0,
644 &vars->maxiobytes,
645 &vars->flags);
646 #if 0
647 uint32_t msDelay = (131071 & random());
648 HIBLOG("sleep %d\n", msDelay);
649 IOSleep(msDelay);
650 #endif
651 clock_get_uptime(&endTime);
652 SUB_ABSOLUTETIME(&endTime, &startTime);
653 absolutetime_to_nanoseconds(endTime, &nsec);
654
655 if (!vars->fileRef) {
656 err = kIOReturnNoSpace;
657 }
658
659 HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
660 if (kIOReturnSuccess != err) {
661 break;
662 }
663
664 HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
665 (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
666 vars->maxiobytes, kIOPolledFileSSD & vars->flags);
667 assert(!vars->block0);
668 if (extentsData->getLength() < sizeof(IOPolledFileExtent)) {
669 err = kIOReturnNoSpace;
670 break;
671 }
672
673 vars->fileSize = ctx.size;
674 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
675
676 part = IOCopyMediaForDev(image_dev);
677 if (!part) {
678 err = kIOReturnNotFound;
679 break;
680 }
681
682 if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) {
683 break;
684 }
685
686 if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) {
687 vars->blockSize = num->unsigned32BitValue();
688 }
689 if (vars->blockSize < 4096) {
690 vars->blockSize = 4096;
691 }
692
693 HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
694 major(image_dev), minor(image_dev), (long)vars->blockSize,
695 vars->pollers->pollers->getCount());
696
697 OSString * keyUUID = NULL;
698 #if defined(__i386__) || defined(__x86_64__)
699 if (volumeCryptKey) {
700 err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
701 }
702 #elif defined(__arm64__)
703 uint32_t swSeed = 0;
704 if (volumeCryptKey) {
705 if (flags & kIOPolledFileHibernate) {
706 err = IOGetHibernationCryptKey(volumeCryptKey, keySize, &swSeed);
707 if (kIOReturnSuccess != err) {
708 HIBLOG("error 0x%x from IOGetHibernationCryptKey\n", err);
709 break;
710 }
711 } else {
712 *keySize = 0;
713 }
714 }
715 #else
716 if (volumeCryptKey) {
717 HIBLOG("IOPolledFileOpen: unable to get volumeCryptKey\n");
718 err = kIOReturnNotFound;
719 break;
720 }
721 #endif
722
723 *fileVars = vars;
724 vars->fileExtents = extentsData;
725
726 // make imagePath
727 OSData * data = NULL;
728 if (imagePath) {
729 #if defined(__i386__) || defined(__x86_64__)
730 char str2[24 + sizeof(uuid_string_t) + 2];
731
732 if (keyUUID) {
733 snprintf(str2, sizeof(str2), "%qx:%s",
734 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
735 } else {
736 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
737 }
738
739 err = IOService::getPlatform()->callPlatformFunction(
740 gIOCreateEFIDevicePathSymbol, false,
741 (void *) part, (void *) str2,
742 (void *) (uintptr_t) true, (void *) &data);
743 #elif defined(__arm64__)
744 char str2[26];
745 snprintf(str2, sizeof(str2), "%qx:%x", vars->extentMap[0].start, swSeed);
746 data = OSData::withBytes(str2, (unsigned int) strlen(str2));
747 err = kIOReturnSuccess;
748 #else
749 err = kIOReturnNotFound;
750 #endif
751 if (kIOReturnSuccess != err) {
752 HIBLOG("error 0x%x getting path\n", err);
753 break;
754 }
755 *imagePath = data;
756 }
757
758 // Release key UUID if we have one
759 if (keyUUID) {
760 keyUUID->release();
761 keyUUID = NULL; // Just in case
762 }
763 }while (false);
764
765 if (kIOReturnSuccess != err) {
766 HIBLOG("error 0x%x opening polled file\n", err);
767 IOPolledFileClose(&vars, 0, NULL, 0, 0, 0);
768 if (extentsData) {
769 extentsData->release();
770 }
771 }
772
773 if (part) {
774 part->release();
775 }
776
777 return err;
778 }
779
780 IOReturn
781 IOPolledFileOpen(const char * filename,
782 uint32_t flags,
783 uint64_t setFileSize, uint64_t fsFreeSize,
784 void * write_file_addr, size_t write_file_len,
785 IOPolledFileIOVars ** fileVars,
786 OSSharedPtr<OSData>& imagePath,
787 uint8_t * volumeCryptKey, size_t * keySize)
788 {
789 OSData* imagePathRaw = NULL;
790 IOReturn result = IOPolledFileOpen(filename, flags, setFileSize, fsFreeSize, write_file_addr, write_file_len,
791 fileVars, &imagePathRaw, volumeCryptKey, keySize);
792 imagePath.reset(imagePathRaw, OSNoRetain);
793 return result;
794 }
795
796 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
797
798 IOReturn
799 IOPolledFileClose(IOPolledFileIOVars ** pVars,
800 off_t write_offset, void * addr, size_t write_length,
801 off_t discard_offset, off_t discard_end)
802 {
803 IOPolledFileIOVars * vars;
804
805 vars = *pVars;
806 if (!vars) {
807 return kIOReturnSuccess;
808 }
809
810 if (vars->fileRef) {
811 kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
812 discard_offset, discard_end);
813 vars->fileRef = NULL;
814 }
815 if (vars->fileExtents) {
816 vars->fileExtents->release();
817 vars->fileExtents = NULL;
818 }
819 if (vars->pollers) {
820 vars->pollers->release();
821 vars->pollers = NULL;
822 }
823
824 if (vars->allocated) {
825 IODelete(vars, IOPolledFileIOVars, 1);
826 } else {
827 bzero(vars, sizeof(IOPolledFileIOVars));
828 }
829 *pVars = NULL;
830
831 return kIOReturnSuccess;
832 }
833
834 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
835
836 IOReturn
837 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
838 uint32_t openState)
839 {
840 IOReturn err;
841
842 err = kIOReturnSuccess;
843 do{
844 if (!vars->pollers->openCount) {
845 err = IOPolledFilePollersProbe(vars->pollers);
846 if (kIOReturnSuccess != err) {
847 break;
848 }
849 }
850 err = IOPolledFilePollersOpen(vars, openState, false);
851 if (kIOReturnSuccess != err) {
852 break;
853 }
854 if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) {
855 vars->pollers->openCount++;
856 }
857 vars->pollers->io = false;
858 vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
859 vars->bufferHalf = 0;
860 vars->bufferOffset = 0;
861 assert(vars->pollers->ioBuffer->getLength() <= UINT_MAX);
862 vars->bufferSize = (typeof(vars->bufferSize))(vars->pollers->ioBuffer->getLength() >> 1);
863
864 if (vars->maxiobytes < vars->bufferSize) {
865 vars->bufferSize = (typeof(vars->bufferSize))vars->maxiobytes;
866 }
867 }while (false);
868
869 if (kIOReturnSuccess != err) {
870 HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
871 }
872
873 return err;
874 }
875
876
877 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
878
879 IOReturn
880 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
881 {
882 IOPolledFileExtent * extentMap;
883
884 extentMap = vars->extentMap;
885
886 vars->position = position;
887
888 if (position > vars->fileSize) {
889 HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
890 return kIOReturnNoSpace;
891 }
892
893 while (position >= extentMap->length) {
894 position -= extentMap->length;
895 extentMap++;
896 }
897
898 vars->currentExtent = extentMap;
899 vars->extentRemaining = extentMap->length - position;
900 vars->extentPosition = vars->position - position;
901
902 if (vars->bufferSize <= vars->extentRemaining) {
903 vars->bufferLimit = vars->bufferSize;
904 } else {
905 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
906 }
907
908 return kIOReturnSuccess;
909 }
910
911 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
912
913 IOReturn
914 IOPolledFileWrite(IOPolledFileIOVars * vars,
915 const uint8_t * bytes, IOByteCount size,
916 IOPolledFileCryptVars * cryptvars)
917 {
918 IOReturn err = kIOReturnSuccess;
919 IOByteCount copy, original_size = size;
920 bool flush = false;
921
922 do{
923 if (!bytes && !size) {
924 // seek to end of block & flush
925 size = vars->position & (vars->blockSize - 1);
926 if (size) {
927 size = vars->blockSize - size;
928 }
929 flush = true;
930 }
931
932 copy = vars->bufferLimit - vars->bufferOffset;
933 if (copy > size) {
934 copy = size;
935 } else {
936 flush = true;
937 }
938
939 if (bytes) {
940 #if KASAN
941 /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
942 * avoid triggering global redzone sanitizer violations when accessing
943 * interstices between 'C' structures
944 */
945 __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
946 #else
947 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
948 #endif
949 bytes += copy;
950 } else {
951 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
952 }
953
954 size -= copy;
955 vars->bufferOffset += copy;
956 vars->position += copy;
957
958 if (flush && vars->bufferOffset) {
959 uint64_t offset = (vars->position - vars->bufferOffset
960 - vars->extentPosition + vars->currentExtent->start);
961 uint32_t length = (vars->bufferOffset);
962
963 #if CRYPTO
964 if (cryptvars && vars->encryptStart
965 && (vars->position > vars->encryptStart)
966 && ((vars->position - length) < vars->encryptEnd)) {
967 AbsoluteTime startTime, endTime;
968
969 uint64_t encryptLen, encryptStart;
970 encryptLen = vars->position - vars->encryptStart;
971 if (encryptLen > length) {
972 encryptLen = length;
973 }
974 encryptStart = length - encryptLen;
975 if (vars->position > vars->encryptEnd) {
976 encryptLen -= (vars->position - vars->encryptEnd);
977 }
978
979 clock_get_uptime(&startTime);
980
981 assert(encryptLen <= UINT_MAX);
982 // encrypt the buffer
983 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
984 &cryptvars->aes_iv[0],
985 (unsigned int) (encryptLen / AES_BLOCK_SIZE),
986 vars->buffer + vars->bufferHalf + encryptStart,
987 &cryptvars->ctx.encrypt);
988
989 clock_get_uptime(&endTime);
990 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
991 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
992 vars->cryptBytes += encryptLen;
993
994 // save initial vector for following encrypts
995 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
996 &cryptvars->aes_iv[0],
997 AES_BLOCK_SIZE);
998 }
999 #endif /* CRYPTO */
1000
1001 err = IOPolledFilePollersIODone(vars->pollers, true);
1002 if (kIOReturnSuccess != err) {
1003 break;
1004 }
1005
1006 if (vars->position & (vars->blockSize - 1)) {
1007 HIBLOG("misaligned file pos %qx\n", vars->position);
1008 }
1009 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1010
1011 err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
1012 if (kIOReturnSuccess != err) {
1013 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
1014 vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
1015 break;
1016 }
1017 vars->pollers->io = true;
1018
1019 vars->extentRemaining -= vars->bufferOffset;
1020 if (!vars->extentRemaining) {
1021 vars->currentExtent++;
1022 vars->extentRemaining = vars->currentExtent->length;
1023 vars->extentPosition = vars->position;
1024 }
1025
1026 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1027 vars->bufferOffset = 0;
1028 if (vars->bufferSize <= vars->extentRemaining) {
1029 vars->bufferLimit = vars->bufferSize;
1030 } else {
1031 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
1032 }
1033
1034 if (!vars->extentRemaining) {
1035 err = kIOReturnOverrun;
1036 break;
1037 }
1038
1039 flush = false;
1040 }
1041 }while (size);
1042
1043 return err;
1044 }
1045
1046 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1047
1048 IOReturn
1049 IOPolledFileFlush(IOPolledFileIOVars * vars)
1050 {
1051 // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
1052 IOReturn err = kIOReturnSuccess;
1053
1054 err = IOPolledFilePollersIODone(vars->pollers, true);
1055 if (kIOReturnSuccess != err) {
1056 return err;
1057 }
1058
1059 err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
1060 if (kIOReturnSuccess != err) {
1061 HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
1062 vars, vars->pollers, err);
1063 return err;
1064 }
1065 vars->pollers->io = true;
1066
1067 return err;
1068 }
1069
1070 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1071
1072 IOReturn
1073 IOPolledFileRead(IOPolledFileIOVars * vars,
1074 uint8_t * bytes, IOByteCount size,
1075 IOPolledFileCryptVars * cryptvars)
1076 {
1077 IOReturn err = kIOReturnSuccess;
1078 IOByteCount copy;
1079
1080 // bytesWritten += size;
1081
1082 do{
1083 copy = vars->bufferLimit - vars->bufferOffset;
1084 if (copy > size) {
1085 copy = size;
1086 }
1087
1088 if (bytes) {
1089 #if KASAN
1090 __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1091 #else
1092 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1093 #endif
1094 bytes += copy;
1095 }
1096 size -= copy;
1097 vars->bufferOffset += copy;
1098 // vars->position += copy;
1099
1100 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) {
1101 if (!vars->pollers->io) {
1102 cryptvars = NULL;
1103 }
1104 err = IOPolledFilePollersIODone(vars->pollers, true);
1105 if (kIOReturnSuccess != err) {
1106 break;
1107 }
1108
1109 if (vars->position & (vars->blockSize - 1)) {
1110 HIBLOG("misaligned file pos %qx\n", vars->position);
1111 }
1112
1113 vars->position += vars->lastRead;
1114 vars->extentRemaining -= vars->lastRead;
1115 vars->bufferLimit = vars->lastRead;
1116
1117 if (!vars->extentRemaining) {
1118 vars->currentExtent++;
1119 vars->extentRemaining = vars->currentExtent->length;
1120 vars->extentPosition = vars->position;
1121 if (!vars->extentRemaining) {
1122 err = kIOReturnOverrun;
1123 break;
1124 }
1125 }
1126
1127 uint32_t length;
1128 uint32_t lastReadLength = vars->lastRead;
1129 uint64_t offset = (vars->position
1130 - vars->extentPosition + vars->currentExtent->start);
1131 if (vars->extentRemaining <= vars->bufferSize) {
1132 length = ((uint32_t) vars->extentRemaining);
1133 } else {
1134 length = vars->bufferSize;
1135 }
1136 if ((length + vars->position) > vars->readEnd) {
1137 length = ((uint32_t) (vars->readEnd - vars->position));
1138 }
1139
1140 vars->lastRead = length;
1141 if (length) {
1142 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1143 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1144 if (kIOReturnSuccess != err) {
1145 break;
1146 }
1147 vars->pollers->io = true;
1148 }
1149
1150 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1151 vars->bufferOffset = 0;
1152
1153 #if CRYPTO
1154 if (cryptvars) {
1155 uint8_t thisVector[AES_BLOCK_SIZE];
1156 AbsoluteTime startTime, endTime;
1157
1158 // save initial vector for following decrypts
1159 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1160 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1161 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1162
1163 // decrypt the buffer
1164 clock_get_uptime(&startTime);
1165
1166 assert(lastReadLength <= UINT_MAX);
1167 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1168 &thisVector[0],
1169 (unsigned int) (lastReadLength / AES_BLOCK_SIZE),
1170 vars->buffer + vars->bufferHalf,
1171 &cryptvars->ctx.decrypt);
1172
1173 clock_get_uptime(&endTime);
1174 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1175 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1176 vars->cryptBytes += lastReadLength;
1177 }
1178 #endif /* CRYPTO */
1179 }
1180 }while (size);
1181
1182 return err;
1183 }
1184
1185 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU