- vm_object_t object = NULL;
- vm_object_t shadow_object = NULL;
- vm_object_offset_t offset;
- vm_object_offset_t shadow_offset;
- vm_object_offset_t target_offset;
- upl_size_t i;
- wpl_array_t lite_list;
- vm_page_t t,m;
- int occupied;
- boolean_t shadow_internal;
-
- if (upl == UPL_NULL)
- return KERN_INVALID_ARGUMENT;
-
- if (upl->flags & UPL_IO_WIRE) {
- boolean_t empty;
- return upl_commit_range(upl,
- 0, upl->size, 0,
- NULL, 0, &empty);
- }
-
- upl_lock(upl);
- if(upl->flags & UPL_DEVICE_MEMORY) {
- upl_unlock(upl);
- return KERN_SUCCESS;
- }
-
- object = upl->map_object;
-
- if (object == NULL) {
- panic("upl_abort: upl object is not backed by an object");
- upl_unlock(upl);
- return KERN_INVALID_ARGUMENT;
- }
-
- if(object->pageout) {
- shadow_object = object->shadow;
- shadow_offset = object->shadow_offset;
- } else {
- shadow_object = object;
- shadow_offset = upl->offset - object->paging_offset;
- }
-
- if(upl->flags & UPL_INTERNAL) {
- lite_list = (wpl_array_t)
- ((((uintptr_t)upl) + sizeof(struct upl))
- + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
- } else {
- lite_list = (wpl_array_t)
- (((uintptr_t)upl) + sizeof(struct upl));
- }
- offset = 0;
-
- if (object != shadow_object)
- vm_object_lock(object);
- vm_object_lock(shadow_object);
-
- shadow_internal = shadow_object->internal;
-
- for(i = 0; i<(upl->size); i+=PAGE_SIZE, offset += PAGE_SIZE_64) {
- m = VM_PAGE_NULL;
- target_offset = offset + shadow_offset;
- if(upl->flags & UPL_LITE) {
- int pg_num;
- pg_num = offset/PAGE_SIZE;
- if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
- lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
- m = vm_page_lookup(
- shadow_object, target_offset);
- }
- }
- if(object->pageout) {
- if ((t = vm_page_lookup(object, offset)) != NULL) {
- t->pageout = FALSE;
- VM_PAGE_FREE(t);
- if(m == NULL) {
- m = vm_page_lookup(
- shadow_object, target_offset);
- }
- if(m != VM_PAGE_NULL)
- vm_object_paging_end(m->object);
- }
- }
- if(m != VM_PAGE_NULL) {
- vm_page_lock_queues();
- if(m->absent) {
- boolean_t must_free = TRUE;
-
- /* COPYOUT = FALSE case */
- /* check for error conditions which must */
- /* be passed back to the pages customer */
- if(error & UPL_ABORT_RESTART) {
- m->restart = TRUE;
- m->absent = FALSE;
- vm_object_absent_release(m->object);
- m->page_error = KERN_MEMORY_ERROR;
- m->error = TRUE;
- must_free = FALSE;
- } else if(error & UPL_ABORT_UNAVAILABLE) {
- m->restart = FALSE;
- m->unusual = TRUE;
- must_free = FALSE;
- } else if(error & UPL_ABORT_ERROR) {
- m->restart = FALSE;
- m->absent = FALSE;
- vm_object_absent_release(m->object);
- m->page_error = KERN_MEMORY_ERROR;
- m->error = TRUE;
- must_free = FALSE;
- }
-
- /*
- * ENCRYPTED SWAP:
- * If the page was already encrypted,
- * we don't really need to decrypt it
- * now. It will get decrypted later,
- * on demand, as soon as someone needs
- * to access its contents.
- */
-
- m->cleaning = FALSE;
- m->overwriting = FALSE;
- PAGE_WAKEUP_DONE(m);
-
- if (must_free == TRUE) {
- vm_page_free(m);
- } else {
- vm_page_activate(m);
- }
- vm_page_unlock_queues();
- continue;
- }
- /*
- * Handle the trusted pager throttle.
- */
- if (m->laundry) {
- vm_pageout_throttle_up(m);
- }
- if(m->pageout) {
- assert(m->busy);
- assert(m->wire_count == 1);
- m->pageout = FALSE;
- vm_page_unwire(m);
- }
- m->dump_cleaning = FALSE;
- m->cleaning = FALSE;
- m->overwriting = FALSE;
-#if MACH_PAGEMAP
- vm_external_state_clr(
- m->object->existence_map, m->offset);
-#endif /* MACH_PAGEMAP */
- if(error & UPL_ABORT_DUMP_PAGES) {
- vm_page_free(m);
- pmap_disconnect(m->phys_page);
- } else {
- PAGE_WAKEUP_DONE(m);
- }
- vm_page_unlock_queues();
- }
- }
- occupied = 1;
- if (upl->flags & UPL_DEVICE_MEMORY) {
- occupied = 0;
- } else if (upl->flags & UPL_LITE) {
- int pg_num;
- int j;
- pg_num = upl->size/PAGE_SIZE;
- pg_num = (pg_num + 31) >> 5;
- occupied = 0;
- for(j= 0; j<pg_num; j++) {
- if(lite_list[j] != 0) {
- occupied = 1;
- break;
- }
- }
- } else {
- if(queue_empty(&upl->map_object->memq)) {
- occupied = 0;
- }
- }
-
- if(occupied == 0) {
- if(object == shadow_object)
- vm_object_paging_end(shadow_object);
- }
- vm_object_unlock(shadow_object);
- if (object != shadow_object)
- vm_object_unlock(object);
-
- upl_unlock(upl);
- return KERN_SUCCESS;
-}
-
-/* an option on commit should be wire */
-kern_return_t
-upl_commit(
- upl_t upl,
- upl_page_info_t *page_list,
- mach_msg_type_number_t count)
-{
- if (upl == UPL_NULL)
- return KERN_INVALID_ARGUMENT;
-
- if(upl->flags & (UPL_LITE | UPL_IO_WIRE)) {
- boolean_t empty;
- return upl_commit_range(upl, 0, upl->size, 0,
- page_list, count, &empty);
- }
-
- if (count == 0)
- page_list = NULL;
-
- upl_lock(upl);
- if (upl->flags & UPL_DEVICE_MEMORY)
- page_list = NULL;
-
- if (upl->flags & UPL_ENCRYPTED) {
- /*
- * ENCRYPTED SWAP:
- * This UPL was encrypted, but we don't need
- * to decrypt here. We'll decrypt each page
- * later, on demand, as soon as someone needs
- * to access the page's contents.
- */
- }
-
- if ((upl->flags & UPL_CLEAR_DIRTY) ||
- (upl->flags & UPL_PAGE_SYNC_DONE) || page_list) {
- vm_object_t shadow_object = upl->map_object->shadow;
- vm_object_t object = upl->map_object;
- vm_object_offset_t target_offset;
- upl_size_t xfer_end;
- int entry;
-
- vm_page_t t, m;
- upl_page_info_t *p;
-
- if (object != shadow_object)
- vm_object_lock(object);
- vm_object_lock(shadow_object);
-
- entry = 0;
- target_offset = object->shadow_offset;
- xfer_end = upl->size + object->shadow_offset;
-
- while(target_offset < xfer_end) {
-
- if ((t = vm_page_lookup(object,
- target_offset - object->shadow_offset))
- == NULL) {
- target_offset += PAGE_SIZE_64;
- entry++;
- continue;
- }
-
- m = vm_page_lookup(shadow_object, target_offset);
- if(m != VM_PAGE_NULL) {
- /*
- * ENCRYPTED SWAP:
- * If this page was encrypted, we
- * don't need to decrypt it here.
- * We'll decrypt it later, on demand,
- * as soon as someone needs to access
- * its contents.
- */
-
- if (upl->flags & UPL_CLEAR_DIRTY) {
- pmap_clear_modify(m->phys_page);
- m->dirty = FALSE;
- }
- /* It is a part of the semantic of */
- /* COPYOUT_FROM UPLs that a commit */
- /* implies cache sync between the */
- /* vm page and the backing store */
- /* this can be used to strip the */
- /* precious bit as well as clean */
- if (upl->flags & UPL_PAGE_SYNC_DONE)
- m->precious = FALSE;
-
- if(page_list) {
- p = &(page_list[entry]);
- if(page_list[entry].phys_addr &&
- p->pageout && !m->pageout) {
- vm_page_lock_queues();
- m->busy = TRUE;
- m->pageout = TRUE;
- vm_page_wire(m);
- vm_page_unlock_queues();
- } else if (page_list[entry].phys_addr &&
- !p->pageout && m->pageout &&
- !m->dump_cleaning) {
- vm_page_lock_queues();
- m->pageout = FALSE;
- m->absent = FALSE;
- m->overwriting = FALSE;
- vm_page_unwire(m);
- PAGE_WAKEUP_DONE(m);
- vm_page_unlock_queues();
- }
- page_list[entry].phys_addr = 0;
- }
- }
- target_offset += PAGE_SIZE_64;
- entry++;
- }
- vm_object_unlock(shadow_object);
- if (object != shadow_object)
- vm_object_unlock(object);
-
- }
- if (upl->flags & UPL_DEVICE_MEMORY) {
- vm_object_lock(upl->map_object->shadow);
- if(upl->map_object == upl->map_object->shadow)
- vm_object_paging_end(upl->map_object->shadow);
- vm_object_unlock(upl->map_object->shadow);
- }
- upl_unlock(upl);
- return KERN_SUCCESS;
projects / apple / xnu.git / blobdiff