+
+ if(isVectorUPL && subupl_size)
+ goto process_upl_to_encrypt;
+}
+
+#else /* CRYPTO */
+void
+upl_encrypt(
+ __unused upl_t upl,
+ __unused upl_offset_t crypt_offset,
+ __unused upl_size_t crypt_size)
+{
+}
+
+void
+vm_page_encrypt(
+ __unused vm_page_t page,
+ __unused vm_map_offset_t kernel_mapping_offset)
+{
+}
+
+void
+vm_page_decrypt(
+ __unused vm_page_t page,
+ __unused vm_map_offset_t kernel_mapping_offset)
+{
+}
+
+#endif /* CRYPTO */
+
+void
+vm_pageout_queue_steal(vm_page_t page, boolean_t queues_locked)
+{
+ boolean_t pageout;
+
+ pageout = page->pageout;
+
+ page->list_req_pending = FALSE;
+ page->cleaning = FALSE;
+ page->pageout = FALSE;
+
+ if (!queues_locked) {
+ vm_page_lockspin_queues();
+ }
+
+ /*
+ * need to drop the laundry count...
+ * we may also need to remove it
+ * from the I/O paging queue...
+ * vm_pageout_throttle_up handles both cases
+ *
+ * the laundry and pageout_queue flags are cleared...
+ */
+ vm_pageout_throttle_up(page);
+
+ if (pageout == TRUE) {
+ /*
+ * toss the wire count we picked up
+ * when we intially set this page up
+ * to be cleaned...
+ */
+ vm_page_unwire(page, TRUE);
+ }
+ vm_page_steal_pageout_page++;
+
+ if (!queues_locked) {
+ vm_page_unlock_queues();
+ }
+}
+
+upl_t
+vector_upl_create(vm_offset_t upl_offset)
+{
+ int vector_upl_size = sizeof(struct _vector_upl);
+ int i=0;
+ upl_t upl;
+ vector_upl_t vector_upl = (vector_upl_t)kalloc(vector_upl_size);
+
+ upl = upl_create(0,UPL_VECTOR,0);
+ upl->vector_upl = vector_upl;
+ upl->offset = upl_offset;
+ vector_upl->size = 0;
+ vector_upl->offset = upl_offset;
+ vector_upl->invalid_upls=0;
+ vector_upl->num_upls=0;
+ vector_upl->pagelist = NULL;
+
+ for(i=0; i < MAX_VECTOR_UPL_ELEMENTS ; i++) {
+ vector_upl->upl_iostates[i].size = 0;
+ vector_upl->upl_iostates[i].offset = 0;
+
+ }
+ return upl;
+}
+
+void
+vector_upl_deallocate(upl_t upl)
+{
+ if(upl) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ if(vector_upl->invalid_upls != vector_upl->num_upls)
+ panic("Deallocating non-empty Vectored UPL\n");
+ kfree(vector_upl->pagelist,(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE)));
+ vector_upl->invalid_upls=0;
+ vector_upl->num_upls = 0;
+ vector_upl->pagelist = NULL;
+ vector_upl->size = 0;
+ vector_upl->offset = 0;
+ kfree(vector_upl, sizeof(struct _vector_upl));
+ vector_upl = (vector_upl_t)0xdeadbeef;
+ }
+ else
+ panic("vector_upl_deallocate was passed a non-vectored upl\n");
+ }
+ else
+ panic("vector_upl_deallocate was passed a NULL upl\n");
+}
+
+boolean_t
+vector_upl_is_valid(upl_t upl)
+{
+ if(upl && ((upl->flags & UPL_VECTOR)==UPL_VECTOR)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl == NULL || vector_upl == (vector_upl_t)0xdeadbeef || vector_upl == (vector_upl_t)0xfeedbeef)
+ return FALSE;
+ else
+ return TRUE;
+ }
+ return FALSE;
+}
+
+boolean_t
+vector_upl_set_subupl(upl_t upl,upl_t subupl, uint32_t io_size)
+{
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ if(subupl) {
+ if(io_size) {
+ if(io_size < PAGE_SIZE)
+ io_size = PAGE_SIZE;
+ subupl->vector_upl = (void*)vector_upl;
+ vector_upl->upl_elems[vector_upl->num_upls++] = subupl;
+ vector_upl->size += io_size;
+ upl->size += io_size;
+ }
+ else {
+ uint32_t i=0,invalid_upls=0;
+ for(i = 0; i < vector_upl->num_upls; i++) {
+ if(vector_upl->upl_elems[i] == subupl)
+ break;
+ }
+ if(i == vector_upl->num_upls)
+ panic("Trying to remove sub-upl when none exists");
+
+ vector_upl->upl_elems[i] = NULL;
+ invalid_upls = hw_atomic_add(&(vector_upl)->invalid_upls, 1);
+ if(invalid_upls == vector_upl->num_upls)
+ return TRUE;
+ else
+ return FALSE;
+ }
+ }
+ else
+ panic("vector_upl_set_subupl was passed a NULL upl element\n");
+ }
+ else
+ panic("vector_upl_set_subupl was passed a non-vectored upl\n");
+ }
+ else
+ panic("vector_upl_set_subupl was passed a NULL upl\n");
+
+ return FALSE;
+}
+
+void
+vector_upl_set_pagelist(upl_t upl)
+{
+ if(vector_upl_is_valid(upl)) {
+ uint32_t i=0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ vm_offset_t pagelist_size=0, cur_upl_pagelist_size=0;
+
+ vector_upl->pagelist = (upl_page_info_array_t)kalloc(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE));
+
+ for(i=0; i < vector_upl->num_upls; i++) {
+ cur_upl_pagelist_size = sizeof(struct upl_page_info) * vector_upl->upl_elems[i]->size/PAGE_SIZE;
+ bcopy(UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(vector_upl->upl_elems[i]), (char*)vector_upl->pagelist + pagelist_size, cur_upl_pagelist_size);
+ pagelist_size += cur_upl_pagelist_size;
+ if(vector_upl->upl_elems[i]->highest_page > upl->highest_page)
+ upl->highest_page = vector_upl->upl_elems[i]->highest_page;
+ }
+ assert( pagelist_size == (sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE)) );
+ }
+ else
+ panic("vector_upl_set_pagelist was passed a non-vectored upl\n");
+ }
+ else
+ panic("vector_upl_set_pagelist was passed a NULL upl\n");
+
+}
+
+upl_t
+vector_upl_subupl_byindex(upl_t upl, uint32_t index)
+{
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ if(index < vector_upl->num_upls)
+ return vector_upl->upl_elems[index];
+ }
+ else
+ panic("vector_upl_subupl_byindex was passed a non-vectored upl\n");
+ }
+ return NULL;
+}
+
+upl_t
+vector_upl_subupl_byoffset(upl_t upl, upl_offset_t *upl_offset, upl_size_t *upl_size)
+{
+ if(vector_upl_is_valid(upl)) {
+ uint32_t i=0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ upl_t subupl = NULL;
+ vector_upl_iostates_t subupl_state;
+
+ for(i=0; i < vector_upl->num_upls; i++) {
+ subupl = vector_upl->upl_elems[i];
+ subupl_state = vector_upl->upl_iostates[i];
+ if( *upl_offset <= (subupl_state.offset + subupl_state.size - 1)) {
+ /* We could have been passed an offset/size pair that belongs
+ * to an UPL element that has already been committed/aborted.
+ * If so, return NULL.
+ */
+ if(subupl == NULL)
+ return NULL;
+ if((subupl_state.offset + subupl_state.size) < (*upl_offset + *upl_size)) {
+ *upl_size = (subupl_state.offset + subupl_state.size) - *upl_offset;
+ if(*upl_size > subupl_state.size)
+ *upl_size = subupl_state.size;
+ }
+ if(*upl_offset >= subupl_state.offset)
+ *upl_offset -= subupl_state.offset;
+ else if(i)
+ panic("Vector UPL offset miscalculation\n");
+ return subupl;
+ }
+ }
+ }
+ else
+ panic("vector_upl_subupl_byoffset was passed a non-vectored UPL\n");
+ }
+ return NULL;
+}
+
+void
+vector_upl_get_submap(upl_t upl, vm_map_t *v_upl_submap, vm_offset_t *submap_dst_addr)
+{
+ *v_upl_submap = NULL;
+
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ *v_upl_submap = vector_upl->submap;
+ *submap_dst_addr = vector_upl->submap_dst_addr;
+ }
+ else
+ panic("vector_upl_get_submap was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_get_submap was passed a null UPL\n");
+}
+
+void
+vector_upl_set_submap(upl_t upl, vm_map_t submap, vm_offset_t submap_dst_addr)
+{
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ vector_upl->submap = submap;
+ vector_upl->submap_dst_addr = submap_dst_addr;
+ }
+ else
+ panic("vector_upl_get_submap was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_get_submap was passed a NULL UPL\n");
+}
+
+void
+vector_upl_set_iostate(upl_t upl, upl_t subupl, upl_offset_t offset, upl_size_t size)
+{
+ if(vector_upl_is_valid(upl)) {
+ uint32_t i = 0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ for(i = 0; i < vector_upl->num_upls; i++) {
+ if(vector_upl->upl_elems[i] == subupl)
+ break;
+ }
+
+ if(i == vector_upl->num_upls)
+ panic("setting sub-upl iostate when none exists");
+
+ vector_upl->upl_iostates[i].offset = offset;
+ if(size < PAGE_SIZE)
+ size = PAGE_SIZE;
+ vector_upl->upl_iostates[i].size = size;
+ }
+ else
+ panic("vector_upl_set_iostate was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_set_iostate was passed a NULL UPL\n");
+}
+
+void
+vector_upl_get_iostate(upl_t upl, upl_t subupl, upl_offset_t *offset, upl_size_t *size)
+{
+ if(vector_upl_is_valid(upl)) {
+ uint32_t i = 0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if(vector_upl) {
+ for(i = 0; i < vector_upl->num_upls; i++) {
+ if(vector_upl->upl_elems[i] == subupl)
+ break;
+ }
+
+ if(i == vector_upl->num_upls)
+ panic("getting sub-upl iostate when none exists");
+
+ *offset = vector_upl->upl_iostates[i].offset;
+ *size = vector_upl->upl_iostates[i].size;
+ }
+ else
+ panic("vector_upl_get_iostate was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_get_iostate was passed a NULL UPL\n");
+}
+
+void
+vector_upl_get_iostate_byindex(upl_t upl, uint32_t index, upl_offset_t *offset, upl_size_t *size)
+{
+ if(vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if(vector_upl) {
+ if(index < vector_upl->num_upls) {
+ *offset = vector_upl->upl_iostates[index].offset;
+ *size = vector_upl->upl_iostates[index].size;
+ }
+ else
+ *offset = *size = 0;
+ }
+ else
+ panic("vector_upl_get_iostate_byindex was passed a non-vectored UPL\n");
+ }
+ else
+ panic("vector_upl_get_iostate_byindex was passed a NULL UPL\n");
+}
+
+upl_page_info_t *
+upl_get_internal_vectorupl_pagelist(upl_t upl)
+{
+ return ((vector_upl_t)(upl->vector_upl))->pagelist;
+}
+
+void *
+upl_get_internal_vectorupl(upl_t upl)
+{
+ return upl->vector_upl;
projects / apple / xnu.git / blobdiff