+extern vm_page_t vm_pages;
+extern vm_page_t vm_page_array_beginning_addr;
+extern vm_page_t vm_page_array_ending_addr;
+
+
+
+
+struct vm_page_with_ppnum {
+ struct vm_page vm_page_with_ppnum;
+};
+typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
+
+
+#define VM_PAGE_GET_PHYS_PAGE(page) (page)->phys_page
+#define VM_PAGE_SET_PHYS_PAGE(page, ppnum) \
+ MACRO_BEGIN \
+ (page)->phys_page = ppnum; \
+ MACRO_END
+
+
+
+
+#define DEBUG_ENCRYPTED_SWAP 1
+#if DEBUG_ENCRYPTED_SWAP
+#define ASSERT_PAGE_DECRYPTED(page) \
+ MACRO_BEGIN \
+ if ((page)->encrypted) { \
+ panic("VM page %p should not be encrypted here\n", \
+ (page)); \
+ } \
+ MACRO_END
+#else /* DEBUG_ENCRYPTED_SWAP */
+#define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
+#endif /* DEBUG_ENCRYPTED_SWAP */
+
+
+
+#if defined(__LP64__)
+
+#define VM_VPLQ_ALIGNMENT 128
+#define VM_PACKED_POINTER_ALIGNMENT 64 /* must be a power of 2 */
+#define VM_PACKED_POINTER_SHIFT 6
+
+#define VM_PACKED_FROM_VM_PAGES_ARRAY 0x80000000
+
+static inline vm_page_packed_t vm_page_pack_ptr(uintptr_t p)
+{
+ vm_page_packed_t packed_ptr;
+
+ if (!p)
+ return ((vm_page_packed_t)0);
+
+ if (p >= (uintptr_t)(vm_page_array_beginning_addr) && p < (uintptr_t)(vm_page_array_ending_addr)) {
+ packed_ptr = ((vm_page_packed_t)(((vm_page_t)p - vm_page_array_beginning_addr)));
+ assert(! (packed_ptr & VM_PACKED_FROM_VM_PAGES_ARRAY));
+ packed_ptr |= VM_PACKED_FROM_VM_PAGES_ARRAY;
+ return packed_ptr;
+ }
+
+ assert((p & (VM_PACKED_POINTER_ALIGNMENT - 1)) == 0);
+
+ packed_ptr = ((vm_page_packed_t)(((uintptr_t)(p - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)) >> VM_PACKED_POINTER_SHIFT));
+ assert(packed_ptr != 0);
+ assert(! (packed_ptr & VM_PACKED_FROM_VM_PAGES_ARRAY));
+ return packed_ptr;
+}
+
+
+static inline uintptr_t vm_page_unpack_ptr(uintptr_t p)
+{
+ if (!p)
+ return ((uintptr_t)0);
+
+ if (p & VM_PACKED_FROM_VM_PAGES_ARRAY)
+ return ((uintptr_t)(&vm_pages[(uint32_t)(p & ~VM_PACKED_FROM_VM_PAGES_ARRAY)]));
+ return (((p << VM_PACKED_POINTER_SHIFT) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS));
+}
+
+
+#define VM_PAGE_PACK_PTR(p) vm_page_pack_ptr((uintptr_t)(p))
+#define VM_PAGE_UNPACK_PTR(p) vm_page_unpack_ptr((uintptr_t)(p))
+
+#define VM_PAGE_OBJECT(p) ((vm_object_t)(VM_PAGE_UNPACK_PTR(p->vm_page_object)))
+#define VM_PAGE_PACK_OBJECT(o) ((vm_page_object_t)(VM_PAGE_PACK_PTR(o)))
+
+
+#define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
+MACRO_BEGIN \
+ (p)->snext = 0; \
+MACRO_END
+
+
+#define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) VM_PAGE_PACK_PTR(p)
+
+
+static __inline__ void
+vm_page_enqueue_tail(
+ vm_page_queue_t que,
+ vm_page_queue_entry_t elt)
+{
+ vm_page_queue_entry_t old_tail;
+
+ old_tail = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(que->prev);
+ elt->next = VM_PAGE_PACK_PTR(que);
+ elt->prev = que->prev;
+ old_tail->next = VM_PAGE_PACK_PTR(elt);
+ que->prev = VM_PAGE_PACK_PTR(elt);
+}
+
+
+static __inline__ void
+vm_page_remque(
+ vm_page_queue_entry_t elt)
+{
+ vm_page_queue_entry_t next_elt, prev_elt;
+
+ next_elt = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(elt->next);
+
+ /* next_elt may equal prev_elt (and the queue head) if elt was the only element */
+ prev_elt = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(elt->prev);
+
+ next_elt->prev = VM_PAGE_PACK_PTR(prev_elt);
+ prev_elt->next = VM_PAGE_PACK_PTR(next_elt);
+
+ elt->next = 0;
+ elt->prev = 0;
+}
+
+
+/*
+ * Macro: vm_page_queue_init
+ * Function:
+ * Initialize the given queue.
+ * Header:
+ * void vm_page_queue_init(q)
+ * vm_page_queue_t q; \* MODIFIED *\
+ */
+#define vm_page_queue_init(q) \
+MACRO_BEGIN \
+ assert((((uintptr_t)q) & (VM_PACKED_POINTER_ALIGNMENT-1)) == 0); \
+ assert((VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR((uintptr_t)q))) == (uintptr_t)q); \
+ (q)->next = VM_PAGE_PACK_PTR(q); \
+ (q)->prev = VM_PAGE_PACK_PTR(q); \
+MACRO_END
+
+
+/*
+ * Macro: vm_page_queue_enter
+ * Function:
+ * Insert a new element at the tail of the queue.
+ * Header:
+ * void vm_page_queue_enter(q, elt, type, field)
+ * queue_t q;
+ * <type> elt;
+ * <type> is what's in our queue
+ * <field> is the chain field in (*<type>)
+ * Note:
+ * This should only be used with Method 2 queue iteration (element chains)
+ */
+#define vm_page_queue_enter(head, elt, type, field) \
+MACRO_BEGIN \
+ vm_page_queue_entry_t __prev; \
+ \
+ __prev = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((head)->prev)); \
+ if ((head) == __prev) { \
+ (head)->next = VM_PAGE_PACK_PTR(elt); \
+ } \
+ else { \
+ ((type)(void *)__prev)->field.next = VM_PAGE_PACK_PTR(elt); \
+ } \
+ (elt)->field.prev = VM_PAGE_PACK_PTR(__prev); \
+ (elt)->field.next = VM_PAGE_PACK_PTR(head); \
+ (head)->prev = VM_PAGE_PACK_PTR(elt); \
+MACRO_END
+
+
+/*
+ * Macro: vm_page_queue_enter_first
+ * Function:
+ * Insert a new element at the head of the queue.
+ * Header:
+ * void queue_enter_first(q, elt, type, field)
+ * queue_t q;
+ * <type> elt;
+ * <type> is what's in our queue
+ * <field> is the chain field in (*<type>)
+ * Note:
+ * This should only be used with Method 2 queue iteration (element chains)
+ */
+#define vm_page_queue_enter_first(head, elt, type, field) \
+MACRO_BEGIN \
+ vm_page_queue_entry_t __next; \
+ \
+ __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((head)->next)); \
+ if ((head) == __next) { \
+ (head)->prev = VM_PAGE_PACK_PTR(elt); \
+ } \
+ else { \
+ ((type)(void *)__next)->field.prev = VM_PAGE_PACK_PTR(elt); \
+ } \
+ (elt)->field.next = VM_PAGE_PACK_PTR(__next); \
+ (elt)->field.prev = VM_PAGE_PACK_PTR(head); \
+ (head)->next = VM_PAGE_PACK_PTR(elt); \
+MACRO_END
+
+
+/*
+ * Macro: vm_page_queue_remove
+ * Function:
+ * Remove an arbitrary item from the queue.
+ * Header:
+ * void vm_page_queue_remove(q, qe, type, field)
+ * arguments as in vm_page_queue_enter
+ * Note:
+ * This should only be used with Method 2 queue iteration (element chains)
+ */
+#define vm_page_queue_remove(head, elt, type, field) \
+MACRO_BEGIN \
+ vm_page_queue_entry_t __next, __prev; \
+ \
+ __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((elt)->field.next)); \
+ __prev = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((elt)->field.prev)); \
+ \
+ if ((head) == __next) \
+ (head)->prev = VM_PAGE_PACK_PTR(__prev); \
+ else \
+ ((type)(void *)__next)->field.prev = VM_PAGE_PACK_PTR(__prev); \
+ \
+ if ((head) == __prev) \
+ (head)->next = VM_PAGE_PACK_PTR(__next); \
+ else \
+ ((type)(void *)__prev)->field.next = VM_PAGE_PACK_PTR(__next); \
+ \
+ (elt)->field.next = 0; \
+ (elt)->field.prev = 0; \
+MACRO_END
+
+
+/*
+ * Macro: vm_page_queue_remove_first
+ * Function:
+ * Remove and return the entry at the head of
+ * the queue.
+ * Header:
+ * vm_page_queue_remove_first(head, entry, type, field)
+ * entry is returned by reference
+ * Note:
+ * This should only be used with Method 2 queue iteration (element chains)
+ */
+#define vm_page_queue_remove_first(head, entry, type, field) \
+MACRO_BEGIN \
+ vm_page_queue_entry_t __next; \
+ \
+ (entry) = (type)(void *) VM_PAGE_UNPACK_PTR(((head)->next)); \
+ __next = ((vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR((entry)->field.next)); \
+ \
+ if ((head) == __next) \
+ (head)->prev = VM_PAGE_PACK_PTR(head); \
+ else \
+ ((type)(void *)(__next))->field.prev = VM_PAGE_PACK_PTR(head); \
+ (head)->next = VM_PAGE_PACK_PTR(__next); \
+ \
+ (entry)->field.next = 0; \
+ (entry)->field.prev = 0; \
+MACRO_END
+
+
+/*
+ * Macro: vm_page_queue_end
+ * Function:
+ * Tests whether a new entry is really the end of
+ * the queue.
+ * Header:
+ * boolean_t vm_page_queue_end(q, qe)
+ * vm_page_queue_t q;
+ * vm_page_queue_entry_t qe;
+ */
+#define vm_page_queue_end(q, qe) ((q) == (qe))
+
+
+/*
+ * Macro: vm_page_queue_empty
+ * Function:
+ * Tests whether a queue is empty.
+ * Header:
+ * boolean_t vm_page_queue_empty(q)
+ * vm_page_queue_t q;
+ */
+#define vm_page_queue_empty(q) vm_page_queue_end((q), ((vm_page_queue_entry_t)vm_page_queue_first(q)))
+
+
+
+/*
+ * Macro: vm_page_queue_first
+ * Function:
+ * Returns the first entry in the queue,
+ * Header:
+ * uintpr_t vm_page_queue_first(q)
+ * vm_page_queue_t q; \* IN *\
+ */
+#define vm_page_queue_first(q) (VM_PAGE_UNPACK_PTR((q)->next))
+
+
+
+/*
+ * Macro: vm_page_queue_last
+ * Function:
+ * Returns the last entry in the queue.
+ * Header:
+ * vm_page_queue_entry_t queue_last(q)
+ * queue_t q; \* IN *\
+ */
+#define vm_page_queue_last(q) (VM_PAGE_UNPACK_PTR((q)->prev))
+
projects / apple / xnu.git / blobdiff