2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
23 #include <sys/errno.h>
24 #include <kern/host.h>
25 #include <mach/mach_types.h>
26 #include <vm/vm_map.h>
27 #include <vm/vm_kern.h>
28 #include <vm/vm_pageout.h>
29 #include <mach/kern_return.h>
30 #include <mach/memory_object_types.h>
31 #include <mach/port.h>
32 #include <mach/policy.h>
33 #include <ipc/ipc_port.h>
34 #include <ipc/ipc_space.h>
35 #include <kern/thread.h>
36 #include <vm/vm_pageout.h>
39 /* BSD VM COMPONENT INTERFACES */
59 return(map
->hdr
.nentries
);
69 return(vm_map_first_entry(map
)->vme_start
);
79 return(vm_map_last_entry(map
)->vme_end
);
86 /* until component support available */
87 int vnode_pager_workaround
;
89 typedef int vnode_port_t
;
91 typedef struct vnode_pager
{
92 ipc_port_t pager
; /* pager */
93 ipc_port_t pager_handle
; /* pager handle */
94 ipc_port_t vm_obj_handle
; /* memory object's control handle */
95 vnode_port_t vnode_handle
; /* vnode handle */
98 typedef struct vnode_port_entry
{
99 queue_chain_t links
; /* queue links */
100 ipc_port_t name
; /* port name */
101 vnode_pager_t pager_rec
; /* pager record */
102 } *vnode_port_entry_t
;
105 #define VNODE_PORT_HASH_COUNT 127
106 #define vnode_port_hash(name_port) \
107 (((int)(name_port) & 0xffffff) % VNODE_PORT_HASH_COUNT)
109 queue_head_t vnode_port_hashtable
[VNODE_PORT_HASH_COUNT
];
110 zone_t vnode_port_hash_zone
;
111 decl_simple_lock_data(,vnode_port_hash_lock
)
115 trigger_name_to_port(
119 vnode_pager_bootstrap(
123 vnode_pager_alloc_map(
143 vnode_pager_data_request(
151 vnode_pager_data_return(
161 vnode_pager_no_senders(
163 mach_port_mscount_t
);
166 vnode_pager_terminate(
171 vnode_pager_cluster_read(
177 vnode_pager_cluster_write(
183 memory_object_change_attributes(
185 memory_object_flavor_t
,
186 memory_object_info_t
,
187 mach_msg_type_number_t
,
189 mach_msg_type_name_t
);
215 vnode_port_hash_init(void);
218 vnode_port_hash_insert(
223 vnode_port_hash_lookup(
227 vnode_port_hash_delete(
231 vnode_pager_release_from_cache(
234 zone_t vnode_pager_zone
;
237 #define VNODE_PAGER_NULL ((vnode_pager_t) 0)
239 /* TODO: Should be set dynamically by vnode_pager_init() */
240 #define CLUSTER_SHIFT 1
242 /* TODO: Should be set dynamically by vnode_pager_bootstrap() */
243 #define MAX_VNODE 10000
249 #define PAGER_ALL 0xffffffff
250 #define PAGER_INIT 0x00000001
251 #define PAGER_PAGEIN 0x00000002
253 #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
255 #define PAGER_DEBUG(LEVEL, A)
259 * Routine: macx_triggers
261 * Syscall interface to set the call backs for low and
269 mach_port_t trigger_name
)
272 ipc_port_t default_pager_port
= MACH_PORT_NULL
;
273 ipc_port_t trigger_port
;
275 kr
= host_default_memory_manager(host_priv_self(),
276 &default_pager_port
, 0);
277 if(kr
!= KERN_SUCCESS
) {
280 trigger_port
= trigger_name_to_port(trigger_name
);
281 if(trigger_port
== NULL
) {
284 /* trigger_port is locked and active */
285 ip_unlock(trigger_port
);
286 default_pager_triggers(default_pager_port
,
287 hi_water
, low_water
, flags
, trigger_port
);
288 ipc_port_make_send(trigger_port
);
291 * Set thread scheduling priority and policy for the current thread
292 * it is assumed for the time being that the thread setting the alert
293 * is the same one which will be servicing it.
296 struct policy_timeshare_base fifo_base
;
297 struct policy_timeshare_limit fifo_limit
;
299 processor_set_t pset
;
300 policy_limit_t limit
;
302 pset
= (current_thread())->processor_set
;
303 base
= (policy_base_t
) &fifo_base
;
304 limit
= (policy_limit_t
) &fifo_limit
;
305 fifo_limit
.max_priority
= fifo_base
.base_priority
= MAXPRI_STANDARD
;
306 thread_set_policy((current_thread())->top_act
, pset
, POLICY_FIFO
, base
, POLICY_TIMESHARE_BASE_COUNT
, limit
, POLICY_TIMESHARE_LIMIT_COUNT
);
309 current_thread()->vm_privilege
= TRUE
;
316 trigger_name_to_port(
317 mach_port_t trigger_name
)
319 ipc_port_t trigger_port
;
322 if (trigger_name
== 0)
325 space
= current_space();
326 if(ipc_port_translate_receive(space
, (mach_port_name_t
)trigger_name
,
327 &trigger_port
) != KERN_SUCCESS
)
336 vnode_pager_bootstrap(void)
338 register vm_size_t size
;
340 size
= (vm_size_t
) sizeof(struct vnode_pager
);
341 vnode_pager_zone
= zinit(size
, (vm_size_t
) MAX_VNODE
*size
,
342 PAGE_SIZE
, "vnode pager structures");
343 vnode_port_hash_init();
356 vnode_pager_t vnode_object
;
361 (vnode_object
= vnode_port_hash_lookup(pager
))) {
362 if (vnode_object
->vnode_handle
== vp
)
366 vnode_object
= vnode_object_create(vp
);
367 if (vnode_object
== VNODE_PAGER_NULL
)
368 panic("vnode_pager_setup: vnode_object_create() failed");
370 vnode_object
->pager
= ipc_port_alloc_kernel();
371 assert (vnode_object
->pager
!= IP_NULL
);
372 pager_mux_hash_insert(vnode_object
->pager
,
373 (rpc_subsystem_t
)&vnode_pager_workaround
);
375 vnode_object
->pager_handle
= ipc_port_make_send(vnode_object
->pager
);
377 vnode_port_hash_insert(vnode_object
->pager_handle
, vnode_object
);
379 ipc_port_make_sonce(vnode_object
->pager
);
380 ip_lock(vnode_object
->pager
); /* unlocked in nsrequest below */
381 ipc_port_nsrequest(vnode_object
->pager
, 1, vnode_object
->pager
, &previous
);
383 PAGER_DEBUG(PAGER_INIT
, ("vnode_pager_setup: vp %x pager %x vnode_pager %x\n", vp
, vnode_object
->pager_handle
, vnode_object
));
385 ubc_setpager( vp
, vnode_object
->pager_handle
);
386 return(vnode_object
->pager_handle
);
397 vnode_pager_t vnode_object
;
401 (vnode_object
= vnode_port_hash_lookup(pager
))) {
402 if (vnode_object
->vnode_handle
== vp
)
403 return(vnode_object
->vm_obj_handle
);
415 vnode_pager_init(ipc_port_t pager
,
416 ipc_port_t pager_request
,
419 vnode_pager_t vnode_object
;
421 memory_object_attr_info_data_t attributes
;
422 vm_object_t vm_object
;
425 PAGER_DEBUG(PAGER_ALL
, ("vnode_pager_init: %x, %x, %x\n", pager
, pager_request
, pg_size
));
427 vnode_object
= vnode_port_hash_lookup(pager
);
428 if (vnode_object
== VNODE_PAGER_NULL
)
429 panic("vnode_pager_init: lookup failed");
431 vnode_object
->vm_obj_handle
= pager_request
;
433 vm_object
= vm_object_lookup(pager_request
);
435 if (vm_object
== VM_OBJECT_NULL
)
436 panic("vnode_pager_init: vm_object_lookup() failed");
438 attributes
.copy_strategy
= MEMORY_OBJECT_COPY_DELAY
;
439 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
440 attributes
.cluster_size
= (1 << (PAGE_SHIFT
));
441 attributes
.may_cache_object
= TRUE
;
442 attributes
.temporary
= TRUE
;
444 kr
= memory_object_change_attributes(
446 MEMORY_OBJECT_ATTRIBUTE_INFO
,
447 (memory_object_info_t
) &attributes
,
448 MEMORY_OBJECT_ATTR_INFO_COUNT
,
450 if (kr
!= KERN_SUCCESS
)
451 panic("vnode_pager_init: memory_object_change_attributes() failed");
453 return(KERN_SUCCESS
);
460 vnode_pager_data_return(
462 ipc_port_t control_port
,
463 vm_object_offset_t offset
,
467 boolean_t kernel_copy
)
469 register vnode_pager_t vnode_object
;
471 vnode_object
= vnode_port_hash_lookup(mem_obj
);
472 if (vnode_object
== VNODE_PAGER_NULL
)
473 panic("vnode_pager_data_return: lookup failed");
475 vnode_pager_cluster_write(vnode_object
, offset
, data_cnt
);
484 vnode_pager_data_request(
486 ipc_port_t mem_obj_control
,
487 vm_object_offset_t offset
,
489 vm_prot_t protection_required
)
491 register vnode_pager_t vnode_object
;
493 PAGER_DEBUG(PAGER_ALL
, ("vnode_pager_data_request: %x, %x, %x, %x, %x\n", mem_obj
, mem_obj_control
, offset
, length
, protection_required
));
495 vnode_object
= vnode_port_hash_lookup(mem_obj
);
497 PAGER_DEBUG(PAGER_PAGEIN
, ("vnode_pager_data_request: %x, %x, %x, %x, %x, vnode_object %x\n", mem_obj
, mem_obj_control
, offset
, length
, protection_required
, vnode_object
));
499 if (vnode_object
== VNODE_PAGER_NULL
)
500 panic("vnode_pager_data_request: lookup failed");
502 vnode_pager_cluster_read(vnode_object
, offset
, length
);
511 vnode_pager_no_senders(
513 mach_port_mscount_t mscount
)
515 register vnode_pager_t vnode_object
;
517 PAGER_DEBUG(PAGER_ALL
, ("vnode_pager_nosenders: %x, %x\n", mem_obj
, mscount
));
519 vnode_object
= vnode_port_hash_lookup(mem_obj
);
520 if (vnode_object
== VNODE_PAGER_NULL
)
521 panic("vnode_pager_no_senders: lookup failed");
523 assert(vnode_object
->pager_handle
== mem_obj
);
525 pager_mux_hash_delete((ipc_port_t
) vnode_object
->pager_handle
);
526 ipc_port_dealloc_kernel(vnode_object
->pager
);
527 vnode_port_hash_delete(vnode_object
->pager_handle
);
528 if (vnode_object
->vnode_handle
!= (vnode_port_t
) NULL
) {
529 vnode_pager_vrele(vnode_object
->vnode_handle
);
531 zfree(vnode_pager_zone
, (vm_offset_t
) vnode_object
);
540 vnode_pager_terminate(
542 ipc_port_t mem_obj_control
)
544 register vnode_pager_t vnode_object
;
546 PAGER_DEBUG(PAGER_ALL
, ("vnode_pager_terminate: %x, %x\n", mem_obj
, mem_obj_control
));
548 vnode_object
= vnode_port_hash_lookup(mem_obj
);
549 if (vnode_object
== VNODE_PAGER_NULL
)
550 panic("vnode_pager_terminate: lookup failed");
552 assert(vnode_object
->pager_handle
== mem_obj
);
554 /* release extra send right created by the fact that the caller */
555 /* of vnode_pager_setup does not establish a mapping between a */
556 /* cache object and the mem_obj (AMO). When a subsequent vm_map */
557 /* is done, vm_map will bump the send right count */
558 ipc_port_release_send(mem_obj
);
560 /* release a send right because terminate is called directly and */
561 /* not through IPC, the right won't disappear quietly */
562 ipc_port_release_send(mem_obj
);
564 ipc_port_dealloc_kernel(mem_obj_control
);
566 return(KERN_SUCCESS
);
573 vnode_pager_synchronize(
575 ipc_port_t pager_request
,
576 vm_object_offset_t offset
,
578 vm_sync_t sync_flags
)
580 memory_object_synchronize_completed(vm_object_lookup(pager_request
), offset
, length
);
582 return (KERN_SUCCESS
);
589 vnode_pager_cluster_write(
590 vnode_pager_t vnode_object
,
591 vm_object_offset_t offset
,
599 if (cnt
& PAGE_MASK
) {
600 panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
602 size
= (cnt
< (PAGE_SIZE
*32)) ? cnt
: (PAGE_SIZE
*32); /* effective min */
606 kret
= vnode_pageout(vnode_object
->vnode_handle
, (upl_t
)NULL
, (vm_offset_t
)NULL
, offset
, size
, 0, &local_error
);
608 if (local_error
!= 0) {
617 return(KERN_FAILURE
);
619 return(KERN_SUCCESS
);
628 vnode_pager_cluster_read(
629 vnode_pager_t vnode_object
,
630 vm_object_offset_t offset
,
638 if(cnt
& PAGE_MASK
) {
639 panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
646 kret
= vnode_pagein(vnode_object
->vnode_handle
, (upl_t
)NULL
, (vm_offset_t
)NULL
, offset
, size
, 0, &local_error
);
648 if (local_error
!= 0) {
656 return(KERN_FAILURE
);
658 return(KERN_SUCCESS
);
667 vnode_pager_release_from_cache(
670 memory_object_free_from_cache(
671 &realhost
, (int)&vnode_pager_workaround
, cnt
);
681 register vnode_pager_t vnode_object
;
683 vnode_object
= (struct vnode_pager
*) zalloc(vnode_pager_zone
);
684 if (vnode_object
== VNODE_PAGER_NULL
)
685 return(VNODE_PAGER_NULL
);
686 vnode_object
->pager_handle
= IP_NULL
;
687 vnode_object
->vm_obj_handle
= IP_NULL
;
688 vnode_object
->vnode_handle
= vp
;
690 return(vnode_object
);
697 vnode_port_hash_init(void)
699 register vm_size_t size
;
703 size
= (vm_size_t
) sizeof(struct vnode_port_entry
);
705 vnode_port_hash_zone
= zinit(size
,
706 (vm_size_t
) MAX_VNODE
* size
,
707 PAGE_SIZE
, "vnode_pager port hash");
709 for (i
= 0; i
< VNODE_PORT_HASH_COUNT
; i
++)
710 queue_init(&vnode_port_hashtable
[i
]);
712 simple_lock_init(&vnode_port_hash_lock
,ETAP_NO_TRACE
);
719 vnode_port_hash_insert(
720 ipc_port_t name_port
,
723 register vnode_port_entry_t new_entry
;
725 new_entry
= (vnode_port_entry_t
) zalloc(vnode_port_hash_zone
);
727 * TODO: Delete the following check once MAX_VNODE is removed
730 panic("vnode_port_hash_insert: no space");
731 new_entry
->name
= name_port
;
732 new_entry
->pager_rec
= rec
;
734 simple_lock(&vnode_port_hash_lock
);
735 queue_enter(&vnode_port_hashtable
[vnode_port_hash(name_port
)],
736 new_entry
, vnode_port_entry_t
, links
);
737 simple_unlock(&vnode_port_hash_lock
);
744 vnode_port_hash_lookup(
745 ipc_port_t name_port
)
747 register queue_t bucket
;
748 register vnode_port_entry_t entry
;
751 bucket
= (queue_t
) &vnode_port_hashtable
[vnode_port_hash(name_port
)];
753 simple_lock(&vnode_port_hash_lock
);
754 entry
= (vnode_port_entry_t
) queue_first(bucket
);
755 while (!queue_end(bucket
,&entry
->links
)) {
756 if (entry
->name
== name_port
) {
757 rec
= entry
->pager_rec
;
758 simple_unlock(&vnode_port_hash_lock
);
761 entry
= (vnode_port_entry_t
)queue_next(&entry
->links
);
763 simple_unlock(&vnode_port_hash_lock
);
764 return(VNODE_PAGER_NULL
);
771 vnode_port_hash_delete(
772 ipc_port_t name_port
)
774 register queue_t bucket
;
775 register vnode_port_entry_t entry
;
777 bucket
= (queue_t
) &vnode_port_hashtable
[vnode_port_hash(name_port
)];
779 simple_lock(&vnode_port_hash_lock
);
780 entry
= (vnode_port_entry_t
) queue_first(bucket
);
781 while (!queue_end(bucket
,&entry
->links
)) {
782 if (entry
->name
== name_port
) {
783 queue_remove(bucket
, entry
, vnode_port_entry_t
,links
);
784 simple_unlock(&vnode_port_hash_lock
);
785 zfree(vnode_port_hash_zone
, (vm_offset_t
) entry
);
788 entry
= (vnode_port_entry_t
)queue_next(&entry
->links
);
790 simple_unlock(&vnode_port_hash_lock
);