osfmk/default_pager/default_pager_internal.h

   1 /*
   2  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56
  57 /*
  58  *      Default pager.
  59  *              General definitions.
  60  */
  61
  62 #ifndef _DEFAULT_PAGER_INTERNAL_H_
  63 #define _DEFAULT_PAGER_INTERNAL_H_
  64
  65 #include <default_pager/diag.h>
  66 #include <default_pager/default_pager_types.h>
  67 #include <mach/mach_types.h>
  68 #include <ipc/ipc_port.h>
  69 #include <ipc/ipc_types.h>
  70 #include <ipc/ipc_space.h>
  71 #include <kern/lock.h>
  72 #include <kern/kalloc.h>
  73 #include <kern/thread.h>
  74 #include <vm/vm_kern.h>
  75 #include <device/device_types.h>
  76
  77 /*
  78  * Default option settings.
  79  */
  80 #ifndef PARALLEL
  81 #define PARALLEL        1
  82 #endif
  83
  84 #ifndef CHECKSUM
  85 #define CHECKSUM        0
  86 #endif
  87
  88 #define MACH_PORT_FACE mach_port_t
  89
  90 #if CONFIG_FREEZE
  91 #define RECLAIM_SWAP    1
  92 #else
  93 #define RECLAIM_SWAP    0
  94 #endif
  95
  96 #define USE_PRECIOUS    0
  97
  98 #ifdef  USER_PAGER
  99 #define UP(stuff)       stuff
 100 #else   /* USER_PAGER */
 101 #define UP(stuff)
 102 #endif  /* USER_PAGER */
 103
 104 #define dprintf(args)                                           \
 105         do {                                                    \
 106                 printf("%s[KERNEL]: ", my_name);                \
 107                 printf args;                                    \
 108         } while (0)
 109
 110 /*
 111  * Debug.
 112  */
 113 __private_extern__ char my_name[];
 114
 115 #define DEFAULT_PAGER_DEBUG     0
 116
 117 #if     DEFAULT_PAGER_DEBUG
 118
 119 extern int      debug_mask;
 120 #define DEBUG_MSG_EXTERNAL      0x00000001
 121 #define DEBUG_MSG_INTERNAL      0x00000002
 122 #define DEBUG_MO_EXTERNAL       0x00000100
 123 #define DEBUG_MO_INTERNAL       0x00000200
 124 #define DEBUG_VS_EXTERNAL       0x00010000
 125 #define DEBUG_VS_INTERNAL       0x00020000
 126 #define DEBUG_BS_EXTERNAL       0x01000000
 127 #define DEBUG_BS_INTERNAL       0x02000000
 128
 129 #define DP_DEBUG(level, args)                                           \
 130         do {                                                            \
 131                 if (debug_mask & (level))                               \
 132                         dprintf(args);                                  \
 133         } while (0)
 134
 135 #define ASSERT(expr)                                                    \
 136         do {                                                            \
 137                 if (!(expr))                                            \
 138 #ifndef MACH_KERNEL
 139                         panic("%s[%d]%s: assertion failed in %s line %d: %s",\
 140                               my_name, dp_thread_id(), here,            \
 141                               __FILE__, __LINE__, # expr);              \
 142 #else
 143                   panic("%s[KERNEL]: assertion failed in %s line %d: %s",\
 144                               my_name, __FILE__, __LINE__, # expr); \
 145 #endif
 146         } while (0)
 147
 148 #else   /* DEFAULT_PAGER_DEBUG */
 149
 150 #define DP_DEBUG(level, args) do {} while(0)
 151 #define ASSERT(clause) do {} while(0)
 152
 153 #endif  /* DEFAULT_PAGER_DEBUG */
 154
 155 #ifndef MACH_KERNEL
 156 extern char *mach_error_string(kern_return_t);
 157 #endif
 158
 159 #define PAGER_SUCCESS   0
 160 #define PAGER_FULL      1
 161 #define PAGER_ERROR     2
 162
 163 /*
 164  * VM and IPC globals.
 165  */
 166 #ifdef MACH_KERNEL
 167 #define vm_page_size PAGE_SIZE
 168 #define vm_page_mask PAGE_MASK
 169 #define vm_page_shift PAGE_SHIFT
 170 #else
 171 extern vm_object_size_t vm_page_size;
 172 extern unsigned long long       vm_page_mask;
 173 extern int              vm_page_shift;
 174 #endif
 175
 176 #ifndef MACH_KERNEL
 177 #define ptoa(p) ((p)*vm_page_size)
 178 #define atop(a) ((a)/vm_page_size)
 179 #endif
 180 #define howmany(a,b)    ((((a) % (b)) == 0) ? ((a) / (b)) : (((a) / (b)) + 1))
 181
 182 extern memory_object_default_t  default_pager_object;
 183
 184 #ifdef MACH_KERNEL
 185 extern lck_mtx_t                dpt_lock;       /* Lock for the dpt array */
 186 extern int      default_pager_internal_count;
 187 extern MACH_PORT_FACE   default_pager_host_port;
 188 /* extern task_t                default_pager_self; */  /* dont need or want */
 189 extern MACH_PORT_FACE   default_pager_internal_set;
 190 extern MACH_PORT_FACE   default_pager_external_set;
 191 extern MACH_PORT_FACE   default_pager_default_set;
 192 #else
 193 extern mach_port_t      default_pager_host_port;
 194 extern task_port_t      default_pager_self;
 195 extern mach_port_t      default_pager_internal_set;
 196 extern mach_port_t      default_pager_external_set;
 197 extern mach_port_t      default_pager_default_set;
 198 #endif
 199
 200 typedef vm32_offset_t dp_offset_t;
 201 typedef vm32_size_t dp_size_t;
 202 typedef vm32_address_t dp_address_t;
 203
 204 typedef struct default_pager_thread {
 205 #ifndef MACH_KERNEL
 206         cthread_t       dpt_thread;     /* Server thread. */
 207 #endif
 208         vm_offset_t     dpt_buffer;     /* Read buffer. */
 209         boolean_t       dpt_internal;   /* Do we handle internal objects? */
 210 #ifndef MACH_KERNEL
 211         int             dpt_id;         /* thread id for printf */
 212 #else
 213         int             checked_out;
 214 #endif
 215         boolean_t       dpt_initialized_p; /* Thread is ready for requests.  */
 216 } default_pager_thread_t;
 217
 218 #ifdef MACH_KERNEL
 219 extern default_pager_thread_t   **dpt_array;
 220 #endif
 221
 222 /*
 223  * Global statistics.
 224  */
 225 struct global_stats {
 226         unsigned int    gs_pageout_calls;       /* # pageout calls */
 227         unsigned int    gs_pagein_calls;        /* # pagein calls */
 228         unsigned int    gs_pages_in;            /* # pages paged in (total) */
 229         unsigned int    gs_pages_out;           /* # pages paged out (total) */
 230         unsigned int    gs_pages_unavail;       /* # zero-fill pages */
 231         unsigned int    gs_pages_init;          /* # page init requests */
 232         unsigned int    gs_pages_init_writes;   /* # page init writes */
 233         VSTATS_LOCK_DECL(gs_lock)
 234 };
 235 extern struct global_stats global_stats;
 236 #define GSTAT(clause)   VSTATS_ACTION(&global_stats.gs_lock, (clause))
 237
 238 /*
 239  * Cluster related definitions.
 240  * Clusters are sized in number of pages per cluster.
 241  * Cluster sizes must be powers of two.
 242  *
 243  * These numbers are related to the struct vs_map,
 244  * defined below.
 245  */
 246 #define MAX_CLUSTER_SIZE 8
 247 #define MAX_CLUSTER_SHIFT 3
 248 #define NO_CLSIZE 0
 249
 250 /*
 251  * bit map related macros
 252  */
 253 #define NBBY            8       /* bits per byte XXX */
 254 #define BYTEMASK        0xff
 255 #define setbit(a,i)     (*(((char *)(a)) + ((i)/NBBY)) |= 1<<((i)%NBBY))
 256 #define clrbit(a,i)     (*(((char *)(a)) + ((i)/NBBY)) &= ~(1<<((i)%NBBY)))
 257 #define isset(a,i)      (*(((char *)(a)) + ((i)/NBBY)) & (1<<((i)%NBBY)))
 258 #define isclr(a,i)      ((*(((char *)(a)) + ((i)/NBBY)) & (1<<((i)%NBBY))) == 0)
 259
 260 /*
 261  *      Default Pager.
 262  *              Backing Store Management.
 263  */
 264
 265 #define BS_MAXPRI       4
 266 #define BS_MINPRI       0
 267 #define BS_NOPRI        -1
 268 #define BS_FULLPRI      -2
 269
 270 /*
 271  * Quick way to access the emergency segment backing store structures
 272  * without a full-blown search.
 273  */
 274 extern MACH_PORT_FACE           emergency_segment_backing_store;
 275
 276 /*
 277  * Mapping between backing store port and backing store object.
 278  */
 279 struct backing_store {
 280         queue_chain_t   bs_links;       /* link in backing_store_list */
 281         lck_mtx_t               bs_lock;        /* lock for the structure */
 282         MACH_PORT_FACE  bs_port;        /* backing store port */
 283         int             bs_priority;
 284         int             bs_clsize;      /* cluster size in pages */
 285
 286         /* statistics */
 287         unsigned int    bs_pages_free;          /* # unallocated pages */
 288         unsigned int    bs_pages_total;         /* # pages (total) */
 289         unsigned int    bs_pages_in;            /* # page read requests */
 290         unsigned int    bs_pages_in_fail;       /* # page read errors */
 291         unsigned int    bs_pages_out;           /* # page write requests */
 292         unsigned int    bs_pages_out_fail;      /* # page write errors */
 293 };
 294 typedef struct backing_store    *backing_store_t;
 295 #define BACKING_STORE_NULL      ((backing_store_t) 0)
 296 #define BS_STAT(bs, clause)     VSTATS_ACTION(&(bs)->bs_lock, (clause))
 297
 298 #ifdef MACH_KERNEL
 299 #define BS_LOCK_INIT(bs)        lck_mtx_init(&(bs)->bs_lock, &default_pager_lck_grp, &default_pager_lck_attr)
 300 #define BS_LOCK_DESTROY(bs)     lck_mtx_destroy(&(bs)->bs_lock, &default_pager_lck_grp)
 301 #define BS_LOCK(bs)                     lck_mtx_lock(&(bs)->bs_lock)
 302 #define BS_UNLOCK(bs)           lck_mtx_unlock(&(bs)->bs_lock)
 303
 304 struct backing_store_list_head {
 305         queue_head_t    bsl_queue;
 306         lck_mtx_t       bsl_lock;
 307 #endif
 308 };
 309 extern struct backing_store_list_head   backing_store_list;
 310 extern int      backing_store_release_trigger_disable;
 311
 312 #define BSL_LOCK_INIT()         lck_mtx_init(&backing_store_list.bsl_lock, &default_pager_lck_grp, &default_pager_lck_attr)
 313 #define BSL_LOCK_DESTROY()      lck_mtx_destroy(&backing_store_list.bsl_lock, &default_pager_lck_grp)
 314 #define BSL_LOCK()                      lck_mtx_lock(&backing_store_list.bsl_lock)
 315 #define BSL_UNLOCK()            lck_mtx_unlock(&backing_store_list.bsl_lock)
 316
 317 /*
 318  *      Paging segment management.
 319  *      Controls allocation of blocks within paging area.
 320  */
 321 struct paging_segment {
 322         /* device management */
 323         union {
 324                 MACH_PORT_FACE  dev;            /* Port to device */
 325                 struct vnode    *vnode;         /* vnode for bs file */
 326         } storage_type;
 327         unsigned int    ps_segtype;     /* file type or partition */
 328         MACH_PORT_FACE  ps_device;      /* Port to device */
 329         dp_offset_t     ps_offset;      /* Offset of segment within device */
 330         dp_offset_t     ps_recnum;      /* Number of device records in segment*/
 331         unsigned int    ps_pgnum;       /* Number of pages in segment */
 332         unsigned int    ps_record_shift;/* Bit shift: pages to device records */
 333
 334         /* clusters and pages */
 335         unsigned int    ps_clshift;     /* Bit shift: clusters to pages */
 336         unsigned int    ps_ncls;        /* Number of clusters in segment */
 337         unsigned int    ps_clcount;     /* Number of free clusters */
 338         unsigned int    ps_pgcount;     /* Number of free pages */
 339         unsigned int    ps_hint;        /* Hint of where to look next. */
 340         unsigned int    ps_special_clusters; /* Clusters that might come in while we've
 341                                         * released the locks doing a ps_delete.
 342                                         */
 343
 344         /* bitmap */
 345         lck_mtx_t               ps_lock;        /* Lock for contents of struct */
 346         unsigned char   *ps_bmap;       /* Map of used clusters */
 347
 348         /* backing store */
 349         backing_store_t ps_bs;          /* Backing store segment belongs to */
 350 #define PS_CAN_USE              0x1
 351 #define PS_GOING_AWAY           0x2
 352 #define PS_EMERGENCY_SEGMENT    0x4
 353         unsigned int    ps_state;
 354 };
 355
 356 #define IS_PS_OK_TO_USE(ps)             ((ps->ps_state & PS_CAN_USE) == PS_CAN_USE)
 357 #define IS_PS_GOING_AWAY(ps)            ((ps->ps_state & PS_GOING_AWAY) == PS_GOING_AWAY)
 358 #define IS_PS_EMERGENCY_SEGMENT(ps)     ((ps->ps_state & PS_EMERGENCY_SEGMENT) == PS_EMERGENCY_SEGMENT)
 359
 360 #define ps_vnode        storage_type.vnode
 361 #define ps_device       storage_type.dev
 362 #define PS_PARTITION 1
 363 #define PS_FILE 2
 364
 365 typedef struct paging_segment *paging_segment_t;
 366
 367 #define PAGING_SEGMENT_NULL     ((paging_segment_t) 0)
 368
 369 #define PS_LOCK_INIT(ps)        lck_mtx_init(&(ps)->ps_lock, &default_pager_lck_grp, &default_pager_lck_attr)
 370 #define PS_LOCK_DESTROY(ps)     lck_mtx_destroy(&(ps)->ps_lock, &default_pager_lck_grp)
 371 #define PS_LOCK(ps)                     lck_mtx_lock(&(ps)->ps_lock)
 372 #define PS_UNLOCK(ps)           lck_mtx_unlock(&(ps)->ps_lock)
 373
 374 typedef unsigned int    pseg_index_t;
 375
 376 #define INVALID_PSEG_INDEX      ((pseg_index_t)-1)
 377 #define EMERGENCY_PSEG_INDEX            ((pseg_index_t) 0)
 378 /*
 379  * MAX_PSEG_INDEX value is related to struct vs_map below.
 380  * "0" is reserved for empty map entries (no segment).
 381  */
 382 #define MAX_PSEG_INDEX  63      /* 0 is reserved for empty map */
 383 #define MAX_NUM_PAGING_SEGMENTS MAX_PSEG_INDEX
 384
 385 /* paging segments array */
 386 extern paging_segment_t paging_segments[MAX_NUM_PAGING_SEGMENTS];
 387 extern lck_mtx_t paging_segments_lock;
 388 extern int      paging_segment_count;   /* number of active paging segments */
 389 extern int      paging_segment_max;     /* highest used paging segment index */
 390 extern int ps_select_array[DEFAULT_PAGER_BACKING_STORE_MAXPRI+1];
 391
 392 #define PSL_LOCK_INIT()         lck_mtx_init(&paging_segments_lock, &default_pager_lck_grp, &default_pager_lck_attr)
 393 #define PSL_LOCK_DESTROY()      lck_mtx_destroy(&paging_segments_lock, &default_pager_lck_grp)
 394 #define PSL_LOCK()              lck_mtx_lock(&paging_segments_lock)
 395 #define PSL_UNLOCK()    lck_mtx_unlock(&paging_segments_lock)
 396
 397 /*
 398  * Vstruct manipulation.  The vstruct is the pager's internal
 399  * representation of vm objects it manages.  There is one vstruct allocated
 400  * per vm object.
 401  *
 402  * The following data structures are defined for vstruct and vm object
 403  * management.
 404  */
 405
 406 /*
 407  * vs_map
 408  * A structure used only for temporary objects.  It is the element
 409  * contained in the vs_clmap structure, which contains information
 410  * about which clusters and pages in an object are present on backing
 411  * store (a paging file).
 412  * Note that this structure and its associated constants may change
 413  * with minimal impact on code.  The only function which knows the
 414  * internals of this structure is ps_clmap().
 415  *
 416  * If it is necessary to change the maximum number of paging segments
 417  * or pages in a cluster, then this structure is the one most
 418  * affected.   The constants and structures which *may* change are:
 419  *      MAX_CLUSTER_SIZE
 420  *      MAX_CLUSTER_SHIFT
 421  *      MAX_NUM_PAGING_SEGMENTS
 422  *      VSTRUCT_DEF_CLSHIFT
 423  *      struct vs_map and associated macros and constants (VSM_*)
 424  *        (only the macro definitions need change, the exported (inside the
 425  *         pager only) interfaces remain the same; the constants are for
 426  *         internal vs_map manipulation only).
 427  *      struct clbmap (below).
 428  */
 429 struct vs_map {
 430         unsigned int    vsmap_entry:23,         /* offset in paging segment */
 431                         vsmap_psindex:8,        /* paging segment */
 432                         vsmap_error:1,
 433                         vsmap_bmap:16,
 434                         vsmap_alloc:16;
 435 };
 436
 437 typedef struct vs_map *vs_map_t;
 438
 439
 440 #define VSM_ENTRY_NULL  0x7fffff
 441
 442 /*
 443  * Exported macros for manipulating the vs_map structure --
 444  * checking status, getting and setting bits.
 445  */
 446 #define VSCLSIZE(vs)            (1U << (vs)->vs_clshift)
 447 #define VSM_ISCLR(vsm)          (((vsm).vsmap_entry == VSM_ENTRY_NULL) &&   \
 448                                         ((vsm).vsmap_error == 0))
 449 #define VSM_ISERR(vsm)          ((vsm).vsmap_error)
 450 #define VSM_SETCLOFF(vsm, val)  ((vsm).vsmap_entry = (val))
 451 #define VSM_SETERR(vsm, err)    ((vsm).vsmap_error = 1,   \
 452                                         (vsm).vsmap_entry = (err))
 453 #define VSM_GETERR(vsm)         ((vsm).vsmap_entry)
 454 #define VSM_SETPG(vsm, page)    ((vsm).vsmap_bmap |= (1 << (page)))
 455 #define VSM_CLRPG(vsm, page)    ((vsm).vsmap_bmap &= ~(1 << (page)))
 456 #define VSM_SETPS(vsm, psindx)  ((vsm).vsmap_psindex = (psindx))
 457 #define VSM_PSINDEX(vsm)        ((vsm).vsmap_psindex)
 458 #define VSM_PS(vsm)             paging_segments[(vsm).vsmap_psindex]
 459 #define VSM_BMAP(vsm)           ((vsm).vsmap_bmap)
 460 #define VSM_CLOFF(vsm)          ((vsm).vsmap_entry)
 461 #define VSM_CLR(vsm)            ((vsm).vsmap_entry = VSM_ENTRY_NULL,   \
 462                                         (vsm).vsmap_psindex = 0,   \
 463                                         (vsm).vsmap_error = 0,     \
 464                                         (vsm).vsmap_bmap = 0,      \
 465                                         (vsm).vsmap_alloc = 0)
 466 #define VSM_ALLOC(vsm)          ((vsm).vsmap_alloc)
 467 #define VSM_SETALLOC(vsm, page) ((vsm).vsmap_alloc |= (1 << (page)))
 468 #define VSM_CLRALLOC(vsm, page) ((vsm).vsmap_alloc &= ~(1 << (page)))
 469
 470 /*
 471  * Constants and macros for dealing with vstruct maps,
 472  * which comprise vs_map structures, which
 473  * map vm objects to backing storage (paging files and clusters).
 474  */
 475 #define CLMAP_THRESHOLD 512     /* bytes */
 476 #define CLMAP_ENTRIES           (CLMAP_THRESHOLD/(int)sizeof(struct vs_map))
 477 #define CLMAP_SIZE(ncls)        (ncls*(int)sizeof(struct vs_map))
 478
 479 #define INDIRECT_CLMAP_ENTRIES(ncls) (((ncls-1)/CLMAP_ENTRIES) + 1)
 480 #define INDIRECT_CLMAP_SIZE(ncls) (INDIRECT_CLMAP_ENTRIES(ncls) * (int)sizeof(struct vs_map *))
 481 #define INDIRECT_CLMAP(size)    (CLMAP_SIZE(size) > CLMAP_THRESHOLD)
 482
 483 #define RMAPSIZE(blocks)        (howmany(blocks,NBBY))
 484
 485 #define CL_FIND 1
 486 #define CL_ALLOC 2
 487
 488 /*
 489  * clmap
 490  *
 491  * A cluster map returned by ps_clmap.  It is an abstracted cluster of
 492  * pages.  It gives the caller information about the cluster
 493  * desired.  On read it tells the caller if a cluster is mapped, and if so,
 494  * which of its pages are valid.  It should not be referenced directly,
 495  * except by  ps_clmap; macros should be used.  If the number of pages
 496  * in a cluster needs to be more than 32, then the struct clbmap must
 497  * become larger.
 498  */
 499 struct clbmap {
 500         unsigned int    clb_map;
 501 };
 502
 503 struct clmap {
 504         paging_segment_t cl_ps;         /* paging segment backing cluster */
 505         int             cl_numpages;    /* number of valid pages */
 506         struct clbmap   cl_bmap;        /* map of pages in cluster */
 507         int             cl_error;       /* cluster error value */
 508         struct clbmap   cl_alloc;       /* map of allocated pages in cluster */
 509 };
 510
 511 #define  CLMAP_ERROR(clm)       (clm).cl_error
 512 #define  CLMAP_PS(clm)          (clm).cl_ps
 513 #define  CLMAP_NPGS(clm)        (clm).cl_numpages
 514 #define  CLMAP_ISSET(clm,i)     ((1<<(i))&((clm).cl_bmap.clb_map))
 515 #define  CLMAP_ALLOC(clm)       (clm).cl_alloc.clb_map
 516 /*
 517  * Shift off unused bits in a partial cluster
 518  */
 519 #define  CLMAP_SHIFT(clm,vs)    \
 520         (clm)->cl_bmap.clb_map >>= (VSCLSIZE(vs) - (clm)->cl_numpages)
 521 #define  CLMAP_SHIFTALLOC(clm,vs)       \
 522         (clm)->cl_alloc.clb_map >>= (VSCLSIZE(vs) - (clm)->cl_numpages)
 523
 524 typedef struct vstruct_alias {
 525         memory_object_pager_ops_t name;
 526         struct vstruct *vs;
 527 } vstruct_alias_t;
 528
 529 #define DPT_LOCK_INIT(lock)             lck_mtx_init(&(lock), &default_pager_lck_grp, &default_pager_lck_attr)
 530 #define DPT_LOCK_DESTROY(lock)          lck_mtx_destroy(&(lock), &default_pager_lck_grp)
 531 #define DPT_LOCK(lock)                  lck_mtx_lock(&(lock))
 532 #define DPT_UNLOCK(lock)                lck_mtx_unlock(&(lock))
 533 #define DPT_SLEEP(lock, e, i)   lck_mtx_sleep(&(lock), LCK_SLEEP_DEFAULT, (event_t)(e), i)
 534 #define VS_LOCK_TYPE                    hw_lock_data_t
 535 #define VS_LOCK_INIT(vs)                hw_lock_init(&(vs)->vs_lock)
 536 #define VS_TRY_LOCK(vs)                 (VS_LOCK(vs),TRUE)
 537 #define VS_LOCK(vs)                             hw_lock_lock(&(vs)->vs_lock)
 538 #define VS_UNLOCK(vs)                   hw_lock_unlock(&(vs)->vs_lock)
 539 #define VS_MAP_LOCK_TYPE                lck_mtx_t
 540 #define VS_MAP_LOCK_INIT(vs)    lck_mtx_init(&(vs)->vs_map_lock, &default_pager_lck_grp, &default_pager_lck_attr)
 541 #define VS_MAP_LOCK_DESTROY(vs) lck_mtx_destroy(&(vs)->vs_map_lock, &default_pager_lck_grp)
 542 #define VS_MAP_LOCK(vs)                 lck_mtx_lock(&(vs)->vs_map_lock)
 543 #define VS_MAP_TRY_LOCK(vs)             lck_mtx_try_lock(&(vs)->vs_map_lock)
 544 #define VS_MAP_UNLOCK(vs)               lck_mtx_unlock(&(vs)->vs_map_lock)
 545
 546
 547 /*
 548  * VM Object Structure:  This is the structure used to manage
 549  * default pager object associations with their control counter-
 550  * parts (VM objects).
 551  *
 552  * The start of this structure MUST match a "struct memory_object".
 553  */
 554 typedef struct vstruct {
 555         struct ipc_object_header        vs_pager_header;        /* fake ip_kotype() */
 556         memory_object_pager_ops_t vs_pager_ops; /* == &default_pager_ops */
 557         memory_object_control_t vs_control;     /* our mem obj control ref */
 558         VS_LOCK_TYPE            vs_lock;        /* data for the lock */
 559
 560         /* JMM - Could combine these first two in a single pending count now */
 561         unsigned int            vs_next_seqno;  /* next sequence num to issue */
 562         unsigned int            vs_seqno;       /* Pager port sequence number */
 563         unsigned int            vs_readers;     /* Reads in progress */
 564         unsigned int            vs_writers;     /* Writes in progress */
 565
 566         unsigned int
 567         /* boolean_t */         vs_waiting_seqno:1,     /* to wait on seqno */
 568         /* boolean_t */         vs_waiting_read:1,      /* waiting on reader? */
 569         /* boolean_t */         vs_waiting_write:1,     /* waiting on writer? */
 570         /* boolean_t */         vs_waiting_async:1,     /* waiting on async? */
 571         /* boolean_t */         vs_indirect:1,          /* map indirect? */
 572         /* boolean_t */         vs_xfer_pending:1;      /* xfer out of seg? */
 573
 574         unsigned int            vs_async_pending;/* pending async write count */
 575         unsigned int            vs_errors;      /* Pageout error count */
 576         unsigned int            vs_references;  /* references */
 577
 578         queue_chain_t           vs_links;       /* Link in pager-wide list */
 579
 580         unsigned int            vs_clshift;     /* Bit shift: clusters->pages */
 581         unsigned int            vs_size;        /* Object size in clusters */
 582         lck_mtx_t               vs_map_lock;    /* to protect map below */
 583         union {
 584                 struct vs_map   *vsu_dmap;      /* Direct map of clusters */
 585                 struct vs_map   **vsu_imap;     /* Indirect map of clusters */
 586         } vs_un;
 587 } *vstruct_t;
 588
 589 #define vs_dmap vs_un.vsu_dmap
 590 #define vs_imap vs_un.vsu_imap
 591
 592 #define VSTRUCT_NULL    ((vstruct_t) 0)
 593
 594 __private_extern__ void vs_async_wait(vstruct_t);
 595
 596 #if PARALLEL
 597 __private_extern__ void vs_lock(vstruct_t);
 598 __private_extern__ void vs_unlock(vstruct_t);
 599 __private_extern__ void vs_start_read(vstruct_t);
 600 __private_extern__ void vs_finish_read(vstruct_t);
 601 __private_extern__ void vs_wait_for_readers(vstruct_t);
 602 __private_extern__ void vs_start_write(vstruct_t);
 603 __private_extern__ void vs_finish_write(vstruct_t);
 604 __private_extern__ void vs_wait_for_writers(vstruct_t);
 605 __private_extern__ void vs_wait_for_sync_writers(vstruct_t);
 606 #else   /* PARALLEL */
 607 #define vs_lock(vs)
 608 #define vs_unlock(vs)
 609 #define vs_start_read(vs)
 610 #define vs_wait_for_readers(vs)
 611 #define vs_finish_read(vs)
 612 #define vs_start_write(vs)
 613 #define vs_wait_for_writers(vs)
 614 #define vs_wait_for_sync_writers(vs)
 615 #define vs_finish_write(vs)
 616 #endif /* PARALLEL */
 617
 618 /*
 619  * Data structures and variables dealing with asynchronous
 620  * completion of paging operations.
 621  */
 622 /*
 623  * vs_async
 624  *      A structure passed to ps_write_device for asynchronous completions.
 625  *      It contains enough information to complete the write and
 626  *      inform the VM of its completion.
 627  */
 628 struct vs_async {
 629         struct vs_async *vsa_next;      /* pointer to next structure */
 630         vstruct_t       vsa_vs;         /* the vstruct for the object */
 631         vm_offset_t     vsa_addr;       /* the vaddr of the data moved */
 632         vm_offset_t     vsa_offset;     /* the object offset of the data */
 633         vm_size_t       vsa_size;       /* the number of bytes moved */
 634         paging_segment_t vsa_ps;        /* the paging segment used */
 635         int             vsa_flags;      /* flags */
 636         int             vsa_error;      /* error, if there is one */
 637         MACH_PORT_FACE  reply_port;     /* associated reply port */
 638 };
 639
 640 /*
 641  * flags values.
 642  */
 643 #define VSA_READ        0x0001
 644 #define VSA_WRITE       0x0002
 645 #define VSA_TRANSFER    0x0004
 646
 647 /*
 648  * List of all vstructs.  A specific vstruct is
 649  * found directly via its port, this list is
 650  * only used for monitoring purposes by the
 651  * default_pager_object* calls
 652  */
 653 struct vstruct_list_head {
 654         queue_head_t    vsl_queue;
 655         lck_mtx_t               vsl_lock;
 656         int             vsl_count;      /* saves code */
 657 };
 658
 659 __private_extern__ struct vstruct_list_head     vstruct_list;
 660
 661 __private_extern__ void vstruct_list_insert(vstruct_t vs);
 662 __private_extern__ void vstruct_list_delete(vstruct_t vs);
 663
 664
 665 extern lck_grp_t                default_pager_lck_grp;
 666 extern lck_attr_t               default_pager_lck_attr;
 667
 668 #define VSL_LOCK_INIT()         lck_mtx_init(&vstruct_list.vsl_lock, &default_pager_lck_grp, &default_pager_lck_attr)
 669 #define VSL_LOCK_DESTROY()      lck_mtx_destroy(&vstruct_list.vsl_lock, &default_pager_lck_grp)
 670 #define VSL_LOCK()                      lck_mtx_lock(&vstruct_list.vsl_lock)
 671 #define VSL_LOCK_TRY()          lck_mtx_try_lock(&vstruct_list.vsl_lock)
 672 #define VSL_UNLOCK()            lck_mtx_unlock(&vstruct_list.vsl_lock)
 673 #define VSL_SLEEP(e,i)          lck_mtx_sleep(&vstruct_list.vsl_lock, LCK_SLEEP_DEFAULT, (e), (i))
 674
 675 #ifdef MACH_KERNEL
 676 __private_extern__ zone_t       vstruct_zone;
 677 #endif
 678
 679 /*
 680  * Create port alias for vstruct address.
 681  *
 682  * We assume that the last two bits of a vstruct address will be zero due to
 683  * memory allocation restrictions, hence are available for use as a sanity
 684  * check.
 685  */
 686 #ifdef MACH_KERNEL
 687
 688 extern const struct memory_object_pager_ops default_pager_ops;
 689
 690 #define mem_obj_is_vs(_mem_obj_)                                        \
 691         (((_mem_obj_) != NULL) &&                                       \
 692          ((_mem_obj_)->mo_pager_ops == &default_pager_ops))
 693 #define mem_obj_to_vs(_mem_obj_)                                        \
 694         ((vstruct_t)(_mem_obj_))
 695 #define vs_to_mem_obj(_vs_) ((memory_object_t)(_vs_))
 696 #define vs_lookup(_mem_obj_, _vs_)                                      \
 697         do {                                                            \
 698         if (!mem_obj_is_vs(_mem_obj_))                                  \
 699                 panic("bad dp memory object");                          \
 700         _vs_ = mem_obj_to_vs(_mem_obj_);                                \
 701         } while (0)
 702 #define vs_lookup_safe(_mem_obj_, _vs_)                                 \
 703         do {                                                            \
 704         if (!mem_obj_is_vs(_mem_obj_))                                  \
 705                 _vs_ = VSTRUCT_NULL;                                    \
 706         else                                                            \
 707                 _vs_ = mem_obj_to_vs(_mem_obj_);                        \
 708         } while (0)
 709 #else
 710
 711 #define vs_to_port(_vs_)        (((vm_offset_t)(_vs_))+1)
 712 #define port_to_vs(_port_)      ((vstruct_t)(((vm_offset_t)(_port_))&~3))
 713 #define port_is_vs(_port_)      ((((vm_offset_t)(_port_))&3) == 1)
 714
 715 #define vs_lookup(_port_, _vs_)                                         \
 716         do {                                                            \
 717                 if (!MACH_PORT_VALID(_port_) || !port_is_vs(_port_)     \
 718                     || port_to_vs(_port_)->vs_mem_obj != (_port_))      \
 719                         Panic("bad pager port");                        \
 720                 _vs_ = port_to_vs(_port_);                              \
 721         } while (0)
 722 #endif
 723
 724 /*
 725  * Cross-module routines declaration.
 726  */
 727 #ifndef MACH_KERNEL
 728 extern int              dp_thread_id(void);
 729 #endif
 730 extern boolean_t        device_reply_server(mach_msg_header_t *,
 731                                             mach_msg_header_t *);
 732 #ifdef MACH_KERNEL
 733 extern boolean_t        default_pager_no_senders(memory_object_t,
 734                                                  mach_port_mscount_t);
 735 #else
 736 extern void             default_pager_no_senders(memory_object_t,
 737                                                  mach_port_seqno_t,
 738                                                  mach_port_mscount_t);
 739 #endif
 740
 741 extern int              local_log2(unsigned int);
 742 extern void             bs_initialize(void);
 743 extern void             bs_global_info(uint64_t *,
 744                                        uint64_t *);
 745 extern boolean_t        bs_add_device(char *,
 746                                       MACH_PORT_FACE);
 747 extern vstruct_t        ps_vstruct_create(dp_size_t);
 748 extern void             ps_vstruct_dealloc(vstruct_t);
 749 extern void             ps_vstruct_reclaim(vstruct_t,
 750                                            boolean_t,
 751                                            boolean_t);
 752 extern kern_return_t    pvs_cluster_read(vstruct_t,
 753                                          dp_offset_t,
 754                                          dp_size_t,
 755                                          void *);
 756 extern kern_return_t    vs_cluster_write(vstruct_t,
 757                                          upl_t,
 758                                          upl_offset_t,
 759                                          upl_size_t,
 760                                          boolean_t,
 761                                          int);
 762 extern dp_offset_t      ps_clmap(vstruct_t,
 763                                  dp_offset_t,
 764                                  struct clmap *,
 765                                  int,
 766                                  dp_size_t,
 767                                  int);
 768 extern vm_size_t        ps_vstruct_allocated_size(vstruct_t);
 769 extern unsigned int     ps_vstruct_allocated_pages(vstruct_t,
 770                                                    default_pager_page_t *,
 771                                                    unsigned int);
 772 extern boolean_t        bs_set_default_clsize(unsigned int);
 773
 774 extern boolean_t        verbose;
 775
 776 extern thread_call_t    default_pager_backing_store_monitor_callout;
 777 extern void             default_pager_backing_store_monitor(thread_call_param_t, thread_call_param_t);
 778
 779 extern ipc_port_t       max_pages_trigger_port;
 780 extern unsigned int     dp_pages_free;
 781 extern unsigned int     maximum_pages_free;
 782
 783 /* Do we know yet if swap files need to be encrypted ? */
 784 extern boolean_t        dp_encryption_inited;
 785 /* Should we encrypt data before writing to swap ? */
 786 extern boolean_t        dp_encryption;
 787
 788 extern boolean_t        dp_isssd;
 789
 790 #endif  /* _DEFAULT_PAGER_INTERNAL_H_ */