tests/vm_phys_footprint_legacy.c

   1 #define ENTITLED 1
   2
   3 #include <darwintest.h>
   4 #include <darwintest_utils.h>
   5
   6 #include <mach/mach_error.h>
   7 #include <mach/mach_init.h>
   8 #include <mach/mach_port.h>
   9 #include <mach/mach_vm.h>
  10 #include <mach/task.h>
  11 #include <mach/task_info.h>
  12 #include <mach/vm_map.h>
  13
  14 #include <sys/mman.h>
  15
  16 #include <Kernel/kern/ledger.h>
  17 extern int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3);
  18
  19 #if ENTITLED && defined(__arm64__)
  20 #define LEGACY_FOOTPRINT 1
  21 #else /* ENTITLED && __arm64__ */
  22 #define LEGACY_FOOTPRINT 0
  23 #endif /* ENTITLED && __arm64__ */
  24
  25 #define MEM_SIZE (100 * 1024 * 1024) /* 100 MB */
  26
  27 static int64_t ledger_count = -1;
  28 static int footprint_index = -1;
  29 static int pagetable_index = -1;
  30 static struct ledger_entry_info *lei = NULL;
  31
  32 static void
  33 ledger_init(void)
  34 {
  35         static int                      ledger_inited = 0;
  36         struct ledger_info              li;
  37         struct ledger_template_info     *templateInfo;
  38         int64_t                         templateCnt;
  39         int                             i;
  40
  41         if (ledger_inited) {
  42                 return;
  43         }
  44         ledger_inited = 1;
  45
  46         T_SETUPBEGIN;
  47         T_QUIET;
  48         T_WITH_ERRNO;
  49         T_ASSERT_EQ(ledger(LEDGER_INFO,
  50             (caddr_t)(uintptr_t)getpid(),
  51             (caddr_t)&li,
  52             NULL),
  53             0,
  54             "ledger(LEDGER_INFO)");
  55
  56         templateCnt = li.li_entries;
  57         templateInfo = malloc((size_t)li.li_entries * sizeof(struct ledger_template_info));
  58         T_QUIET;
  59         T_WITH_ERRNO;
  60         T_ASSERT_NE(templateInfo, NULL, "malloc()");
  61
  62         ledger_count = li.li_entries;
  63         footprint_index = -1;
  64         pagetable_index = -1;
  65         T_QUIET;
  66         T_WITH_ERRNO;
  67         T_ASSERT_GE(ledger(LEDGER_TEMPLATE_INFO,
  68             (caddr_t)templateInfo,
  69             (caddr_t)&templateCnt,
  70             NULL),
  71             0,
  72             "ledger(LEDGER_TEMPLATE_INFO)");
  73         for (i = 0; i < templateCnt; i++) {
  74                 if (!strncmp(templateInfo[i].lti_name,
  75                     "phys_footprint",
  76                     strlen("phys_footprint"))) {
  77                         footprint_index = i;
  78                 } else if (!strncmp(templateInfo[i].lti_name,
  79                     "page_table",
  80                     strlen("page_table"))) {
  81                         pagetable_index = i;
  82                 }
  83         }
  84         free(templateInfo);
  85
  86         lei = (struct ledger_entry_info *)
  87             malloc((size_t)ledger_count * sizeof(*lei));
  88         T_QUIET;
  89         T_WITH_ERRNO;
  90         T_ASSERT_NE(lei, NULL, "malloc(ledger_entry_info)");
  91
  92         T_QUIET;
  93         T_ASSERT_NE(footprint_index, -1, "no footprint_index");
  94         T_QUIET;
  95         T_ASSERT_NE(pagetable_index, -1, "no pagetable_index");
  96
  97         T_SETUPEND;
  98 }
  99
 100 static void
 101 get_ledger_info(
 102         uint64_t        *phys_footprint,
 103         uint64_t        *page_table)
 104 {
 105         int64_t count;
 106
 107         count = ledger_count;
 108         T_QUIET;
 109         T_WITH_ERRNO;
 110         T_ASSERT_GE(ledger(LEDGER_ENTRY_INFO,
 111             (caddr_t)(uintptr_t)getpid(),
 112             (caddr_t)lei,
 113             (caddr_t)&count),
 114             0,
 115             "ledger(LEDGER_ENTRY_INFO)");
 116         T_QUIET;
 117         T_ASSERT_GT(count, (int64_t)footprint_index, "no entry for footprint");
 118         T_QUIET;
 119         T_ASSERT_GT(count, (int64_t)pagetable_index, "no entry for pagetable");
 120         if (phys_footprint) {
 121                 *phys_footprint = (uint64_t)(lei[footprint_index].lei_balance);
 122         }
 123         if (page_table) {
 124                 *page_table = (uint64_t)(lei[pagetable_index].lei_balance);
 125         }
 126 }
 127
 128 static mach_vm_address_t
 129 pre_warm(
 130         mach_vm_size_t  vm_size)
 131 {
 132         kern_return_t           kr;
 133         mach_vm_address_t       vm_addr;
 134         unsigned char           BigBufOnStack[100 * 1024];
 135         uint64_t                footprint, page_table;
 136
 137         /* make sure ledgers are ready to be queried */
 138         ledger_init();
 139
 140         T_SETUPBEGIN;
 141
 142         /*
 143          * Touch a few pages ahead on the stack, to make
 144          * sure we don't see a footprint increase due to
 145          * an extra stack page later.
 146          */
 147         memset(BigBufOnStack, 0xb, sizeof(BigBufOnStack));
 148         T_QUIET;
 149         T_EXPECT_EQ(BigBufOnStack[0], 0xb,
 150             "BigBufOnStack[0] == 0x%x",
 151             BigBufOnStack[0]);
 152         T_QUIET;
 153         T_EXPECT_EQ(BigBufOnStack[sizeof(BigBufOnStack) - 1], 0xb,
 154             "BigBufOnStack[%lu] == 0x%x",
 155             sizeof(BigBufOnStack),
 156             BigBufOnStack[sizeof(BigBufOnStack) - 1]);
 157
 158         /*
 159          * Pre-allocate, touch and then release the same amount
 160          * of memory we'll be allocating later during the test,
 161          * to account for any memory overhead (page tables, global
 162          * variables, ...).
 163          */
 164         vm_addr = 0;
 165         kr = mach_vm_allocate(mach_task_self(),
 166             &vm_addr,
 167             vm_size,
 168             VM_FLAGS_ANYWHERE);
 169         T_QUIET;
 170         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate(%lld) error 0x%x (%s)",
 171             vm_size, kr, mach_error_string(kr));
 172         memset((char *)(uintptr_t)vm_addr, 'p', (size_t)vm_size);
 173         kr = mach_vm_deallocate(mach_task_self(),
 174             vm_addr,
 175             vm_size);
 176         T_QUIET;
 177         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
 178             kr, mach_error_string(kr));
 179
 180         /*
 181          * Exercise the ledger code to make sure it's ready to run
 182          * without any extra memory overhead later.
 183          */
 184         get_ledger_info(&footprint, &page_table);
 185
 186         T_SETUPEND;
 187
 188         /*
 189          * Return the start of the virtual range we pre-warmed, so that the
 190          * test can check that it's using the same range.
 191          */
 192         return vm_addr;
 193 }
 194
 195 T_DECL(legacy_phys_footprint_anonymous,
 196     "phys_footprint for anonymous memory",
 197     T_META_NAMESPACE("xnu.vm"),
 198     T_META_LTEPHASE(LTE_POSTINIT))
 199 {
 200         uint64_t                footprint_before, pagetable_before;
 201         uint64_t                footprint_after, pagetable_after;
 202         uint64_t                footprint_expected;
 203         kern_return_t           kr;
 204         mach_vm_address_t       pre_vm_addr, vm_addr;
 205         mach_vm_size_t          vm_size, dirty_size;
 206
 207         /* pre-warm to account for page table expansion */
 208         pre_vm_addr = pre_warm(MEM_SIZE);
 209
 210         /* allocating virtual memory... */
 211         get_ledger_info(&footprint_before, &pagetable_before);
 212         vm_addr = 0;
 213         vm_size = MEM_SIZE;
 214         kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
 215             VM_FLAGS_ANYWHERE);
 216         T_QUIET;
 217         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
 218             kr, mach_error_string(kr));
 219         T_QUIET;
 220         T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
 221         /* ... should not change footprint */
 222         get_ledger_info(&footprint_after, &pagetable_after);
 223         footprint_expected = footprint_before;
 224         footprint_expected += (pagetable_after - pagetable_before);
 225         T_LOG("virtual allocation does not change phys_footprint");
 226         T_EXPECT_EQ(footprint_after, footprint_expected,
 227             "virtual allocation of %lld bytes: "
 228             "footprint %lld -> %lld expected %lld delta %lld",
 229             vm_size, footprint_before, footprint_after,
 230             footprint_expected, footprint_after - footprint_expected);
 231
 232         /* touching memory... */
 233         get_ledger_info(&footprint_before, &pagetable_before);
 234         dirty_size = vm_size / 2;
 235         memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
 236         /* ... should increase footprint */
 237         get_ledger_info(&footprint_after, &pagetable_after);
 238         footprint_expected = footprint_before + dirty_size;
 239         footprint_expected += (pagetable_after - pagetable_before);
 240         T_LOG("modifying anonymous memory increases phys_footprint");
 241         T_EXPECT_EQ(footprint_after, footprint_expected,
 242             "touched %lld bytes: "
 243             "footprint %lld -> %lld expected %lld delta %lld",
 244             dirty_size, footprint_before, footprint_after,
 245             footprint_expected, footprint_after - footprint_expected);
 246
 247         /* deallocating memory... */
 248         get_ledger_info(&footprint_before, &pagetable_before);
 249         kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
 250         T_QUIET;
 251         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
 252             kr, mach_error_string(kr));
 253         /* ... should decrease footprint */
 254         get_ledger_info(&footprint_after, &pagetable_after);
 255         footprint_expected = footprint_before - dirty_size;
 256         footprint_expected += (pagetable_after - pagetable_before);
 257         T_LOG("deallocating dirty anonymous memory decreases phys_footprint");
 258         T_EXPECT_EQ(footprint_after, footprint_expected,
 259             "deallocated %lld dirty bytes: "
 260             "footprint %lld -> %lld expected %lld delta %lld",
 261             dirty_size, footprint_before, footprint_after,
 262             footprint_expected, footprint_after - footprint_expected);
 263 }
 264
 265 #define TEMP_FILE_TEMPLATE "/tmp/phys_footprint_data.XXXXXXXX"
 266 #define TEMP_FILE_SIZE  (1 * 1024 * 1024)
 267
 268 T_DECL(legacy_phys_footprint_file,
 269     "phys_footprint for mapped file",
 270     T_META_NAMESPACE("xnu.vm"),
 271     T_META_LTEPHASE(LTE_POSTINIT))
 272 {
 273         uint64_t                footprint_before, pagetable_before;
 274         uint64_t                footprint_after, pagetable_after;
 275         uint64_t                footprint_expected;
 276         mach_vm_address_t       pre_vm_addr;
 277         int                     fd;
 278         char                    *map_addr;
 279         size_t                  map_size, dirty_size;
 280         ssize_t                 nbytes;
 281         char                    tmp_file_name[PATH_MAX] = TEMP_FILE_TEMPLATE;
 282         char                    *buf;
 283         size_t                  buf_size;
 284
 285         T_SETUPBEGIN;
 286         buf_size = TEMP_FILE_SIZE;
 287         T_QUIET;
 288         T_ASSERT_NOTNULL(buf = (char *)malloc(buf_size),
 289             "allocate %zu-byte buffer", buf_size);
 290         memset(buf, 'f', buf_size);
 291         T_WITH_ERRNO;
 292         T_QUIET;
 293         T_ASSERT_NOTNULL(mktemp(tmp_file_name),
 294             "create temporary file name");
 295         T_WITH_ERRNO;
 296         T_QUIET;
 297         T_ASSERT_GE(fd = open(tmp_file_name, O_CREAT | O_RDWR),
 298             0,
 299             "create temp file");
 300         T_WITH_ERRNO;
 301         T_QUIET;
 302         T_ASSERT_EQ(nbytes = write(fd, buf, buf_size),
 303             (ssize_t)buf_size,
 304             "write %zu bytes", buf_size);
 305         free(buf);
 306         T_SETUPEND;
 307
 308         /* pre-warm to account for page table expansion */
 309         pre_vm_addr = pre_warm(TEMP_FILE_SIZE);
 310
 311         /* mapping a file does not impact footprint... */
 312         get_ledger_info(&footprint_before, &pagetable_before);
 313         map_size = TEMP_FILE_SIZE;
 314         T_WITH_ERRNO;
 315         T_QUIET;
 316         T_ASSERT_NOTNULL(map_addr = (char *)mmap(NULL, map_size,
 317             PROT_READ | PROT_WRITE,
 318             MAP_FILE | MAP_SHARED, fd, 0),
 319             "mmap()");
 320         T_QUIET;
 321         T_EXPECT_EQ((mach_vm_address_t)map_addr, pre_vm_addr,
 322             "pre-warm mishap");
 323         /* ... should not change footprint */
 324         get_ledger_info(&footprint_after, &pagetable_after);
 325         footprint_expected = footprint_before;
 326         footprint_expected += (pagetable_after - pagetable_before);
 327         T_LOG("mapping file does not change phys_footprint");
 328         T_EXPECT_EQ(footprint_after, footprint_expected,
 329             "mapping file with %zu bytes: "
 330             "footprint %lld -> %lld expected %lld delta %lld",
 331             map_size, footprint_before, footprint_after,
 332             footprint_expected, footprint_after - footprint_expected);
 333
 334         /* touching file-backed memory... */
 335         get_ledger_info(&footprint_before, &pagetable_before);
 336         dirty_size = map_size / 2;
 337         memset(map_addr, 'F', dirty_size);
 338         /* ... should not impact footprint */
 339         get_ledger_info(&footprint_after, &pagetable_after);
 340         footprint_expected = footprint_before;
 341         footprint_expected += (pagetable_after - pagetable_before);
 342         T_LOG("modifying file-backed memory does not impact phys_footprint");
 343         T_EXPECT_EQ(footprint_after, footprint_expected,
 344             "touched %zu bytes: "
 345             "footprint %lld -> %lld expected %lld delta %lld",
 346             dirty_size, footprint_before, footprint_after,
 347             footprint_expected, footprint_after - footprint_expected);
 348
 349         /* deallocating file-backed memory... */
 350         get_ledger_info(&footprint_before, &pagetable_before);
 351         T_WITH_ERRNO;
 352         T_QUIET;
 353         T_ASSERT_EQ(munmap(map_addr, map_size),
 354             0,
 355             "unmap file");
 356         /* ... should not impact footprint */
 357         get_ledger_info(&footprint_after, &pagetable_after);
 358         footprint_expected = footprint_before;
 359         footprint_expected += (pagetable_after - pagetable_before);
 360         T_LOG("unmapping file-backed memory does not impact phys_footprint");
 361         T_EXPECT_EQ(footprint_after, footprint_expected,
 362             "unmapped %zu dirty bytes: "
 363             "footprint %lld -> %lld expected %lld delta %lld",
 364             dirty_size, footprint_before, footprint_after,
 365             footprint_expected, footprint_after - footprint_expected);
 366 }
 367
 368 T_DECL(legacy_phys_footprint_purgeable,
 369     "phys_footprint for purgeable memory",
 370     T_META_NAMESPACE("xnu.vm"),
 371     T_META_LTEPHASE(LTE_POSTINIT))
 372 {
 373         uint64_t                footprint_before, pagetable_before;
 374         uint64_t                footprint_after, pagetable_after;
 375         uint64_t                footprint_expected;
 376         kern_return_t           kr;
 377         mach_vm_address_t       pre_vm_addr, vm_addr;
 378         mach_vm_size_t          vm_size, dirty_size;
 379         int                     state;
 380
 381         /* pre-warm to account for page table expansion */
 382         pre_vm_addr = pre_warm(MEM_SIZE);
 383
 384         /* allocating purgeable virtual memory... */
 385         get_ledger_info(&footprint_before, &pagetable_before);
 386         vm_addr = 0;
 387         vm_size = MEM_SIZE;
 388         kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
 389             VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
 390         T_QUIET;
 391         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
 392             kr, mach_error_string(kr));
 393         T_QUIET;
 394         T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
 395         /* ... should not change footprint */
 396         get_ledger_info(&footprint_after, &pagetable_after);
 397         footprint_expected = footprint_before;
 398         footprint_expected += (pagetable_after - pagetable_before);
 399         T_LOG("purgeable virtual allocation does not change phys_footprint");
 400         T_EXPECT_EQ(footprint_after, footprint_expected,
 401             "purgeable virtual allocation of %lld bytes: "
 402             "footprint %lld -> %lld expected %lld delta %lld",
 403             vm_size, footprint_before, footprint_after,
 404             footprint_expected, footprint_after - footprint_expected);
 405
 406         /* touching memory... */
 407         get_ledger_info(&footprint_before, &pagetable_before);
 408         dirty_size = vm_size / 2;
 409         memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
 410         /* ... should increase footprint */
 411         get_ledger_info(&footprint_after, &pagetable_after);
 412         footprint_expected = footprint_before + dirty_size;
 413         footprint_expected += (pagetable_after - pagetable_before);
 414         T_LOG("modifying anonymous memory increases phys_footprint");
 415         T_EXPECT_EQ(footprint_after, footprint_expected,
 416             "touched %lld bytes: "
 417             "footprint %lld -> %lld expected %lld delta %lld",
 418             dirty_size, footprint_before, footprint_after,
 419             footprint_expected, footprint_after - footprint_expected);
 420
 421         /* making it volatile... */
 422         get_ledger_info(&footprint_before, &pagetable_before);
 423         state = VM_PURGABLE_VOLATILE;
 424         T_QUIET;
 425         T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
 426             vm_addr,
 427             VM_PURGABLE_SET_STATE,
 428             &state),
 429             KERN_SUCCESS,
 430             "vm_purgable_control(VOLATILE)");
 431         T_QUIET;
 432         T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
 433             "memory was non-volatile");
 434         /* ... should decrease footprint */
 435         get_ledger_info(&footprint_after, &pagetable_after);
 436         footprint_expected = footprint_before - dirty_size;
 437         footprint_expected += (pagetable_after - pagetable_before);
 438         T_LOG("making volatile decreases phys_footprint");
 439         T_EXPECT_EQ(footprint_after, footprint_expected,
 440             "made volatile %lld dirty bytes: "
 441             "footprint %lld -> %lld expected %lld delta %lld",
 442             dirty_size, footprint_before, footprint_after,
 443             footprint_expected, footprint_after - footprint_expected);
 444
 445         /* making it non-volatile... */
 446         get_ledger_info(&footprint_before, &pagetable_before);
 447         state = VM_PURGABLE_NONVOLATILE;
 448         T_QUIET;
 449         T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
 450             vm_addr,
 451             VM_PURGABLE_SET_STATE,
 452             &state),
 453             KERN_SUCCESS,
 454             "vm_purgable_control(NONVOLATILE)");
 455         T_QUIET;
 456         T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
 457             "memory was volatile");
 458         /* ... should increase footprint */
 459         get_ledger_info(&footprint_after, &pagetable_after);
 460         footprint_expected = footprint_before + dirty_size;
 461         footprint_expected += (pagetable_after - pagetable_before);
 462         T_LOG("making non-volatile increases phys_footprint");
 463         T_EXPECT_EQ(footprint_after, footprint_expected,
 464             "made non-volatile %lld dirty bytes: "
 465             "footprint %lld -> %lld expected %lld delta %lld",
 466             dirty_size, footprint_before, footprint_after,
 467             footprint_expected, footprint_after - footprint_expected);
 468
 469         /* deallocating memory... */
 470         get_ledger_info(&footprint_before, &pagetable_before);
 471         kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
 472         T_QUIET;
 473         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
 474             kr, mach_error_string(kr));
 475         /* ... should decrease footprint */
 476         get_ledger_info(&footprint_after, &pagetable_after);
 477         footprint_expected = footprint_before - dirty_size;
 478         footprint_expected += (pagetable_after - pagetable_before);
 479         T_LOG("deallocating memory decreases phys_footprint");
 480         T_EXPECT_EQ(footprint_after, footprint_expected,
 481             "deallocated %lld dirty bytes: "
 482             "footprint %lld -> %lld expected %lld delta %lld",
 483             dirty_size, footprint_before, footprint_after,
 484             footprint_expected, footprint_after - footprint_expected);
 485 }
 486
 487 T_DECL(legacy_phys_footprint_purgeable_ownership,
 488     "phys_footprint for owned purgeable memory",
 489     T_META_NAMESPACE("xnu.vm"),
 490     T_META_LTEPHASE(LTE_POSTINIT))
 491 {
 492         uint64_t                footprint_before, pagetable_before;
 493         uint64_t                footprint_after, pagetable_after;
 494         uint64_t                footprint_expected;
 495         kern_return_t           kr;
 496         mach_vm_address_t       pre_vm_addr, vm_addr;
 497         mach_vm_size_t          vm_size, dirty_size, me_size;
 498         int                     state;
 499         mach_port_t             me_port;
 500
 501         /* pre-warm to account for page table expansion */
 502         pre_vm_addr = pre_warm(MEM_SIZE);
 503
 504         /* allocating purgeable virtual memory... */
 505         get_ledger_info(&footprint_before, &pagetable_before);
 506         vm_addr = 0;
 507         vm_size = MEM_SIZE;
 508         kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
 509             VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
 510         T_QUIET;
 511         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
 512             kr, mach_error_string(kr));
 513         T_QUIET;
 514         T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
 515         /* ... should not change footprint */
 516         get_ledger_info(&footprint_after, &pagetable_after);
 517         footprint_expected = footprint_before;
 518         footprint_expected += (pagetable_after - pagetable_before);
 519         T_LOG("purgeable virtual allocation does not change phys_footprint");
 520         T_EXPECT_EQ(footprint_after, footprint_expected,
 521             "purgeable virtual allocation of %lld bytes: "
 522             "footprint %lld -> %lld expected %lld delta %lld",
 523             vm_size, footprint_before, footprint_after,
 524             footprint_expected, footprint_after - footprint_expected);
 525
 526         /* touching memory... */
 527         get_ledger_info(&footprint_before, &pagetable_before);
 528         dirty_size = vm_size / 2;
 529         memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
 530         /* ... should increase footprint */
 531         get_ledger_info(&footprint_after, &pagetable_after);
 532         footprint_expected = footprint_before + dirty_size;
 533         footprint_expected += (pagetable_after - pagetable_before);
 534         T_LOG("modifying anonymous memory increases phys_footprint");
 535         T_EXPECT_EQ(footprint_after, footprint_expected,
 536             "touched %lld bytes: "
 537             "footprint %lld -> %lld expected %lld delta %lld",
 538             dirty_size, footprint_before, footprint_after,
 539             footprint_expected, footprint_after - footprint_expected);
 540
 541         /* making it volatile... */
 542         get_ledger_info(&footprint_before, &pagetable_before);
 543         state = VM_PURGABLE_VOLATILE;
 544         T_QUIET;
 545         T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
 546             vm_addr,
 547             VM_PURGABLE_SET_STATE,
 548             &state),
 549             KERN_SUCCESS,
 550             "vm_purgable_control(VOLATILE)");
 551         T_QUIET;
 552         T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
 553             "memory was non-volatile");
 554         /* ... should decrease footprint */
 555         get_ledger_info(&footprint_after, &pagetable_after);
 556         footprint_expected = footprint_before - dirty_size;
 557         footprint_expected += (pagetable_after - pagetable_before);
 558         T_LOG("making volatile decreases phys_footprint");
 559         T_EXPECT_EQ(footprint_after, footprint_expected,
 560             "made volatile %lld dirty bytes: "
 561             "footprint %lld -> %lld expected %lld delta %lld",
 562             dirty_size, footprint_before, footprint_after,
 563             footprint_expected, footprint_after - footprint_expected);
 564
 565         /* making it non-volatile... */
 566         get_ledger_info(&footprint_before, &pagetable_before);
 567         state = VM_PURGABLE_NONVOLATILE;
 568         T_QUIET;
 569         T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
 570             vm_addr,
 571             VM_PURGABLE_SET_STATE,
 572             &state),
 573             KERN_SUCCESS,
 574             "vm_purgable_control(NONVOLATILE)");
 575         T_QUIET;
 576         T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
 577             "memory was volatile");
 578         /* ... should increase footprint */
 579         get_ledger_info(&footprint_after, &pagetable_after);
 580         footprint_expected = footprint_before + dirty_size;
 581         footprint_expected += (pagetable_after - pagetable_before);
 582         T_LOG("making non-volatile increases phys_footprint");
 583         T_EXPECT_EQ(footprint_after, footprint_expected,
 584             "made non-volatile %lld dirty bytes: "
 585             "footprint %lld -> %lld expected %lld delta %lld",
 586             dirty_size, footprint_before, footprint_after,
 587             footprint_expected, footprint_after - footprint_expected);
 588
 589         /* making a memory entry... */
 590         get_ledger_info(&footprint_before, &pagetable_before);
 591         me_size = vm_size;
 592         me_port = MACH_PORT_NULL;
 593         kr = mach_make_memory_entry_64(mach_task_self(),
 594             &me_size,
 595             vm_addr,
 596             VM_PROT_READ | VM_PROT_WRITE,
 597             &me_port,
 598             MACH_PORT_NULL);
 599         T_QUIET;
 600         T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
 601             kr, mach_error_string(kr));
 602         T_QUIET;
 603         T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
 604         /* ... should not change footprint */
 605         get_ledger_info(&footprint_after, &pagetable_after);
 606         footprint_expected = footprint_before;
 607         footprint_expected += (pagetable_after - pagetable_before);
 608         T_LOG("making a memory entry does not change phys_footprint");
 609         T_EXPECT_EQ(footprint_after, footprint_expected,
 610             "making a memory entry of %lld bytes: "
 611             "footprint %lld -> %lld expected %lld delta %lld",
 612             vm_size, footprint_before, footprint_after,
 613             footprint_expected, footprint_after - footprint_expected);
 614
 615         /* deallocating memory while holding memory entry... */
 616         get_ledger_info(&footprint_before, &pagetable_before);
 617         kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
 618         T_QUIET;
 619         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
 620             kr, mach_error_string(kr));
 621         /* ... should not change footprint */
 622         get_ledger_info(&footprint_after, &pagetable_after);
 623         footprint_expected = footprint_before;
 624         footprint_expected += (pagetable_after - pagetable_before);
 625         T_LOG("deallocating owned memory while holding memory entry "
 626             "does not change phys_footprint");
 627         T_EXPECT_EQ(footprint_after, footprint_expected,
 628             "deallocated %lld dirty bytes: "
 629             "footprint %lld -> %lld expected %lld delta %lld",
 630             dirty_size, footprint_before, footprint_after,
 631             footprint_expected, footprint_after - footprint_expected);
 632
 633         /* releasing the memory entry... */
 634         kr = mach_port_deallocate(mach_task_self(), me_port);
 635         T_QUIET;
 636         T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
 637             kr, mach_error_string(kr));
 638         /* ... should decrease footprint */
 639         get_ledger_info(&footprint_after, &pagetable_after);
 640         footprint_expected = footprint_before - dirty_size;
 641         footprint_expected += (pagetable_after - pagetable_before);
 642         T_LOG("releasing memory entry decreases phys_footprint");
 643         T_EXPECT_EQ(footprint_after, footprint_expected,
 644             "made volatile %lld dirty bytes: "
 645             "footprint %lld -> %lld expected %lld delta %lld",
 646             dirty_size, footprint_before, footprint_after,
 647             footprint_expected, footprint_after - footprint_expected);
 648 }
 649
 650 #ifdef MAP_MEM_LEDGER_TAGGED
 651 T_DECL(legacy_phys_footprint_ledger_purgeable_owned,
 652     "phys_footprint for ledger-tagged purgeable memory ownership",
 653     T_META_NAMESPACE("xnu.vm"),
 654     T_META_LTEPHASE(LTE_POSTINIT))
 655 {
 656         uint64_t                footprint_before, pagetable_before;
 657         uint64_t                footprint_after, pagetable_after;
 658         uint64_t                footprint_expected;
 659         kern_return_t           kr;
 660         mach_vm_address_t       pre_vm_addr, vm_addr;
 661         mach_vm_size_t          vm_size, dirty_size, me_size;
 662         int                     state;
 663         mach_port_t             me_port;
 664
 665         /* pre-warm to account for page table expansion */
 666         pre_vm_addr = pre_warm(MEM_SIZE);
 667
 668         /* making a memory entry... */
 669         get_ledger_info(&footprint_before, &pagetable_before);
 670         vm_size = MEM_SIZE;
 671         me_size = vm_size;
 672         me_port = MACH_PORT_NULL;
 673         kr = mach_make_memory_entry_64(mach_task_self(),
 674             &me_size,
 675             0,
 676             (MAP_MEM_NAMED_CREATE |
 677             MAP_MEM_LEDGER_TAGGED |
 678             MAP_MEM_PURGABLE |
 679             VM_PROT_READ | VM_PROT_WRITE),
 680             &me_port,
 681             MACH_PORT_NULL);
 682         T_QUIET;
 683         T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
 684             kr, mach_error_string(kr));
 685         T_QUIET;
 686         T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
 687         /* ... should not change footprint */
 688         get_ledger_info(&footprint_after, &pagetable_after);
 689         footprint_expected = footprint_before;
 690         footprint_expected += (pagetable_after - pagetable_before);
 691         T_LOG("making a memory entry does not change phys_footprint");
 692         T_EXPECT_EQ(footprint_after, footprint_expected,
 693             "making a memory entry of %lld bytes: "
 694             "footprint %lld -> %lld expected %lld delta %lld",
 695             vm_size, footprint_before, footprint_after,
 696             footprint_expected, footprint_after - footprint_expected);
 697
 698         /* mapping ledger-tagged virtual memory... */
 699         get_ledger_info(&footprint_before, &pagetable_before);
 700         vm_addr = 0;
 701         kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
 702             0, /* mask */
 703             VM_FLAGS_ANYWHERE,
 704             me_port,
 705             0, /* offset */
 706             FALSE, /* copy */
 707             VM_PROT_READ | VM_PROT_WRITE,
 708             VM_PROT_READ | VM_PROT_WRITE,
 709             VM_INHERIT_DEFAULT);
 710         T_QUIET;
 711         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
 712             kr, mach_error_string(kr));
 713         T_QUIET;
 714         T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
 715         /* ... should not change footprint */
 716         get_ledger_info(&footprint_after, &pagetable_after);
 717         footprint_expected = footprint_before;
 718         footprint_expected += (pagetable_after - pagetable_before);
 719         T_LOG("mapping ledger-tagged memory does not change phys_footprint");
 720         T_EXPECT_EQ(footprint_after, footprint_expected,
 721             "ledger-tagged mapping of %lld bytes: "
 722             "footprint %lld -> %lld expected %lld delta %lld",
 723             vm_size, footprint_before, footprint_after,
 724             footprint_expected, footprint_after - footprint_expected);
 725
 726         /* touching memory... */
 727         get_ledger_info(&footprint_before, &pagetable_before);
 728         dirty_size = vm_size / 2;
 729         memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
 730         /* ... should increase footprint */
 731         get_ledger_info(&footprint_after, &pagetable_after);
 732         footprint_expected = footprint_before + dirty_size;
 733         footprint_expected += (pagetable_after - pagetable_before);
 734         T_LOG("modifying ledger-tagged memory increases phys_footprint");
 735         T_EXPECT_EQ(footprint_after, footprint_expected,
 736             "touched %lld bytes: "
 737             "footprint %lld -> %lld expected %lld delta %lld",
 738             dirty_size, footprint_before, footprint_after,
 739             footprint_expected, footprint_after - footprint_expected);
 740
 741         /* making it volatile... */
 742         get_ledger_info(&footprint_before, &pagetable_before);
 743         state = VM_PURGABLE_VOLATILE;
 744         T_QUIET;
 745         T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
 746             vm_addr,
 747             VM_PURGABLE_SET_STATE,
 748             &state),
 749             KERN_SUCCESS,
 750             "vm_purgable_control(VOLATILE)");
 751         T_QUIET;
 752         T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
 753             "memory was non-volatile");
 754         /* ... should decrease footprint */
 755         get_ledger_info(&footprint_after, &pagetable_after);
 756         footprint_expected = footprint_before - dirty_size;
 757         footprint_expected += (pagetable_after - pagetable_before);
 758         T_LOG("making volatile decreases phys_footprint");
 759         T_EXPECT_EQ(footprint_after, footprint_expected,
 760             "made volatile %lld dirty bytes: "
 761             "footprint %lld -> %lld expected %lld delta %lld",
 762             dirty_size, footprint_before, footprint_after,
 763             footprint_expected, footprint_after - footprint_expected);
 764
 765         /* making it non-volatile... */
 766         get_ledger_info(&footprint_before, &pagetable_before);
 767         state = VM_PURGABLE_NONVOLATILE;
 768         T_QUIET;
 769         T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
 770             vm_addr,
 771             VM_PURGABLE_SET_STATE,
 772             &state),
 773             KERN_SUCCESS,
 774             "vm_purgable_control(NONVOLATILE)");
 775         T_QUIET;
 776         T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
 777             "memory was volatile");
 778         /* ... should increase footprint */
 779         get_ledger_info(&footprint_after, &pagetable_after);
 780         footprint_expected = footprint_before + dirty_size;
 781         footprint_expected += (pagetable_after - pagetable_before);
 782         T_LOG("making non-volatile increases phys_footprint");
 783         T_EXPECT_EQ(footprint_after, footprint_expected,
 784             "made non-volatile %lld dirty bytes: "
 785             "footprint %lld -> %lld expected %lld delta %lld",
 786             dirty_size, footprint_before, footprint_after,
 787             footprint_expected, footprint_after - footprint_expected);
 788
 789         /* deallocating memory while holding memory entry... */
 790         get_ledger_info(&footprint_before, &pagetable_before);
 791         kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
 792         T_QUIET;
 793         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
 794             kr, mach_error_string(kr));
 795         /* ... should not change footprint */
 796         get_ledger_info(&footprint_after, &pagetable_after);
 797         footprint_expected = footprint_before;
 798         footprint_expected += (pagetable_after - pagetable_before);
 799         T_LOG("deallocating owned memory while holding memory entry "
 800             "does not change phys_footprint");
 801         T_EXPECT_EQ(footprint_after, footprint_expected,
 802             "deallocated %lld dirty bytes: "
 803             "footprint %lld -> %lld expected %lld delta %lld",
 804             dirty_size, footprint_before, footprint_after,
 805             footprint_expected, footprint_after - footprint_expected);
 806
 807         /* releasing the memory entry... */
 808         kr = mach_port_deallocate(mach_task_self(), me_port);
 809         T_QUIET;
 810         T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
 811             kr, mach_error_string(kr));
 812         /* ... should decrease footprint */
 813         get_ledger_info(&footprint_after, &pagetable_after);
 814         footprint_expected = footprint_before - dirty_size;
 815         footprint_expected += (pagetable_after - pagetable_before);
 816         T_LOG("releasing memory entry decreases phys_footprint");
 817         T_EXPECT_EQ(footprint_after, footprint_expected,
 818             "made volatile %lld dirty bytes: "
 819             "footprint %lld -> %lld expected %lld delta %lld",
 820             dirty_size, footprint_before, footprint_after,
 821             footprint_expected, footprint_after - footprint_expected);
 822 }
 823
 824 T_DECL(legacy_phys_footprint_ledger_owned,
 825     "phys_footprint for ledger-tagged memory ownership",
 826     T_META_NAMESPACE("xnu.vm"),
 827     T_META_LTEPHASE(LTE_POSTINIT))
 828 {
 829         uint64_t                footprint_before, pagetable_before;
 830         uint64_t                footprint_after, pagetable_after;
 831         uint64_t                footprint_expected;
 832         kern_return_t           kr;
 833         mach_vm_address_t       pre_vm_addr, vm_addr;
 834         mach_vm_size_t          vm_size, dirty_size, me_size;
 835         int                     state;
 836         mach_port_t             me_port;
 837
 838         /* pre-warm to account for page table expansion */
 839         pre_vm_addr = pre_warm(MEM_SIZE);
 840
 841         /* making a memory entry... */
 842         get_ledger_info(&footprint_before, &pagetable_before);
 843         vm_size = MEM_SIZE;
 844         me_size = vm_size;
 845         me_port = MACH_PORT_NULL;
 846         kr = mach_make_memory_entry_64(mach_task_self(),
 847             &me_size,
 848             0,
 849             (MAP_MEM_NAMED_CREATE |
 850             MAP_MEM_LEDGER_TAGGED |
 851             VM_PROT_READ | VM_PROT_WRITE),
 852             &me_port,
 853             MACH_PORT_NULL);
 854         T_QUIET;
 855         T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
 856             kr, mach_error_string(kr));
 857         T_QUIET;
 858         T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
 859         /* ... should not change footprint */
 860         get_ledger_info(&footprint_after, &pagetable_after);
 861         footprint_expected = footprint_before;
 862         footprint_expected += (pagetable_after - pagetable_before);
 863         T_LOG("making a memory entry does not change phys_footprint");
 864         T_EXPECT_EQ(footprint_after, footprint_expected,
 865             "making a memory entry of %lld bytes: "
 866             "footprint %lld -> %lld expected %lld delta %lld",
 867             vm_size, footprint_before, footprint_after,
 868             footprint_expected, footprint_after - footprint_expected);
 869
 870         /* mapping ledger-tagged virtual memory... */
 871         get_ledger_info(&footprint_before, &pagetable_before);
 872         vm_addr = 0;
 873         kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
 874             0, /* mask */
 875             VM_FLAGS_ANYWHERE,
 876             me_port,
 877             0, /* offset */
 878             FALSE, /* copy */
 879             VM_PROT_READ | VM_PROT_WRITE,
 880             VM_PROT_READ | VM_PROT_WRITE,
 881             VM_INHERIT_DEFAULT);
 882         T_QUIET;
 883         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
 884             kr, mach_error_string(kr));
 885         T_QUIET;
 886         T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
 887         /* ... should not change footprint */
 888         get_ledger_info(&footprint_after, &pagetable_after);
 889         footprint_expected = footprint_before;
 890         footprint_expected += (pagetable_after - pagetable_before);
 891         T_LOG("mapping ledger-tagged memory does not change phys_footprint");
 892         T_EXPECT_EQ(footprint_after, footprint_expected,
 893             "ledger-tagged mapping of %lld bytes: "
 894             "footprint %lld -> %lld expected %lld delta %lld",
 895             vm_size, footprint_before, footprint_after,
 896             footprint_expected, footprint_after - footprint_expected);
 897
 898         /* touching memory... */
 899         get_ledger_info(&footprint_before, &pagetable_before);
 900         dirty_size = vm_size / 2;
 901         memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
 902         /* ... should increase footprint */
 903         get_ledger_info(&footprint_after, &pagetable_after);
 904         footprint_expected = footprint_before + dirty_size;
 905         footprint_expected += (pagetable_after - pagetable_before);
 906         T_LOG("modifying ledger-tagged memory increases phys_footprint");
 907         T_EXPECT_EQ(footprint_after, footprint_expected,
 908             "touched %lld bytes: "
 909             "footprint %lld -> %lld expected %lld delta %lld",
 910             dirty_size, footprint_before, footprint_after,
 911             footprint_expected, footprint_after - footprint_expected);
 912
 913         /* deallocating memory while holding memory entry... */
 914         get_ledger_info(&footprint_before, &pagetable_before);
 915         kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
 916         T_QUIET;
 917         T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
 918             kr, mach_error_string(kr));
 919         /* ... should not change footprint */
 920         get_ledger_info(&footprint_after, &pagetable_after);
 921         footprint_expected = footprint_before;
 922         footprint_expected += (pagetable_after - pagetable_before);
 923         T_LOG("deallocating owned memory while holding memory entry "
 924             "does not change phys_footprint");
 925         T_EXPECT_EQ(footprint_after, footprint_expected,
 926             "deallocated %lld dirty bytes: "
 927             "footprint %lld -> %lld expected %lld delta %lld",
 928             dirty_size, footprint_before, footprint_after,
 929             footprint_expected, footprint_after - footprint_expected);
 930
 931         /* releasing the memory entry... */
 932         kr = mach_port_deallocate(mach_task_self(), me_port);
 933         T_QUIET;
 934         T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
 935             kr, mach_error_string(kr));
 936         /* ... should decrease footprint */
 937         get_ledger_info(&footprint_after, &pagetable_after);
 938         footprint_expected = footprint_before - dirty_size;
 939         footprint_expected += (pagetable_after - pagetable_before);
 940         T_LOG("releasing memory entry decreases phys_footprint");
 941         T_EXPECT_EQ(footprint_after, footprint_expected,
 942             "made volatile %lld dirty bytes: "
 943             "footprint %lld -> %lld expected %lld delta %lld",
 944             dirty_size, footprint_before, footprint_after,
 945             footprint_expected, footprint_after - footprint_expected);
 946 }
 947 #endif /* MAP_MEM_LEDGER_TAGGED */
 948
 949 /* IOSurface code from: CoreImage/CoreImageTests/CIRender/SurfaceUtils.c */
 950 #include <CoreFoundation/CoreFoundation.h>
 951 #include <IOSurface/IOSurface.h>
 952 #include <IOSurface/IOSurfacePrivate.h>
 953 static size_t
 954 bytes_per_element(uint32_t format)
 955 {
 956         size_t bpe = 0;
 957         switch (format) {
 958         case 32:     // kCVPixelFormatType_32ARGB (ARGB8)
 959                 bpe = 4;
 960                 break;
 961         default:
 962                 bpe = 0;
 963                 break;
 964         }
 965         return bpe;
 966 }
 967 static size_t
 968 bytes_per_pixel(uint32_t format)
 969 {
 970         size_t bpe = 0;
 971         switch (format) {
 972         case 32:     // kCVPixelFormatType_32ARGB (ARGB8)
 973                 bpe = 4;
 974                 break;
 975         default:
 976                 bpe = 0;
 977                 break;
 978         }
 979         return bpe;
 980 }
 981 static inline size_t
 982 roundSizeToMultiple(size_t size, size_t mult)
 983 {
 984         return ((size + mult - 1) / mult) * mult;
 985 }
 986 static inline void
 987 setIntValue(CFMutableDictionaryRef dict, const CFStringRef key, int value)
 988 {
 989         CFNumberRef number = CFNumberCreate(0, kCFNumberIntType, &value);
 990         CFDictionarySetValue(dict, key, number);
 991         CFRelease(number);
 992 }
 993 typedef void (^SurfacePlaneBlock)(void *data, size_t planeIndex, size_t width, size_t height, size_t rowbytes);
 994 static IOReturn
 995 SurfaceApplyPlaneBlock(IOSurfaceRef surface, SurfacePlaneBlock block)
 996 {
 997         if (surface == nil || block == nil) {
 998                 return kIOReturnBadArgument;
 999         }
1000
1001         IOReturn result = kIOReturnSuccess;
1002         size_t planeCount = IOSurfaceGetPlaneCount(surface);
1003
1004         if (planeCount == 0) {
1005                 result = IOSurfaceLock(surface, 0, NULL);
1006                 if (result != kIOReturnSuccess) {
1007                         return result;
1008                 }
1009
1010                 void* base = IOSurfaceGetBaseAddress(surface);
1011                 size_t rb = IOSurfaceGetBytesPerRow(surface);
1012                 size_t w = IOSurfaceGetWidth(surface);
1013                 size_t h = IOSurfaceGetHeight(surface);
1014
1015                 if (base && rb && w && h) {
1016                         block(base, 0, w, h, rb);
1017                 }
1018
1019                 IOSurfaceUnlock(surface, 0, NULL);
1020         } else if (planeCount == 2) {
1021                 for (size_t i = 0; i < planeCount; i++) {
1022                         result = IOSurfaceLock(surface, 0, NULL);
1023                         if (result != kIOReturnSuccess) {
1024                                 return result;
1025                         }
1026
1027                         void* base = IOSurfaceGetBaseAddressOfPlane(surface, i);
1028                         size_t rb = IOSurfaceGetBytesPerRowOfPlane(surface, i);
1029                         size_t w = IOSurfaceGetWidthOfPlane(surface, i);
1030                         size_t h = IOSurfaceGetHeightOfPlane(surface, i);
1031
1032                         if (base && rb && w && h) {
1033                                 block(base, i, w, h, rb);
1034                         }
1035
1036                         IOSurfaceUnlock(surface, 0, NULL);
1037                 }
1038         }
1039         return result;
1040 }
1041 static void
1042 ClearSurface(IOSurfaceRef surface)
1043 {
1044         const int zero = 0;
1045         (void) SurfaceApplyPlaneBlock(surface, ^(void *p, size_t i, __unused size_t w, size_t h, size_t rb)
1046         {
1047                 if (i == 0) {
1048                         memset(p, zero, rb * h);
1049                 } else {
1050                         memset(p, 128, rb * h);
1051                 }
1052         });
1053 }
1054 static IOSurfaceRef
1055 CreateSurface(uint32_t pixelsWide, uint32_t pixelsHigh, uint32_t rowBytesAlignment, uint32_t fmt, bool purgeable, bool clear)
1056 {
1057         IOSurfaceRef surface = nil;
1058
1059         if (pixelsWide < 1 || pixelsHigh < 1 || fmt == 0) {
1060                 return nil;
1061         }
1062
1063         size_t bpp = bytes_per_pixel(fmt);
1064         size_t bpe = bytes_per_element(fmt);
1065         if (bpp == 0 || bpe == 0) {
1066                 return nil;
1067         }
1068
1069         size_t rowbytes = pixelsWide * bpp;
1070         if (rowBytesAlignment == 0) {
1071                 rowBytesAlignment = 16;
1072         }
1073         rowbytes = roundSizeToMultiple(rowbytes, rowBytesAlignment);
1074
1075         CFMutableDictionaryRef props = CFDictionaryCreateMutable(0, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1076         setIntValue(props, kIOSurfaceBytesPerRow, (int)rowbytes);
1077         setIntValue(props, kIOSurfaceWidth, (int)pixelsWide);
1078         setIntValue(props, kIOSurfaceHeight, (int)pixelsHigh);
1079         setIntValue(props, kIOSurfacePixelFormat, (int)fmt);
1080 #if TARGET_OS_IPHONE
1081         setIntValue(props, kIOSurfaceNonPurgeable, purgeable);
1082 #else /* TARGET_OS_IPHONE */
1083         (void)purgeable;
1084 #endif /* TARGET_OS_IPHONE */
1085         {
1086                 if (bpe != bpp) { // i.e. a 422 format such as 'yuvf' etc.
1087                         setIntValue(props, kIOSurfaceElementWidth, 2);
1088                         setIntValue(props, kIOSurfaceElementHeight, 1);
1089                 }
1090                 setIntValue(props, kIOSurfaceBytesPerElement, (int)bpe);
1091         }
1092
1093         surface = IOSurfaceCreate(props);
1094
1095         if (clear) {
1096                 ClearSurface(surface);
1097         }
1098
1099         CFRelease(props);
1100         return surface;
1101 }
1102 T_DECL(legacy_phys_footprint_purgeable_iokit,
1103     "phys_footprint for purgeable IOKit memory",
1104     T_META_NAMESPACE("xnu.vm"),
1105     T_META_LTEPHASE(LTE_POSTINIT))
1106 {
1107         uint64_t        footprint_before, pagetable_before;
1108         uint64_t        footprint_after, pagetable_after;
1109         uint64_t        footprint_expected;
1110         IOSurfaceRef    surface;
1111         uint32_t        old_state;
1112         uint64_t        surface_size;
1113
1114         T_SETUPBEGIN;
1115         ledger_init();
1116         surface = CreateSurface(1024, 1024, 0, 32, true, true);
1117         IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1118         IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1119         CFRelease(surface);
1120         T_SETUPEND;
1121
1122         surface_size = 1024 * 1024 * 4;
1123
1124         /* create IOsurface: footprint grows */
1125         get_ledger_info(&footprint_before, &pagetable_before);
1126         surface = CreateSurface(1024, 1024, 0, 32, true, true);
1127         get_ledger_info(&footprint_after, &pagetable_after);
1128 #if LEGACY_FOOTPRINT
1129         footprint_expected = footprint_before;
1130         footprint_expected += (pagetable_after - pagetable_before);
1131         T_LOG("LEGACY FOOTPRINT: creating IOSurface: no footprint impact");
1132         T_EXPECT_EQ(footprint_after, footprint_expected,
1133             "create IOSurface %lld bytes: "
1134             "footprint %lld -> %lld expected %lld delta %lld",
1135             surface_size, footprint_before, footprint_after,
1136             footprint_expected, footprint_after - footprint_expected);
1137 #else /* LEGACY_FOOTPRINT */
1138         footprint_expected = footprint_before + surface_size;
1139         footprint_expected += (pagetable_after - pagetable_before);
1140         T_LOG("creating IOSurface increases phys_footprint");
1141         T_EXPECT_EQ(footprint_after, footprint_expected,
1142             "create IOSurface %lld bytes: "
1143             "footprint %lld -> %lld expected %lld delta %lld",
1144             surface_size, footprint_before, footprint_after,
1145             footprint_expected, footprint_after - footprint_expected);
1146 #endif /* LEGACY_FOOTPRINT */
1147
1148         /* make IOSurface volatile: footprint shrinks */
1149         get_ledger_info(&footprint_before, &pagetable_before);
1150         IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1151         get_ledger_info(&footprint_after, &pagetable_after);
1152 #if LEGACY_FOOTPRINT
1153         footprint_expected = footprint_before;
1154         footprint_expected += (pagetable_after - pagetable_before);
1155         T_LOG("LEGACY FOOTPRINT: volatile IOSurface: no footprint impact");
1156         T_EXPECT_EQ(footprint_after, footprint_expected,
1157             "volatile IOSurface %lld bytes: "
1158             "footprint %lld -> %lld expected %lld delta %lld",
1159             surface_size, footprint_before, footprint_after,
1160             footprint_expected, footprint_after - footprint_expected);
1161 #else /* LEGACY_FOOTPRINT */
1162         footprint_expected = footprint_before - surface_size;
1163         footprint_expected += (pagetable_after - pagetable_before);
1164         T_LOG("making IOSurface volatile decreases phys_footprint");
1165         T_EXPECT_EQ(footprint_after, footprint_expected,
1166             "made volatile %lld bytes: "
1167             "footprint %lld -> %lld expected %lld delta %lld",
1168             surface_size, footprint_before, footprint_after,
1169             footprint_expected, footprint_after - footprint_expected);
1170 #endif /* LEGACY_FOOTPRINT */
1171
1172         /* make IOSurface non-volatile: footprint grows */
1173         get_ledger_info(&footprint_before, &pagetable_before);
1174         IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1175         get_ledger_info(&footprint_after, &pagetable_after);
1176 #if LEGACY_FOOTPRINT
1177         footprint_expected = footprint_before;
1178         footprint_expected += (pagetable_after - pagetable_before);
1179         T_LOG("LEGACY FOOTPRINT: non-volatile IOSurface: no footprint impact");
1180         T_EXPECT_EQ(footprint_after, footprint_expected,
1181             "non-volatile IOSurface %lld bytes: "
1182             "footprint %lld -> %lld expected %lld delta %lld",
1183             surface_size, footprint_before, footprint_after,
1184             footprint_expected, footprint_after - footprint_expected);
1185 #else /* LEGACY_FOOTPRINT */
1186         footprint_expected = footprint_before + surface_size;
1187         footprint_expected += (pagetable_after - pagetable_before);
1188         T_LOG("making IOSurface non-volatile increases phys_footprint");
1189         T_EXPECT_EQ(footprint_after, footprint_expected,
1190             "made non-volatile %lld bytes: "
1191             "footprint %lld -> %lld expected %lld delta %lld",
1192             surface_size, footprint_before, footprint_after,
1193             footprint_expected, footprint_after - footprint_expected);
1194 #endif /* LEGACY_FOOTPRINT */
1195
1196         /* accessing IOSurface re-mapping: no footprint impact */
1197
1198         /* deallocating IOSurface re-mapping: no footprint impact */
1199
1200         /* release IOSurface: footprint shrinks */
1201         get_ledger_info(&footprint_before, &pagetable_before);
1202         CFRelease(surface);
1203         get_ledger_info(&footprint_after, &pagetable_after);
1204 #if LEGACY_FOOTPRINT
1205         footprint_expected = footprint_before;
1206         footprint_expected += (pagetable_after - pagetable_before);
1207         T_LOG("LEGACY FOOTPRINT: release IOSurface: no footprint impact");
1208         T_EXPECT_EQ(footprint_after, footprint_expected,
1209             "releasing IOSurface %lld bytes: "
1210             "footprint %lld -> %lld expected %lld delta %lld",
1211             surface_size, footprint_before, footprint_after,
1212             footprint_expected, footprint_after - footprint_expected);
1213 #else /* LEGACY_FOOTPRINT */
1214         footprint_expected = footprint_before - surface_size;
1215         footprint_expected += (pagetable_after - pagetable_before);
1216         T_LOG("releasing IOSurface decreases phys_footprint");
1217         T_EXPECT_EQ(footprint_after, footprint_expected,
1218             "released IOSurface %lld bytes: "
1219             "footprint %lld -> %lld expected %lld delta %lld",
1220             surface_size, footprint_before, footprint_after,
1221             footprint_expected, footprint_after - footprint_expected);
1222 #endif /* LEGACY_FOOTPRINT */
1223 }