xnu-7195.101.1.tar.gz

[apple/xnu.git] / tests / vm_phys_footprint.c

#include <darwintest.h>
#include <darwintest_utils.h>

#include <mach/mach_error.h>
#include <mach/mach_init.h>
#include <mach/mach_port.h>
#include <mach/mach_vm.h>
#include <mach/task.h>
#include <mach/task_info.h>
#include <mach/vm_map.h>

#include <sys/mman.h>
#include <sys/types.h>
#include <sys/sysctl.h>

#include <TargetConditionals.h>

#include <Kernel/kern/ledger.h>
extern int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3);

T_GLOBAL_META(T_META_RUN_CONCURRENTLY(true));
boolean_t legacy_footprint;

#if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
#define TEST_VM_NAMESPACE "xnu.vm_legacy"
#else /* ENTITLED && __arm64__ */
#define TEST_VM_NAMESPACE "xnu.vm"
#endif /* ENTITLED && __arm64__ */

#define MEM_SIZE (100 * 1024 * 1024) /* 100 MB */

static int64_t ledger_count = -1;
static int footprint_index = -1;
static int pagetable_index = -1;
static struct ledger_entry_info *lei = NULL;

static void
ledger_init(void)
{
        static int                      ledger_inited = 0;
        struct ledger_info              li;
        struct ledger_template_info     *templateInfo;
        int64_t                         templateCnt;
        int                             i;
        int                             legacy_footprint_entitlement_mode;
        size_t                          oldlen;

        if (ledger_inited) {
                return;
        }
        ledger_inited = 1;

        T_SETUPBEGIN;

        legacy_footprint = FALSE;
#if LEGACY_FOOTPRINT_ENTITLED
        int ret;

        T_QUIET;
        T_WITH_ERRNO;
        oldlen = sizeof(legacy_footprint_entitlement_mode);
        ret = sysctlbyname("kern.legacy_footprint_entitlement_mode",
            &legacy_footprint_entitlement_mode,
            &oldlen,
            NULL,
            0);
        if (ret == 0 && legacy_footprint_entitlement_mode == 2) {
                legacy_footprint = TRUE;
        }
#endif /* LEGACY_FOOTPRINT_ENTITLED */

        T_QUIET;
        T_WITH_ERRNO;
        T_ASSERT_EQ(ledger(LEDGER_INFO,
            (caddr_t)(uintptr_t)getpid(),
            (caddr_t)&li,
            NULL),
            0,
            "ledger(LEDGER_INFO)");

        templateCnt = li.li_entries;
        templateInfo = malloc((size_t)li.li_entries * sizeof(struct ledger_template_info));
        T_QUIET;
        T_WITH_ERRNO;
        T_ASSERT_NE(templateInfo, NULL, "malloc()");

        ledger_count = li.li_entries;
        footprint_index = -1;
        pagetable_index = -1;
        T_QUIET;
        T_WITH_ERRNO;
        T_ASSERT_GE(ledger(LEDGER_TEMPLATE_INFO,
            (caddr_t)templateInfo,
            (caddr_t)&templateCnt,
            NULL),
            0,
            "ledger(LEDGER_TEMPLATE_INFO)");
        for (i = 0; i < templateCnt; i++) {
                if (!strncmp(templateInfo[i].lti_name,
                    "phys_footprint",
                    strlen("phys_footprint"))) {
                        footprint_index = i;
                } else if (!strncmp(templateInfo[i].lti_name,
                    "page_table",
                    strlen("page_table"))) {
                        pagetable_index = i;
                }
        }
        free(templateInfo);

        lei = (struct ledger_entry_info *)
            malloc((size_t)ledger_count * sizeof(*lei));
        T_QUIET;
        T_WITH_ERRNO;
        T_ASSERT_NE(lei, NULL, "malloc(ledger_entry_info)");

        T_QUIET;
        T_ASSERT_NE(footprint_index, -1, "no footprint_index");
        T_QUIET;
        T_ASSERT_NE(pagetable_index, -1, "no pagetable_index");

        T_SETUPEND;
}

static void
get_ledger_info(
        uint64_t        *phys_footprint,
        uint64_t        *page_table)
{
        int64_t count;

        count = ledger_count;
        T_QUIET;
        T_WITH_ERRNO;
        T_ASSERT_GE(ledger(LEDGER_ENTRY_INFO,
            (caddr_t)(uintptr_t)getpid(),
            (caddr_t)lei,
            (caddr_t)&count),
            0,
            "ledger(LEDGER_ENTRY_INFO)");
        T_QUIET;
        T_ASSERT_GT(count, (int64_t)footprint_index, "no entry for footprint");
        T_QUIET;
        T_ASSERT_GT(count, (int64_t)pagetable_index, "no entry for pagetable");
        if (phys_footprint) {
                *phys_footprint = (uint64_t)(lei[footprint_index].lei_balance);
        }
        if (page_table) {
                *page_table = (uint64_t)(lei[pagetable_index].lei_balance);
        }
}

static mach_vm_address_t
pre_warm(
        mach_vm_size_t  vm_size)
{
        kern_return_t           kr;
        mach_vm_address_t       vm_addr;
        unsigned char           BigBufOnStack[100 * 1024];
        uint64_t                footprint, page_table;

        /* make sure ledgers are ready to be queried */
        ledger_init();

        T_SETUPBEGIN;

        /*
         * Touch a few pages ahead on the stack, to make
         * sure we don't see a footprint increase due to
         * an extra stack page later.
         */
        memset(BigBufOnStack, 0xb, sizeof(BigBufOnStack));
        T_QUIET;
        T_EXPECT_EQ(BigBufOnStack[0], 0xb,
            "BigBufOnStack[0] == 0x%x",
            BigBufOnStack[0]);
        T_QUIET;
        T_EXPECT_EQ(BigBufOnStack[sizeof(BigBufOnStack) - 1], 0xb,
            "BigBufOnStack[%lu] == 0x%x",
            sizeof(BigBufOnStack),
            BigBufOnStack[sizeof(BigBufOnStack) - 1]);

        /*
         * Pre-allocate, touch and then release the same amount
         * of memory we'll be allocating later during the test,
         * to account for any memory overhead (page tables, global
         * variables, ...).
         */
        vm_addr = 0;
        kr = mach_vm_allocate(mach_task_self(),
            &vm_addr,
            vm_size,
            VM_FLAGS_ANYWHERE);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate(%lld) error 0x%x (%s)",
            vm_size, kr, mach_error_string(kr));
        memset((char *)(uintptr_t)vm_addr, 'p', (size_t)vm_size);
        kr = mach_vm_deallocate(mach_task_self(),
            vm_addr,
            vm_size);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));

        /*
         * Exercise the ledger code to make sure it's ready to run
         * without any extra memory overhead later.
         */
        get_ledger_info(&footprint, &page_table);

        T_SETUPEND;

        /*
         * Return the start of the virtual range we pre-warmed, so that the
         * test can check that it's using the same range.
         */
        return vm_addr;
}

T_DECL(phys_footprint_anonymous,
    "phys_footprint for anonymous memory",
    T_META_NAMESPACE(TEST_VM_NAMESPACE),
    T_META_LTEPHASE(LTE_POSTINIT))
{
        uint64_t                footprint_before, pagetable_before;
        uint64_t                footprint_after, pagetable_after;
        uint64_t                footprint_expected;
        kern_return_t           kr;
        mach_vm_address_t       pre_vm_addr, vm_addr;
        mach_vm_size_t          vm_size, dirty_size;

        /* pre-warm to account for page table expansion */
        pre_vm_addr = pre_warm(MEM_SIZE);

        /* allocating virtual memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        vm_addr = 0;
        vm_size = MEM_SIZE;
        kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
            VM_FLAGS_ANYWHERE);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        T_QUIET;
        T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("virtual allocation does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "virtual allocation of %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            vm_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* touching memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        dirty_size = vm_size / 2;
        memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
        /* ... should increase footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("modifying anonymous memory increases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "touched %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* deallocating memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        /* ... should decrease footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("deallocating dirty anonymous memory decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "deallocated %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);
}

#define TEMP_FILE_TEMPLATE "/tmp/phys_footprint_data.XXXXXXXX"
#define TEMP_FILE_SIZE  (1 * 1024 * 1024)

T_DECL(phys_footprint_file,
    "phys_footprint for mapped file",
    T_META_NAMESPACE(TEST_VM_NAMESPACE),
    T_META_LTEPHASE(LTE_POSTINIT))
{
        uint64_t                footprint_before, pagetable_before;
        uint64_t                footprint_after, pagetable_after;
        uint64_t                footprint_expected;
        mach_vm_address_t       pre_vm_addr;
        int                     fd;
        char                    *map_addr;
        size_t                  map_size, dirty_size;
        ssize_t                 nbytes;
        char                    tmp_file_name[PATH_MAX] = TEMP_FILE_TEMPLATE;
        char                    *buf;
        size_t                  buf_size;

        T_SETUPBEGIN;
        buf_size = TEMP_FILE_SIZE;
        T_QUIET;
        T_ASSERT_NOTNULL(buf = (char *)malloc(buf_size),
            "allocate %zu-byte buffer", buf_size);
        memset(buf, 'f', buf_size);
        T_WITH_ERRNO;
        T_QUIET;
        T_ASSERT_NOTNULL(mktemp(tmp_file_name),
            "create temporary file name");
        T_WITH_ERRNO;
        T_QUIET;
        T_ASSERT_GE(fd = open(tmp_file_name, O_CREAT | O_RDWR),
            0,
            "create temp file");
        T_WITH_ERRNO;
        T_QUIET;
        T_ASSERT_EQ(nbytes = write(fd, buf, buf_size),
            (ssize_t)buf_size,
            "write %zu bytes", buf_size);
        free(buf);
        T_SETUPEND;

        /* pre-warm to account for page table expansion */
        pre_vm_addr = pre_warm(TEMP_FILE_SIZE);

        /* mapping a file does not impact footprint... */
        get_ledger_info(&footprint_before, &pagetable_before);
        map_size = TEMP_FILE_SIZE;
        T_WITH_ERRNO;
        T_QUIET;
        T_ASSERT_NOTNULL(map_addr = (char *)mmap(NULL, map_size,
            PROT_READ | PROT_WRITE,
            MAP_FILE | MAP_SHARED, fd, 0),
            "mmap()");
        T_QUIET;
        T_EXPECT_EQ((mach_vm_address_t)map_addr, pre_vm_addr,
            "pre-warm mishap");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("mapping file does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "mapping file with %zu bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            map_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* touching file-backed memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        dirty_size = map_size / 2;
        memset(map_addr, 'F', dirty_size);
        /* ... should not impact footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("modifying file-backed memory does not impact phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "touched %zu bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* deallocating file-backed memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        T_WITH_ERRNO;
        T_QUIET;
        T_ASSERT_EQ(munmap(map_addr, map_size),
            0,
            "unmap file");
        /* ... should not impact footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("unmapping file-backed memory does not impact phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "unmapped %zu dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);
}

T_DECL(phys_footprint_purgeable,
    "phys_footprint for purgeable memory",
    T_META_NAMESPACE(TEST_VM_NAMESPACE),
    T_META_LTEPHASE(LTE_POSTINIT))
{
        uint64_t                footprint_before, pagetable_before;
        uint64_t                footprint_after, pagetable_after;
        uint64_t                footprint_expected;
        kern_return_t           kr;
        mach_vm_address_t       pre_vm_addr, vm_addr;
        mach_vm_size_t          vm_size, dirty_size;
        int                     state;

        /* pre-warm to account for page table expansion */
        pre_vm_addr = pre_warm(MEM_SIZE);

        /* allocating purgeable virtual memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        vm_addr = 0;
        vm_size = MEM_SIZE;
        kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
            VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        T_QUIET;
        T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("purgeable virtual allocation does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "purgeable virtual allocation of %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            vm_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* touching memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        dirty_size = vm_size / 2;
        memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
        /* ... should increase footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("modifying anonymous memory increases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "touched %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* making it volatile... */
        get_ledger_info(&footprint_before, &pagetable_before);
        state = VM_PURGABLE_VOLATILE;
        T_QUIET;
        T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
            vm_addr,
            VM_PURGABLE_SET_STATE,
            &state),
            KERN_SUCCESS,
            "vm_purgable_control(VOLATILE)");
        T_QUIET;
        T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
            "memory was non-volatile");
        /* ... should decrease footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making volatile decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* making it non-volatile... */
        get_ledger_info(&footprint_before, &pagetable_before);
        state = VM_PURGABLE_NONVOLATILE;
        T_QUIET;
        T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
            vm_addr,
            VM_PURGABLE_SET_STATE,
            &state),
            KERN_SUCCESS,
            "vm_purgable_control(NONVOLATILE)");
        T_QUIET;
        T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
            "memory was volatile");
        /* ... should increase footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making non-volatile increases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made non-volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* deallocating memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        /* ... should decrease footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("deallocating memory decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "deallocated %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);
}

T_DECL(phys_footprint_purgeable_ownership,
    "phys_footprint for owned purgeable memory",
    T_META_NAMESPACE(TEST_VM_NAMESPACE),
    T_META_LTEPHASE(LTE_POSTINIT))
{
        uint64_t                footprint_before, pagetable_before;
        uint64_t                footprint_after, pagetable_after;
        uint64_t                footprint_expected;
        kern_return_t           kr;
        mach_vm_address_t       pre_vm_addr, vm_addr;
        mach_vm_size_t          vm_size, dirty_size, me_size;
        int                     state;
        mach_port_t             me_port;

        /* pre-warm to account for page table expansion */
        pre_vm_addr = pre_warm(MEM_SIZE);

        /* allocating purgeable virtual memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        vm_addr = 0;
        vm_size = MEM_SIZE;
        kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
            VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        T_QUIET;
        T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("purgeable virtual allocation does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "purgeable virtual allocation of %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            vm_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* touching memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        dirty_size = vm_size / 2;
        memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
        /* ... should increase footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("modifying anonymous memory increases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "touched %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* making it volatile... */
        get_ledger_info(&footprint_before, &pagetable_before);
        state = VM_PURGABLE_VOLATILE;
        T_QUIET;
        T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
            vm_addr,
            VM_PURGABLE_SET_STATE,
            &state),
            KERN_SUCCESS,
            "vm_purgable_control(VOLATILE)");
        T_QUIET;
        T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
            "memory was non-volatile");
        /* ... should decrease footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making volatile decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* making it non-volatile... */
        get_ledger_info(&footprint_before, &pagetable_before);
        state = VM_PURGABLE_NONVOLATILE;
        T_QUIET;
        T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
            vm_addr,
            VM_PURGABLE_SET_STATE,
            &state),
            KERN_SUCCESS,
            "vm_purgable_control(NONVOLATILE)");
        T_QUIET;
        T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
            "memory was volatile");
        /* ... should increase footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making non-volatile increases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made non-volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* making a memory entry... */
        get_ledger_info(&footprint_before, &pagetable_before);
        me_size = vm_size;
        me_port = MACH_PORT_NULL;
        kr = mach_make_memory_entry_64(mach_task_self(),
            &me_size,
            vm_addr,
            VM_PROT_READ | VM_PROT_WRITE,
            &me_port,
            MACH_PORT_NULL);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
            kr, mach_error_string(kr));
        T_QUIET;
        T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making a memory entry does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "making a memory entry of %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            vm_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* deallocating memory while holding memory entry... */
        get_ledger_info(&footprint_before, &pagetable_before);
        kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("deallocating owned memory while holding memory entry "
            "does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "deallocated %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* releasing the memory entry... */
        kr = mach_port_deallocate(mach_task_self(), me_port);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        /* ... should decrease footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("releasing memory entry decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);
}

#ifdef MAP_MEM_LEDGER_TAGGED
T_DECL(phys_footprint_ledger_purgeable_owned,
    "phys_footprint for ledger-tagged purgeable memory ownership",
    T_META_NAMESPACE(TEST_VM_NAMESPACE),
    T_META_LTEPHASE(LTE_POSTINIT))
{
        uint64_t                footprint_before, pagetable_before;
        uint64_t                footprint_after, pagetable_after;
        uint64_t                footprint_expected;
        kern_return_t           kr;
        mach_vm_address_t       pre_vm_addr, vm_addr;
        mach_vm_size_t          vm_size, dirty_size, me_size;
        int                     state;
        mach_port_t             me_port;

        /* pre-warm to account for page table expansion */
        pre_vm_addr = pre_warm(MEM_SIZE);

        /* making a memory entry... */
        get_ledger_info(&footprint_before, &pagetable_before);
        vm_size = MEM_SIZE;
        me_size = vm_size;
        me_port = MACH_PORT_NULL;
        kr = mach_make_memory_entry_64(mach_task_self(),
            &me_size,
            0,
            (MAP_MEM_NAMED_CREATE |
            MAP_MEM_LEDGER_TAGGED |
            MAP_MEM_PURGABLE |
            VM_PROT_READ | VM_PROT_WRITE),
            &me_port,
            MACH_PORT_NULL);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
            kr, mach_error_string(kr));
        T_QUIET;
        T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making a memory entry does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "making a memory entry of %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            vm_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* mapping ledger-tagged virtual memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        vm_addr = 0;
        kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
            0, /* mask */
            VM_FLAGS_ANYWHERE,
            me_port,
            0, /* offset */
            FALSE, /* copy */
            VM_PROT_READ | VM_PROT_WRITE,
            VM_PROT_READ | VM_PROT_WRITE,
            VM_INHERIT_DEFAULT);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
            kr, mach_error_string(kr));
        T_QUIET;
        T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("mapping ledger-tagged memory does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "ledger-tagged mapping of %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            vm_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* touching memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        dirty_size = vm_size / 2;
        memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
        /* ... should increase footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("modifying ledger-tagged memory increases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "touched %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* making it volatile... */
        get_ledger_info(&footprint_before, &pagetable_before);
        state = VM_PURGABLE_VOLATILE;
        T_QUIET;
        T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
            vm_addr,
            VM_PURGABLE_SET_STATE,
            &state),
            KERN_SUCCESS,
            "vm_purgable_control(VOLATILE)");
        T_QUIET;
        T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
            "memory was non-volatile");
        /* ... should decrease footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making volatile decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* making it non-volatile... */
        get_ledger_info(&footprint_before, &pagetable_before);
        state = VM_PURGABLE_NONVOLATILE;
        T_QUIET;
        T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
            vm_addr,
            VM_PURGABLE_SET_STATE,
            &state),
            KERN_SUCCESS,
            "vm_purgable_control(NONVOLATILE)");
        T_QUIET;
        T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
            "memory was volatile");
        /* ... should increase footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making non-volatile increases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made non-volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* deallocating memory while holding memory entry... */
        get_ledger_info(&footprint_before, &pagetable_before);
        kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("deallocating owned memory while holding memory entry "
            "does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "deallocated %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* releasing the memory entry... */
        kr = mach_port_deallocate(mach_task_self(), me_port);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        /* ... should decrease footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("releasing memory entry decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);
}

T_DECL(phys_footprint_ledger_owned,
    "phys_footprint for ledger-tagged memory ownership",
    T_META_NAMESPACE(TEST_VM_NAMESPACE),
    T_META_LTEPHASE(LTE_POSTINIT))
{
        uint64_t                footprint_before, pagetable_before;
        uint64_t                footprint_after, pagetable_after;
        uint64_t                footprint_expected;
        kern_return_t           kr;
        mach_vm_address_t       pre_vm_addr, vm_addr;
        mach_vm_size_t          vm_size, dirty_size, me_size;
        mach_port_t             me_port;

        /* pre-warm to account for page table expansion */
        pre_vm_addr = pre_warm(MEM_SIZE);

        /* making a memory entry... */
        get_ledger_info(&footprint_before, &pagetable_before);
        vm_size = MEM_SIZE;
        me_size = vm_size;
        me_port = MACH_PORT_NULL;
        kr = mach_make_memory_entry_64(mach_task_self(),
            &me_size,
            0,
            (MAP_MEM_NAMED_CREATE |
            MAP_MEM_LEDGER_TAGGED |
            VM_PROT_READ | VM_PROT_WRITE),
            &me_port,
            MACH_PORT_NULL);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
            kr, mach_error_string(kr));
        T_QUIET;
        T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making a memory entry does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "making a memory entry of %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            vm_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* mapping ledger-tagged virtual memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        vm_addr = 0;
        kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
            0, /* mask */
            VM_FLAGS_ANYWHERE,
            me_port,
            0, /* offset */
            FALSE, /* copy */
            VM_PROT_READ | VM_PROT_WRITE,
            VM_PROT_READ | VM_PROT_WRITE,
            VM_INHERIT_DEFAULT);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
            kr, mach_error_string(kr));
        T_QUIET;
        T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("mapping ledger-tagged memory does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "ledger-tagged mapping of %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            vm_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* touching memory... */
        get_ledger_info(&footprint_before, &pagetable_before);
        dirty_size = vm_size / 2;
        memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
        /* ... should increase footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("modifying ledger-tagged memory increases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "touched %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* deallocating memory while holding memory entry... */
        get_ledger_info(&footprint_before, &pagetable_before);
        kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        /* ... should not change footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("deallocating owned memory while holding memory entry "
            "does not change phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "deallocated %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* releasing the memory entry... */
        kr = mach_port_deallocate(mach_task_self(), me_port);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        /* ... should decrease footprint */
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - dirty_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("releasing memory entry decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made volatile %lld dirty bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            dirty_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);
}
#endif /* MAP_MEM_LEDGER_TAGGED */

/* IOSurface code from: CoreImage/CoreImageTests/CIRender/SurfaceUtils.c */
#include <CoreFoundation/CoreFoundation.h>
#include <IOSurface/IOSurface.h>
#include <IOSurface/IOSurfacePrivate.h>
static size_t
bytes_per_element(uint32_t format)
{
        size_t bpe = 0;
        switch (format) {
        case 32:     // kCVPixelFormatType_32ARGB (ARGB8)
                bpe = 4;
                break;
        default:
                bpe = 0;
                break;
        }
        return bpe;
}
static size_t
bytes_per_pixel(uint32_t format)
{
        size_t bpe = 0;
        switch (format) {
        case 32:     // kCVPixelFormatType_32ARGB (ARGB8)
                bpe = 4;
                break;
        default:
                bpe = 0;
                break;
        }
        return bpe;
}
static inline size_t
roundSizeToMultiple(size_t size, size_t mult)
{
        return ((size + mult - 1) / mult) * mult;
}
static inline void
setIntValue(CFMutableDictionaryRef dict, const CFStringRef key, int value)
{
        CFNumberRef number = CFNumberCreate(0, kCFNumberIntType, &value);
        CFDictionarySetValue(dict, key, number);
        CFRelease(number);
}
static inline void
setBoolValue(CFMutableDictionaryRef dict, const CFStringRef key, bool value)
{
        CFDictionarySetValue(dict, key, value ? kCFBooleanTrue : kCFBooleanFalse);
}
typedef void (^SurfacePlaneBlock)(void *data, size_t planeIndex, size_t width, size_t height, size_t rowbytes);
static IOReturn
SurfaceApplyPlaneBlock(IOSurfaceRef surface, SurfacePlaneBlock block)
{
        if (surface == nil || block == nil) {
                return kIOReturnBadArgument;
        }

        IOReturn result = kIOReturnSuccess;
        size_t planeCount = IOSurfaceGetPlaneCount(surface);

        if (planeCount == 0) {
                result = IOSurfaceLock(surface, 0, NULL);
                if (result != kIOReturnSuccess) {
                        return result;
                }

                void* base = IOSurfaceGetBaseAddress(surface);
                size_t rb = IOSurfaceGetBytesPerRow(surface);
                size_t w = IOSurfaceGetWidth(surface);
                size_t h = IOSurfaceGetHeight(surface);

                if (base && rb && w && h) {
                        block(base, 0, w, h, rb);
                }

                IOSurfaceUnlock(surface, 0, NULL);
        } else if (planeCount == 2) {
                for (size_t i = 0; i < planeCount; i++) {
                        result = IOSurfaceLock(surface, 0, NULL);
                        if (result != kIOReturnSuccess) {
                                return result;
                        }

                        void* base = IOSurfaceGetBaseAddressOfPlane(surface, i);
                        size_t rb = IOSurfaceGetBytesPerRowOfPlane(surface, i);
                        size_t w = IOSurfaceGetWidthOfPlane(surface, i);
                        size_t h = IOSurfaceGetHeightOfPlane(surface, i);

                        if (base && rb && w && h) {
                                block(base, i, w, h, rb);
                        }

                        IOSurfaceUnlock(surface, 0, NULL);
                }
        }
        return result;
}
static void
ClearSurface(IOSurfaceRef surface)
{
        const int zero = 0;
        (void) SurfaceApplyPlaneBlock(surface, ^(void *p, size_t i, __unused size_t w, size_t h, size_t rb)
        {
                if (i == 0) {
                        memset(p, zero, rb * h);
                } else {
                        memset(p, 128, rb * h);
                }
        });
}
static size_t
SurfaceGetMemorySize(IOSurfaceRef surface)
{
        size_t planeCount = IOSurfaceGetPlaneCount(surface);

        if (planeCount == 0) {
                size_t rb = IOSurfaceGetBytesPerRow(surface);
                size_t h = IOSurfaceGetHeight(surface);
                return rb * h;
        } else if (planeCount == 2) {
                size_t rb0 = IOSurfaceGetBytesPerRowOfPlane(surface, 0);
                size_t h0 = IOSurfaceGetHeightOfPlane(surface, 0);
                size_t rb1 = IOSurfaceGetBytesPerRowOfPlane(surface, 1);
                size_t h1 = IOSurfaceGetHeightOfPlane(surface, 1);
                return rb0 * h0 + rb1 * h1;
        }
        return 0;
}
static IOSurfaceRef
CreateSurface(uint32_t pixelsWide, uint32_t pixelsHigh, uint32_t rowBytesAlignment, uint32_t fmt, bool purgeable, bool clear)
{
        IOSurfaceRef surface = nil;

        if (pixelsWide < 1 || pixelsHigh < 1 || fmt == 0) {
                return nil;
        }

        size_t bpp = bytes_per_pixel(fmt);
        size_t bpe = bytes_per_element(fmt);
        if (bpp == 0 || bpe == 0) {
                return nil;
        }

        size_t rowbytes = pixelsWide * bpp;
        if (rowBytesAlignment == 0) {
                rowBytesAlignment = 16;
        }
        rowbytes = roundSizeToMultiple(rowbytes, rowBytesAlignment);

        CFMutableDictionaryRef props = CFDictionaryCreateMutable(0, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
        setIntValue(props, kIOSurfaceBytesPerRow, (int)rowbytes);
        setIntValue(props, kIOSurfaceWidth, (int)pixelsWide);
        setIntValue(props, kIOSurfaceHeight, (int)pixelsHigh);
        setIntValue(props, kIOSurfacePixelFormat, (int)fmt);
#if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
        setBoolValue(props, kIOSurfaceNonPurgeable, !purgeable);
#else /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
        (void)purgeable;
#endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
        {
                if (bpe != bpp) { // i.e. a 422 format such as 'yuvf' etc.
                        setIntValue(props, kIOSurfaceElementWidth, 2);
                        setIntValue(props, kIOSurfaceElementHeight, 1);
                }
                setIntValue(props, kIOSurfaceBytesPerElement, (int)bpe);
        }

        surface = IOSurfaceCreate(props);

        if (clear) {
                ClearSurface(surface);
        }

        CFRelease(props);
        return surface;
}
T_DECL(phys_footprint_purgeable_iokit,
    "phys_footprint for purgeable IOKit memory",
    T_META_NAMESPACE(TEST_VM_NAMESPACE),
    T_META_LTEPHASE(LTE_POSTINIT))
{
        uint64_t        footprint_before, pagetable_before;
        uint64_t        footprint_after, pagetable_after;
        uint64_t        footprint_expected, footprint_delta_slop;
        int64_t         footprint_delta;
        IOSurfaceRef    surface;
        uint32_t        old_state;
        uint64_t        surface_size;

        T_SETUPBEGIN;
        footprint_delta_slop = 8 * vm_kernel_page_size;
        ledger_init();
        surface = CreateSurface(1024, 1024, 0, 32, true, true);
        IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
        IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
        CFRelease(surface);
        T_SETUPEND;

        surface_size = 1024 * 1024 * 4;

        /* create IOsurface: footprint grows */
        get_ledger_info(&footprint_before, &pagetable_before);
        surface = CreateSurface(1024, 1024, 0, 32, true, true);
        get_ledger_info(&footprint_after, &pagetable_after);
        if (legacy_footprint) {
                footprint_expected = footprint_before;
                footprint_expected += (pagetable_after - pagetable_before);
                footprint_delta = (int64_t)(footprint_after - footprint_expected);
                T_LOG("LEGACY FOOTPRINT: creating purgeable IOSurface: no footprint impact");
                T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
                    "create purgeable IOSurface %lld bytes: "
                    "footprint %lld -> %lld expected %lld delta %lld",
                    surface_size, footprint_before, footprint_after,
                    footprint_expected, footprint_delta);
        } else {
                footprint_expected = footprint_before + surface_size;
                footprint_expected += (pagetable_after - pagetable_before);
                footprint_delta = (int64_t)(footprint_after - footprint_expected);
                T_LOG("creating purgeable IOSurface increases phys_footprint");
                T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
                    "create purgeable IOSurface %lld bytes: "
                    "footprint %lld -> %lld expected %lld delta %lld",
                    surface_size, footprint_before, footprint_after,
                    footprint_expected, footprint_delta);
        }

        /* make IOSurface volatile: footprint shrinks */
        get_ledger_info(&footprint_before, &pagetable_before);
        IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
        get_ledger_info(&footprint_after, &pagetable_after);
        if (legacy_footprint) {
                footprint_expected = footprint_before;
                footprint_expected += (pagetable_after - pagetable_before);
                T_LOG("LEGACY FOOTPRINT: volatile IOSurface: no footprint impact");
                T_EXPECT_EQ(footprint_after, footprint_expected,
                    "volatile IOSurface %lld bytes: "
                    "footprint %lld -> %lld expected %lld delta %lld",
                    surface_size, footprint_before, footprint_after,
                    footprint_expected, footprint_after - footprint_expected);
        } else {
                footprint_expected = footprint_before - surface_size;
                footprint_expected += (pagetable_after - pagetable_before);
                T_LOG("making IOSurface volatile decreases phys_footprint");
                T_EXPECT_EQ(footprint_after, footprint_expected,
                    "made volatile %lld bytes: "
                    "footprint %lld -> %lld expected %lld delta %lld",
                    surface_size, footprint_before, footprint_after,
                    footprint_expected, footprint_after - footprint_expected);
        }

        /* make IOSurface non-volatile: footprint grows */
        get_ledger_info(&footprint_before, &pagetable_before);
        IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
        get_ledger_info(&footprint_after, &pagetable_after);
        if (legacy_footprint) {
                footprint_expected = footprint_before;
                footprint_expected += (pagetable_after - pagetable_before);
                T_LOG("LEGACY FOOTPRINT: non-volatile IOSurface: no footprint impact");
                T_EXPECT_EQ(footprint_after, footprint_expected,
                    "non-volatile IOSurface %lld bytes: "
                    "footprint %lld -> %lld expected %lld delta %lld",
                    surface_size, footprint_before, footprint_after,
                    footprint_expected, footprint_after - footprint_expected);
        } else {
                footprint_expected = footprint_before + surface_size;
                footprint_expected += (pagetable_after - pagetable_before);
                T_LOG("making IOSurface non-volatile increases phys_footprint");
                T_EXPECT_EQ(footprint_after, footprint_expected,
                    "made non-volatile %lld bytes: "
                    "footprint %lld -> %lld expected %lld delta %lld",
                    surface_size, footprint_before, footprint_after,
                    footprint_expected, footprint_after - footprint_expected);
        }

        /* accessing IOSurface re-mapping: no footprint impact */

        /* deallocating IOSurface re-mapping: no footprint impact */

        /* release IOSurface: footprint shrinks */
        get_ledger_info(&footprint_before, &pagetable_before);
        CFRelease(surface);
        get_ledger_info(&footprint_after, &pagetable_after);
        if (legacy_footprint) {
                footprint_expected = footprint_before;
                footprint_expected += (pagetable_after - pagetable_before);
                T_LOG("LEGACY FOOTPRINT: release IOSurface: no footprint impact");
                T_EXPECT_EQ(footprint_after, footprint_expected,
                    "releasing IOSurface %lld bytes: "
                    "footprint %lld -> %lld expected %lld delta %lld",
                    surface_size, footprint_before, footprint_after,
                    footprint_expected, footprint_after - footprint_expected);
        } else {
                footprint_expected = footprint_before - surface_size;
                footprint_expected += (pagetable_after - pagetable_before);
                T_LOG("releasing IOSurface decreases phys_footprint");
                T_EXPECT_EQ(footprint_after, footprint_expected,
                    "released IOSurface %lld bytes: "
                    "footprint %lld -> %lld expected %lld delta %lld",
                    surface_size, footprint_before, footprint_after,
                    footprint_expected, footprint_after - footprint_expected);
        }
}

#if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
T_DECL(phys_footprint_nonpurgeable_iokit,
    "phys_footprint for non-purgeable IOKit memory",
    T_META_NAMESPACE(TEST_VM_NAMESPACE),
    T_META_LTEPHASE(LTE_POSTINIT))
{
        uint64_t        footprint_before, pagetable_before;
        uint64_t        footprint_after, pagetable_after;
        uint64_t        footprint_expected, footprint_delta_slop;
        int64_t         footprint_delta;
        IOSurfaceRef    surface;
        uint64_t        surface_size;
        void            *map_base;
        size_t          map_size;
        mach_vm_address_t remap_addr;
        kern_return_t kr;
        vm_prot_t       cur_prot, max_prot;
        uint32_t        old_state;


        T_SETUPBEGIN;
        ledger_init();
        surface = CreateSurface(1024, 1024, 0, 32, false, true);
        CFRelease(surface);
        footprint_delta_slop = 8 * vm_kernel_page_size;
        T_SETUPEND;

        surface_size = 1024 * 1024 * 4;

        /* create IOsurface: footprint grows */
        get_ledger_info(&footprint_before, &pagetable_before);
        surface = CreateSurface(1024, 1024, 0, 32, false, true);
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + surface_size;
        footprint_expected += (pagetable_after - pagetable_before);
        footprint_delta = (int64_t)(footprint_after - footprint_expected);
        T_LOG("creating non-purgeable IOSurface increases phys_footprint");
        T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
            "create non-purgeable IOSurface %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            surface_size, footprint_before, footprint_after,
            footprint_expected, footprint_delta);

        /* make IOSurface volatile: fail and no footprint impact */
        get_ledger_info(&footprint_before, &pagetable_before);
        IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("making non-purgeable IOSurface volatile: no footprint impact");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "made volatile %lld non-purgeable bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            surface_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* re-mapping IOSurface: no footprint impact */
        get_ledger_info(&footprint_before, &pagetable_before);
        map_base = IOSurfaceGetBaseAddress(surface);
        map_size = SurfaceGetMemorySize(surface);
//      T_EXPECT_EQ(map_size, surface_size, "map_size %lld surface_size %lld",
//                  map_size, surface_size);
        remap_addr = 0;
        kr = mach_vm_remap(mach_task_self(),
            &remap_addr,
            (mach_vm_size_t)surface_size,
            0,
            VM_FLAGS_ANYWHERE,
            mach_task_self(),
            (mach_vm_address_t)map_base,
            FALSE,                /* copy */
            &cur_prot,
            &max_prot,
            VM_INHERIT_DEFAULT);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_remap() error 0x%x (%s)",
            kr, mach_error_string(kr));
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("re-mapping IOSurface does not impact phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "remapping IOSurface %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            surface_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* accessing IOSurface re-mapping: footprint grows */
        get_ledger_info(&footprint_before, &pagetable_before);
        memset((char *)(uintptr_t)remap_addr, 'p', (size_t)surface_size);
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before + surface_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("accessing re-mapped IOSurface grows phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "accessing remapped IOSurface %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            surface_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* deallocating IOSurface re-mapping: footprint shrinks */
        get_ledger_info(&footprint_before, &pagetable_before);
        kr = mach_vm_deallocate(mach_task_self(),
            remap_addr,
            (mach_vm_size_t)surface_size);
        T_QUIET;
        T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
            kr, mach_error_string(kr));
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - surface_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("deallocating re-mapping of IOSurface shrinks phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "deallocating remapped IOSurface %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            surface_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);

        /* release IOSurface: footprint shrinks */
        get_ledger_info(&footprint_before, &pagetable_before);
        CFRelease(surface);
        get_ledger_info(&footprint_after, &pagetable_after);
        footprint_expected = footprint_before - surface_size;
        footprint_expected += (pagetable_after - pagetable_before);
        T_LOG("releasing IOSurface decreases phys_footprint");
        T_EXPECT_EQ(footprint_after, footprint_expected,
            "released IOSurface %lld bytes: "
            "footprint %lld -> %lld expected %lld delta %lld",
            surface_size, footprint_before, footprint_after,
            footprint_expected, footprint_after - footprint_expected);
}
#endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */