/*
- * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2010-2018 Apple Computer, Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <libkern/OSAtomic.h>
#include <mach/mach_types.h>
+#include <os/overflow.h>
+
+#include <vm/pmap.h>
/*
* Ledger entry flags. Bits in second nibble (masked by 0xF0) are used for
#define LF_REFILL_INPROGRESS 0x0800 /* the ledger is being refilled */
#define LF_CALLED_BACK 0x1000 /* callback was called for balance in deficit */
#define LF_WARNED 0x2000 /* callback was called for balance warning */
-#define LF_TRACKING_MAX 0x4000 /* track max balance over user-specfied time */
+#define LF_TRACKING_MAX 0x4000 /* track max balance. Exclusive w.r.t refill */
#define LF_PANIC_ON_NEGATIVE 0x8000 /* panic if it goes negative */
#define LF_TRACK_CREDIT_ONLY 0x10000 /* only update "credit" */
int lt_table_size;
volatile uint32_t lt_inuse;
lck_mtx_t lt_lock;
+ zone_t lt_zone;
+ bool lt_initialized;
struct entry_template *lt_entries;
};
splx(s); \
}
-/*
- * Use 2 "tocks" to track the rolling maximum balance of a ledger entry.
- */
-#define NTOCKS 2
-/*
- * The explicit alignment is to ensure that atomic operations don't panic
- * on ARM.
- */
-struct ledger_entry {
- volatile uint32_t le_flags;
- ledger_amount_t le_limit;
- ledger_amount_t le_warn_level;
- volatile ledger_amount_t le_credit __attribute__((aligned(8)));
- volatile ledger_amount_t le_debit __attribute__((aligned(8)));
- union {
- struct {
- /*
- * XXX - the following two fields can go away if we move all of
- * the refill logic into process policy
- */
- uint64_t le_refill_period;
- uint64_t le_last_refill;
- } le_refill;
- struct _le_peak {
- uint32_t le_max; /* Lower 32-bits of observed max balance */
- uint32_t le_time; /* time when this peak was observed */
- } le_peaks[NTOCKS];
- } _le;
-} __attribute__((aligned(8)));
-
-struct ledger {
- uint64_t l_id;
- int32_t l_refs;
- int32_t l_size;
- struct ledger_template *l_template;
- struct ledger_entry l_entries[0] __attribute__((aligned(8)));
-};
-
static int ledger_cnt = 0;
/* ledger ast helper functions */
static uint32_t ledger_check_needblock(ledger_t l, uint64_t now);
static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
-static void ledger_entry_check_new_balance(ledger_t ledger, int entry,
- struct ledger_entry *le);
+static void ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
+ int entry, struct ledger_entry *le);
#if 0
static void
template->lt_cnt = 0;
template->lt_table_size = 1;
template->lt_inuse = 0;
+ template->lt_zone = NULL;
lck_mtx_init(&template->lt_lock, &ledger_lck_grp, LCK_ATTR_NULL);
template->lt_entries = (struct entry_template *)
int idx;
struct entry_template *et;
- if ((key == NULL) || (strlen(key) >= LEDGER_NAME_MAX))
+ if ((key == NULL) || (strlen(key) >= LEDGER_NAME_MAX) || (template->lt_zone != NULL))
return (-1);
template_lock(template);
/* If the table is full, attempt to double its size */
if (template->lt_cnt == template->lt_table_size) {
struct entry_template *new_entries, *old_entries;
- int old_cnt, old_sz;
+ int old_cnt, old_sz, new_sz = 0;
spl_t s;
old_cnt = template->lt_table_size;
- old_sz = (int)(old_cnt * sizeof (struct entry_template));
- new_entries = kalloc(old_sz * 2);
+ old_sz = old_cnt * (int)(sizeof(struct entry_template));
+ /* double old_sz allocation, but check for overflow */
+ if (os_mul_overflow(old_sz, 2, &new_sz)) {
+ template_unlock(template);
+ return -1;
+ }
+ new_entries = kalloc(new_sz);
if (new_entries == NULL) {
template_unlock(template);
- return (-1);
+ return -1;
}
memcpy(new_entries, template->lt_entries, old_sz);
memset(((char *)new_entries) + old_sz, 0, old_sz);
+ /* assume: if the sz didn't overflow, neither will the count */
template->lt_table_size = old_cnt * 2;
old_entries = template->lt_entries;
return (idx);
}
+/*
+ * Complete the initialization of ledger template
+ * by initializing ledger zone. After initializing
+ * the ledger zone, adding an entry in the ledger
+ * template would fail.
+ */
+void
+ledger_template_complete(ledger_template_t template)
+{
+ size_t ledger_size;
+ ledger_size = sizeof(struct ledger) + (template->lt_cnt * sizeof(struct ledger_entry));
+ template->lt_zone = zinit(ledger_size, CONFIG_TASK_MAX * ledger_size,
+ ledger_size,
+ template->lt_name);
+ template->lt_initialized = true;
+}
+
+/*
+ * Like ledger_template_complete, except we'll ask
+ * the pmap layer to manage allocations for us.
+ * Meant for ledgers that should be owned by the
+ * pmap layer.
+ */
+void
+ledger_template_complete_secure_alloc(ledger_template_t template)
+{
+ size_t ledger_size;
+ ledger_size = sizeof(struct ledger) + (template->lt_cnt * sizeof(struct ledger_entry));
+ pmap_ledger_alloc_init(ledger_size);
+ template->lt_initialized = true;
+}
+
/*
* Create a new ledger based on the specified template. As part of the
* ledger creation we need to allocate space for a table of ledger entries.
ledger_instantiate(ledger_template_t template, int entry_type)
{
ledger_t ledger;
- size_t cnt, sz;
+ size_t cnt;
int i;
template_lock(template);
cnt = template->lt_cnt;
template_unlock(template);
- sz = sizeof(*ledger) + (cnt * sizeof(struct ledger_entry));
+ if (template->lt_zone) {
+ ledger = (ledger_t)zalloc(template->lt_zone);
+ } else {
+ ledger = pmap_ledger_alloc();
+ }
- ledger = (ledger_t)kalloc(sz);
if (ledger == NULL) {
ledger_template_dereference(template);
return LEDGER_NULL;
/* Just released the last reference. Free it. */
if (v == 1) {
- kfree(ledger,
- sizeof(*ledger) + ledger->l_size * sizeof(struct ledger_entry));
+ if (ledger->l_template->lt_zone) {
+ zfree(ledger->l_template->lt_zone, ledger);
+ } else {
+ pmap_ledger_free(ledger);
+ }
}
return (KERN_SUCCESS);
balance = le->le_credit - le->le_debit;
due = periods * le->le_limit;
+
if (balance - due < 0)
due = balance;
- assert(due >= 0);
+ assertf(due >= 0,"now=%llu, ledger=%p, entry=%d, balance=%lld, due=%lld", now, ledger, entry, balance, due);
OSAddAtomic64(due, &le->le_debit);
ledger_limit_entry_wakeup(le);
}
-/*
- * In tenths of a second, the length of one lookback period (a "tock") for
- * ledger rolling maximum calculations. The effective lookback window will be this times
- * NTOCKS.
- *
- * Use a tock length of 2.5 seconds to get a total lookback period of 5 seconds.
- *
- * XXX Could make this caller-definable, at the point that rolling max tracking
- * is enabled for the entry.
- */
-#define TOCKLEN 25
-
-/*
- * How many sched_tick's are there in one tock (one of our lookback periods)?
- *
- * X sched_ticks 2.5 sec N sched_ticks
- * --------------- = ---------- * -------------
- * tock tock sec
- *
- * where N sched_ticks/sec is calculated via 1 << SCHED_TICK_SHIFT (see sched_prim.h)
- *
- * This should give us 20 sched_tick's in one 2.5 second-long tock.
- */
-#define SCHED_TICKS_PER_TOCK ((TOCKLEN * (1 << SCHED_TICK_SHIFT)) / 10)
-
-/*
- * Rolling max timestamps use their own unit (let's call this a "tock"). One tock is the
- * length of one lookback period that we use for our rolling max calculation.
- *
- * Calculate the current time in tocks from sched_tick (which runs at a some
- * fixed rate).
- */
-#define CURRENT_TOCKSTAMP() (sched_tick / SCHED_TICKS_PER_TOCK)
-
-/*
- * Does the given tockstamp fall in either the current or the previous tocks?
- */
-#define TOCKSTAMP_IS_STALE(now, tock) ((((now) - (tock)) < NTOCKS) ? FALSE : TRUE)
-
void
-ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *le)
+ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
+ int entry, struct ledger_entry *le)
{
- ledger_amount_t credit, debit;
-
if (le->le_flags & LF_TRACKING_MAX) {
ledger_amount_t balance = le->le_credit - le->le_debit;
- uint32_t now = CURRENT_TOCKSTAMP();
- struct _le_peak *p = &le->_le.le_peaks[now % NTOCKS];
- if (!TOCKSTAMP_IS_STALE(now, p->le_time) || (balance > p->le_max)) {
- /*
- * The current balance is greater than the previously
- * observed peak for the current time block, *or* we
- * haven't yet recorded a peak for the current time block --
- * so this is our new peak.
- *
- * (We only track the lower 32-bits of a balance for rolling
- * max purposes.)
- */
- p->le_max = (uint32_t)balance;
- p->le_time = now;
+ if (balance > le->_le._le_max.le_lifetime_max){
+ le->_le._le_max.le_lifetime_max = balance;
}
+
+#if CONFIG_LEDGER_INTERVAL_MAX
+ if (balance > le->_le._le_max.le_interval_max) {
+ le->_le._le_max.le_interval_max = balance;
+ }
+#endif /* LEDGER_CONFIG_INTERVAL_MAX */
}
/* Check to see whether we're due a refill */
if (le->le_flags & LF_REFILL_SCHEDULED) {
+ assert(!(le->le_flags & LF_TRACKING_MAX));
+
uint64_t now = mach_absolute_time();
if ((now - le->_le.le_refill.le_last_refill) > le->_le.le_refill.le_refill_period)
ledger_refill(now, ledger, entry);
if ((le->le_flags & LEDGER_ACTION_BLOCK) ||
(!(le->le_flags & LF_CALLED_BACK) &&
entry_get_callback(ledger, entry))) {
- set_astledger(current_thread());
+ act_set_astledger_async(thread);
}
} else {
/*
* set the AST so it can be done before returning
* to userland.
*/
- set_astledger(current_thread());
+ act_set_astledger_async(thread);
}
} else {
/*
* know the ledger balance is now back below
* the warning level.
*/
- set_astledger(current_thread());
+ act_set_astledger_async(thread);
}
}
}
}
- credit = le->le_credit;
- debit = le->le_debit;
if ((le->le_flags & LF_PANIC_ON_NEGATIVE) &&
- ((credit < debit) ||
- (le->le_credit < le->le_debit))) {
- panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld/%lld debit:%lld/%lld balance:%lld/%lld\n",
+ (le->le_credit < le->le_debit)) {
+ panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:%lld balance:%lld\n",
ledger, entry, le,
- credit, le->le_credit,
- debit, le->le_debit,
- credit - debit, le->le_credit - le->le_debit);
+ le->le_credit,
+ le->le_debit,
+ le->le_credit - le->le_debit);
}
}
void
-ledger_check_new_balance(ledger_t ledger, int entry)
+ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry)
{
struct ledger_entry *le;
assert(entry > 0 && entry <= ledger->l_size);
le = &ledger->l_entries[entry];
- ledger_entry_check_new_balance(ledger, entry, le);
+ ledger_entry_check_new_balance(thread, ledger, entry, le);
}
-
/*
- * Add value to an entry in a ledger.
+ * Add value to an entry in a ledger for a specific thread.
*/
kern_return_t
-ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
+ledger_credit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
{
ledger_amount_t old, new;
struct ledger_entry *le;
old = OSAddAtomic64(amount, &le->le_credit);
new = old + amount;
- lprintf(("%p Credit %lld->%lld\n", current_thread(), old, new));
- ledger_entry_check_new_balance(ledger, entry, le);
+ lprintf(("%p Credit %lld->%lld\n", thread, old, new));
+
+ if (thread) {
+ ledger_entry_check_new_balance(thread, ledger, entry, le);
+ }
return (KERN_SUCCESS);
}
+/*
+ * Add value to an entry in a ledger.
+ */
+kern_return_t
+ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
+{
+ return ledger_credit_thread(current_thread(), ledger, entry, amount);
+}
+
+/*
+ * Add value to an entry in a ledger; do not check balance after update.
+ */
+kern_return_t
+ledger_credit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
+{
+ return ledger_credit_thread(NULL, ledger, entry, amount);
+}
+
/* Add all of one ledger's values into another.
* They must have been created from the same template.
* This is not done atomically. Another thread (if not otherwise synchronized)
ledger_rollup(ledger_t to_ledger, ledger_t from_ledger)
{
int i;
- struct ledger_entry *from_le, *to_le;
assert(to_ledger->l_template == from_ledger->l_template);
for (i = 0; i < to_ledger->l_size; i++) {
- if (ENTRY_VALID(from_ledger, i) && ENTRY_VALID(to_ledger, i)) {
- from_le = &from_ledger->l_entries[i];
- to_le = &to_ledger->l_entries[i];
- OSAddAtomic64(from_le->le_credit, &to_le->le_credit);
- OSAddAtomic64(from_le->le_debit, &to_le->le_debit);
- }
+ ledger_rollup_entry(to_ledger, from_ledger, i);
+ }
+
+ return (KERN_SUCCESS);
+}
+
+/* Add one ledger entry value to another.
+ * They must have been created from the same template.
+ * Since the credit and debit values are added one
+ * at a time, other thread might read the a bogus value.
+ */
+kern_return_t
+ledger_rollup_entry(ledger_t to_ledger, ledger_t from_ledger, int entry)
+{
+ struct ledger_entry *from_le, *to_le;
+
+ assert(to_ledger->l_template == from_ledger->l_template);
+ if (ENTRY_VALID(from_ledger, entry) && ENTRY_VALID(to_ledger, entry)) {
+ from_le = &from_ledger->l_entries[entry];
+ to_le = &to_ledger->l_entries[entry];
+ OSAddAtomic64(from_le->le_credit, &to_le->le_credit);
+ OSAddAtomic64(from_le->le_debit, &to_le->le_debit);
}
return (KERN_SUCCESS);
ledger_zero_balance(ledger_t ledger, int entry)
{
struct ledger_entry *le;
+ ledger_amount_t debit, credit;
if (!ENTRY_VALID(ledger, entry))
return (KERN_INVALID_VALUE);
le = &ledger->l_entries[entry];
top:
+ debit = le->le_debit;
+ credit = le->le_credit;
+
if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
assert(le->le_debit == 0);
- if (!OSCompareAndSwap64(le->le_credit, 0, &le->le_credit)) {
+ if (!OSCompareAndSwap64(credit, 0, &le->le_credit)) {
goto top;
}
lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, 0));
- } else if (le->le_credit > le->le_debit) {
- if (!OSCompareAndSwap64(le->le_debit, le->le_credit, &le->le_debit))
+ } else if (credit > debit) {
+ if (!OSCompareAndSwap64(debit, credit, &le->le_debit))
goto top;
lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_debit, le->le_credit));
- } else if (le->le_credit < le->le_debit) {
- if (!OSCompareAndSwap64(le->le_credit, le->le_debit, &le->le_credit))
+ } else if (credit < debit) {
+ if (!OSCompareAndSwap64(credit, debit, &le->le_credit))
goto top;
lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, le->le_debit));
}
}
le->le_limit = limit;
- le->_le.le_refill.le_last_refill = 0;
+ if (le->le_flags & LF_REFILL_SCHEDULED) {
+ assert(!(le->le_flags & LF_TRACKING_MAX));
+ le->_le.le_refill.le_last_refill = 0;
+ }
flag_clear(&le->le_flags, LF_CALLED_BACK);
flag_clear(&le->le_flags, LF_WARNED);
ledger_limit_entry_wakeup(le);
return (KERN_SUCCESS);
}
+#if CONFIG_LEDGER_INTERVAL_MAX
kern_return_t
-ledger_get_maximum(ledger_t ledger, int entry,
- ledger_amount_t *max_observed_balance)
+ledger_get_interval_max(ledger_t ledger, int entry,
+ ledger_amount_t *max_interval_balance, int reset)
{
- struct ledger_entry *le;
- uint32_t now = CURRENT_TOCKSTAMP();
- int i;
-
+ struct ledger_entry *le;
le = &ledger->l_entries[entry];
if (!ENTRY_VALID(ledger, entry) || !(le->le_flags & LF_TRACKING_MAX)) {
return (KERN_INVALID_VALUE);
}
- /*
- * Start with the current balance; if neither of the recorded peaks are
- * within recent history, we use this.
- */
- *max_observed_balance = le->le_credit - le->le_debit;
-
- for (i = 0; i < NTOCKS; i++) {
- if (!TOCKSTAMP_IS_STALE(now, le->_le.le_peaks[i].le_time) &&
- (le->_le.le_peaks[i].le_max > *max_observed_balance)) {
- /*
- * The peak for this time block isn't stale, and it
- * is greater than the current balance -- so use it.
- */
- *max_observed_balance = le->_le.le_peaks[i].le_max;
- }
+ *max_interval_balance = le->_le._le_max.le_interval_max;
+ lprintf(("ledger_get_interval_max: %lld%s\n", *max_interval_balance,
+ (reset) ? " --> 0" : ""));
+
+ if (reset) {
+ le->_le._le_max.le_interval_max = 0;
+ }
+
+ return (KERN_SUCCESS);
+}
+#endif /* CONFIG_LEDGER_INTERVAL_MAX */
+
+kern_return_t
+ledger_get_lifetime_max(ledger_t ledger, int entry,
+ ledger_amount_t *max_lifetime_balance)
+{
+ struct ledger_entry *le;
+ le = &ledger->l_entries[entry];
+
+ if (!ENTRY_VALID(ledger, entry) || !(le->le_flags & LF_TRACKING_MAX)) {
+ return (KERN_INVALID_VALUE);
}
-
- lprintf(("ledger_get_maximum: %lld\n", *max_observed_balance));
+
+ *max_lifetime_balance = le->_le._le_max.le_lifetime_max;
+ lprintf(("ledger_get_lifetime_max: %lld\n", *max_lifetime_balance));
return (KERN_SUCCESS);
}
return (KERN_INVALID_VALUE);
}
+ /* Refill is incompatible with max tracking. */
+ if (template->lt_entries[entry].et_flags & LF_REFILL_SCHEDULED) {
+ return (KERN_INVALID_VALUE);
+ }
+
template->lt_entries[entry].et_flags |= LF_TRACKING_MAX;
- template_unlock(template);
+ template_unlock(template);
return (KERN_SUCCESS);
}
}
kern_return_t
-ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
+ledger_debit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
{
struct ledger_entry *le;
ledger_amount_t old, new;
}
lprintf(("%p Debit %lld->%lld\n", thread, old, new));
- ledger_entry_check_new_balance(ledger, entry, le);
+ if (thread) {
+ ledger_entry_check_new_balance(thread, ledger, entry, le);
+ }
+
return (KERN_SUCCESS);
+}
+kern_return_t
+ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
+{
+ return ledger_debit_thread(current_thread(), ledger, entry, amount);
+}
+
+kern_return_t
+ledger_debit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
+{
+ return ledger_debit_thread(NULL, ledger, entry, amount);
}
void
/* We're over the limit, so refill if we are eligible and past due. */
if (le->le_flags & LF_REFILL_SCHEDULED) {
+ assert(!(le->le_flags & LF_TRACKING_MAX));
+
if ((le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period) > now) {
ledger_refill(now, l, i);
if (limit_exceeded(le) == FALSE)
((le->le_flags & LEDGER_ACTION_BLOCK) == 0))
continue;
+ assert(!(le->le_flags & LF_TRACKING_MAX));
+
/* Prepare to sleep until the resource is refilled */
- ret = assert_wait_deadline(le, TRUE,
+ ret = assert_wait_deadline(le, THREAD_INTERRUPTIBLE,
le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period);
if (ret != THREAD_WAITING)
return(KERN_SUCCESS);
return (KERN_SUCCESS);
}
+kern_return_t
+ledger_get_panic_on_negative(ledger_t ledger, int entry, int *panic_on_negative)
+{
+ struct ledger_entry *le;
+
+ if (!ENTRY_VALID(ledger, entry))
+ return (KERN_INVALID_ARGUMENT);
+
+ le = &ledger->l_entries[entry];
+
+ if (le->le_flags & LF_PANIC_ON_NEGATIVE) {
+ *panic_on_negative = TRUE;
+ } else {
+ *panic_on_negative = FALSE;
+ }
+
+ return (KERN_SUCCESS);
+}
+
kern_return_t
ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
{
projects / apple / xnu.git / blobdiff