* @OSF_COPYRIGHT@
*/
-#include <kern/lock.h>
+#include <kern/kern_types.h>
#include <kern/ledger.h>
#include <kern/kalloc.h>
#include <kern/task.h>
+#include <kern/thread.h>
#include <kern/processor.h>
#include <kern/machine.h>
#include <kern/queue.h>
+#include <kern/policy_internal.h>
+
#include <sys/errno.h>
#include <libkern/OSAtomic.h>
#include <mach/mach_types.h>
+#include <os/overflow.h>
/*
* Ledger entry flags. Bits in second nibble (masked by 0xF0) are used for
#define LF_CALLED_BACK 0x1000 /* callback was called for balance in deficit */
#define LF_WARNED 0x2000 /* callback was called for balance warning */
#define LF_TRACKING_MAX 0x4000 /* track max balance over user-specfied time */
+#define LF_PANIC_ON_NEGATIVE 0x8000 /* panic if it goes negative */
+#define LF_TRACK_CREDIT_ONLY 0x10000 /* only update "credit" */
/* Determine whether a ledger entry exists and has been initialized and active */
#define ENTRY_VALID(l, e) \
(((l) != NULL) && ((e) >= 0) && ((e) < (l)->l_size) && \
(((l)->l_entries[e].le_flags & LF_ENTRY_ACTIVE) == LF_ENTRY_ACTIVE))
+#define ASSERT(a) assert(a)
+
#ifdef LEDGER_DEBUG
int ledger_debug = 0;
-#define ASSERT(a) assert(a)
#define lprintf(a) if (ledger_debug) { \
printf("%lld ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \
printf a ; \
}
#else
#define lprintf(a)
-#define ASSERT(a)
#endif
struct ledger_callback {
} __attribute__((aligned(8)));
struct ledger {
- int l_id;
+ uint64_t l_id;
+ int32_t l_refs;
+ int32_t l_size;
struct ledger_template *l_template;
- int l_refs;
- int l_size;
- struct ledger_entry *l_entries;
+ struct ledger_entry l_entries[0] __attribute__((aligned(8)));
};
static int ledger_cnt = 0;
static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
+static void ledger_entry_check_new_balance(ledger_t ledger, int entry,
+ struct ledger_entry *le);
+
#if 0
static void
debug_callback(const void *p0, __unused const void *p1)
/* If the table is full, attempt to double its size */
if (template->lt_cnt == template->lt_table_size) {
struct entry_template *new_entries, *old_entries;
- int old_cnt, old_sz;
+ int old_cnt, old_sz, new_sz = 0;
spl_t s;
old_cnt = template->lt_table_size;
- old_sz = (int)(old_cnt * sizeof (struct entry_template));
- new_entries = kalloc(old_sz * 2);
+ old_sz = old_cnt * (int)(sizeof(struct entry_template));
+ /* double old_sz allocation, but check for overflow */
+ if (os_mul_overflow(old_sz, 2, &new_sz)) {
+ template_unlock(template);
+ return -1;
+ }
+ new_entries = kalloc(new_sz);
if (new_entries == NULL) {
template_unlock(template);
- return (-1);
+ return -1;
}
memcpy(new_entries, template->lt_entries, old_sz);
memset(((char *)new_entries) + old_sz, 0, old_sz);
+ /* assume: if the sz didn't overflow, neither will the count */
template->lt_table_size = old_cnt * 2;
old_entries = template->lt_entries;
template_lock(template);
for (idx = 0; idx < template->lt_cnt; idx++)
- if (template->lt_entries[idx].et_key &&
+ if (template->lt_entries != NULL &&
(strcmp(key, template->lt_entries[idx].et_key) == 0))
break;
ledger_instantiate(ledger_template_t template, int entry_type)
{
ledger_t ledger;
- size_t sz;
+ size_t cnt, sz;
int i;
- ledger = (ledger_t)kalloc(sizeof (struct ledger));
- if (ledger == NULL)
- return (LEDGER_NULL);
-
- ledger->l_template = template;
- ledger->l_id = ledger_cnt++;
- ledger->l_refs = 1;
-
template_lock(template);
template->lt_refs++;
- ledger->l_size = template->lt_cnt;
+ cnt = template->lt_cnt;
template_unlock(template);
- sz = ledger->l_size * sizeof (struct ledger_entry);
- ledger->l_entries = kalloc(sz);
- if (sz && (ledger->l_entries == NULL)) {
+ sz = sizeof(*ledger) + (cnt * sizeof(struct ledger_entry));
+
+ ledger = (ledger_t)kalloc(sz);
+ if (ledger == NULL) {
ledger_template_dereference(template);
- kfree(ledger, sizeof(struct ledger));
- return (LEDGER_NULL);
+ return LEDGER_NULL;
}
+ ledger->l_template = template;
+ ledger->l_id = ledger_cnt++;
+ ledger->l_refs = 1;
+ ledger->l_size = (int32_t)cnt;
+
template_lock(template);
assert(ledger->l_size <= template->lt_cnt);
for (i = 0; i < ledger->l_size; i++) {
/* Just released the last reference. Free it. */
if (v == 1) {
- kfree(ledger->l_entries,
- ledger->l_size * sizeof (struct ledger_entry));
- kfree(ledger, sizeof (*ledger));
+ kfree(ledger,
+ sizeof(*ledger) + ledger->l_size * sizeof(struct ledger_entry));
}
return (KERN_SUCCESS);
{
ledger_amount_t balance;
- assert((le->le_credit >= 0) && (le->le_debit >= 0));
+ if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+ assert(le->le_debit == 0);
+ } else {
+ assert((le->le_credit >= 0) && (le->le_debit >= 0));
+ }
/*
* XXX - Currently, we only support warnings for ledgers which
{
ledger_amount_t balance;
- assert((le->le_credit >= 0) && (le->le_debit >= 0));
+ if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+ assert(le->le_debit == 0);
+ } else {
+ assert((le->le_credit >= 0) && (le->le_debit >= 0));
+ }
balance = le->le_credit - le->le_debit;
if ((le->le_limit <= 0) && (balance < le->le_limit))
struct ledger_entry *le;
ledger_amount_t balance, due;
+ assert(entry >= 0 && entry < ledger->l_size);
+
le = &ledger->l_entries[entry];
assert(le->le_limit != LEDGER_LIMIT_INFINITY);
+ if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+ assert(le->le_debit == 0);
+ return;
+ }
+
/*
* If another thread is handling the refill already, we're not
* needed.
balance = le->le_credit - le->le_debit;
due = periods * le->le_limit;
+
if (balance - due < 0)
due = balance;
- assert(due >= 0);
+ assertf(due >= 0,"now=%llu, ledger=%p, entry=%d, balance=%lld, due=%lld", now, ledger, entry, balance, due);
OSAddAtomic64(due, &le->le_debit);
*/
#define TOCKSTAMP_IS_STALE(now, tock) ((((now) - (tock)) < NTOCKS) ? FALSE : TRUE)
-static void
-ledger_check_new_balance(ledger_t ledger, int entry)
+void
+ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *le)
{
- struct ledger_entry *le;
-
- le = &ledger->l_entries[entry];
+ ledger_amount_t credit, debit;
if (le->le_flags & LF_TRACKING_MAX) {
ledger_amount_t balance = le->le_credit - le->le_debit;
}
}
}
+
+ credit = le->le_credit;
+ debit = le->le_debit;
+ if ((le->le_flags & LF_PANIC_ON_NEGATIVE) &&
+ ((credit < debit) ||
+ (le->le_credit < le->le_debit))) {
+ panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld/%lld debit:%lld/%lld balance:%lld/%lld\n",
+ ledger, entry, le,
+ credit, le->le_credit,
+ debit, le->le_debit,
+ credit - debit, le->le_credit - le->le_debit);
+ }
}
+void
+ledger_check_new_balance(ledger_t ledger, int entry)
+{
+ struct ledger_entry *le;
+ assert(entry > 0 && entry <= ledger->l_size);
+ le = &ledger->l_entries[entry];
+ ledger_entry_check_new_balance(ledger, entry, le);
+}
+
+
/*
* Add value to an entry in a ledger.
*/
old = OSAddAtomic64(amount, &le->le_credit);
new = old + amount;
lprintf(("%p Credit %lld->%lld\n", current_thread(), old, new));
- ledger_check_new_balance(ledger, entry);
+ ledger_entry_check_new_balance(ledger, entry, le);
+
+ return (KERN_SUCCESS);
+}
+
+/* Add all of one ledger's values into another.
+ * They must have been created from the same template.
+ * This is not done atomically. Another thread (if not otherwise synchronized)
+ * may see bogus values when comparing one entry to another.
+ * As each entry's credit & debit are modified one at a time, the warning/limit
+ * may spuriously trip, or spuriously fail to trip, or another thread (if not
+ * otherwise synchronized) may see a bogus balance.
+ */
+kern_return_t
+ledger_rollup(ledger_t to_ledger, ledger_t from_ledger)
+{
+ int i;
+
+ assert(to_ledger->l_template == from_ledger->l_template);
+
+ for (i = 0; i < to_ledger->l_size; i++) {
+ ledger_rollup_entry(to_ledger, from_ledger, i);
+ }
+
+ return (KERN_SUCCESS);
+}
+
+/* Add one ledger entry value to another.
+ * They must have been created from the same template.
+ * Since the credit and debit values are added one
+ * at a time, other thread might read the a bogus value.
+ */
+kern_return_t
+ledger_rollup_entry(ledger_t to_ledger, ledger_t from_ledger, int entry)
+{
+ struct ledger_entry *from_le, *to_le;
+
+ assert(to_ledger->l_template == from_ledger->l_template);
+ if (ENTRY_VALID(from_ledger, entry) && ENTRY_VALID(to_ledger, entry)) {
+ from_le = &from_ledger->l_entries[entry];
+ to_le = &to_ledger->l_entries[entry];
+ OSAddAtomic64(from_le->le_credit, &to_le->le_credit);
+ OSAddAtomic64(from_le->le_debit, &to_le->le_debit);
+ }
return (KERN_SUCCESS);
}
ledger_zero_balance(ledger_t ledger, int entry)
{
struct ledger_entry *le;
+ ledger_amount_t debit, credit;
if (!ENTRY_VALID(ledger, entry))
return (KERN_INVALID_VALUE);
le = &ledger->l_entries[entry];
top:
- if (le->le_credit > le->le_debit) {
- if (!OSCompareAndSwap64(le->le_debit, le->le_credit, &le->le_debit))
+ debit = le->le_debit;
+ credit = le->le_credit;
+
+ if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+ assert(le->le_debit == 0);
+ if (!OSCompareAndSwap64(credit, 0, &le->le_credit)) {
+ goto top;
+ }
+ lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, 0));
+ } else if (credit > debit) {
+ if (!OSCompareAndSwap64(debit, credit, &le->le_debit))
goto top;
lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_debit, le->le_credit));
- } else if (le->le_credit < le->le_debit) {
- if (!OSCompareAndSwap64(le->le_credit, le->le_debit, &le->le_credit))
+ } else if (credit < debit) {
+ if (!OSCompareAndSwap64(credit, debit, &le->le_credit))
goto top;
lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, le->le_debit));
}
return (KERN_SUCCESS);
}
+kern_return_t
+ledger_panic_on_negative(ledger_template_t template, int entry)
+{
+ template_lock(template);
+
+ if ((entry < 0) || (entry >= template->lt_cnt)) {
+ template_unlock(template);
+ return (KERN_INVALID_VALUE);
+ }
+
+ template->lt_entries[entry].et_flags |= LF_PANIC_ON_NEGATIVE;
+
+ template_unlock(template);
+
+ return (KERN_SUCCESS);
+}
+
+kern_return_t
+ledger_track_credit_only(ledger_template_t template, int entry)
+{
+ template_lock(template);
+
+ if ((entry < 0) || (entry >= template->lt_cnt)) {
+ template_unlock(template);
+ return (KERN_INVALID_VALUE);
+ }
+
+ template->lt_entries[entry].et_flags |= LF_TRACK_CREDIT_ONLY;
+
+ template_unlock(template);
+
+ return (KERN_SUCCESS);
+}
+
/*
* Add a callback to be executed when the resource goes into deficit.
*/
return (KERN_SUCCESS);
}
-void
-set_astledger(thread_t thread)
-{
- spl_t s = splsched();
-
- if (thread == current_thread()) {
- thread_ast_set(thread, AST_LEDGER);
- ast_propagate(thread->ast);
- } else {
- processor_t p;
-
- thread_lock(thread);
- thread_ast_set(thread, AST_LEDGER);
- p = thread->last_processor;
- if ((p != PROCESSOR_NULL) && (p->state == PROCESSOR_RUNNING) &&
- (p->active_thread == thread))
- cause_ast_check(p);
- thread_unlock(thread);
- }
-
- splx(s);
-}
-
kern_return_t
ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
{
le = &ledger->l_entries[entry];
- old = OSAddAtomic64(amount, &le->le_debit);
- new = old + amount;
-
+ if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+ assert(le->le_debit == 0);
+ old = OSAddAtomic64(-amount, &le->le_credit);
+ new = old - amount;
+ } else {
+ old = OSAddAtomic64(amount, &le->le_debit);
+ new = old + amount;
+ }
lprintf(("%p Debit %lld->%lld\n", thread, old, new));
- ledger_check_new_balance(ledger, entry);
+
+ ledger_entry_check_new_balance(ledger, entry, le);
return (KERN_SUCCESS);
}
return (KERN_SUCCESS);
}
+kern_return_t
+ledger_reset_callback_state(ledger_t ledger, int entry)
+{
+ struct ledger_entry *le;
+
+ if (!ENTRY_VALID(ledger, entry))
+ return (KERN_INVALID_ARGUMENT);
+
+ le = &ledger->l_entries[entry];
+
+ flag_clear(&le->le_flags, LF_CALLED_BACK);
+
+ return (KERN_SUCCESS);
+}
+
+kern_return_t
+ledger_disable_panic_on_negative(ledger_t ledger, int entry)
+{
+ struct ledger_entry *le;
+
+ if (!ENTRY_VALID(ledger, entry))
+ return (KERN_INVALID_ARGUMENT);
+
+ le = &ledger->l_entries[entry];
+
+ flag_clear(&le->le_flags, LF_PANIC_ON_NEGATIVE);
+
+ return (KERN_SUCCESS);
+}
+
kern_return_t
ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
{
le = &ledger->l_entries[entry];
- assert((le->le_credit >= 0) && (le->le_debit >= 0));
+ if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+ assert(le->le_debit == 0);
+ } else {
+ assert((le->le_credit >= 0) && (le->le_debit >= 0));
+ }
*balance = le->le_credit - le->le_debit;
assert(ledger != NULL);
assert(lei != NULL);
- assert(entry < ledger->l_size);
-
- struct ledger_entry *le = &ledger->l_entries[entry];
- ledger_fill_entry_info(le, lei, now);
+ if (entry >= 0 && entry < ledger->l_size) {
+ struct ledger_entry *le = &ledger->l_entries[entry];
+ ledger_fill_entry_info(le, lei, now);
+ }
}
int
projects / apple / xnu.git / blobdiff