xnu-3789.41.3.tar.gz

[apple/xnu.git] / osfmk / kern / ledger.c

diff --git a/osfmk/kern/ledger.c b/osfmk/kern/ledger.c
index 13146ce6f78e502b42e04ec8225658772de78b1a..a7e9f41737994c6d819e1816349f78845cd0fe38 100644 (file)
--- a/osfmk/kern/ledger.c
+++ b/osfmk/kern/ledger.c
@@ -29,14 +29,17 @@
  * @OSF_COPYRIGHT@
  */
 
  * @OSF_COPYRIGHT@
  */
 

-#include <kern/lock.h>
+#include <kern/kern_types.h>

 #include <kern/ledger.h>
 #include <kern/kalloc.h>
 #include <kern/task.h>
 #include <kern/ledger.h>
 #include <kern/kalloc.h>
 #include <kern/task.h>

+#include <kern/thread.h>

 
 #include <kern/processor.h>
 #include <kern/machine.h>
 #include <kern/queue.h>
 
 #include <kern/processor.h>
 #include <kern/machine.h>
 #include <kern/queue.h>

+#include <kern/policy_internal.h>
+

 #include <sys/errno.h>
 
 #include <libkern/OSAtomic.h>
 #include <sys/errno.h>
 
 #include <libkern/OSAtomic.h>


@@ -54,23 +57,25 @@
 #define        LF_CALLED_BACK          0x1000  /* callback was called for balance in deficit */
 #define        LF_WARNED               0x2000  /* callback was called for balance warning */ 
 #define        LF_TRACKING_MAX         0x4000  /* track max balance over user-specfied time */
 #define        LF_CALLED_BACK          0x1000  /* callback was called for balance in deficit */
 #define        LF_WARNED               0x2000  /* callback was called for balance warning */ 
 #define        LF_TRACKING_MAX         0x4000  /* track max balance over user-specfied time */

+#define LF_PANIC_ON_NEGATIVE   0x8000  /* panic if it goes negative */
+#define LF_TRACK_CREDIT_ONLY   0x10000 /* only update "credit" */

 
 /* Determine whether a ledger entry exists and has been initialized and active */
 #define        ENTRY_VALID(l, e)                                       \
        (((l) != NULL) && ((e) >= 0) && ((e) < (l)->l_size) &&  \
        (((l)->l_entries[e].le_flags & LF_ENTRY_ACTIVE) == LF_ENTRY_ACTIVE))
 
 
 /* Determine whether a ledger entry exists and has been initialized and active */
 #define        ENTRY_VALID(l, e)                                       \
        (((l) != NULL) && ((e) >= 0) && ((e) < (l)->l_size) &&  \
        (((l)->l_entries[e].le_flags & LF_ENTRY_ACTIVE) == LF_ENTRY_ACTIVE))
 

+#define ASSERT(a) assert(a)
+

 #ifdef LEDGER_DEBUG
 int ledger_debug = 0;
 
 #ifdef LEDGER_DEBUG
 int ledger_debug = 0;
 

-#define ASSERT(a) assert(a)

 #define        lprintf(a) if (ledger_debug) {                                  \
        printf("%lld  ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \
        printf a ;                                                      \
 }
 #else
 #define        lprintf(a)
 #define        lprintf(a) if (ledger_debug) {                                  \
        printf("%lld  ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \
        printf a ;                                                      \
 }
 #else
 #define        lprintf(a)

-#define        ASSERT(a)

 #endif
 
 struct ledger_callback {
 #endif
 
 struct ledger_callback {


@@ -154,11 +159,11 @@ struct ledger_entry {
 } __attribute__((aligned(8)));
 
 struct ledger {
 } __attribute__((aligned(8)));
 
 struct ledger {

-       int                     l_id;
+       uint64_t                l_id;
+       int32_t                 l_refs;
+       int32_t                 l_size;

        struct ledger_template  *l_template;
        struct ledger_template  *l_template;

-       int                     l_refs;
-       int                     l_size;
-       struct ledger_entry     *l_entries;
+       struct ledger_entry     l_entries[0] __attribute__((aligned(8)));

 };
 
 static int ledger_cnt = 0;
 };
 
 static int ledger_cnt = 0;


@@ -168,6 +173,9 @@ static kern_return_t ledger_perform_blocking(ledger_t l);
 static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
 static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
 
 static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
 static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
 

+static void ledger_entry_check_new_balance(ledger_t ledger, int entry,
+                                          struct ledger_entry *le);
+

 #if 0
 static void
 debug_callback(const void *p0, __unused const void *p1)
 #if 0
 static void
 debug_callback(const void *p0, __unused const void *p1)


@@ -321,7 +329,7 @@ ledger_key_lookup(ledger_template_t template, const char *key)
 
        template_lock(template);
        for (idx = 0; idx < template->lt_cnt; idx++)
 
        template_lock(template);
        for (idx = 0; idx < template->lt_cnt; idx++)

-               if (template->lt_entries[idx].et_key &&
+               if (template->lt_entries != NULL &&

                    (strcmp(key, template->lt_entries[idx].et_key) == 0))
                        break;
 
                    (strcmp(key, template->lt_entries[idx].et_key) == 0))
                        break;
 


@@ -343,30 +351,27 @@ ledger_t
 ledger_instantiate(ledger_template_t template, int entry_type)
 {
        ledger_t ledger;
 ledger_instantiate(ledger_template_t template, int entry_type)
 {
        ledger_t ledger;

-       size_t sz;
+       size_t cnt, sz;

        int i;
 
        int i;
 

-       ledger = (ledger_t)kalloc(sizeof (struct ledger));
-       if (ledger == NULL)
-               return (LEDGER_NULL);
-
-       ledger->l_template = template;
-       ledger->l_id = ledger_cnt++;
-       ledger->l_refs = 1;
-

        template_lock(template);
        template->lt_refs++;
        template_lock(template);
        template->lt_refs++;

-       ledger->l_size = template->lt_cnt;
+       cnt = template->lt_cnt;

        template_unlock(template);
 
        template_unlock(template);
 

-       sz = ledger->l_size * sizeof (struct ledger_entry);
-       ledger->l_entries = kalloc(sz);
-       if (sz && (ledger->l_entries == NULL)) {
+       sz = sizeof(*ledger) + (cnt * sizeof(struct ledger_entry));
+
+       ledger = (ledger_t)kalloc(sz);
+       if (ledger == NULL) {

                ledger_template_dereference(template);
                ledger_template_dereference(template);

-               kfree(ledger, sizeof(struct ledger));
-               return (LEDGER_NULL);
+               return LEDGER_NULL;

        }
 
        }
 

+       ledger->l_template = template;
+       ledger->l_id = ledger_cnt++;
+       ledger->l_refs = 1;
+       ledger->l_size = (int32_t)cnt;
+

        template_lock(template);
        assert(ledger->l_size <= template->lt_cnt);
        for (i = 0; i < ledger->l_size; i++) {
        template_lock(template);
        assert(ledger->l_size <= template->lt_cnt);
        for (i = 0; i < ledger->l_size; i++) {


@@ -445,9 +450,8 @@ ledger_dereference(ledger_t ledger)
 
        /* Just released the last reference.  Free it. */
        if (v == 1) {
 
        /* Just released the last reference.  Free it. */
        if (v == 1) {

-               kfree(ledger->l_entries,
-                   ledger->l_size * sizeof (struct ledger_entry));
-               kfree(ledger, sizeof (*ledger));
+               kfree(ledger,
+                     sizeof(*ledger) + ledger->l_size * sizeof(struct ledger_entry));

        }
 
        return (KERN_SUCCESS);
        }
 
        return (KERN_SUCCESS);


@@ -461,7 +465,11 @@ warn_level_exceeded(struct ledger_entry *le)
 {
        ledger_amount_t balance;
 
 {
        ledger_amount_t balance;
 

-       assert((le->le_credit >= 0) && (le->le_debit >= 0));
+       if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+               assert(le->le_debit == 0);
+       } else {
+               assert((le->le_credit >= 0) && (le->le_debit >= 0));
+       }

 
        /*
         * XXX - Currently, we only support warnings for ledgers which
 
        /*
         * XXX - Currently, we only support warnings for ledgers which


@@ -481,7 +489,11 @@ limit_exceeded(struct ledger_entry *le)
 {
        ledger_amount_t balance;
 
 {
        ledger_amount_t balance;
 

-       assert((le->le_credit >= 0) && (le->le_debit >= 0));
+       if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+               assert(le->le_debit == 0);
+       } else {
+               assert((le->le_credit >= 0) && (le->le_debit >= 0));
+       }

 
        balance = le->le_credit - le->le_debit;
        if ((le->le_limit <= 0) && (balance < le->le_limit))
 
        balance = le->le_credit - le->le_debit;
        if ((le->le_limit <= 0) && (balance < le->le_limit))


@@ -533,10 +545,17 @@ ledger_refill(uint64_t now, ledger_t ledger, int entry)
        struct ledger_entry *le;
        ledger_amount_t balance, due;
 
        struct ledger_entry *le;
        ledger_amount_t balance, due;
 

+       assert(entry >= 0 && entry < ledger->l_size);
+

        le = &ledger->l_entries[entry];
 
        assert(le->le_limit != LEDGER_LIMIT_INFINITY);
 
        le = &ledger->l_entries[entry];
 
        assert(le->le_limit != LEDGER_LIMIT_INFINITY);
 

+       if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+               assert(le->le_debit == 0);
+               return;
+       }
+

        /*
         * If another thread is handling the refill already, we're not
         * needed.
        /*
         * If another thread is handling the refill already, we're not
         * needed.


@@ -650,12 +669,10 @@ ledger_refill(uint64_t now, ledger_t ledger, int entry)
  */
 #define TOCKSTAMP_IS_STALE(now, tock) ((((now) - (tock)) < NTOCKS) ? FALSE : TRUE)
 
  */
 #define TOCKSTAMP_IS_STALE(now, tock) ((((now) - (tock)) < NTOCKS) ? FALSE : TRUE)
 

-static void
-ledger_check_new_balance(ledger_t ledger, int entry)
+void
+ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *le)

 {
 {

-       struct ledger_entry *le;
-
-       le = &ledger->l_entries[entry];
+       ledger_amount_t credit, debit;

 
        if (le->le_flags & LF_TRACKING_MAX) {
                ledger_amount_t balance = le->le_credit - le->le_debit;
 
        if (le->le_flags & LF_TRACKING_MAX) {
                ledger_amount_t balance = le->le_credit - le->le_debit;


@@ -746,8 +763,30 @@ ledger_check_new_balance(ledger_t ledger, int entry)
                        }
                }
        }
                        }
                }
        }

+
+       credit = le->le_credit;
+       debit = le->le_debit;
+       if ((le->le_flags & LF_PANIC_ON_NEGATIVE) &&
+           ((credit < debit) ||
+            (le->le_credit < le->le_debit))) {
+               panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld/%lld debit:%lld/%lld balance:%lld/%lld\n",
+                     ledger, entry, le,
+                     credit, le->le_credit,
+                     debit, le->le_debit,
+                     credit - debit, le->le_credit - le->le_debit);
+       }

 }
 
 }
 

+void
+ledger_check_new_balance(ledger_t ledger, int entry)
+{
+       struct ledger_entry *le;
+       assert(entry > 0 && entry <= ledger->l_size);
+       le = &ledger->l_entries[entry];
+       ledger_entry_check_new_balance(ledger, entry, le);
+}
+
+

 /*
  * Add value to an entry in a ledger.
  */
 /*
  * Add value to an entry in a ledger.
  */


@@ -768,7 +807,50 @@ ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
        old = OSAddAtomic64(amount, &le->le_credit);
        new = old + amount;
        lprintf(("%p Credit %lld->%lld\n", current_thread(), old, new));
        old = OSAddAtomic64(amount, &le->le_credit);
        new = old + amount;
        lprintf(("%p Credit %lld->%lld\n", current_thread(), old, new));

-       ledger_check_new_balance(ledger, entry);
+       ledger_entry_check_new_balance(ledger, entry, le);
+
+       return (KERN_SUCCESS);
+}
+
+/* Add all of one ledger's values into another.
+ * They must have been created from the same template.
+ * This is not done atomically. Another thread (if not otherwise synchronized)
+ * may see bogus values when comparing one entry to another.
+ * As each entry's credit & debit are modified one at a time, the warning/limit
+ * may spuriously trip, or spuriously fail to trip, or another thread (if not
+ * otherwise synchronized) may see a bogus balance.
+ */
+kern_return_t
+ledger_rollup(ledger_t to_ledger, ledger_t from_ledger)
+{
+       int i;
+
+       assert(to_ledger->l_template == from_ledger->l_template);
+
+       for (i = 0; i < to_ledger->l_size; i++) {
+               ledger_rollup_entry(to_ledger, from_ledger, i);
+       }
+
+       return (KERN_SUCCESS);
+}
+
+/* Add one ledger entry value to another.
+ * They must have been created from the same template.
+ * Since the credit and debit values are added one
+ * at a time, other thread might read the a bogus value.
+ */
+kern_return_t
+ledger_rollup_entry(ledger_t to_ledger, ledger_t from_ledger, int entry)
+{
+       struct ledger_entry *from_le, *to_le;
+
+       assert(to_ledger->l_template == from_ledger->l_template);
+       if (ENTRY_VALID(from_ledger, entry) && ENTRY_VALID(to_ledger, entry)) {
+               from_le = &from_ledger->l_entries[entry];
+               to_le   =   &to_ledger->l_entries[entry];
+               OSAddAtomic64(from_le->le_credit, &to_le->le_credit);
+               OSAddAtomic64(from_le->le_debit,  &to_le->le_debit);
+       }

 
        return (KERN_SUCCESS);
 }
 
        return (KERN_SUCCESS);
 }


@@ -789,7 +871,13 @@ ledger_zero_balance(ledger_t ledger, int entry)
        le = &ledger->l_entries[entry];
 
 top:
        le = &ledger->l_entries[entry];
 
 top:

-       if (le->le_credit > le->le_debit) {
+       if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+               assert(le->le_debit == 0);
+               if (!OSCompareAndSwap64(le->le_credit, 0, &le->le_credit)) {
+                       goto top;
+               }
+               lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, 0));
+       } else if (le->le_credit > le->le_debit) {

                if (!OSCompareAndSwap64(le->le_debit, le->le_credit, &le->le_debit))
                        goto top;
                lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_debit, le->le_credit));
                if (!OSCompareAndSwap64(le->le_debit, le->le_credit, &le->le_debit))
                        goto top;
                lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_debit, le->le_credit));


@@ -921,6 +1009,40 @@ ledger_track_maximum(ledger_template_t template, int entry,
        return (KERN_SUCCESS);
 }
 
        return (KERN_SUCCESS);
 }
 

+kern_return_t
+ledger_panic_on_negative(ledger_template_t template, int entry)
+{
+       template_lock(template);
+
+       if ((entry < 0) || (entry >= template->lt_cnt)) {
+               template_unlock(template);
+               return (KERN_INVALID_VALUE);
+       }
+
+       template->lt_entries[entry].et_flags |= LF_PANIC_ON_NEGATIVE;
+
+       template_unlock(template);
+
+       return (KERN_SUCCESS);
+}
+
+kern_return_t
+ledger_track_credit_only(ledger_template_t template, int entry)
+{
+       template_lock(template);
+
+       if ((entry < 0) || (entry >= template->lt_cnt)) {
+               template_unlock(template);
+               return (KERN_INVALID_VALUE);
+       }
+
+       template->lt_entries[entry].et_flags |= LF_TRACK_CREDIT_ONLY;
+
+       template_unlock(template);
+
+       return (KERN_SUCCESS);
+}
+

 /*
  * Add a callback to be executed when the resource goes into deficit.
  */
 /*
  * Add a callback to be executed when the resource goes into deficit.
  */


@@ -1099,29 +1221,6 @@ ledger_set_action(ledger_t ledger, int entry, int action)
        return (KERN_SUCCESS);
 }
 
        return (KERN_SUCCESS);
 }
 

-void
-set_astledger(thread_t thread)
-{
-       spl_t s = splsched();
-
-       if (thread == current_thread()) {
-               thread_ast_set(thread, AST_LEDGER);
-               ast_propagate(thread->ast);
-       } else {
-               processor_t p;
-
-               thread_lock(thread);
-               thread_ast_set(thread, AST_LEDGER);
-               p = thread->last_processor;
-               if ((p != PROCESSOR_NULL) && (p->state == PROCESSOR_RUNNING) &&
-                  (p->active_thread == thread))
-                       cause_ast_check(p);
-               thread_unlock(thread);
-       }
-       
-       splx(s);
-}
-

 kern_return_t
 ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
 {
 kern_return_t
 ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
 {


@@ -1136,11 +1235,17 @@ ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
 
        le = &ledger->l_entries[entry];
 
 
        le = &ledger->l_entries[entry];
 

-       old = OSAddAtomic64(amount, &le->le_debit);
-       new = old + amount;
-
+       if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+               assert(le->le_debit == 0);
+               old = OSAddAtomic64(-amount, &le->le_credit);
+               new = old - amount;
+       } else {
+               old = OSAddAtomic64(amount, &le->le_debit);
+               new = old + amount;
+       }

        lprintf(("%p Debit %lld->%lld\n", thread, old, new));
        lprintf(("%p Debit %lld->%lld\n", thread, old, new));

-       ledger_check_new_balance(ledger, entry);
+
+       ledger_entry_check_new_balance(ledger, entry, le);

        return (KERN_SUCCESS);
 
 }
        return (KERN_SUCCESS);
 
 }


@@ -1377,6 +1482,36 @@ ledger_get_entries(ledger_t ledger, int entry, ledger_amount_t *credit,
        return (KERN_SUCCESS);
 }
 
        return (KERN_SUCCESS);
 }
 

+kern_return_t
+ledger_reset_callback_state(ledger_t ledger, int entry)
+{
+       struct ledger_entry *le;
+
+       if (!ENTRY_VALID(ledger, entry))
+               return (KERN_INVALID_ARGUMENT);
+
+       le = &ledger->l_entries[entry];
+
+       flag_clear(&le->le_flags, LF_CALLED_BACK);
+
+       return (KERN_SUCCESS);
+}
+
+kern_return_t
+ledger_disable_panic_on_negative(ledger_t ledger, int entry)
+{
+       struct ledger_entry *le;
+
+       if (!ENTRY_VALID(ledger, entry))
+               return (KERN_INVALID_ARGUMENT);
+
+       le = &ledger->l_entries[entry];
+
+       flag_clear(&le->le_flags, LF_PANIC_ON_NEGATIVE);
+
+       return (KERN_SUCCESS);
+}
+

 kern_return_t
 ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
 {
 kern_return_t
 ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
 {


@@ -1387,7 +1522,11 @@ ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
 
        le = &ledger->l_entries[entry];
 
 
        le = &ledger->l_entries[entry];
 

-       assert((le->le_credit >= 0) && (le->le_debit >= 0));
+       if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
+               assert(le->le_debit == 0);
+       } else {
+               assert((le->le_credit >= 0) && (le->le_debit >= 0));
+       }

 
        *balance = le->le_credit - le->le_debit;
 
 
        *balance = le->le_credit - le->le_debit;
 


@@ -1491,11 +1630,11 @@ ledger_get_entry_info(ledger_t                  ledger,
 
        assert(ledger != NULL);
        assert(lei != NULL);
 
        assert(ledger != NULL);
        assert(lei != NULL);

-       assert(entry < ledger->l_size);
-
-       struct ledger_entry *le = &ledger->l_entries[entry];

 
 

-       ledger_fill_entry_info(le, lei, now);
+       if (entry >= 0 && entry < ledger->l_size) {
+               struct ledger_entry *le = &ledger->l_entries[entry];
+               ledger_fill_entry_info(le, lei, now);
+       }

 }
 
 int
 }
 
 int