xnu-6153.11.26.tar.gz

[apple/xnu.git] / libkern / os / refcnt.c

diff --git a/libkern/os/refcnt.c b/libkern/os/refcnt.c
index 67deb068f441cabc21abc2e6ed1fbbcb14f631d4..0cbcdf745ef39a8374475491d525f45aaece3181 100644 (file)
--- a/libkern/os/refcnt.c
+++ b/libkern/os/refcnt.c
@@ -1,16 +1,22 @@
+#if KERNEL

 #include <kern/assert.h>
 #include <kern/debug.h>
 #include <pexpert/pexpert.h>
 #include <kern/btlog.h>
 #include <kern/backtrace.h>
 #include <libkern/libkern.h>
 #include <kern/assert.h>
 #include <kern/debug.h>
 #include <pexpert/pexpert.h>
 #include <kern/btlog.h>
 #include <kern/backtrace.h>
 #include <libkern/libkern.h>

+#endif
+

 #include "refcnt.h"
 
 #define OS_REFCNT_MAX_COUNT     ((os_ref_count_t)0x0FFFFFFFUL)
 
 #if OS_REFCNT_DEBUG
 #include "refcnt.h"
 
 #define OS_REFCNT_MAX_COUNT     ((os_ref_count_t)0x0FFFFFFFUL)
 
 #if OS_REFCNT_DEBUG

-os_refgrp_decl(static, global_ref_group, "all", NULL);
-static bool ref_debug_enable = false;
+extern struct os_refgrp global_ref_group;
+os_refgrp_decl(, global_ref_group, "all", NULL);
+
+extern bool ref_debug_enable;
+bool ref_debug_enable = false;

 static const size_t ref_log_nrecords = 1000000;
 
 #define REFLOG_BTDEPTH   10
 static const size_t ref_log_nrecords = 1000000;
 
 #define REFLOG_BTDEPTH   10


@@ -22,77 +28,75 @@ static const size_t ref_log_nrecords = 1000000;
 # define __debug_only __unused
 #endif /* OS_REFCNT_DEBUG */
 
 # define __debug_only __unused
 #endif /* OS_REFCNT_DEBUG */
 

-static const char *
-ref_grp_name(struct os_refcnt __debug_only *rc)
+void
+os_ref_panic_live(void *rc)

 {
 {

-#if OS_REFCNT_DEBUG
-       if (rc && rc->ref_group && rc->ref_group->grp_name) {
-               return rc->ref_group->grp_name;
-       }
-#endif
-       return "<null>";
+       panic("os_refcnt: unexpected release of final reference (rc=%p)\n", rc);
+       __builtin_unreachable();

 }
 
 }
 

-__attribute__((cold, noinline, not_tail_called, noreturn))
+__abortlike

 static void
 static void

-os_ref_panic_underflow(struct os_refcnt *rc)
+os_ref_panic_underflow(void *rc)

 {
 {

-       panic("os_refcnt: underflow (rc=%p, grp=%s)\n", rc, ref_grp_name(rc));
+       panic("os_refcnt: underflow (rc=%p)\n", rc);

        __builtin_unreachable();
 }
 
        __builtin_unreachable();
 }
 

-static inline void
-os_ref_check_underflow(struct os_refcnt *rc, os_ref_count_t count)
+__abortlike
+static void
+os_ref_panic_resurrection(void *rc)

 {
 {

-       if (__improbable(count == 0)) {
-               os_ref_panic_underflow(rc);
-       }
+       panic("os_refcnt: attempted resurrection (rc=%p)\n", rc);
+       __builtin_unreachable();

 }
 
 }
 

-__attribute__((cold, noinline, not_tail_called, noreturn))
+__abortlike

 static void
 static void

-os_ref_panic_resurrection(struct os_refcnt *rc)
+os_ref_panic_overflow(void *rc)

 {
 {

-       panic("os_refcnt: used unsafely when zero (rc=%p, grp=%s)\n", rc, ref_grp_name(rc));
+       panic("os_refcnt: overflow (rc=%p)\n", rc);

        __builtin_unreachable();
 }
 
 static inline void
        __builtin_unreachable();
 }
 
 static inline void

-os_ref_assert_referenced(struct os_refcnt *rc, os_ref_count_t count)
+os_ref_check_underflow(void *rc, os_ref_count_t count)

 {
        if (__improbable(count == 0)) {
 {
        if (__improbable(count == 0)) {

-               os_ref_panic_resurrection(rc);
+               os_ref_panic_underflow(rc);

        }
 }
 
        }
 }
 

-__attribute__((cold, noinline, not_tail_called, noreturn))
-static void
-os_ref_panic_overflow(struct os_refcnt *rc)
+static inline void
+os_ref_check_overflow(os_ref_atomic_t *rc, os_ref_count_t count)

 {
 {

-       panic("os_refcnt: overflow (rc=%p, grp=%s)\n", rc, ref_grp_name(rc));
-       __builtin_unreachable();
+       if (__improbable(count >= OS_REFCNT_MAX_COUNT)) {
+               os_ref_panic_overflow(rc);
+       }

 }
 
 static inline void
 }
 
 static inline void

-os_ref_check_overflow(struct os_refcnt *rc, os_ref_count_t count)
+os_ref_assert_referenced(void *rc, os_ref_count_t count)

 {
 {

-       if (__improbable(count >= OS_REFCNT_MAX_COUNT)) {
-               os_ref_panic_overflow(rc);
+       if (__improbable(count == 0)) {
+               os_ref_panic_resurrection(rc);

        }
 }
 
        }
 }
 

-static void
-os_ref_check_retain(struct os_refcnt *rc, os_ref_count_t count)
+static inline void
+os_ref_check_retain(os_ref_atomic_t *rc, os_ref_count_t count)

 {
        os_ref_assert_referenced(rc, count);
        os_ref_check_overflow(rc, count);
 }
 
 #if OS_REFCNT_DEBUG
 {
        os_ref_assert_referenced(rc, count);
        os_ref_check_overflow(rc, count);
 }
 
 #if OS_REFCNT_DEBUG

+#if KERNEL
+__attribute__((cold, noinline))

 static void
 ref_log_op(struct os_refgrp *grp, void *elem, int op)
 {
 static void
 ref_log_op(struct os_refgrp *grp, void *elem, int op)
 {

-       if (!ref_debug_enable || grp == NULL) {
+       if (grp == NULL) {

                return;
        }
 
                return;
        }
 


@@ -102,10 +106,11 @@ ref_log_op(struct os_refgrp *grp, void *elem, int op)
        }
 
        uintptr_t bt[REFLOG_BTDEPTH];
        }
 
        uintptr_t bt[REFLOG_BTDEPTH];

-       uint32_t nframes = backtrace(bt, REFLOG_BTDEPTH);
+       uint32_t nframes = backtrace(bt, REFLOG_BTDEPTH, NULL);

        btlog_add_entry((btlog_t *)grp->grp_log, elem, op, (void **)bt, nframes);
 }
 
        btlog_add_entry((btlog_t *)grp->grp_log, elem, op, (void **)bt, nframes);
 }
 

+__attribute__((cold, noinline))

 static void
 ref_log_drop(struct os_refgrp *grp, void *elem)
 {
 static void
 ref_log_drop(struct os_refgrp *grp, void *elem)
 {


@@ -121,6 +126,7 @@ ref_log_drop(struct os_refgrp *grp, void *elem)
        btlog_remove_entries_for_element(grp->grp_log, elem);
 }
 
        btlog_remove_entries_for_element(grp->grp_log, elem);
 }
 

+__attribute__((cold, noinline))

 static void
 ref_log_init(struct os_refgrp *grp)
 {
 static void
 ref_log_init(struct os_refgrp *grp)
 {


@@ -145,17 +151,30 @@ ref_log_init(struct os_refgrp *grp)
                if (strcmp(g, grp->grp_name) == 0) {
                        /* enable logging on this refgrp */
                        grp->grp_log = btlog_create(ref_log_nrecords, REFLOG_BTDEPTH, true);
                if (strcmp(g, grp->grp_name) == 0) {
                        /* enable logging on this refgrp */
                        grp->grp_log = btlog_create(ref_log_nrecords, REFLOG_BTDEPTH, true);

-                       assert(grp->grp_log);

                        return;
                }
        }
 }
                        return;
                }
        }
 }

+#else
+
+#ifndef ref_log_init
+# define ref_log_init(...) do {} while (0)
+#endif
+#ifndef ref_log_op
+# define ref_log_op(...) do {} while (0)
+#endif
+#ifndef ref_log_drop
+# define ref_log_drop(...) do {} while (0)
+#endif
+
+#endif /* KERNEL */

 
 /*
  * attach a new refcnt to a group
  */
 
 /*
  * attach a new refcnt to a group
  */

+__attribute__((cold, noinline))

 static void
 static void

-ref_attach_to_group(struct os_refcnt *rc, struct os_refgrp *grp, os_ref_count_t init_count)
+ref_attach_to_group(os_ref_atomic_t *rc, struct os_refgrp *grp, os_ref_count_t init_count)

 {
        if (grp == NULL) {
                return;
 {
        if (grp == NULL) {
                return;


@@ -181,7 +200,7 @@ ref_attach_to_group(struct os_refcnt *rc, struct os_refgrp *grp, os_ref_count_t
        ref_attach_to_group(rc, grp->grp_parent, init_count);
 }
 
        ref_attach_to_group(rc, grp->grp_parent, init_count);
 }
 

-static inline void
+static void

 ref_retain_group(struct os_refgrp *grp)
 {
        if (grp) {
 ref_retain_group(struct os_refgrp *grp)
 {
        if (grp) {


@@ -191,7 +210,8 @@ ref_retain_group(struct os_refgrp *grp)
        }
 }
 
        }
 }
 

-static inline void
+__attribute__((cold, noinline))
+static void

 ref_release_group(struct os_refgrp *grp, bool final)
 {
        if (grp) {
 ref_release_group(struct os_refgrp *grp, bool final)
 {
        if (grp) {


@@ -204,48 +224,57 @@ ref_release_group(struct os_refgrp *grp, bool final)
                ref_release_group(grp->grp_parent, final);
        }
 }
                ref_release_group(grp->grp_parent, final);
        }
 }

-#endif

 
 

-#undef os_ref_init_count
-void
-os_ref_init_count(struct os_refcnt *rc, struct os_refgrp __debug_only *grp, os_ref_count_t count)
+__attribute__((cold, noinline))
+static void
+ref_init_debug(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp, os_ref_count_t count)

 {
 {

-       atomic_init(&rc->ref_count, count);
+       ref_attach_to_group(rc, grp, count);

 
 

-#if OS_REFCNT_DEBUG
-       assert(count > 0);
-       if (grp) {
-               rc->ref_group = grp;
-       } else {
-               rc->ref_group = &global_ref_group;
+       for (os_ref_count_t i = 0; i < count; i++) {
+               ref_log_op(grp, (void *)rc, REFLOG_RETAIN);

        }
        }

+}

 
 

-       ref_attach_to_group(rc, rc->ref_group, count);
+__attribute__((cold, noinline))
+static void
+ref_retain_debug(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp)
+{
+       ref_retain_group(grp);
+       ref_log_op(grp, (void *)rc, REFLOG_RETAIN);
+}
+#endif

 
 

-       for (os_ref_count_t i = 0; i < count; i++) {
-               ref_log_op(rc->ref_group, (void *)rc, REFLOG_RETAIN);
+void
+os_ref_init_count_internal(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp, os_ref_count_t count)
+{
+       os_ref_check_underflow(rc, count);
+       atomic_init(rc, count);
+
+#if OS_REFCNT_DEBUG
+       if (__improbable(ref_debug_enable && grp)) {
+               ref_init_debug(rc, grp, count);

        }
 #endif
 }
 
 void
        }
 #endif
 }
 
 void

-os_ref_retain(struct os_refcnt *rc)
+os_ref_retain_internal(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp)

 {
 {

-       os_ref_count_t old = atomic_fetch_add_explicit(&rc->ref_count, 1, memory_order_relaxed);
+       os_ref_count_t old = atomic_fetch_add_explicit(rc, 1, memory_order_relaxed);

        os_ref_check_retain(rc, old);
 
 #if OS_REFCNT_DEBUG
        os_ref_check_retain(rc, old);
 
 #if OS_REFCNT_DEBUG

-       if (__improbable(ref_debug_enable)) {
-               ref_retain_group(rc->ref_group);
-               ref_log_op(rc->ref_group, (void *)rc, REFLOG_RETAIN);
+       if (__improbable(grp && ref_debug_enable)) {
+               ref_retain_debug(rc, grp);

        }
 #endif
 }
 
 bool
        }
 #endif
 }
 
 bool

-os_ref_retain_try(struct os_refcnt *rc)
+os_ref_retain_try_internal(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp)

 {
 {

-       os_ref_count_t cur = os_ref_get_count(rc);
+       os_ref_count_t cur = os_ref_get_count_internal(rc);

 
        while (1) {
                if (__improbable(cur == 0)) {
 
        while (1) {
                if (__improbable(cur == 0)) {


@@ -254,83 +283,275 @@ os_ref_retain_try(struct os_refcnt *rc)
 
                os_ref_check_retain(rc, cur);
 
 
                os_ref_check_retain(rc, cur);
 

-               if (atomic_compare_exchange_weak_explicit(&rc->ref_count, &cur, cur + 1,
+               if (atomic_compare_exchange_weak_explicit(rc, &cur, cur + 1,

                    memory_order_relaxed, memory_order_relaxed)) {
                    memory_order_relaxed, memory_order_relaxed)) {

-#if OS_REFCNT_DEBUG
-                       if (__improbable(ref_debug_enable)) {
-                               ref_retain_group(rc->ref_group);
-                               ref_log_op(rc->ref_group, (void *)rc, REFLOG_RETAIN);
-                       }
-#endif
-                       return true;
+                       break;

                }
        }
                }
        }

+
+#if OS_REFCNT_DEBUG
+       if (__improbable(grp && ref_debug_enable)) {
+               ref_retain_debug(rc, grp);
+       }
+#endif
+
+       return true;

 }
 
 }
 

-os_ref_count_t
-os_ref_release_explicit(struct os_refcnt *rc, memory_order release_order, memory_order dealloc_order)
+__attribute__((always_inline))
+static inline os_ref_count_t
+_os_ref_release_inline(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp,
+    memory_order release_order, memory_order dealloc_order)

 {
 {

+       os_ref_count_t val;
+

 #if OS_REFCNT_DEBUG
 #if OS_REFCNT_DEBUG

-       /*
-        * Care not to use 'rc' after the decrement because it might be deallocated
-        * under us.
-        */
-       struct os_refgrp *grp = rc->ref_group;
-       if (__improbable(ref_debug_enable)) {
+       if (__improbable(grp && ref_debug_enable)) {
+               /*
+                * Care not to use 'rc' after the decrement because it might be deallocated
+                * under us.
+                */

                ref_log_op(grp, (void *)rc, REFLOG_RELEASE);
        }
 #endif
 
                ref_log_op(grp, (void *)rc, REFLOG_RELEASE);
        }
 #endif
 

-       os_ref_count_t val = atomic_fetch_sub_explicit(&rc->ref_count, 1, release_order);
+       val = atomic_fetch_sub_explicit(rc, 1, release_order);

        os_ref_check_underflow(rc, val);
        if (__improbable(--val == 0)) {
        os_ref_check_underflow(rc, val);
        if (__improbable(--val == 0)) {

-               atomic_load_explicit(&rc->ref_count, dealloc_order);
+               atomic_load_explicit(rc, dealloc_order);
+       }
+

 #if OS_REFCNT_DEBUG
 #if OS_REFCNT_DEBUG

-               if (__improbable(ref_debug_enable)) {
+       if (__improbable(grp && ref_debug_enable)) {
+               if (val == 0) {

                        ref_log_drop(grp, (void *)rc); /* rc is only used as an identifier */
                }
                        ref_log_drop(grp, (void *)rc); /* rc is only used as an identifier */
                }

+               ref_release_group(grp, !val);
+       }

 #endif
 #endif

+
+       return val;
+}
+
+__attribute__((noinline))
+os_ref_count_t
+os_ref_release_internal(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp,
+    memory_order release_order, memory_order dealloc_order)
+{
+       // Legacy exported interface with bad codegen due to the barriers
+       // not being immediate
+       //
+       // Also serves as the debug function
+       return _os_ref_release_inline(rc, grp, release_order, dealloc_order);
+}
+
+os_ref_count_t
+os_ref_release_barrier_internal(os_ref_atomic_t *rc,
+    struct os_refgrp * __debug_only grp)
+{
+#if OS_REFCNT_DEBUG
+       if (__improbable(grp && ref_debug_enable)) {
+               return os_ref_release_internal(rc, grp,
+                          memory_order_release, memory_order_acquire);

        }
        }

+#endif
+       return _os_ref_release_inline(rc, NULL,
+                  memory_order_release, memory_order_acquire);
+}

 
 

+os_ref_count_t
+os_ref_release_relaxed_internal(os_ref_atomic_t *rc,
+    struct os_refgrp * __debug_only grp)
+{

 #if OS_REFCNT_DEBUG
 #if OS_REFCNT_DEBUG

-       if (__improbable(ref_debug_enable)) {
+       if (__improbable(grp && ref_debug_enable)) {
+               return os_ref_release_internal(rc, grp,
+                          memory_order_relaxed, memory_order_relaxed);
+       }
+#endif
+       return _os_ref_release_inline(rc, NULL,
+                  memory_order_relaxed, memory_order_relaxed);
+}
+
+void
+os_ref_retain_locked_internal(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp)
+{
+       os_ref_count_t val = os_ref_get_count_internal(rc);
+       os_ref_check_retain(rc, val);
+       atomic_store_explicit(rc, ++val, memory_order_relaxed);
+
+#if OS_REFCNT_DEBUG
+       if (__improbable(grp && ref_debug_enable)) {
+               ref_retain_debug(rc, grp);
+       }
+#endif
+}
+
+os_ref_count_t
+os_ref_release_locked_internal(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp)
+{
+       os_ref_count_t val = os_ref_get_count_internal(rc);
+       os_ref_check_underflow(rc, val);
+       atomic_store_explicit(rc, --val, memory_order_relaxed);
+
+#if OS_REFCNT_DEBUG
+       if (__improbable(grp && ref_debug_enable)) {

                ref_release_group(grp, !val);
                ref_release_group(grp, !val);

+               ref_log_op(grp, (void *)rc, REFLOG_RELEASE);
+               if (val == 0) {
+                       ref_log_drop(grp, (void *)rc);
+               }

        }
 #endif
 
        return val;
 }
 
        }
 #endif
 
        return val;
 }
 

+/*
+ * Bitwise API
+ */
+
+os_ref_count_t
+os_ref_get_count_mask(os_ref_atomic_t *rc, os_ref_count_t bits)
+{
+       os_ref_count_t ret;
+       ret = os_ref_get_count_raw(rc);
+       return ret >> bits;
+}
+
+#undef os_ref_init_count_mask

 void
 void

-os_ref_retain_locked(struct os_refcnt *rc)
+os_ref_init_count_mask(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp,
+    os_ref_count_t init_count, os_ref_count_t init_bits, os_ref_count_t b)

 {
 {

-       os_ref_count_t val = atomic_load_explicit(&rc->ref_count, memory_order_relaxed);
-       os_ref_check_retain(rc, val);
-       atomic_store_explicit(&rc->ref_count, ++val, memory_order_relaxed);
+       assert(init_bits < (1U << b));
+       os_ref_check_underflow(rc, init_count);
+       atomic_init(rc, (init_count << b) | init_bits);
+
+#if OS_REFCNT_DEBUG
+       if (__improbable(ref_debug_enable && grp)) {
+               ref_init_debug(rc, grp, init_count);
+       }
+#endif
+}
+
+#undef os_ref_retain_mask
+void
+os_ref_retain_mask(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp, os_ref_count_t bits)
+{
+       os_ref_count_t old = atomic_fetch_add_explicit(rc, 1U << bits, memory_order_relaxed);
+       os_ref_check_overflow(rc, old);
+       os_ref_assert_referenced(rc, old >> bits);

 
 #if OS_REFCNT_DEBUG
 
 #if OS_REFCNT_DEBUG

-       if (__improbable(ref_debug_enable)) {
-               ref_retain_group(rc->ref_group);
-               ref_log_op(rc->ref_group, (void *)rc, REFLOG_RETAIN);
+       if (__improbable(grp && ref_debug_enable)) {
+               ref_retain_debug(rc, grp);

        }
 #endif
 }
 
        }
 #endif
 }
 

+#undef os_ref_release_mask_internal

 os_ref_count_t
 os_ref_count_t

-os_ref_release_locked(struct os_refcnt *rc)
+os_ref_release_mask_internal(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp, os_ref_count_t bits,
+    memory_order release_order, memory_order dealloc_order)

 {
 {

-       os_ref_count_t val = atomic_load_explicit(&rc->ref_count, memory_order_relaxed);
+#if OS_REFCNT_DEBUG
+       if (__improbable(grp && ref_debug_enable)) {
+               /*
+                * Care not to use 'rc' after the decrement because it might be deallocated
+                * under us.
+                */
+               ref_log_op(grp, (void *)rc, REFLOG_RELEASE);
+       }
+#endif
+
+       os_ref_count_t val = atomic_fetch_sub_explicit(rc, 1U << bits, release_order);
+       val >>= bits;

        os_ref_check_underflow(rc, val);
        os_ref_check_underflow(rc, val);

-       atomic_store_explicit(&rc->ref_count, --val, memory_order_relaxed);
+       if (__improbable(--val == 0)) {
+               atomic_load_explicit(rc, dealloc_order);
+       }
+
+#if OS_REFCNT_DEBUG
+       if (__improbable(grp && ref_debug_enable)) {
+               if (val == 0) {
+                       ref_log_drop(grp, (void *)rc); /* rc is only used as an identifier */
+               }
+               ref_release_group(grp, !val);
+       }
+#endif
+
+       return val;
+}
+
+#undef os_ref_retain_try_mask
+bool
+os_ref_retain_try_mask(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp, os_ref_count_t bits)
+{
+       os_ref_count_t cur = os_ref_get_count_internal(rc);
+
+       while (1) {
+               if (__improbable((cur >> bits) == 0)) {
+                       return false;
+               }
+
+               os_ref_check_overflow(rc, cur);
+
+               os_ref_count_t next = cur + (1U << bits);
+               if (atomic_compare_exchange_weak_explicit(rc, &cur, next,
+                   memory_order_relaxed, memory_order_relaxed)) {
+                       break;
+               }
+       }
+
+#if OS_REFCNT_DEBUG
+       if (__improbable(grp && ref_debug_enable)) {
+               ref_retain_debug(rc, grp);
+       }
+#endif
+
+       return true;
+}
+
+#undef os_ref_retain_locked_mask
+void
+os_ref_retain_locked_mask(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp, os_ref_count_t bits)
+{
+       os_ref_count_t val = os_ref_get_count_internal(rc);
+
+       os_ref_check_overflow(rc, val);
+       os_ref_assert_referenced(rc, val >> bits);
+
+       val += (1U << bits);
+       atomic_store_explicit(rc, val, memory_order_relaxed);
+
+#if OS_REFCNT_DEBUG
+       if (__improbable(grp && ref_debug_enable)) {
+               ref_retain_debug(rc, grp);
+       }
+#endif
+}
+
+#undef os_ref_release_locked_mask
+os_ref_count_t
+os_ref_release_locked_mask(os_ref_atomic_t *rc, struct os_refgrp * __debug_only grp, os_ref_count_t bits)
+{
+       os_ref_count_t val = os_ref_get_count_internal(rc);
+       os_ref_check_underflow(rc, val >> bits);
+       val -= (1U << bits);
+       atomic_store_explicit(rc, val, memory_order_relaxed);
+
+       val >>= bits;

 
 #if OS_REFCNT_DEBUG
 
 #if OS_REFCNT_DEBUG

-       if (__improbable(ref_debug_enable)) {
-               ref_release_group(rc->ref_group, !val);
-               ref_log_op(rc->ref_group, (void *)rc, REFLOG_RELEASE);
+       if (__improbable(grp && ref_debug_enable)) {
+               ref_release_group(grp, !val);
+               ref_log_op(grp, (void *)rc, REFLOG_RELEASE);

                if (val == 0) {
                if (val == 0) {

-                       ref_log_drop(rc->ref_group, (void *)rc);
+                       ref_log_drop(grp, (void *)rc);

                }
        }
 #endif
                }
        }
 #endif

+

        return val;
 }
        return val;
 }