bsd/pgo/profile_runtime.c

   1 /*
   2  * Copyright (c) 2014 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <machine/machine_routines.h>
  30 #include <sys/sysproto.h>
  31 #include <sys/malloc.h>
  32 #include <sys/systm.h>
  33 #include <sys/pgo.h>
  34 #include <sys/kauth.h>
  35 #include <security/mac_framework.h>
  36 #include <libkern/OSKextLib.h>
  37
  38
  39 #ifdef PROFILE
  40
  41 /* These __llvm functions are defined in InstrProfiling.h in compiler_rt.  That
  42  * is a internal header, so we need to re-prototype them here.  */
  43
  44 uint64_t __llvm_profile_get_size_for_buffer(void);
  45 int __llvm_profile_write_buffer(char *Buffer);
  46 uint64_t __llvm_profile_get_size_for_buffer_internal(const char *DataBegin,
  47     const char *DataEnd,
  48     const char *CountersBegin,
  49     const char *CountersEnd,
  50     const char *NamesBegin,
  51     const char *NamesEnd);
  52 int __llvm_profile_write_buffer_internal(char *Buffer,
  53     const char *DataBegin,
  54     const char *DataEnd,
  55     const char *CountersBegin,
  56     const char *CountersEnd,
  57     const char *NamesBegin,
  58     const char *NamesEnd);
  59
  60 extern char __pgo_hib_DataStart __asm("section$start$__HIB$__llvm_prf_data");
  61 extern char __pgo_hib_DataEnd   __asm("section$end$__HIB$__llvm_prf_data");
  62 extern char __pgo_hib_NamesStart __asm("section$start$__HIB$__llvm_prf_names");
  63 extern char __pgo_hib_NamesEnd   __asm("section$end$__HIB$__llvm_prf_names");
  64 extern char __pgo_hib_CountersStart __asm("section$start$__HIB$__llvm_prf_cnts");
  65 extern char __pgo_hib_CountersEnd   __asm("section$end$__HIB$__llvm_prf_cnts");
  66
  67
  68 static uint64_t
  69 get_size_for_buffer(int flags)
  70 {
  71         if (flags & PGO_HIB) {
  72                 return __llvm_profile_get_size_for_buffer_internal(
  73                         &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
  74                         &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
  75                         &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
  76         } else {
  77                 return __llvm_profile_get_size_for_buffer();
  78         }
  79 }
  80
  81
  82 static int
  83 write_buffer(int flags, char *buffer)
  84 {
  85         if (flags & PGO_HIB) {
  86                 return __llvm_profile_write_buffer_internal(
  87                         buffer,
  88                         &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
  89                         &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
  90                         &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
  91         } else {
  92                 return __llvm_profile_write_buffer(buffer);
  93         }
  94 }
  95
  96
  97 #endif
  98
  99 /* this variable is used to signal to the debugger that we'd like it to reset
 100  * the counters */
 101 int kdp_pgo_reset_counters = 0;
 102
 103 /* called in debugger context */
 104 kern_return_t
 105 do_pgo_reset_counters()
 106 {
 107 #ifdef PROFILE
 108         memset(&__pgo_hib_CountersStart, 0,
 109             ((uintptr_t)(&__pgo_hib_CountersEnd)) - ((uintptr_t)(&__pgo_hib_CountersStart)));
 110 #endif
 111         OSKextResetPgoCounters();
 112         kdp_pgo_reset_counters = 0;
 113         return KERN_SUCCESS;
 114 }
 115
 116 static kern_return_t
 117 kextpgo_trap()
 118 {
 119         return DebuggerTrapWithState(DBOP_RESET_PGO_COUNTERS, NULL, NULL, NULL, 0, NULL, FALSE, 0);
 120 }
 121
 122 static kern_return_t
 123 pgo_reset_counters()
 124 {
 125         kern_return_t r;
 126         boolean_t istate;
 127
 128         OSKextResetPgoCountersLock();
 129
 130         istate = ml_set_interrupts_enabled(FALSE);
 131
 132         kdp_pgo_reset_counters = 1;
 133         r = kextpgo_trap();
 134
 135         ml_set_interrupts_enabled(istate);
 136
 137         OSKextResetPgoCountersUnlock();
 138         return r;
 139 }
 140
 141
 142 /*
 143  * returns:
 144  *   EPERM  unless you are root
 145  *   EINVAL for invalid args.
 146  *   ENOSYS for not implemented
 147  *   ERANGE for integer overflow
 148  *   ENOENT if kext not found
 149  *   ENOTSUP kext does not support PGO
 150  *   EIO llvm returned an error.  shouldn't ever happen.
 151  */
 152
 153 int
 154 grab_pgo_data(struct proc *p,
 155     struct grab_pgo_data_args *uap,
 156     register_t *retval)
 157 {
 158         char *buffer = NULL;
 159         int err = 0;
 160
 161         (void) p;
 162
 163         if (!kauth_cred_issuser(kauth_cred_get())) {
 164                 err = EPERM;
 165                 goto out;
 166         }
 167
 168 #if CONFIG_MACF
 169         err = mac_system_check_info(kauth_cred_get(), "kern.profiling_data");
 170         if (err) {
 171                 goto out;
 172         }
 173 #endif
 174
 175         if (uap->flags & ~PGO_ALL_FLAGS ||
 176             uap->size < 0 ||
 177             (uap->size > 0 && uap->buffer == 0)) {
 178                 err = EINVAL;
 179                 goto out;
 180         }
 181
 182         if (uap->flags & PGO_RESET_ALL) {
 183                 if (uap->flags != PGO_RESET_ALL || uap->uuid || uap->buffer || uap->size) {
 184                         err = EINVAL;
 185                 } else {
 186                         kern_return_t r = pgo_reset_counters();
 187                         switch (r) {
 188                         case KERN_SUCCESS:
 189                                 err = 0;
 190                                 break;
 191                         case KERN_OPERATION_TIMED_OUT:
 192                                 err = ETIMEDOUT;
 193                                 break;
 194                         default:
 195                                 err = EIO;
 196                                 break;
 197                         }
 198                 }
 199                 goto out;
 200         }
 201
 202         *retval = 0;
 203
 204         if (uap->uuid) {
 205                 uuid_t uuid;
 206                 err = copyin(uap->uuid, &uuid, sizeof(uuid));
 207                 if (err) {
 208                         goto out;
 209                 }
 210
 211                 if (uap->buffer == 0 && uap->size == 0) {
 212                         uint64_t size64;
 213
 214                         if (uap->flags & PGO_WAIT_FOR_UNLOAD) {
 215                                 err = EINVAL;
 216                                 goto out;
 217                         }
 218
 219                         err = OSKextGrabPgoData(uuid, &size64, NULL, 0, 0, !!(uap->flags & PGO_METADATA));
 220                         if (size64 == 0 && err == 0) {
 221                                 err = EIO;
 222                         }
 223                         if (err) {
 224                                 goto out;
 225                         }
 226
 227                         ssize_t size = size64;
 228                         if (((uint64_t) size) != size64 ||
 229                             size < 0) {
 230                                 err = ERANGE;
 231                                 goto out;
 232                         }
 233
 234                         *retval = size;
 235                         err = 0;
 236                         goto out;
 237                 } else if (!uap->buffer || uap->size <= 0) {
 238                         err = EINVAL;
 239                         goto out;
 240                 } else {
 241                         uint64_t size64 = 0;
 242
 243                         err = OSKextGrabPgoData(uuid, &size64, NULL, 0,
 244                             false,
 245                             !!(uap->flags & PGO_METADATA));
 246
 247                         if (size64 == 0 && err == 0) {
 248                                 err = EIO;
 249                         }
 250                         if (err) {
 251                                 goto out;
 252                         }
 253
 254                         if (uap->size < 0 || (uint64_t)uap->size < size64) {
 255                                 err = EINVAL;
 256                                 goto out;
 257                         }
 258
 259                         MALLOC(buffer, char *, size64, M_TEMP, M_WAITOK | M_ZERO);
 260                         if (!buffer) {
 261                                 err = ENOMEM;
 262                                 goto out;
 263                         }
 264
 265                         err = OSKextGrabPgoData(uuid, &size64, buffer, size64,
 266                             !!(uap->flags & PGO_WAIT_FOR_UNLOAD),
 267                             !!(uap->flags & PGO_METADATA));
 268                         if (err) {
 269                                 goto out;
 270                         }
 271
 272                         ssize_t size = size64;
 273                         if (((uint64_t) size) != size64 ||
 274                             size < 0) {
 275                                 err = ERANGE;
 276                                 goto out;
 277                         }
 278
 279                         err = copyout(buffer, uap->buffer, size);
 280                         if (err) {
 281                                 goto out;
 282                         }
 283
 284                         *retval = size;
 285                         goto out;
 286                 }
 287         }
 288
 289
 290 #ifdef PROFILE
 291
 292         uint64_t size64 = get_size_for_buffer(uap->flags);
 293         ssize_t size = size64;
 294
 295         if (uap->flags & (PGO_WAIT_FOR_UNLOAD | PGO_METADATA)) {
 296                 err = EINVAL;
 297                 goto out;
 298         }
 299
 300         if (((uint64_t) size) != size64 ||
 301             size < 0) {
 302                 err = ERANGE;
 303                 goto out;
 304         }
 305
 306
 307         if (uap->buffer == 0 && uap->size == 0) {
 308                 *retval = size;
 309                 err = 0;
 310                 goto out;
 311         } else if (uap->size < size) {
 312                 err = EINVAL;
 313                 goto out;
 314         } else {
 315                 MALLOC(buffer, char *, size, M_TEMP, M_WAITOK | M_ZERO);
 316                 if (!buffer) {
 317                         err = ENOMEM;
 318                         goto out;
 319                 }
 320
 321                 err = write_buffer(uap->flags, buffer);
 322                 if (err) {
 323                         err = EIO;
 324                         goto out;
 325                 }
 326
 327                 err = copyout(buffer, uap->buffer, size);
 328                 if (err) {
 329                         goto out;
 330                 }
 331
 332                 *retval = size;
 333                 goto out;
 334         }
 335
 336 #else
 337
 338         *retval = -1;
 339         err = ENOSYS;
 340         goto out;
 341
 342 #endif
 343
 344 out:
 345         if (buffer) {
 346                 FREE(buffer, M_TEMP);
 347         }
 348         if (err) {
 349                 *retval = -1;
 350         }
 351         return err;
 352 }