bsd/pgo/profile_runtime.c

   1 /*
   2  * Copyright (c) 2014 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <machine/machine_routines.h>
  30 #include <sys/sysproto.h>
  31 #include <sys/malloc.h>
  32 #include <sys/systm.h>
  33 #include <sys/pgo.h>
  34 #include <sys/kauth.h>
  35 #include <security/mac_framework.h>
  36 #include <libkern/OSKextLib.h>
  37
  38
  39 #ifdef PROFILE
  40
  41 /* These __llvm functions are defined in InstrProfiling.h in compiler_rt.  That
  42  * is a internal header, so we need to re-prototype them here.  */
  43
  44 uint64_t __llvm_profile_get_size_for_buffer(void);
  45 int __llvm_profile_write_buffer(char *Buffer);
  46 uint64_t __llvm_profile_get_size_for_buffer_internal(const char *DataBegin,
  47                                                      const char *DataEnd,
  48                                                      const char *CountersBegin,
  49                                                      const char *CountersEnd ,
  50                                                      const char *NamesBegin,
  51                                                      const char *NamesEnd);
  52 int __llvm_profile_write_buffer_internal(char *Buffer,
  53                                          const char *DataBegin,
  54                                          const char *DataEnd,
  55                                          const char *CountersBegin,
  56                                          const char *CountersEnd ,
  57                                          const char *NamesBegin,
  58                                          const char *NamesEnd);
  59
  60 extern char __pgo_hib_DataStart __asm("section$start$__HIB$__llvm_prf_data");
  61 extern char __pgo_hib_DataEnd   __asm("section$end$__HIB$__llvm_prf_data");
  62 extern char __pgo_hib_NamesStart __asm("section$start$__HIB$__llvm_prf_names");
  63 extern char __pgo_hib_NamesEnd   __asm("section$end$__HIB$__llvm_prf_names");
  64 extern char __pgo_hib_CountersStart __asm("section$start$__HIB$__llvm_prf_cnts");
  65 extern char __pgo_hib_CountersEnd   __asm("section$end$__HIB$__llvm_prf_cnts");
  66
  67
  68 static uint64_t get_size_for_buffer(int flags)
  69 {
  70         if (flags & PGO_HIB) {
  71                 return __llvm_profile_get_size_for_buffer_internal(
  72                         &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
  73                         &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
  74                         &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
  75         } else {
  76                 return __llvm_profile_get_size_for_buffer();
  77         }
  78 }
  79
  80
  81 static int write_buffer(int flags, char *buffer)
  82 {
  83         if (flags & PGO_HIB) {
  84                 return __llvm_profile_write_buffer_internal(
  85                         buffer,
  86                         &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
  87                         &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
  88                         &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
  89         } else {
  90                 return __llvm_profile_write_buffer(buffer);
  91         }
  92 }
  93
  94
  95 #endif
  96
  97 /* this variable is used to signal to the debugger that we'd like it to reset
  98  * the counters */
  99 int kdp_pgo_reset_counters = 0;
 100
 101 /* called in debugger context */
 102 kern_return_t do_pgo_reset_counters()
 103 {
 104 #ifdef PROFILE
 105     memset(&__pgo_hib_CountersStart, 0,
 106            ((uintptr_t)(&__pgo_hib_CountersEnd)) - ((uintptr_t)(&__pgo_hib_CountersStart)));
 107 #endif
 108     OSKextResetPgoCounters();
 109     kdp_pgo_reset_counters = 0;
 110     return KERN_SUCCESS;
 111 }
 112
 113 static kern_return_t
 114 kextpgo_trap()
 115 {
 116     return DebuggerTrapWithState(DBOP_RESET_PGO_COUNTERS, NULL, NULL, NULL, 0, NULL, FALSE, 0);
 117 }
 118
 119 static kern_return_t
 120 pgo_reset_counters()
 121 {
 122     kern_return_t r;
 123     boolean_t istate;
 124
 125     OSKextResetPgoCountersLock();
 126
 127     istate = ml_set_interrupts_enabled(FALSE);
 128
 129     kdp_pgo_reset_counters = 1;
 130     r = kextpgo_trap();
 131
 132     ml_set_interrupts_enabled(istate);
 133
 134     OSKextResetPgoCountersUnlock();
 135     return r;
 136 }
 137
 138
 139 /*
 140  * returns:
 141  *   EPERM  unless you are root
 142  *   EINVAL for invalid args.
 143  *   ENOSYS for not implemented
 144  *   ERANGE for integer overflow
 145  *   ENOENT if kext not found
 146  *   ENOTSUP kext does not support PGO
 147  *   EIO llvm returned an error.  shouldn't ever happen.
 148  */
 149
 150 int grab_pgo_data(struct proc *p,
 151                   struct grab_pgo_data_args *uap,
 152                   register_t *retval)
 153 {
 154         char *buffer = NULL;
 155         int err = 0;
 156
 157         (void) p;
 158
 159         if (!kauth_cred_issuser(kauth_cred_get())) {
 160                 err = EPERM;
 161                 goto out;
 162         }
 163
 164 #if CONFIG_MACF
 165         err = mac_system_check_info(kauth_cred_get(), "kern.profiling_data");
 166         if (err) {
 167                 goto out;
 168         }
 169 #endif
 170
 171         if ( uap->flags & ~PGO_ALL_FLAGS ||
 172              uap->size < 0 ||
 173              (uap->size > 0 && uap->buffer == 0))
 174         {
 175                 err = EINVAL;
 176                 goto out;
 177         }
 178
 179         if ( uap->flags & PGO_RESET_ALL ) {
 180             if (uap->flags != PGO_RESET_ALL || uap->uuid || uap->buffer || uap->size ) {
 181                 err = EINVAL;
 182             } else {
 183                 kern_return_t r = pgo_reset_counters();
 184                 switch (r) {
 185                 case KERN_SUCCESS:
 186                     err = 0;
 187                     break;
 188                 case KERN_OPERATION_TIMED_OUT:
 189                     err = ETIMEDOUT;
 190                     break;
 191                 default:
 192                     err = EIO;
 193                     break;
 194                 }
 195             }
 196             goto out;
 197         }
 198
 199         *retval = 0;
 200
 201         if (uap->uuid) {
 202                 uuid_t uuid;
 203                 err = copyin(uap->uuid, &uuid, sizeof(uuid));
 204                 if (err) {
 205                         goto out;
 206                 }
 207
 208                 if (uap->buffer == 0 && uap->size == 0) {
 209                     uint64_t size64;
 210
 211                     if (uap->flags & PGO_WAIT_FOR_UNLOAD) {
 212                         err = EINVAL;
 213                         goto out;
 214                     }
 215
 216                     err = OSKextGrabPgoData(uuid, &size64, NULL, 0, 0, !!(uap->flags & PGO_METADATA));
 217                     if (size64 == 0 && err == 0) {
 218                         err = EIO;
 219                     }
 220                     if (err) {
 221                         goto out;
 222                     }
 223
 224                     ssize_t size = size64;
 225                     if ( ((uint64_t) size) != size64  ||
 226                          size < 0 )
 227                     {
 228                         err = ERANGE;
 229                         goto out;
 230                     }
 231
 232                     *retval = size;
 233                     err = 0;
 234                     goto out;
 235
 236                 } else if (!uap->buffer || uap->size <= 0) {
 237
 238                     err = EINVAL;
 239                     goto out;
 240
 241                 } else {
 242
 243                     uint64_t size64 = 0 ;
 244
 245                     err = OSKextGrabPgoData(uuid, &size64, NULL, 0,
 246                                             false,
 247                                             !!(uap->flags & PGO_METADATA));
 248
 249                     if (size64 == 0 && err == 0) {
 250                         err = EIO;
 251                     }
 252                     if (err) {
 253                         goto out;
 254                     }
 255
 256                     if (uap->size < 0 || (uint64_t)uap->size < size64) {
 257                         err = EINVAL;
 258                         goto out;
 259                     }
 260
 261                     MALLOC(buffer, char *, size64, M_TEMP, M_WAITOK | M_ZERO);
 262                     if (!buffer) {
 263                         err = ENOMEM;
 264                         goto out;
 265                     }
 266
 267                     err = OSKextGrabPgoData(uuid, &size64, buffer, size64,
 268                                             !!(uap->flags & PGO_WAIT_FOR_UNLOAD),
 269                                             !!(uap->flags & PGO_METADATA));
 270                     if (err) {
 271                         goto out;
 272                     }
 273
 274                     ssize_t size = size64;
 275                     if ( ((uint64_t) size) != size64  ||
 276                          size < 0 )
 277                     {
 278                         err = ERANGE;
 279                         goto out;
 280                     }
 281
 282                     err = copyout(buffer, uap->buffer, size);
 283                     if (err) {
 284                         goto out;
 285                     }
 286
 287                     *retval = size;
 288                     goto out;
 289                 }
 290         }
 291
 292
 293 #ifdef PROFILE
 294
 295         uint64_t size64 = get_size_for_buffer(uap->flags);
 296         ssize_t size = size64;
 297
 298         if (uap->flags & (PGO_WAIT_FOR_UNLOAD | PGO_METADATA)) {
 299             err = EINVAL;
 300             goto out;
 301         }
 302
 303         if ( ((uint64_t) size) != size64  ||
 304              size < 0 )
 305         {
 306                 err = ERANGE;
 307                 goto out;
 308         }
 309
 310
 311         if (uap->buffer == 0 && uap->size == 0) {
 312                 *retval = size;
 313                 err = 0;
 314                 goto out;
 315         } else if (uap->size < size) {
 316                 err = EINVAL;
 317                 goto out;
 318         } else {
 319                 MALLOC(buffer, char *, size, M_TEMP, M_WAITOK | M_ZERO);
 320                 if (!buffer) {
 321                         err = ENOMEM;
 322                         goto out;
 323                 }
 324
 325                 err = write_buffer(uap->flags, buffer);
 326                 if (err)
 327                 {
 328                     err = EIO;
 329                     goto out;
 330                 }
 331
 332                 err = copyout(buffer, uap->buffer, size);
 333                 if (err) {
 334                         goto out;
 335                 }
 336
 337                 *retval = size;
 338                 goto out;
 339         }
 340
 341 #else
 342
 343         *retval = -1;
 344         err = ENOSYS;
 345         goto out;
 346
 347 #endif
 348
 349 out:
 350         if (buffer) {
 351                 FREE(buffer, M_TEMP);
 352         }
 353         if (err) {
 354                 *retval = -1;
 355         }
 356         return err;
 357 }