osfmk/kern/restartable.c

   1 /*
   2  * Copyright (c) 2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <mach/mach_types.h>
  30 #include <mach/task.h>
  31
  32 #include <kern/ast.h>
  33 #include <kern/kalloc.h>
  34 #include <kern/kern_types.h>
  35 #include <kern/mach_param.h>
  36 #include <kern/machine.h>
  37 #include <kern/misc_protos.h>
  38 #include <kern/processor.h>
  39 #include <kern/queue.h>
  40 #include <kern/restartable.h>
  41 #include <kern/task.h>
  42 #include <kern/thread.h>
  43 #include <kern/waitq.h>
  44
  45 #include <os/hash.h>
  46 #include <os/refcnt.h>
  47
  48 /**
  49  * @file osfmk/kern/restartable.c
  50  *
  51  * @brief
  52  * This module implements restartable userspace functions.
  53  *
  54  * @discussion
  55  * task_restartable_ranges_register() allows task to configure
  56  * the restartable ranges, only once per task,
  57  * before it has made its second thread.
  58  *
  59  * task_restartable_ranges_synchronize() can later be used to trigger
  60  * restarts for threads with a PC in a restartable region.
  61  *
  62  * It is implemented with an AST (AST_RESET_PCS) that will cause threads
  63  * as they return to userspace to reset PCs in a restartable region
  64  * to the recovery offset of this region.
  65  *
  66  * Because signal delivery would mask the proper saved PC for threads,
  67  * sigreturn also forcefully sets the AST and will go through the logic
  68  * every single time.
  69  */
  70
  71 typedef int (*cmpfunc_t)(const void *a, const void *b);
  72 extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
  73
  74 struct restartable_ranges {
  75         queue_chain_t            rr_link;
  76         os_refcnt_t              rr_ref;
  77         uint32_t                 rr_count;
  78         uint32_t                 rr_hash;
  79         task_restartable_range_t rr_ranges[];
  80 };
  81
  82 #if DEBUG || DEVELOPMENT
  83 #define RR_HASH_SIZE   256
  84 #else
  85 // Release kernel userspace should have shared caches and a single registration
  86 #define RR_HASH_SIZE    16
  87 #endif
  88
  89 static queue_head_t rr_hash[RR_HASH_SIZE];
  90 LCK_GRP_DECLARE(rr_lock_grp, "restartable ranges");
  91 LCK_SPIN_DECLARE(rr_spinlock, &rr_lock_grp);
  92
  93 #define rr_lock()   lck_spin_lock_grp(&rr_spinlock, &rr_lock_grp)
  94 #define rr_unlock() lck_spin_unlock(&rr_spinlock);
  95
  96 #pragma mark internals
  97
  98 /**
  99  * @function _ranges_cmp
 100  *
 101  * @brief
 102  * Compares two ranges together.
 103  */
 104 static int
 105 _ranges_cmp(const void *_r1, const void *_r2)
 106 {
 107         const task_restartable_range_t *r1 = _r1;
 108         const task_restartable_range_t *r2 = _r2;
 109
 110         if (r1->location != r2->location) {
 111                 return r1->location < r2->location ? -1 : 1;
 112         }
 113         if (r1->length == r2->length) {
 114                 return 0;
 115         }
 116         return r1->length < r2->length ? -1 : 1;
 117 }
 118
 119 /**
 120  * @function _ranges_validate
 121  *
 122  * @brief
 123  * Validates an array of PC ranges for wraps and intersections.
 124  *
 125  * @discussion
 126  * This sorts and modifies the input.
 127  *
 128  * The ranges must:
 129  * - not wrap around,
 130  * - have a length/recovery offset within a page of the range start
 131  *
 132  * @returns
 133  * - KERN_SUCCESS:          ranges are valid
 134  * - KERN_INVALID_ARGUMENT: ranges are invalid
 135  */
 136 static kern_return_t
 137 _ranges_validate(task_t task, task_restartable_range_t *ranges, uint32_t count)
 138 {
 139         qsort(ranges, count, sizeof(task_restartable_range_t), _ranges_cmp);
 140         uint64_t limit = task_has_64Bit_data(task) ? UINT64_MAX : UINT32_MAX;
 141         uint64_t end, recovery;
 142
 143         for (size_t i = 0; i < count; i++) {
 144                 if (ranges[i].length > TASK_RESTARTABLE_OFFSET_MAX ||
 145                     ranges[i].recovery_offs > TASK_RESTARTABLE_OFFSET_MAX) {
 146                         return KERN_INVALID_ARGUMENT;
 147                 }
 148                 if (ranges[i].flags) {
 149                         return KERN_INVALID_ARGUMENT;
 150                 }
 151                 if (os_add_overflow(ranges[i].location, ranges[i].length, &end)) {
 152                         return KERN_INVALID_ARGUMENT;
 153                 }
 154                 if (os_add_overflow(ranges[i].location, ranges[i].recovery_offs, &recovery)) {
 155                         return KERN_INVALID_ARGUMENT;
 156                 }
 157                 if (ranges[i].location > limit || end > limit || recovery > limit) {
 158                         return KERN_INVALID_ARGUMENT;
 159                 }
 160                 if (i + 1 < count && end > ranges[i + 1].location) {
 161                         return KERN_INVALID_ARGUMENT;
 162                 }
 163         }
 164
 165         return KERN_SUCCESS;
 166 }
 167
 168 /**
 169  * @function _ranges_lookup
 170  *
 171  * @brief
 172  * Lookup the left side of a range for a given PC within a set of ranges.
 173  *
 174  * @returns
 175  * - 0: no PC range found
 176  * - the left-side of the range.
 177  */
 178 __attribute__((always_inline))
 179 static mach_vm_address_t
 180 _ranges_lookup(struct restartable_ranges *rr, mach_vm_address_t pc)
 181 {
 182         task_restartable_range_t *ranges = rr->rr_ranges;
 183         uint32_t l = 0, r = rr->rr_count;
 184
 185         if (pc <= ranges[0].location) {
 186                 return 0;
 187         }
 188         if (pc >= ranges[r - 1].location + ranges[r - 1].length) {
 189                 return 0;
 190         }
 191
 192         while (l < r) {
 193                 uint32_t i = (r + l) / 2;
 194                 mach_vm_address_t location = ranges[i].location;
 195
 196                 if (pc <= location) {
 197                         /* if the PC is exactly at pc_start, no reset is needed */
 198                         r = i;
 199                 } else if (location + ranges[i].length <= pc) {
 200                         /* if the PC is exactly at the end, it's out of the function */
 201                         l = i + 1;
 202                 } else {
 203                         /* else it's strictly in the range, return the recovery pc */
 204                         return location + ranges[i].recovery_offs;
 205                 }
 206         }
 207
 208         return 0;
 209 }
 210
 211 /**
 212  * @function _restartable_ranges_dispose
 213  *
 214  * @brief
 215  * Helper to dispose of a range that has reached a 0 refcount.
 216  */
 217 __attribute__((noinline))
 218 static void
 219 _restartable_ranges_dispose(struct restartable_ranges *rr, bool hash_remove)
 220 {
 221         if (hash_remove) {
 222                 rr_lock();
 223                 remqueue(&rr->rr_link);
 224                 rr_unlock();
 225         }
 226         kfree(rr, sizeof(*rr) + rr->rr_count * sizeof(task_restartable_range_t));
 227 }
 228
 229 /**
 230  * @function _restartable_ranges_equals
 231  *
 232  * @brief
 233  * Helper to compare two restartable ranges.
 234  */
 235 static bool
 236 _restartable_ranges_equals(
 237         const struct restartable_ranges *rr1,
 238         const struct restartable_ranges *rr2)
 239 {
 240         size_t rr1_size = rr1->rr_count * sizeof(task_restartable_range_t);
 241         return rr1->rr_hash == rr2->rr_hash &&
 242                rr1->rr_count == rr2->rr_count &&
 243                memcmp(rr1->rr_ranges, rr2->rr_ranges, rr1_size) == 0;
 244 }
 245
 246 /**
 247  * @function _restartable_ranges_create
 248  *
 249  * @brief
 250  * Helper to create a uniqued restartable range.
 251  *
 252  * @returns
 253  * - KERN_SUCCESS
 254  * - KERN_INVALID_ARGUMENT: the validation of the new ranges failed.
 255  * - KERN_RESOURCE_SHORTAGE: too many ranges, out of memory
 256  */
 257 static kern_return_t
 258 _restartable_ranges_create(task_t task, task_restartable_range_t *ranges,
 259     uint32_t count, struct restartable_ranges **rr_storage)
 260 {
 261         struct restartable_ranges *rr, *rr_found, *rr_base;
 262         queue_head_t *head;
 263         uint32_t base_count, total_count;
 264         size_t base_size, size;
 265         kern_return_t kr;
 266
 267         rr_base = *rr_storage;
 268         base_count = rr_base ? rr_base->rr_count : 0;
 269         base_size = sizeof(task_restartable_range_t) * base_count;
 270         size = sizeof(task_restartable_range_t) * count;
 271
 272         if (os_add_overflow(base_count, count, &total_count)) {
 273                 return KERN_INVALID_ARGUMENT;
 274         }
 275         if (total_count > 1024) {
 276                 return KERN_RESOURCE_SHORTAGE;
 277         }
 278
 279         rr = kalloc(sizeof(*rr) + base_size + size);
 280         if (rr == NULL) {
 281                 return KERN_RESOURCE_SHORTAGE;
 282         }
 283
 284         queue_chain_init(rr->rr_link);
 285         os_ref_init(&rr->rr_ref, NULL);
 286         rr->rr_count = total_count;
 287         if (base_size) {
 288                 memcpy(rr->rr_ranges, rr_base->rr_ranges, base_size);
 289         }
 290         memcpy(rr->rr_ranges + base_count, ranges, size);
 291         kr = _ranges_validate(task, rr->rr_ranges, total_count);
 292         if (kr) {
 293                 _restartable_ranges_dispose(rr, false);
 294                 return kr;
 295         }
 296         rr->rr_hash = os_hash_jenkins(rr->rr_ranges,
 297             rr->rr_count * sizeof(task_restartable_range_t));
 298
 299         head = &rr_hash[rr->rr_hash % RR_HASH_SIZE];
 300
 301         rr_lock();
 302         queue_iterate(head, rr_found, struct restartable_ranges *, rr_link) {
 303                 if (_restartable_ranges_equals(rr, rr_found) &&
 304                 os_ref_retain_try(&rr_found->rr_ref)) {
 305                         goto found;
 306                 }
 307         }
 308
 309         enqueue_tail(head, &rr->rr_link);
 310         rr_found = rr;
 311
 312 found:
 313         if (rr_base && os_ref_release_relaxed(&rr_base->rr_ref) == 0) {
 314                 remqueue(&rr_base->rr_link);
 315         } else {
 316                 rr_base = NULL;
 317         }
 318         rr_unlock();
 319
 320         *rr_storage = rr_found;
 321
 322         if (rr_found != rr) {
 323                 _restartable_ranges_dispose(rr, false);
 324         }
 325         if (rr_base) {
 326                 _restartable_ranges_dispose(rr_base, false);
 327         }
 328         return KERN_SUCCESS;
 329 }
 330
 331 #pragma mark extern interfaces
 332
 333 void
 334 restartable_ranges_release(struct restartable_ranges *rr)
 335 {
 336         if (os_ref_release_relaxed(&rr->rr_ref) == 0) {
 337                 _restartable_ranges_dispose(rr, true);
 338         }
 339 }
 340
 341 void
 342 thread_reset_pcs_ast(thread_t thread)
 343 {
 344         task_t task = thread->task;
 345         struct restartable_ranges *rr;
 346         mach_vm_address_t pc;
 347
 348         /*
 349          * Because restartable_ranges are set while the task only has on thread
 350          * and can't be mutated outside of this, no lock is required to read this.
 351          */
 352         rr = task->restartable_ranges;
 353         if (rr) {
 354                 /* pairs with the barrier in task_restartable_ranges_synchronize() */
 355                 os_atomic_thread_fence(acquire);
 356
 357                 pc = _ranges_lookup(rr, machine_thread_pc(thread));
 358
 359                 if (pc) {
 360                         machine_thread_reset_pc(thread, pc);
 361                 }
 362         }
 363 }
 364
 365 void
 366 restartable_init(void)
 367 {
 368         for (size_t i = 0; i < RR_HASH_SIZE; i++) {
 369                 queue_head_init(rr_hash[i]);
 370         }
 371 }
 372
 373 #pragma mark MiG interfaces
 374
 375 kern_return_t
 376 task_restartable_ranges_register(
 377         task_t                    task,
 378         task_restartable_range_t *ranges,
 379         mach_msg_type_number_t    count)
 380 {
 381         kern_return_t kr;
 382         thread_t th;
 383
 384         if (task != current_task()) {
 385                 return KERN_FAILURE;
 386         }
 387
 388
 389         kr = _ranges_validate(task, ranges, count);
 390
 391         if (kr == KERN_SUCCESS) {
 392                 task_lock(task);
 393
 394                 queue_iterate(&task->threads, th, thread_t, task_threads) {
 395                         if (th != current_thread()) {
 396                                 kr = KERN_NOT_SUPPORTED;
 397                                 break;
 398                         }
 399                 }
 400 #if !DEBUG && !DEVELOPMENT
 401                 /*
 402                  * For security reasons, on release kernels, only allow for this to be
 403                  * configured once.
 404                  *
 405                  * But to be able to test the feature we need to relax this for
 406                  * dev kernels.
 407                  */
 408                 if (task->restartable_ranges) {
 409                         kr = KERN_NOT_SUPPORTED;
 410                 }
 411 #endif
 412                 if (kr == KERN_SUCCESS) {
 413                         kr = _restartable_ranges_create(task, ranges, count,
 414                             &task->restartable_ranges);
 415                 }
 416                 task_unlock(task);
 417         }
 418
 419         return kr;
 420 }
 421
 422 kern_return_t
 423 task_restartable_ranges_synchronize(task_t task)
 424 {
 425         thread_t thread;
 426
 427         if (task != current_task()) {
 428                 return KERN_FAILURE;
 429         }
 430
 431         /* pairs with the barrier in thread_reset_pcs_ast() */
 432         os_atomic_thread_fence(release);
 433
 434         task_lock(task);
 435
 436         if (task->restartable_ranges) {
 437                 queue_iterate(&task->threads, thread, thread_t, task_threads) {
 438                         if (thread != current_thread()) {
 439                                 thread_mtx_lock(thread);
 440                                 act_set_ast_reset_pcs(thread);
 441                                 thread_mtx_unlock(thread);
 442                         }
 443                 }
 444         }
 445
 446         task_unlock(task);
 447
 448         return KERN_SUCCESS;
 449 }