osfmk/i386/mp.h

   1 /*
   2  * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56
  57 /*
  58  */
  59 #ifdef  KERNEL_PRIVATE
  60
  61 #ifndef _I386_MP_H_
  62 #define _I386_MP_H_
  63
  64 #ifndef DEBUG
  65 #include <debug.h>
  66 #endif
  67 //#define       MP_DEBUG 1
  68
  69 #include <i386/apic.h>
  70 #include <i386/mp_events.h>
  71
  72 #define MAX_CPUS        32              /* (8*sizeof(long)) */
  73
  74 #ifndef ASSEMBLER
  75 #include <stdint.h>
  76 #include <sys/cdefs.h>
  77 #include <mach/boolean.h>
  78 #include <mach/kern_return.h>
  79 #include <mach/i386/thread_status.h>
  80 #include <kern/lock.h>
  81
  82 __BEGIN_DECLS
  83
  84 extern kern_return_t intel_startCPU(int slot_num);
  85 extern kern_return_t intel_startCPU_fast(int slot_num);
  86 extern void i386_init_slave(void);
  87 extern void i386_init_slave_fast(void);
  88 extern void smp_init(void);
  89
  90 extern void cpu_interrupt(int cpu);
  91 __END_DECLS
  92
  93 extern  unsigned int    real_ncpus;             /* real number of cpus */
  94 extern  unsigned int    max_ncpus;              /* max number of cpus */
  95 decl_simple_lock_data(extern,kdb_lock)  /* kdb lock             */
  96
  97 __BEGIN_DECLS
  98
  99 extern  void    console_init(void);
 100 extern  void    *console_cpu_alloc(boolean_t boot_cpu);
 101 extern  void    console_cpu_free(void *console_buf);
 102
 103 extern  int     kdb_cpu;                /* current cpu running kdb      */
 104 extern  int     kdb_debug;
 105 extern  int     kdb_active[];
 106
 107 extern  volatile boolean_t mp_kdp_trap;
 108 extern  volatile boolean_t force_immediate_debugger_NMI;
 109 extern  volatile boolean_t pmap_tlb_flush_timeout;
 110
 111 extern  void    mp_kdp_enter(void);
 112 extern  void    mp_kdp_exit(void);
 113
 114 #if MACH_KDB
 115 extern void mp_kdb_exit(void);
 116 #endif
 117
 118 /*
 119  * All cpu rendezvous:
 120  */
 121 extern void mp_rendezvous(
 122                 void (*setup_func)(void *),
 123                 void (*action_func)(void *),
 124                 void (*teardown_func)(void *),
 125                 void *arg);
 126 extern void mp_rendezvous_no_intrs(
 127                 void (*action_func)(void *),
 128                 void *arg);
 129 extern void mp_rendezvous_break_lock(void);
 130
 131 /*
 132  * All cpu broadcast.
 133  * Called from thread context, this blocks until all active cpus have
 134  * run action_func:
 135  */
 136 extern void mp_broadcast(
 137                 void (*action_func)(void *),
 138                 void *arg);
 139
 140 typedef uint32_t cpu_t;
 141 typedef uint32_t cpumask_t;
 142 static inline cpumask_t
 143 cpu_to_cpumask(cpu_t cpu)
 144 {
 145         return (cpu < 32) ? (1 << cpu) : 0;
 146 }
 147 #define CPUMASK_ALL     0xffffffff
 148 #define CPUMASK_SELF    cpu_to_cpumask(cpu_number())
 149 #define CPUMASK_OTHERS  (CPUMASK_ALL & ~CPUMASK_SELF)
 150
 151 /*
 152  * Invoke a function (possibly NULL) on a set of cpus specified by a mask.
 153  * The mask may include the local cpu.
 154  * If the mode is:
 155  *      - ASYNC: other cpus make their calls in parallel.
 156  *      - SYNC: the calls are performed serially in logical cpu order.
 157  * This call returns when the function has been run on all specified cpus.
 158  * The return value is the number of cpus on which the call was made.
 159  * The action function is called with interrupts disabled.
 160  */
 161 extern cpu_t mp_cpus_call(
 162                 cpumask_t       cpus,
 163                 mp_sync_t       mode,
 164                 void            (*action_func)(void *),
 165                 void            *arg);
 166
 167 __END_DECLS
 168
 169 #if MP_DEBUG
 170 typedef struct {
 171         uint64_t        time;
 172         int             cpu;
 173         mp_event_t      event;
 174 } cpu_signal_event_t;
 175
 176 #define LOG_NENTRIES    100
 177 typedef struct {
 178         uint64_t                count[MP_LAST];
 179         int                     next_entry;
 180         cpu_signal_event_t      entry[LOG_NENTRIES];
 181 } cpu_signal_event_log_t;
 182
 183 extern cpu_signal_event_log_t   *cpu_signal[];
 184 extern cpu_signal_event_log_t   *cpu_handle[];
 185
 186 #define DBGLOG(log,_cpu,_event) {                                       \
 187         boolean_t               spl = ml_set_interrupts_enabled(FALSE); \
 188         cpu_signal_event_log_t  *logp = log[cpu_number()];              \
 189         int                     next = logp->next_entry;                \
 190         cpu_signal_event_t      *eventp = &logp->entry[next];           \
 191                                                                         \
 192         logp->count[_event]++;                                          \
 193                                                                         \
 194         eventp->time = rdtsc64();                                       \
 195         eventp->cpu = _cpu;                                             \
 196         eventp->event = _event;                                         \
 197         if (next == (LOG_NENTRIES - 1))                                 \
 198                 logp->next_entry = 0;                                   \
 199         else                                                            \
 200                 logp->next_entry++;                                     \
 201                                                                         \
 202         (void) ml_set_interrupts_enabled(spl);                          \
 203 }
 204
 205 #define DBGLOG_CPU_INIT(cpu)    {                                       \
 206         cpu_signal_event_log_t  **sig_logpp = &cpu_signal[cpu];         \
 207         cpu_signal_event_log_t  **hdl_logpp = &cpu_handle[cpu];         \
 208                                                                         \
 209         if (*sig_logpp == NULL &&                                       \
 210                 kmem_alloc(kernel_map,                                  \
 211                         (vm_offset_t *) sig_logpp,                      \
 212                         sizeof(cpu_signal_event_log_t)) != KERN_SUCCESS)\
 213                 panic("DBGLOG_CPU_INIT cpu_signal allocation failed\n");\
 214         bzero(*sig_logpp, sizeof(cpu_signal_event_log_t));              \
 215         if (*hdl_logpp == NULL &&                                       \
 216                 kmem_alloc(kernel_map,                                  \
 217                         (vm_offset_t *) hdl_logpp,                      \
 218                         sizeof(cpu_signal_event_log_t)) != KERN_SUCCESS)\
 219                 panic("DBGLOG_CPU_INIT cpu_handle allocation failed\n");\
 220         bzero(*sig_logpp, sizeof(cpu_signal_event_log_t));              \
 221 }
 222 #else   /* MP_DEBUG */
 223 #define DBGLOG(log,_cpu,_event)
 224 #define DBGLOG_CPU_INIT(cpu)
 225 #endif  /* MP_DEBUG */
 226
 227 #endif  /* ASSEMBLER */
 228
 229 #define i_bit(bit, word)        ((long)(*(word)) & ((long)1 << (bit)))
 230
 231
 232 /*
 233  *      Device driver synchronization.
 234  *
 235  *      at386_io_lock(op) and at386_io_unlock() are called
 236  *      by device drivers when accessing H/W. The underlying
 237  *      Processing is machine dependant. But the op argument
 238  *      to the at386_io_lock is generic
 239  */
 240
 241 #define MP_DEV_OP_MAX     4
 242 #define MP_DEV_WAIT       MP_DEV_OP_MAX /* Wait for the lock */
 243
 244 /*
 245  * If the caller specifies an op value different than MP_DEV_WAIT, the
 246  * at386_io_lock function must return true if lock was successful else
 247  * false
 248  */
 249
 250 #define MP_DEV_OP_START 0       /* If lock busy, register a pending start op */
 251 #define MP_DEV_OP_INTR  1       /* If lock busy, register a pending intr */
 252 #define MP_DEV_OP_TIMEO 2       /* If lock busy, register a pending timeout */
 253 #define MP_DEV_OP_CALLB 3       /* If lock busy, register a pending callback */
 254
 255 #if     MACH_RT
 256 #define _DISABLE_PREEMPTION                                     \
 257         incl    %gs:CPU_PREEMPTION_LEVEL
 258
 259 #define _ENABLE_PREEMPTION                                      \
 260         decl    %gs:CPU_PREEMPTION_LEVEL                ;       \
 261         jne     9f                                      ;       \
 262         pushl   %eax                                    ;       \
 263         pushl   %ecx                                    ;       \
 264         pushl   %edx                                    ;       \
 265         call    EXT(kernel_preempt_check)               ;       \
 266         popl    %edx                                    ;       \
 267         popl    %ecx                                    ;       \
 268         popl    %eax                                    ;       \
 269 9:
 270
 271 #define _ENABLE_PREEMPTION_NO_CHECK                             \
 272         decl    %gs:CPU_PREEMPTION_LEVEL
 273
 274 #if     MACH_ASSERT
 275 #define DISABLE_PREEMPTION                                      \
 276         pushl   %eax;                                           \
 277         pushl   %ecx;                                           \
 278         pushl   %edx;                                           \
 279         call    EXT(_disable_preemption);                       \
 280         popl    %edx;                                           \
 281         popl    %ecx;                                           \
 282         popl    %eax
 283 #define ENABLE_PREEMPTION                                       \
 284         pushl   %eax;                                           \
 285         pushl   %ecx;                                           \
 286         pushl   %edx;                                           \
 287         call    EXT(_enable_preemption);                        \
 288         popl    %edx;                                           \
 289         popl    %ecx;                                           \
 290         popl    %eax
 291 #define ENABLE_PREEMPTION_NO_CHECK                              \
 292         pushl   %eax;                                           \
 293         pushl   %ecx;                                           \
 294         pushl   %edx;                                           \
 295         call    EXT(_enable_preemption_no_check);               \
 296         popl    %edx;                                           \
 297         popl    %ecx;                                           \
 298         popl    %eax
 299 #define MP_DISABLE_PREEMPTION                                   \
 300         pushl   %eax;                                           \
 301         pushl   %ecx;                                           \
 302         pushl   %edx;                                           \
 303         call    EXT(_mp_disable_preemption);                    \
 304         popl    %edx;                                           \
 305         popl    %ecx;                                           \
 306         popl    %eax
 307 #define MP_ENABLE_PREEMPTION                                    \
 308         pushl   %eax;                                           \
 309         pushl   %ecx;                                           \
 310         pushl   %edx;                                           \
 311         call    EXT(_mp_enable_preemption);                     \
 312         popl    %edx;                                           \
 313         popl    %ecx;                                           \
 314         popl    %eax
 315 #define MP_ENABLE_PREEMPTION_NO_CHECK                           \
 316         pushl   %eax;                                           \
 317         pushl   %ecx;                                           \
 318         pushl   %edx;                                           \
 319         call    EXT(_mp_enable_preemption_no_check);            \
 320         popl    %edx;                                           \
 321         popl    %ecx;                                           \
 322         popl    %eax
 323 #else   /* MACH_ASSERT */
 324 #define DISABLE_PREEMPTION              _DISABLE_PREEMPTION
 325 #define ENABLE_PREEMPTION               _ENABLE_PREEMPTION
 326 #define ENABLE_PREEMPTION_NO_CHECK      _ENABLE_PREEMPTION_NO_CHECK
 327 #define MP_DISABLE_PREEMPTION           _DISABLE_PREEMPTION
 328 #define MP_ENABLE_PREEMPTION            _ENABLE_PREEMPTION
 329 #define MP_ENABLE_PREEMPTION_NO_CHECK   _ENABLE_PREEMPTION_NO_CHECK
 330 #endif  /* MACH_ASSERT */
 331
 332 #else   /* MACH_RT */
 333 #define DISABLE_PREEMPTION
 334 #define ENABLE_PREEMPTION
 335 #define ENABLE_PREEMPTION_NO_CHECK
 336 #define MP_DISABLE_PREEMPTION
 337 #define MP_ENABLE_PREEMPTION
 338 #define MP_ENABLE_PREEMPTION_NO_CHECK
 339 #endif  /* MACH_RT */
 340
 341 #endif /* _I386_MP_H_ */
 342
 343 #endif /* KERNEL_PRIVATE */