]> git.saurik.com Git - apple/xnu.git/blob - osfmk/arm64/proc_reg.h
projects / apple / xnu.git / blob
? search:


10c7aa5671739b8b56bc014cc6e6a69380884d7e
[apple/xnu.git] / osfmk / arm64 / proc_reg.h
1 /*
2 * Copyright (c) 2007-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Processor registers for ARM64
30 */
31 #ifndef _ARM64_PROC_REG_H_
32 #define _ARM64_PROC_REG_H_
33
34 #include <arm/proc_reg.h>
35
36 #if __ARM_KERNEL_PROTECT__
37 /*
38 * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential
39 * architectural or microarchitectural vulnerabilities that could allow cores to
40 * read/access EL1-only mappings while in EL0 mode. This is achieved by
41 * removing as many mappings as possible when the core transitions to EL0 mode
42 * from EL1 mode, and restoring those mappings when the core transitions to EL1
43 * mode from EL0 mode.
44 *
45 * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is
46 * used to map and unmap the ordinary kernel mappings, by contracting and
47 * expanding translation zone size for TTBR1 when exiting and entering EL1,
48 * respectively:
49 *
50 * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is
51 * is in EL0.
52 * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in
53 * EL1.
54 *
55 * T1SZ_USER: T1SZ_BOOT + 1
56 * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT)
57 * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER)
58 * TTBR1_EL1_MAX: (2^64) - 1
59 *
60 * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the
61 * the following TTBR1 layout:
62 *
63 * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX
64 * +---------------------------------------------------------+
65 * | Kernel EL0 Mappings | Kernel EL1 Mappings |
66 * +---------------------------------------------------------+
67 *
68 * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout:
69 *
70 * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX
71 * +---------------------------------------------------------+
72 * | Kernel EL0 Mappings |
73 * +---------------------------------------------------------+
74 *
75 * With the current implementation, both the EL0 and EL1 mappings for the kernel
76 * use otherwise empty translation tables for mapping the exception vectors (so
77 * that we do not need to TLB flush the exception vector address when switching
78 * between EL0 and EL1). The rationale here is that the TLBI would require a
79 * DSB, and DSBs can be extremely expensive.
80 *
81 * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1
82 * ASID. The core switches between ASIDs on EL transitions, so that the TLB
83 * does not need to be fully invalidated on an EL transition.
84 *
85 * Most kernel mappings will be marked non-global in this configuration, as
86 * global mappings would be visible to userspace unless we invalidate them on
87 * eret.
88 */
89 #endif /* __ARM_KERNEL_PROTECT */
90
91 /*
92 * 64-bit Program Status Register (PSR64)
93 *
94 * 31 27 23 22 21 20 19 10 9 5 4 0
95 * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+
96 * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M |
97 * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+
98 *
99 * where:
100 * NZCV: Comparison flags
101 * PAN: Privileged Access Never
102 * SS: Single step
103 * IL: Illegal state
104 * DAIF: Interrupt masks
105 * M: Mode field
106 */
107
108 #define PSR64_NZCV_SHIFT 28
109 #define PSR64_NZCV_MASK (1 << PSR64_NZCV_SHIFT)
110
111 #define PSR64_N_SHIFT 31
112 #define PSR64_N (1 << PSR64_N_SHIFT)
113
114 #define PSR64_Z_SHIFT 30
115 #define PSR64_Z (1 << PSR64_Z_SHIFT)
116
117 #define PSR64_C_SHIFT 29
118 #define PSR64_C (1 << PSR64_C_SHIFT)
119
120 #define PSR64_V_SHIFT 28
121 #define PSR64_V (1 << PSR64_V_SHIFT)
122
123 #define PSR64_PAN_SHIFT 22
124 #define PSR64_PAN (1 << PSR64_PAN_SHIFT)
125
126 #define PSR64_SS_SHIFT 21
127 #define PSR64_SS (1 << PSR64_SS_SHIFT)
128
129 #define PSR64_IL_SHIFT 20
130 #define PSR64_IL (1 << PSR64_IL_SHIFT)
131
132 /*
133 * msr DAIF, Xn and mrs Xn, DAIF transfer into
134 * and out of bits 9:6
135 */
136 #define DAIF_DEBUG_SHIFT 9
137 #define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT)
138
139 #define DAIF_ASYNC_SHIFT 8
140 #define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT)
141
142 #define DAIF_IRQF_SHIFT 7
143 #define DAIF_IRQF (1 << DAIF_IRQF_SHIFT)
144
145 #define DAIF_FIQF_SHIFT 6
146 #define DAIF_FIQF (1 << DAIF_FIQF_SHIFT)
147
148 #define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
149 #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
150
151 #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF))
152
153 /*
154 * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
155 * from bits 3:0.
156 */
157 #define DAIFSC_DEBUGF (1 << 3)
158 #define DAIFSC_ASYNCF (1 << 2)
159 #define DAIFSC_IRQF (1 << 1)
160 #define DAIFSC_FIQF (1 << 0)
161 #define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
162 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
163
164 /*
165 * ARM64_TODO: unify with ARM?
166 */
167 #define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */
168
169 #define PSR64_MODE_MASK 0x1F
170
171 #define PSR64_MODE_USER32_THUMB 0x20
172
173 #define PSR64_MODE_RW_SHIFT 4
174 #define PSR64_MODE_RW_64 0
175 #define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT)
176
177 #define PSR64_MODE_EL_SHIFT 2
178 #define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT)
179 #define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT)
180 #define PSR64_MODE_EL2 (0x2 << PSR64_MODE_EL_SHIFT)
181 #define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT)
182 #define PSR64_MODE_EL0 0
183
184 #define PSR64_MODE_SPX 0x1
185 #define PSR64_MODE_SP0 0
186
187 #define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
188 #define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
189 #define PSR64_KERNEL_STANDARD (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0)
190 #if __ARM_PAN_AVAILABLE__
191 #define PSR64_KERNEL_DEFAULT (PSR64_KERNEL_STANDARD | PSR64_PAN)
192 #else
193 #define PSR64_KERNEL_DEFAULT PSR64_KERNEL_STANDARD
194 #endif
195
196 #define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0)
197 #define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
198
199 #define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32))
200 #define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32))
201
202
203
204 /*
205 * System Control Register (SCTLR)
206 */
207
208 #define SCTLR_RESERVED ((3ULL << 28) | (1ULL << 22) | (1ULL << 20) | (1ULL << 11))
209 #if defined(HAS_APPLE_PAC)
210
211 // 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled
212 #define SCTLR_PACIA_ENABLED_SHIFT 31
213 #define SCTLR_PACIA_ENABLED (1ULL << SCTLR_PACIA_ENABLED_SHIFT)
214 // 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled
215 #define SCTLR_PACIB_ENABLED (1ULL << 30)
216 // 29:28 RES1 11
217 // 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled
218 #define SCTLR_PACDA_ENABLED (1ULL << 27)
219 // 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled
220 #define SCTLR_PACDB_ENABLED (1ULL << 13)
221
222 #define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED)
223 #endif /* defined(HAS_APPLE_PAC) */
224
225 // 26 UCI User Cache Instructions
226 #define SCTLR_UCI_ENABLED (1ULL << 26)
227
228 // 25 EE Exception Endianness
229 #define SCTLR_EE_BIG_ENDIAN (1ULL << 25)
230
231 // 24 E0E EL0 Endianness
232 #define SCTLR_E0E_BIG_ENDIAN (1ULL << 24)
233
234 // 23 SPAN Set PAN
235 #define SCTLR_PAN_UNCHANGED (1ULL << 23)
236
237 // 22 RES1 1
238 // 21 RES0 0
239 // 20 RES1 1
240
241 // 19 WXN Writeable implies eXecute Never
242 #define SCTLR_WXN_ENABLED (1ULL << 19)
243
244 // 18 nTWE Not trap WFE from EL0
245 #define SCTLR_nTWE_WFE_ENABLED (1ULL << 18)
246
247 // 17 RES0 0
248
249 // 16 nTWI Not trap WFI from EL0
250 #define SCTRL_nTWI_WFI_ENABLED (1ULL << 16)
251
252 // 15 UCT User Cache Type register (CTR_EL0)
253 #define SCTLR_UCT_ENABLED (1ULL << 15)
254
255 // 14 DZE User Data Cache Zero (DC ZVA)
256 #define SCTLR_DZE_ENABLED (1ULL << 14)
257
258 // 12 I Instruction cache enable
259 #define SCTLR_I_ENABLED (1ULL << 12)
260
261 // 11 RES1 1
262 // 10 RES0 0
263
264 // 9 UMA User Mask Access
265 #define SCTLR_UMA_ENABLED (1ULL << 9)
266
267 // 8 SED SETEND Disable
268 #define SCTLR_SED_DISABLED (1ULL << 8)
269
270 // 7 ITD IT Disable
271 #define SCTLR_ITD_DISABLED (1ULL << 7)
272
273 // 6 RES0 0
274
275 // 5 CP15BEN CP15 Barrier ENable
276 #define SCTLR_CP15BEN_ENABLED (1ULL << 5)
277
278 // 4 SA0 Stack Alignment check for EL0
279 #define SCTLR_SA0_ENABLED (1ULL << 4)
280
281 // 3 SA Stack Alignment check
282 #define SCTLR_SA_ENABLED (1ULL << 3)
283
284 // 2 C Cache enable
285 #define SCTLR_C_ENABLED (1ULL << 2)
286
287 // 1 A Alignment check
288 #define SCTLR_A_ENABLED (1ULL << 1)
289
290 // 0 M MMU enable
291 #define SCTLR_M_ENABLED (1ULL << 0)
292
293 #define SCTLR_EL1_DEFAULT \
294 (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \
295 SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | \
296 SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED)
297
298 /*
299 * Coprocessor Access Control Register (CPACR)
300 *
301 * 31 28 27 22 21 20 19 0
302 * +---+---+------+------+--------------------+
303 * |000|TTA|000000| FPEN |00000000000000000000|
304 * +---+---+------+------+--------------------+
305 *
306 * where:
307 * TTA: Trace trap
308 * FPEN: Floating point enable
309 */
310 #define CPACR_TTA_SHIFT 28
311 #define CPACR_TTA (1 << CPACR_TTA_SHIFT)
312
313 #define CPACR_FPEN_SHIFT 20
314 #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT)
315 #define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT)
316
317 /*
318 * FPSR: Floating Point Status Register
319 *
320 * 31 30 29 28 27 26 7 6 4 3 2 1 0
321 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
322 * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC|
323 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
324 */
325
326 #define FPSR_N_SHIFT 31
327 #define FPSR_Z_SHIFT 30
328 #define FPSR_C_SHIFT 29
329 #define FPSR_V_SHIFT 28
330 #define FPSR_QC_SHIFT 27
331 #define FPSR_IDC_SHIFT 7
332 #define FPSR_IXC_SHIFT 4
333 #define FPSR_UFC_SHIFT 3
334 #define FPSR_OFC_SHIFT 2
335 #define FPSR_DZC_SHIFT 1
336 #define FPSR_IOC_SHIFT 0
337 #define FPSR_N (1 << FPSR_N_SHIFT)
338 #define FPSR_Z (1 << FPSR_Z_SHIFT)
339 #define FPSR_C (1 << FPSR_C_SHIFT)
340 #define FPSR_V (1 << FPSR_V_SHIFT)
341 #define FPSR_QC (1 << FPSR_QC_SHIFT)
342 #define FPSR_IDC (1 << FPSR_IDC_SHIFT)
343 #define FPSR_IXC (1 << FPSR_IXC_SHIFT)
344 #define FPSR_UFC (1 << FPSR_UFC_SHIFT)
345 #define FPSR_OFC (1 << FPSR_OFC_SHIFT)
346 #define FPSR_DZC (1 << FPSR_DZC_SHIFT)
347 #define FPSR_IOC (1 << FPSR_IOC_SHIFT)
348
349 /*
350 * A mask for all for all of the bits that are not RAZ for FPSR; this
351 * is primarily for converting between a 32-bit view of NEON state
352 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
353 */
354 #define FPSR_MASK \
355 (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \
356 FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC)
357
358 /*
359 * FPCR: Floating Point Control Register
360 *
361 * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0
362 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
363 * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000|
364 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
365 */
366
367 #define FPCR_AHP_SHIFT 26
368 #define FPCR_DN_SHIFT 25
369 #define FPCR_FZ_SHIFT 24
370 #define FPCR_RMODE_SHIFT 22
371 #define FPCR_STRIDE_SHIFT 20
372 #define FPCR_LEN_SHIFT 16
373 #define FPCR_IDE_SHIFT 15
374 #define FPCR_IXE_SHIFT 12
375 #define FPCR_UFE_SHIFT 11
376 #define FPCR_OFE_SHIFT 10
377 #define FPCR_DZE_SHIFT 9
378 #define FPCR_IOE_SHIFT 8
379 #define FPCR_AHP (1 << FPCR_AHP_SHIFT)
380 #define FPCR_DN (1 << FPCR_DN_SHIFT)
381 #define FPCR_FZ (1 << FPCR_FZ_SHIFT)
382 #define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT)
383 #define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT)
384 #define FPCR_LEN (0x7 << FPCR_LEN_SHIFT)
385 #define FPCR_IDE (1 << FPCR_IDE_SHIFT)
386 #define FPCR_IXE (1 << FPCR_IXE_SHIFT)
387 #define FPCR_UFE (1 << FPCR_UFE_SHIFT)
388 #define FPCR_OFE (1 << FPCR_OFE_SHIFT)
389 #define FPCR_DZE (1 << FPCR_DZE_SHIFT)
390 #define FPCR_IOE (1 << FPCR_IOE_SHIFT)
391 #define FPCR_DEFAULT (FPCR_DN)
392 #define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ)
393
394 /*
395 * A mask for all for all of the bits that are not RAZ for FPCR; this
396 * is primarily for converting between a 32-bit view of NEON state
397 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
398 */
399 #define FPCR_MASK \
400 (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \
401 FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE)
402
403 /*
404 * Translation Control Register (TCR)
405 *
406 * Legacy:
407 *
408 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
409 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
410 * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
411 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
412 *
413 * Current (with 16KB granule support):
414 *
415 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
416 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
417 * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
418 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
419 *
420 * TBI1: Top Byte Ignored for TTBR1 region
421 * TBI0: Top Byte Ignored for TTBR0 region
422 * AS: ASID Size
423 * IPS: Physical Address Size limit
424 * TG1: Granule Size for TTBR1 region
425 * SH1: Shareability for TTBR1 region
426 * ORGN1: Outer Cacheability for TTBR1 region
427 * IRGN1: Inner Cacheability for TTBR1 region
428 * EPD1: Translation table walk disable for TTBR1
429 * A1: ASID selection from TTBR1 enable
430 * T1SZ: Virtual address size for TTBR1
431 * TG0: Granule Size for TTBR0 region
432 * SH0: Shareability for TTBR0 region
433 * ORGN0: Outer Cacheability for TTBR0 region
434 * IRGN0: Inner Cacheability for TTBR0 region
435 * T0SZ: Virtual address size for TTBR0
436 */
437
438 #define TCR_T0SZ_SHIFT 0ULL
439 #define TCR_TSZ_BITS 6ULL
440 #define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL)
441
442 #define TCR_IRGN0_SHIFT 8ULL
443 #define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT)
444 #define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT)
445 #define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT)
446 #define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT)
447
448 #define TCR_ORGN0_SHIFT 10ULL
449 #define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT)
450 #define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT)
451 #define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT)
452 #define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT)
453
454 #define TCR_SH0_SHIFT 12ULL
455 #define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT)
456 #define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT)
457 #define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT)
458
459 #define TCR_TG0_GRANULE_SHIFT (14ULL)
460
461 #define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT)
462 #define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT)
463 #define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT)
464
465 #if __ARM_16K_PG__
466 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB)
467 #else
468 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB)
469 #endif
470
471 #define TCR_T1SZ_SHIFT 16ULL
472
473 #define TCR_A1_ASID1 (1ULL << 22ULL)
474 #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL)
475
476 #define TCR_IRGN1_SHIFT 24ULL
477 #define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT)
478 #define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT)
479 #define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT)
480 #define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT)
481
482 #define TCR_ORGN1_SHIFT 26ULL
483 #define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT)
484 #define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT)
485 #define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT)
486 #define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT)
487
488 #define TCR_SH1_SHIFT 28ULL
489 #define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT)
490 #define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT)
491 #define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT)
492
493 #define TCR_TG1_GRANULE_SHIFT 30ULL
494
495 #define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT)
496 #define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT)
497 #define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT)
498
499 #if __ARM_16K_PG__
500 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB)
501 #else
502 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB)
503 #endif
504
505 #define TCR_IPS_SHIFT 32ULL
506 #define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT)
507 #define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT)
508 #define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT)
509 #define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT)
510 #define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT)
511 #define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT)
512
513 #define TCR_AS_16BIT_ASID (1ULL << 36)
514 #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37)
515 #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38)
516 #define TCR_TBID0_TBI_DATA_ONLY (1ULL << 51)
517 #define TCR_TBID1_TBI_DATA_ONLY (1ULL << 52)
518
519 #if defined(HAS_APPLE_PAC)
520 #define TCR_TBID0_ENABLE TCR_TBID0_TBI_DATA_ONLY
521 #else
522 #define TCR_TBID0_ENABLE 0
523 #endif
524
525 /*
526 * Multiprocessor Affinity Register (MPIDR_EL1)
527 *
528 * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+
529 * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0|
530 * +---------------------------------+--+-----+--+-----+----+----+
531 *
532 * where
533 * U: Uniprocessor
534 * MT: Multi-threading at lowest affinity level
535 * Aff2: "1" - PCORE, "0" - ECORE
536 * Aff1: Cluster ID
537 * Aff0: CPU ID
538 */
539 #define MPIDR_AFF0_SHIFT 0
540 #define MPIDR_AFF0_WIDTH 8
541 #define MPIDR_AFF0_MASK (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT)
542 #define MPIDR_AFF1_SHIFT 8
543 #define MPIDR_AFF1_WIDTH 8
544 #define MPIDR_AFF1_MASK (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT)
545 #define MPIDR_AFF2_SHIFT 16
546 #define MPIDR_AFF2_WIDTH 8
547 #define MPIDR_AFF2_MASK (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT)
548
549 /*
550 * TXSZ indicates the size of the range a TTBR covers. Currently,
551 * we support the following:
552 *
553 * 4KB pages, full page L1: 39 bit range.
554 * 4KB pages, sub-page L1: 38 bit range.
555 * 16KB pages, full page L1: 47 bit range.
556 * 16KB pages, sub-page L1: 39 bit range.
557 * 16KB pages, two level page tables: 36 bit range.
558 */
559 #if __ARM_KERNEL_PROTECT__
560 /*
561 * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the
562 * address space is used for the mappings that will remain in place when in EL0.
563 * As a result, 1 bit less of address space is available to the rest of the
564 * the kernel.
565 */
566 #endif /* __ARM_KERNEL_PROTECT__ */
567 #ifdef __ARM_16K_PG__
568 #if __ARM64_PMAP_SUBPAGE_L1__
569 #define T0SZ_BOOT 25ULL
570 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */
571 #define T0SZ_BOOT 17ULL
572 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
573 #else /* __ARM_16K_PG__ */
574 #if __ARM64_PMAP_SUBPAGE_L1__
575 #define T0SZ_BOOT 26ULL
576 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
577 #define T0SZ_BOOT 25ULL
578 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
579 #endif /* __ARM_16K_PG__ */
580
581 #if defined(APPLE_ARM64_ARCH_FAMILY)
582 /* T0SZ must be the same as T1SZ */
583 #define T1SZ_BOOT T0SZ_BOOT
584 #else /* defined(APPLE_ARM64_ARCH_FAMILY) */
585 #ifdef __ARM_16K_PG__
586 #if __ARM64_PMAP_SUBPAGE_L1__
587 #define T1SZ_BOOT 25ULL
588 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */
589 #define T1SZ_BOOT 17ULL
590 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
591 #else /* __ARM_16K_PG__ */
592 #if __ARM64_PMAP_SUBPAGE_L1__
593 #define T1SZ_BOOT 26ULL
594 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
595 #define T1SZ_BOOT 25ULL
596 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/
597 #endif /* __ARM_16K_PG__ */
598 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
599
600 #if __ARM_42BIT_PA_SPACE__
601 #define TCR_IPS_VALUE TCR_IPS_42BITS
602 #else /* !__ARM_42BIT_PA_SPACE__ */
603 #define TCR_IPS_VALUE TCR_IPS_40BITS
604 #endif /* !__ARM_42BIT_PA_SPACE__ */
605
606 #define TCR_EL1_BASE \
607 (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | \
608 TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | \
609 (TCR_TG0_GRANULE_SIZE) | TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \
610 TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) | \
611 TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE))
612
613 #if __ARM_KERNEL_PROTECT__
614 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT))
615 #define T1SZ_USER (T1SZ_BOOT + 1)
616 #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT))
617 #else
618 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT))
619 #endif /* __ARM_KERNEL_PROTECT__ */
620
621
622 /*
623 * Translation Table Base Register (TTBR)
624 *
625 * 63 48 47 x x-1 0
626 * +--------+------------------+------+
627 * | ASID | Base Address | zero |
628 * +--------+------------------+------+
629 *
630 */
631 #define TTBR_ASID_SHIFT 48
632 #define TTBR_ASID_MASK 0xffff000000000000
633
634 #define TTBR_BADDR_MASK 0x0000ffffffffffff
635
636 /*
637 * Memory Attribute Indirection Register
638 *
639 * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0
640 * +-------+-------+-------+-------+-------+-------+-------+-------+
641 * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 |
642 * +-------+-------+-------+-------+-------+-------+-------+-------+
643 *
644 */
645
646 #define MAIR_ATTR_SHIFT(x) (8*(x))
647
648 /* Strongly ordered or device memory attributes */
649 #define MAIR_OUTER_STRONGLY_ORDERED 0x0
650 #define MAIR_OUTER_DEVICE 0x0
651
652 #define MAIR_INNER_STRONGLY_ORDERED 0x0
653 #define MAIR_INNER_DEVICE 0x4
654
655 /* Normal memory attributes */
656 #define MAIR_OUTER_NON_CACHEABLE 0x40
657 #define MAIR_OUTER_WRITE_THROUGH 0x80
658 #define MAIR_OUTER_WRITE_BACK 0xc0
659
660 #define MAIR_INNER_NON_CACHEABLE 0x4
661 #define MAIR_INNER_WRITE_THROUGH 0x8
662 #define MAIR_INNER_WRITE_BACK 0xc
663
664 /* Allocate policy for cacheable memory */
665 #define MAIR_OUTER_WRITE_ALLOCATE 0x10
666 #define MAIR_OUTER_READ_ALLOCATE 0x20
667
668 #define MAIR_INNER_WRITE_ALLOCATE 0x1
669 #define MAIR_INNER_READ_ALLOCATE 0x2
670
671 /* Memory Atribute Encoding */
672
673 /*
674 * Device memory types:
675 * G (gathering): multiple reads/writes can be combined
676 * R (reordering): reads or writes may reach device out of program order
677 * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes)
678 */
679 #define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */
680 #define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */
681 #define MAIR_POSTED_REORDERED 0x08 /* Device Memory, nGRE (reorderable, posted writes) */
682 #define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */
683 #define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
684 #define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */
685 #define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */
686 #define MAIR_INNERWRITEBACK 0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */
687
688
689 /*
690 * ARM 4-level Page Table support - 2*1024TB (2^48) of address space
691 */
692
693
694 /*
695 * Memory Attribute Index
696 */
697 #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */
698 #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */
699 #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */
700 #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */
701 #define CACHE_ATTRINDX_INNERWRITEBACK 0x4 /* inner cache enabled, buffer enabled, write allocate (normal memory) */
702 #define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes (device memory) */
703 #define CACHE_ATTRINDX_POSTED_REORDERED 0x6 /* no cache, reorderable access, posted writes (device memory) */
704 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */
705 #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK
706
707
708 /*
709 * Access protection bit values (TTEs and PTEs), stage 1
710 *
711 * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access)
712 */
713 #define AP_RWNA 0x0 /* priv=read-write, user=no-access */
714 #define AP_RWRW 0x1 /* priv=read-write, user=read-write */
715 #define AP_RONA 0x2 /* priv=read-only, user=no-access */
716 #define AP_RORO 0x3 /* priv=read-only, user=read-only */
717 #define AP_MASK 0x3 /* mask to find ap bits */
718
719 /*
720 * Shareability attributes
721 */
722 #define SH_NONE 0x0 /* Non shareable */
723 #define SH_NONE 0x0 /* Device shareable */
724 #define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */
725 #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */
726 #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */
727
728
729 /*
730 * ARM Page Granule
731 */
732 #ifdef __ARM_16K_PG__
733 #define ARM_PGSHIFT 14
734 #else
735 #define ARM_PGSHIFT 12
736 #endif
737 #define ARM_PGBYTES (1 << ARM_PGSHIFT)
738 #define ARM_PGMASK (ARM_PGBYTES-1)
739
740 /*
741 * L0 Translation table
742 *
743 * 4KB granule size:
744 * Each translation table is 4KB
745 * 512 64-bit entries of 512GB (2^39) of address space.
746 * Covers 256TB (2^48) of address space.
747 *
748 * 16KB granule size:
749 * Each translation table is 16KB
750 * 2 64-bit entries of 128TB (2^47) of address space.
751 * Covers 256TB (2^48) of address space.
752 */
753
754 /* 16K L0 */
755 #define ARM_16K_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */
756 #define ARM_16K_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */
757 #define ARM_16K_TT_L0_SHIFT 47 /* page descriptor shift */
758 #define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */
759
760 /* 4K L0 */
761 #define ARM_4K_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */
762 #define ARM_4K_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */
763 #define ARM_4K_TT_L0_SHIFT 39 /* page descriptor shift */
764 #define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */
765
766 /*
767 * L1 Translation table
768 *
769 * 4KB granule size:
770 * Each translation table is 4KB
771 * 512 64-bit entries of 1GB (2^30) of address space.
772 * Covers 512GB (2^39) of address space.
773 *
774 * 16KB granule size:
775 * Each translation table is 16KB
776 * 2048 64-bit entries of 64GB (2^36) of address space.
777 * Covers 128TB (2^47) of address space.
778 */
779
780 /* 16K L1 */
781 #define ARM_16K_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */
782 #define ARM_16K_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */
783 #define ARM_16K_TT_L1_SHIFT 36 /* page descriptor shift */
784 #ifdef __ARM64_PMAP_SUBPAGE_L1__
785 /* This config supports 512GB per TTBR. */
786 #define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */
787 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
788 #define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */
789 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
790
791 /* 4K L1 */
792 #define ARM_4K_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */
793 #define ARM_4K_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */
794 #define ARM_4K_TT_L1_SHIFT 30 /* page descriptor shift */
795 #ifdef __ARM64_PMAP_SUBPAGE_L1__
796 /* This config supports 256GB per TTBR. */
797 #define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */
798 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
799 #define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */
800 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
801
802 /* some sugar for getting pointers to page tables and entries */
803
804 #define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)
805 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
806 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
807
808 #define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK))
809 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK))
810
811 /*
812 * L2 Translation table
813 *
814 * 4KB granule size:
815 * Each translation table is 4KB
816 * 512 64-bit entries of 2MB (2^21) of address space.
817 * Covers 1GB (2^30) of address space.
818 *
819 * 16KB granule size:
820 * Each translation table is 16KB
821 * 2048 64-bit entries of 32MB (2^25) of address space.
822 * Covers 64GB (2^36) of address space.
823 */
824
825 /* 16K L2 */
826 #define ARM_16K_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */
827 #define ARM_16K_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */
828 #define ARM_16K_TT_L2_SHIFT 25 /* page descriptor shift */
829 #define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */
830
831 /* 4K L2 */
832 #define ARM_4K_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */
833 #define ARM_4K_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */
834 #define ARM_4K_TT_L2_SHIFT 21 /* page descriptor shift */
835 #define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */
836
837 /*
838 * L3 Translation table
839 *
840 * 4KB granule size:
841 * Each translation table is 4KB
842 * 512 64-bit entries of 4KB (2^12) of address space.
843 * Covers 2MB (2^21) of address space.
844 *
845 * 16KB granule size:
846 * Each translation table is 16KB
847 * 2048 64-bit entries of 16KB (2^14) of address space.
848 * Covers 32MB (2^25) of address space.
849 */
850
851 /* 16K L3 */
852 #define ARM_16K_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */
853 #define ARM_16K_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */
854 #define ARM_16K_TT_L3_SHIFT 14 /* page descriptor shift */
855 #define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */
856
857 /* 4K L3 */
858 #define ARM_4K_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */
859 #define ARM_4K_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */
860 #define ARM_4K_TT_L3_SHIFT 12 /* page descriptor shift */
861 #define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */
862
863 #ifdef __ARM_16K_PG__
864
865 /* Native L0 defines */
866 #define ARM_TT_L0_SIZE ARM_16K_TT_L0_SIZE
867 #define ARM_TT_L0_OFFMASK ARM_16K_TT_L0_OFFMASK
868 #define ARM_TT_L0_SHIFT ARM_16K_TT_L0_SHIFT
869 #define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK
870
871 /* Native L1 defines */
872 #define ARM_TT_L1_SIZE ARM_16K_TT_L1_SIZE
873 #define ARM_TT_L1_OFFMASK ARM_16K_TT_L1_OFFMASK
874 #define ARM_TT_L1_SHIFT ARM_16K_TT_L1_SHIFT
875 #define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK
876
877 /* Native L2 defines */
878 #define ARM_TT_L2_SIZE ARM_16K_TT_L2_SIZE
879 #define ARM_TT_L2_OFFMASK ARM_16K_TT_L2_OFFMASK
880 #define ARM_TT_L2_SHIFT ARM_16K_TT_L2_SHIFT
881 #define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK
882
883 /* Native L3 defines */
884 #define ARM_TT_L3_SIZE ARM_16K_TT_L3_SIZE
885 #define ARM_TT_L3_OFFMASK ARM_16K_TT_L3_OFFMASK
886 #define ARM_TT_L3_SHIFT ARM_16K_TT_L3_SHIFT
887 #define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK
888
889 #else /* !__ARM_16K_PG__ */
890
891 /* Native L0 defines */
892 #define ARM_TT_L0_SIZE ARM_4K_TT_L0_SIZE
893 #define ARM_TT_L0_OFFMASK ARM_4K_TT_L0_OFFMASK
894 #define ARM_TT_L0_SHIFT ARM_4K_TT_L0_SHIFT
895 #define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK
896
897 /* Native L1 defines */
898 #define ARM_TT_L1_SIZE ARM_4K_TT_L1_SIZE
899 #define ARM_TT_L1_OFFMASK ARM_4K_TT_L1_OFFMASK
900 #define ARM_TT_L1_SHIFT ARM_4K_TT_L1_SHIFT
901 #define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK
902
903 /* Native L2 defines */
904 #define ARM_TT_L2_SIZE ARM_4K_TT_L2_SIZE
905 #define ARM_TT_L2_OFFMASK ARM_4K_TT_L2_OFFMASK
906 #define ARM_TT_L2_SHIFT ARM_4K_TT_L2_SHIFT
907 #define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK
908
909 /* Native L3 defines */
910 #define ARM_TT_L3_SIZE ARM_4K_TT_L3_SIZE
911 #define ARM_TT_L3_OFFMASK ARM_4K_TT_L3_OFFMASK
912 #define ARM_TT_L3_SHIFT ARM_4K_TT_L3_SHIFT
913 #define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK
914
915 #endif /* !__ARM_16K_PG__ */
916
917 /*
918 * Convenience definitions for:
919 * ARM_TT_LEAF: The last level of the configured page table format.
920 * ARM_TT_TWIG: The second to last level of the configured page table format.
921 * ARM_TT_ROOT: The first level of the configured page table format.
922 *
923 * My apologies to any botanists who may be reading this.
924 */
925 #define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE
926 #define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK
927 #define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT
928 #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK
929
930 #define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE
931 #define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK
932 #define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT
933 #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK
934
935 #define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE
936 #define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK
937 #define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT
938 #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK
939
940 /*
941 * 4KB granule size:
942 *
943 * Level 0 Translation Table Entry
944 *
945 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
946 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
947 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
948 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
949 *
950 * Level 1 Translation Table Entry
951 *
952 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
953 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
954 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
955 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
956 *
957 * Level 1 Translation Block Entry
958 *
959 * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0
960 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
961 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
962 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
963 *
964 * Level 2 Translation Table Entry
965 *
966 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
967 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
968 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
969 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
970 *
971 * Level 2 Translation Block Entry
972 *
973 * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0
974 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
975 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
976 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
977 *
978 * 16KB granule size:
979 *
980 * Level 0 Translation Table Entry
981 *
982 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
983 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
984 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
985 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
986 *
987 * Level 1 Translation Table Entry
988 *
989 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
990 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
991 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
992 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
993 *
994 * Level 2 Translation Table Entry
995 *
996 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
997 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
998 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
999 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1000 *
1001 * Level 2 Translation Block Entry
1002 *
1003 * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0
1004 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1005 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
1006 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1007 *
1008 * where:
1009 * nG: notGlobal bit
1010 * SH: Shareability field
1011 * AP: access protection
1012 * XN: eXecute Never bit
1013 * PXN: Privilege eXecute Never bit
1014 * NS: Non-Secure bit
1015 * HINT: 16 entry continuguous output hint
1016 * AttrIdx: Memory Attribute Index
1017 */
1018
1019 #define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */
1020 #ifdef __ARM_16K_PG__
1021 #define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */
1022 #else
1023 #define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */
1024 #endif
1025
1026 #define ARM_TTE_MAX (TTE_PGENTRIES)
1027
1028 #define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */
1029 #define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */
1030
1031 #define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */
1032
1033 #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
1034 #define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */
1035 #define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */
1036 #define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL
1037 #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
1038
1039 #ifdef __ARM_16K_PG__
1040 /*
1041 * Note that L0/L1 block entries are disallowed for the 16KB granule size; what
1042 * are we doing with these?
1043 */
1044 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */
1045 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */
1046 #define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */
1047 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */
1048 #define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */
1049 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */
1050 #else
1051 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */
1052 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */
1053 #define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */
1054 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */
1055 #define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */
1056 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */
1057 #endif
1058
1059 #define ARM_TTE_BLOCK_APSHIFT 6
1060 #define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */
1061 #define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT)
1062
1063 #define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */
1064 #define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */
1065
1066 #define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */
1067 #define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */
1068
1069 #define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */
1070 #define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */
1071
1072 #define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */
1073 #define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
1074
1075 #define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */
1076 #define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */
1077
1078 #define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */
1079 #define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */
1080
1081 #define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */
1082 #define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */
1083
1084 #define ARM_TTE_BLOCK_WIRED 0x0400000000000000ULL /* value for software wired bit */
1085 #define ARM_TTE_BLOCK_WIREDMASK 0x0400000000000000ULL /* software wired mask */
1086
1087 #define ARM_TTE_BLOCK_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */
1088 #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL /* software writeable mask */
1089
1090 #ifdef __ARM_16K_PG__
1091 /*
1092 * TODO: Do we care about the low bits being unused? It should technically
1093 * work either way, but masking them out should be future proof; it is only a
1094 * matter of time before someone wants to shove something into the free bits.
1095 */
1096 #define ARM_TTE_TABLE_MASK (0x0000ffffffffc000ULL) /* mask for extracting pointer to next table (works at any level) */
1097 #else
1098 #define ARM_TTE_TABLE_MASK (0x0000fffffffff000ULL) /* mask for extracting pointer to next table (works at any level) */
1099 #endif
1100
1101 #define ARM_TTE_TABLE_APSHIFT 61
1102 #define ARM_TTE_TABLE_AP(x) ((x)<<TTE_BLOCK_APSHIFT) /* access protection */
1103
1104 #define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */
1105 #define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */
1106
1107 #define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */
1108 #define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */
1109
1110 #define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */
1111 #define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */
1112
1113 #if __ARM_KERNEL_PROTECT__
1114 #define ARM_TTE_BOOT_BLOCK \
1115 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
1116 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG)
1117 #else /* __ARM_KERNEL_PROTECT__ */
1118 #define ARM_TTE_BOOT_BLOCK \
1119 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
1120 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)
1121 #endif /* __ARM_KERNEL_PROTECT__ */
1122
1123 #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )
1124 /*
1125 * L3 Translation table
1126 *
1127 * 4KB granule size:
1128 * Each translation table is 4KB
1129 * 512 64-bit entries of 4KB (2^12) of address space.
1130 * Covers 2MB (2^21) of address space.
1131 *
1132 * 16KB granule size:
1133 * Each translation table is 16KB
1134 * 2048 64-bit entries of 16KB (2^14) of address space.
1135 * Covers 32MB (2^25) of address space.
1136 */
1137
1138 #ifdef __ARM_16K_PG__
1139 #define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */
1140 #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */
1141 #define ARM_PTE_SHIFT 14 /* page descriptor shift */
1142 #define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */
1143 #else
1144 #define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */
1145 #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */
1146 #define ARM_PTE_SHIFT 12 /* page descriptor shift */
1147 #define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */
1148 #endif
1149
1150 /*
1151 * L3 Page table entries
1152 *
1153 * The following page table entry types are possible:
1154 *
1155 * fault page entry
1156 * 63 2 0
1157 * +------------------------------+--+
1158 * | ignored |00|
1159 * +------------------------------+--+
1160 *
1161 *
1162 * 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0
1163 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1164 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V|
1165 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1166 *
1167 * where:
1168 * nG: notGlobal bit
1169 * SH: Shareability field
1170 * AP: access protection
1171 * XN: eXecute Never bit
1172 * PXN: Privilege eXecute Never bit
1173 * NS: Non-Secure bit
1174 * HINT: 16 entry continuguous output hint
1175 * AttrIdx: Memory Attribute Index
1176 */
1177
1178 #define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */
1179 #ifdef __ARM_16K_PG__
1180 #define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */
1181 #else
1182 #define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */
1183 #endif
1184
1185 #define ARM_PTE_EMPTY 0x0000000000000000ULL /* unassigned - invalid entry */
1186
1187 /* markers for (invalid) PTE for a page sent to compressor */
1188 #define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */
1189 #define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */
1190 #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL
1191
1192 #define ARM_PTE_IS_COMPRESSED(x, p) \
1193 ((((x) & 0x3) == 0) && /* PTE is not valid... */ \
1194 ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \
1195 ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \
1196 (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
1197 (p), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE)))
1198
1199 #define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1200 #define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1201 #define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */
1202 #define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */
1203
1204 #ifdef __ARM_16K_PG__
1205 /* TODO: What does the shift mean here? */
1206 #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFC000ULL /* mask for 16KB page */
1207 #else
1208 #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* mask for 4KB page */
1209 #define ARM_PTE_PAGE_SHIFT 12 /* page shift for 4KB page */
1210 #endif
1211
1212 #define ARM_PTE_AP(x) ((x) << 6) /* access protections */
1213 #define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */
1214 #define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */
1215
1216 #define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */
1217 #define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */
1218
1219 #define ARM_PTE_SH(x) ((x) << 8) /* access shared */
1220 #define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */
1221
1222 #define ARM_PTE_AF 0x0000000000000400ULL /* value for access */
1223 #define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */
1224
1225 #define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */
1226 #define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
1227
1228 #define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */
1229 #define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */
1230
1231 #define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */
1232 #define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */
1233
1234 #if __ARM_16K_PG__
1235 #define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */
1236 #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */
1237 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */
1238 #define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */
1239 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */
1240 #else
1241 #define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */
1242 #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */
1243 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */
1244 #define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */
1245 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */
1246 #endif
1247
1248 #define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */
1249 #define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */
1250
1251 #define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */
1252 #define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */
1253
1254 #define ARM_PTE_WIRED 0x0400000000000000ULL /* value for software wired bit */
1255 #define ARM_PTE_WIRED_MASK 0x0400000000000000ULL /* software wired mask */
1256
1257 #define ARM_PTE_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */
1258 #define ARM_PTE_WRITEABLE_MASK 0x0800000000000000ULL /* software writeable mask */
1259
1260 #if CONFIG_PGTRACE
1261 #define ARM_PTE_PGTRACE 0x0200000000000000ULL /* value for software trace bit */
1262 #define ARM_PTE_PGTRACE_MASK 0x0200000000000000ULL /* software trace mask */
1263 #endif
1264
1265 #define ARM_PTE_BOOT_PAGE_BASE \
1266 (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | \
1267 ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)
1268
1269 #if __ARM_KERNEL_PROTECT__
1270 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG)
1271 #else /* __ARM_KERNEL_PROTECT__ */
1272 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE)
1273 #endif /* __ARM_KERNEL_PROTECT__ */
1274
1275 /*
1276 * TLBI appers to only deal in 4KB page addresses, so give
1277 * it an explicit shift of 12.
1278 */
1279 #define TLBI_ADDR_SHIFT (0)
1280 #define TLBI_ADDR_SIZE (44)
1281 #define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1)
1282 #define TLBI_ASID_SHIFT (48)
1283 #define TLBI_ASID_SIZE (16)
1284 #define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1))
1285
1286 #define RTLBI_ADDR_SIZE (37)
1287 #define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1)
1288 #define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT
1289 #define RTLBI_TG ((uint64_t)(((ARM_TT_L3_SHIFT - 12) >> 1) + 1) << 46)
1290 #define RTLBI_SCALE_SHIFT (44)
1291 #define RTLBI_NUM_SHIFT (39)
1292
1293 /*
1294 * Exception Syndrome Register
1295 *
1296 * 31 26 25 24 0
1297 * +------+--+------------------+
1298 * | EC |IL| ISS |
1299 * +------+--+------------------+
1300 *
1301 * EC - Exception Class
1302 * IL - Instruction Length
1303 * ISS - Instruction Specific Syndrome
1304 *
1305 * Note: The ISS can have many forms. These are defined separately below.
1306 */
1307
1308 #define ESR_EC_SHIFT 26
1309 #define ESR_EC_MASK (0x3FULL << ESR_EC_SHIFT)
1310 #define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
1311
1312 #define ESR_IL_SHIFT 25
1313 #define ESR_IL (1 << ESR_IL_SHIFT)
1314
1315 #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL))
1316
1317 #define ESR_ISS_MASK 0x01FFFFFF
1318 #define ESR_ISS(x) (x & ESR_ISS_MASK)
1319
1320 #ifdef __ASSEMBLER__
1321 /* Define only the classes we need to test in the exception vectors. */
1322 #define ESR_EC_IABORT_EL1 0x21
1323 #define ESR_EC_DABORT_EL1 0x25
1324 #define ESR_EC_SP_ALIGN 0x26
1325 #else
1326 typedef enum {
1327 ESR_EC_UNCATEGORIZED = 0x00,
1328 ESR_EC_WFI_WFE = 0x01,
1329 ESR_EC_MCR_MRC_CP15_TRAP = 0x03,
1330 ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04,
1331 ESR_EC_MCR_MRC_CP14_TRAP = 0x05,
1332 ESR_EC_LDC_STC_CP14_TRAP = 0x06,
1333 ESR_EC_TRAP_SIMD_FP = 0x07,
1334 ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c,
1335 ESR_EC_ILLEGAL_INSTR_SET = 0x0e,
1336 ESR_EC_SVC_32 = 0x11,
1337 ESR_EC_SVC_64 = 0x15,
1338 ESR_EC_MSR_TRAP = 0x18,
1339 ESR_EC_IABORT_EL0 = 0x20,
1340 ESR_EC_IABORT_EL1 = 0x21,
1341 ESR_EC_PC_ALIGN = 0x22,
1342 ESR_EC_DABORT_EL0 = 0x24,
1343 ESR_EC_DABORT_EL1 = 0x25,
1344 ESR_EC_SP_ALIGN = 0x26,
1345 ESR_EC_FLOATING_POINT_32 = 0x28,
1346 ESR_EC_FLOATING_POINT_64 = 0x2C,
1347 ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL.
1348 ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL.
1349 ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL.
1350 ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL.
1351 ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL.
1352 ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL.
1353 ESR_EC_BKPT_AARCH32 = 0x38,
1354 ESR_EC_BRK_AARCH64 = 0x3C,
1355 } esr_exception_class_t;
1356
1357 typedef enum {
1358 FSC_TRANSLATION_FAULT_L0 = 0x04,
1359 FSC_TRANSLATION_FAULT_L1 = 0x05,
1360 FSC_TRANSLATION_FAULT_L2 = 0x06,
1361 FSC_TRANSLATION_FAULT_L3 = 0x07,
1362 FSC_ACCESS_FLAG_FAULT_L1 = 0x09,
1363 FSC_ACCESS_FLAG_FAULT_L2 = 0x0A,
1364 FSC_ACCESS_FLAG_FAULT_L3 = 0x0B,
1365 FSC_PERMISSION_FAULT_L1 = 0x0D,
1366 FSC_PERMISSION_FAULT_L2 = 0x0E,
1367 FSC_PERMISSION_FAULT_L3 = 0x0F,
1368 FSC_SYNC_EXT_ABORT = 0x10,
1369 FSC_ASYNC_EXT_ABORT = 0x11,
1370 FSC_SYNC_EXT_ABORT_TT_L1 = 0x15,
1371 FSC_SYNC_EXT_ABORT_TT_L2 = 0x16,
1372 FSC_SYNC_EXT_ABORT_TT_L3 = 0x17,
1373 FSC_SYNC_PARITY = 0x18,
1374 FSC_ASYNC_PARITY = 0x19,
1375 FSC_SYNC_PARITY_TT_L1 = 0x1D,
1376 FSC_SYNC_PARITY_TT_L2 = 0x1E,
1377 FSC_SYNC_PARITY_TT_L3 = 0x1F,
1378 FSC_ALIGNMENT_FAULT = 0x21,
1379 FSC_DEBUG_FAULT = 0x22
1380 } fault_status_t;
1381 #endif /* ASSEMBLER */
1382
1383 /*
1384 * Software step debug event ISS (EL1)
1385 * 24 23 6 5 0
1386 * +---+-----------------+--+------+
1387 * |ISV|00000000000000000|EX| IFSC |
1388 * +---+-----------------+--+------+
1389 *
1390 * where:
1391 * ISV: Instruction syndrome valid
1392 * EX: Exclusive access
1393 * IFSC: Instruction Fault Status Code
1394 */
1395
1396 #define ISS_SSDE_ISV_SHIFT 24
1397 #define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT)
1398
1399 #define ISS_SSDE_EX_SHIFT 6
1400 #define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT)
1401
1402 #define ISS_SSDE_FSC_MASK 0x3F
1403 #define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK)
1404
1405 /*
1406 * Instruction Abort ISS (EL1)
1407 * 24 10 9 5 0
1408 * +---------------+--+---+------+
1409 * |000000000000000|EA|000| IFSC |
1410 * +---------------+--+---+------+
1411 *
1412 * where:
1413 * EA: External Abort type
1414 * IFSC: Instruction Fault Status Code
1415 */
1416
1417 #define ISS_IA_EA_SHIFT 9
1418 #define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT)
1419
1420 #define ISS_IA_FSC_MASK 0x3F
1421 #define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK)
1422
1423
1424 /*
1425 * Data Abort ISS (EL1)
1426 *
1427 * 24 9 8 7 6 5 0
1428 * +---------------+--+--+-+---+----+
1429 * |000000000000000|EA|CM|0|WnR|DFSC|
1430 * +---------------+--+--+-+---+----+
1431 *
1432 * where:
1433 * EA: External Abort type
1434 * CM: Cache Maintenance operation
1435 * WnR: Write not Read
1436 * DFSC: Data Fault Status Code
1437 */
1438 #define ISS_DA_EA_SHIFT 9
1439 #define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT)
1440
1441 #define ISS_DA_CM_SHIFT 8
1442 #define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT)
1443
1444 #define ISS_DA_WNR_SHIFT 6
1445 #define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT)
1446
1447 #define ISS_DA_FSC_MASK 0x3F
1448 #define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK)
1449
1450 /*
1451 * Floating Point Exception ISS (EL1)
1452 *
1453 * 24 23 22 8 7 4 3 2 1 0
1454 * +-+---+---------------+---+--+---+---+---+---+---+
1455 * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF|
1456 * +-+---+---------------+---+--+---+---+---+---+---+
1457 *
1458 * where:
1459 * TFV: Trapped Fault Valid
1460 * IDF: Input Denormal Exception
1461 * IXF: Input Inexact Exception
1462 * UFF: Underflow Exception
1463 * OFF: Overflow Exception
1464 * DZF: Divide by Zero Exception
1465 * IOF: Invalid Operation Exception
1466 */
1467 #define ISS_FP_TFV_SHIFT 23
1468 #define ISS_FP_TFV (0x1 << ISS_FP_TFV_SHIFT)
1469
1470 #define ISS_FP_IDF_SHIFT 7
1471 #define ISS_FP_IDF (0x1 << ISS_FP_IDF_SHIFT)
1472
1473 #define ISS_FP_IXF_SHIFT 4
1474 #define ISS_FP_IXF (0x1 << ISS_FP_IXF_SHIFT)
1475
1476 #define ISS_FP_UFF_SHIFT 3
1477 #define ISS_FP_UFF (0x1 << ISS_FP_UFF_SHIFT)
1478
1479 #define ISS_FP_OFF_SHIFT 2
1480 #define ISS_FP_OFF (0x1 << ISS_FP_OFF_SHIFT)
1481
1482 #define ISS_FP_DZF_SHIFT 1
1483 #define ISS_FP_DZF (0x1 << ISS_FP_DZF_SHIFT)
1484
1485 #define ISS_FP_IOF_SHIFT 0
1486 #define ISS_FP_IOF (0x1 << ISS_FP_IOF_SHIFT)
1487
1488
1489 /*
1490 * Physical Address Register (EL1)
1491 */
1492 #define PAR_F_SHIFT 0
1493 #define PAR_F (0x1 << PAR_F_SHIFT)
1494
1495 #define PLATFORM_SYSCALL_TRAP_NO 0x80000000
1496
1497 #define ARM64_SYSCALL_CODE_REG_NUM (16)
1498
1499 #define ARM64_CLINE_SHIFT 6
1500
1501 #if defined(APPLE_ARM64_ARCH_FAMILY)
1502 #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */
1503 #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */
1504 #endif
1505
1506 /*
1507 * Timer definitions.
1508 */
1509 #define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */
1510 #define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */
1511 #define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */
1512 #define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */
1513 #define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */
1514 #define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */
1515 #define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to physical timebase + frequency reg enabled */
1516 #define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to virtual timebase + frequency reg enabled */
1517
1518 #define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */
1519 #define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */
1520 #define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */
1521
1522 #define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS
1523 #define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED
1524 #define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE
1525
1526 /*
1527 * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons.
1528 * (cf. osfmk/arm/proc_reg.h)
1529 */
1530 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */
1531
1532 #define MIDR_EL1_REV_SHIFT 0
1533 #define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT)
1534 #define MIDR_EL1_PNUM_SHIFT 4
1535 #define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT)
1536 #define MIDR_EL1_ARCH_SHIFT 16
1537 #define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT)
1538 #define MIDR_EL1_VAR_SHIFT 20
1539 #define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT)
1540 #define MIDR_EL1_IMP_SHIFT 24
1541 #define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT)
1542
1543 /*
1544 * CoreSight debug registers
1545 */
1546 #define CORESIGHT_ED 0
1547 #define CORESIGHT_CTI 1
1548 #define CORESIGHT_PMU 2
1549 #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */
1550
1551 #define CORESIGHT_OFFSET(x) ((x) * 0x10000)
1552 #define CORESIGHT_REGIONS 4
1553 #define CORESIGHT_SIZE 0x1000
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564 /*
1565 * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0
1566 *
1567 * 63 24 23 20 19 16 15 12 11 8 7 4 3 0
1568 * +----------+--------+------+------+------+-----+------+
1569 * | reserved | atomic |crc32 | sha2 | sha1 | aes | res0 |
1570 * +----------+--------+------+------+------+-----+------+
1571 */
1572
1573 #define ID_AA64ISAR0_EL1_FHM_OFFSET 48
1574 #define ID_AA64ISAR0_EL1_FHM_MASK (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET)
1575 #define ID_AA64ISAR0_EL1_FHM_8_2 (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET)
1576
1577 #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20
1578 #define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
1579 #define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
1580
1581 #define ID_AA64ISAR0_EL1_CRC32_OFFSET 16
1582 #define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
1583 #define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
1584
1585 #define ID_AA64ISAR0_EL1_SHA2_OFFSET 12
1586 #define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
1587 #define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
1588
1589 #define ID_AA64ISAR0_EL1_SHA1_OFFSET 8
1590 #define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
1591 #define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
1592
1593 #define ID_AA64ISAR0_EL1_AES_OFFSET 4
1594 #define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET)
1595 #define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET)
1596 #define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET)
1597
1598
1599 #if __APCFG_SUPPORTED__
1600 /*
1601 * APCFG_EL1
1602 *
1603 * 63 2 1 0
1604 * +----------+-+-+
1605 * | reserved |K|R|
1606 * +----------+-+-+
1607 *
1608 * where:
1609 * R: Reserved
1610 * K: ElXEnKey - Enable ARMV8.3 defined {IA,IB,DA,DB} keys when CPU is
1611 * operating in EL1 (or higher) and when under Apple-Mode
1612 */
1613
1614 #define APCFG_EL1_ELXENKEY_OFFSET 1
1615 #define APCFG_EL1_ELXENKEY_MASK (0x1ULL << APCFG_EL1_ELXENKEY_OFFSET)
1616 #define APCFG_EL1_ELXENKEY APCFG_EL1_ELXENKEY_MASK
1617 #endif /* __APCFG_SUPPORTED__ */
1618
1619 #define APSTATE_G_SHIFT (0)
1620 #define APSTATE_P_SHIFT (1)
1621 #define APSTATE_A_SHIFT (2)
1622
1623 #ifdef __APSTS_SUPPORTED__
1624 #define APCTL_EL1_AppleMode (1ULL << 0)
1625 #define APCTL_EL1_KernKeyEn (1ULL << 1)
1626 #define APCTL_EL1_EnAPKey0 (1ULL << 2)
1627 #define APCTL_EL1_EnAPKey1 (1ULL << 3)
1628 #define APSTS_EL1_MKEYVld (1ULL << 0)
1629 #else
1630 #define APCTL_EL1_AppleMode (1ULL << 0)
1631 #define APCTL_EL1_MKEYVld (1ULL << 1)
1632 #define APCTL_EL1_KernKeyEn (1ULL << 2)
1633 #endif
1634
1635
1636
1637 #if defined(HAS_APPLE_PAC)
1638 // The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code
1639 #define PAC_DISCRIMINATOR_RECOVER 0x1e02
1640 #endif
1641
1642 #ifdef __ASSEMBLER__
1643
1644 /*
1645 * Compute CPU version:
1646 * Version is constructed as [4 bits of MIDR variant]:[4 bits of MIDR revision]
1647 *
1648 * Where the "variant" is the major number and the "revision" is the minor number.
1649 *
1650 * For example:
1651 * Cyclone A0 is variant 0, revision 0, i.e. 0.
1652 * Cyclone B0 is variant 1, revision 0, i.e. 0x10
1653 * $0 - register to place value in
1654 */
1655 .macro GET_MIDR_CPU_VERSION
1656 mrs $0, MIDR_EL1 // Read MIDR_EL1 for CPUID
1657 bfi $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #4 // move bits 3:0 (revision) to 19:16 (below variant) to get values adjacent
1658 ubfx $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #8 // And extract the concatenated bitstring to beginning of register
1659 .endmacro
1660
1661 /*
1662 * To apply a workaround for CPU versions less than a given value
1663 * (e.g. earlier than when a fix arrived)
1664 *
1665 * $0 - scratch register1
1666 * $1 - version at which to stop applying workaround
1667 * $2 - label to branch to (at end of workaround)
1668 */
1669 .macro SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL
1670 GET_MIDR_CPU_VERSION $0
1671 cmp $0, $1
1672 b.pl $2 // Unsigned "greater or equal"
1673 .endmacro
1674
1675 /*
1676 * To apply a workaround for CPU versions greater than a given value
1677 * (e.g. starting when a bug was introduced)
1678 *
1679 * $0 - scratch register1
1680 * $1 - version at which to stop applying workaround
1681 * $2 - label to branch to (at end of workaround)
1682 */
1683 .macro SKIP_IF_CPU_VERSION_LESS_THAN
1684 GET_MIDR_CPU_VERSION $0
1685 cmp $0, $1
1686 b.mi $2 // Unsigned "strictly less than"
1687 .endmacro
1688
1689 #endif /* __ASSEMBLER__ */
1690
1691 #define MSR(reg, src) __asm__ volatile ("msr " reg ", %0" :: "r" (src))
1692 #define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest))
1693
1694
1695 #endif /* _ARM64_PROC_REG_H_ */
mirror of XNU