/* * Copyright (c) 2010-2013 Apple Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this * file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_LICENSE_HEADER_END@ */ #include // bool save_xxm = (*((uint32_t*)_COMM_PAGE_CPU_CAPABILITIES) & kHasAVX1_0) != 0; #if __x86_64__ #define RDI_SAVE_RBP -8 #define RSI_SAVE_RBP -16 #define RDX_SAVE_RBP -24 #define RCX_SAVE_RBP -32 #define RBX_SAVE_RBP -40 #define R8_SAVE_RBP -48 #define R9_SAVE_RBP -56 #define R10_SAVE_RBP -64 #define R11_SAVE_RBP -72 #define STATIC_STACK_SIZE 256 // extra padding to allow it to be 64-byte aligned #define XMM0_SAVE_RSP 0x00 #define XMM1_SAVE_RSP 0x10 #define XMM2_SAVE_RSP 0x20 #define XMM3_SAVE_RSP 0x30 #define XMM4_SAVE_RSP 0x40 #define XMM5_SAVE_RSP 0x50 #define XMM6_SAVE_RSP 0x60 #define XMM7_SAVE_RSP 0x70 // returns address of TLV in %rax, all other registers preserved .globl _tlv_get_addr .private_extern _tlv_get_addr _tlv_get_addr: movq 8(%rdi),%rax // get key from descriptor movq %gs:0x0(,%rax,8),%rax // get thread value testq %rax,%rax // if NULL, lazily allocate je LlazyAllocate addq 16(%rdi),%rax // add offset from descriptor ret LlazyAllocate: pushq %rbp movq %rsp,%rbp subq $STATIC_STACK_SIZE,%rsp movq %rdi,RDI_SAVE_RBP(%rbp) # save registers that might be used as parameters movq %rsi,RSI_SAVE_RBP(%rbp) movq %rdx,RDX_SAVE_RBP(%rbp) movq %rcx,RCX_SAVE_RBP(%rbp) movq %rbx,RBX_SAVE_RBP(%rbp) movq %r8, R8_SAVE_RBP(%rbp) movq %r9, R9_SAVE_RBP(%rbp) movq %r10,R10_SAVE_RBP(%rbp) movq %r11,R11_SAVE_RBP(%rbp) cmpl $0, _inited(%rip) jne Linited movl $0x01,%eax cpuid # get cpu features to check on xsave instruction support andl $0x08000000,%ecx # check OSXSAVE bit movl %ecx,_hasXSave(%rip) cmpl $0, %ecx jne LxsaveInfo movl $1, _inited(%rip) jmp Lsse LxsaveInfo: movl $0x0D,%eax movl $0x00,%ecx cpuid # get xsave parameter info movl %eax,_features_lo32(%rip) movl %edx,_features_hi32(%rip) movl %ecx,_bufferSize32(%rip) movl $1, _inited(%rip) Linited: cmpl $0, _hasXSave(%rip) jne Lxsave Lsse: subq $128, %rsp movdqa %xmm0, XMM0_SAVE_RSP(%rsp) movdqa %xmm1, XMM1_SAVE_RSP(%rsp) movdqa %xmm2, XMM2_SAVE_RSP(%rsp) movdqa %xmm3, XMM3_SAVE_RSP(%rsp) movdqa %xmm4, XMM4_SAVE_RSP(%rsp) movdqa %xmm5, XMM5_SAVE_RSP(%rsp) movdqa %xmm6, XMM6_SAVE_RSP(%rsp) movdqa %xmm7, XMM7_SAVE_RSP(%rsp) jmp Lalloc Lxsave: movl _bufferSize32(%rip),%eax movq %rsp, %rdi subq %rax, %rdi # stack alloc buffer andq $-64, %rdi # 64-byte align stack movq %rdi, %rsp # xsave requires buffer to be zero'ed out movq $0, %rcx movq %rdi, %r8 movq %rdi, %r9 addq %rax, %r9 Lz: movq %rcx, (%r8) addq $8, %r8 cmpq %r8,%r9 ja Lz movl _features_lo32(%rip),%eax movl _features_hi32(%rip),%edx # call xsave with buffer on stack and eax:edx flag bits # note: do not use xsaveopt, it assumes you are using the same # buffer as previous xsaves, and this thread is on the same cpu. xsave (%rsp) Lalloc: movq RDI_SAVE_RBP(%rbp),%rdi movq 8(%rdi),%rdi // get key from descriptor call _tlv_allocate_and_initialize_for_key cmpl $0, _hasXSave(%rip) jne Lxrstror movdqa XMM0_SAVE_RSP(%rsp),%xmm0 movdqa XMM1_SAVE_RSP(%rsp),%xmm1 movdqa XMM2_SAVE_RSP(%rsp),%xmm2 movdqa XMM3_SAVE_RSP(%rsp),%xmm3 movdqa XMM4_SAVE_RSP(%rsp),%xmm4 movdqa XMM5_SAVE_RSP(%rsp),%xmm5 movdqa XMM6_SAVE_RSP(%rsp),%xmm6 movdqa XMM7_SAVE_RSP(%rsp),%xmm7 jmp Ldone Lxrstror: movq %rax,%r11 movl _features_lo32(%rip),%eax movl _features_hi32(%rip),%edx # call xsave with buffer on stack and eax:edx flag bits xrstor (%rsp) movq %r11,%rax Ldone: movq RDI_SAVE_RBP(%rbp),%rdi movq RSI_SAVE_RBP(%rbp),%rsi movq RDX_SAVE_RBP(%rbp),%rdx movq RCX_SAVE_RBP(%rbp),%rcx movq RBX_SAVE_RBP(%rbp),%rbx movq R8_SAVE_RBP(%rbp),%r8 movq R9_SAVE_RBP(%rbp),%r9 movq R10_SAVE_RBP(%rbp),%r10 movq R11_SAVE_RBP(%rbp),%r11 movq %rbp,%rsp popq %rbp addq 16(%rdi),%rax // result = buffer + offset ret .data # Cached info from cpuid. _inited: .long 0 _features_lo32: .long 0 _features_hi32: .long 0 _bufferSize32: .long 0 _hasXSave: .long 0 #endif #if __i386__ // returns address of TLV in %eax, all other registers (except %ecx) preserved .globl _tlv_get_addr .private_extern _tlv_get_addr _tlv_get_addr: movl 4(%eax),%ecx // get key from descriptor movl %gs:0x0(,%ecx,4),%ecx // get thread value testl %ecx,%ecx // if NULL, lazily allocate je LlazyAllocate movl 8(%eax),%eax // add offset from descriptor addl %ecx,%eax ret LlazyAllocate: pushl %ebp movl %esp,%ebp pushl %edx // save edx subl $548,%esp movl %eax,-8(%ebp) // save descriptor lea -528(%ebp),%ecx // get 512 byte buffer in frame and $-16, %ecx // 16-byte align buffer for fxsave fxsave (%ecx) movl 4(%eax),%ecx // get key from descriptor movl %ecx,(%esp) // push key parameter, also leaves stack aligned properly call _tlv_allocate_and_initialize_for_key movl -8(%ebp),%ecx // get descriptor movl 8(%ecx),%ecx // get offset from descriptor addl %ecx,%eax // add offset to buffer lea -528(%ebp),%ecx and $-16, %ecx // 16-byte align buffer for fxrstor fxrstor (%ecx) addl $548,%esp popl %edx // restore edx popl %ebp ret #endif #if __arm64__ // Parameters: X0 = descriptor // Result: X0 = address of TLV // Note: all registers except X0, x16, and x17 are preserved .align 2 .globl _tlv_get_addr .private_extern _tlv_get_addr _tlv_get_addr: ldr x16, [x0, #8] // get key from descriptor mrs x17, TPIDRRO_EL0 and x17, x17, #-8 // clear low 3 bits??? ldr x17, [x17, x16, lsl #3] // get thread allocation address for this key cbz x17, LlazyAllocate // if NULL, lazily allocate ldr x16, [x0, #16] // get offset from descriptor add x0, x17, x16 // return allocation+offset ret lr LlazyAllocate: stp fp, lr, [sp, #-16]! mov fp, sp sub sp, sp, #288 stp x1, x2, [sp, #-16]! // save all registers that C function might trash stp x3, x4, [sp, #-16]! stp x5, x6, [sp, #-16]! stp x7, x8, [sp, #-16]! stp x9, x10, [sp, #-16]! stp x11, x12, [sp, #-16]! stp x13, x14, [sp, #-16]! stp x15, x16, [sp, #-16]! stp q0, q1, [sp, #-32]! stp q2, q3, [sp, #-32]! stp q4, q5, [sp, #-32]! stp q6, q7, [sp, #-32]! stp x0, x17, [sp, #-16]! // save descriptor mov x0, x16 // use key from descriptor as parameter bl _tlv_allocate_and_initialize_for_key ldp x16, x17, [sp], #16 // pop descriptor ldr x16, [x16, #16] // get offset from descriptor add x0, x0, x16 // return allocation+offset ldp q6, q7, [sp], #32 ldp q4, q5, [sp], #32 ldp q2, q3, [sp], #32 ldp q0, q1, [sp], #32 ldp x15, x16, [sp], #16 ldp x13, x14, [sp], #16 ldp x11, x12, [sp], #16 ldp x9, x10, [sp], #16 ldp x7, x8, [sp], #16 ldp x5, x6, [sp], #16 ldp x3, x4, [sp], #16 ldp x1, x2, [sp], #16 mov sp, fp ldp fp, lr, [sp], #16 ret lr #endif #if __arm__ // returns address of TLV in r0, all other registers preserved .align 2 .globl _tlv_get_addr .private_extern _tlv_get_addr _tlv_get_addr: push {r1,r2,r3,r7,lr} #if __ARM_ARCH_7K__ sub sp, sp, #12 // align stack to 16 bytes #endif mov r7, r0 // save descriptor in r7 ldr r0, [r7, #4] // get key from descriptor bl _pthread_getspecific // get thread value cmp r0, #0 bne L2 // if NULL, lazily allocate #if __ARM_ARCH_7K__ vpush {d0, d1, d2, d3, d4, d5, d6, d7} #endif ldr r0, [r7, #4] // get key from descriptor bl _tlv_allocate_and_initialize_for_key #if __ARM_ARCH_7K__ vpop {d0, d1, d2, d3, d4, d5, d6, d7} #endif L2: ldr r1, [r7, #8] // get offset from descriptor add r0, r1, r0 // add offset into allocation block #if __ARM_ARCH_7K__ add sp, sp, #12 #endif pop {r1,r2,r3,r7,pc} #endif .subsections_via_symbols