bsd/crypto/aes/i386/aes_key_hw.s

   1 /*      This files defines _aes_encrypt_key_hw and _aes_decrypt_key_hw --- Intel Westmere HW AES-based implementation
   2         of _aes_encrypt_key and _aes_decrypt_key.
   3
   4         These 2 functions SHOULD BE entried ONLY after the AES HW is verified to be available.
   5         They SHOULD NOT be called without AES HW detection. It might cause xnu to crash.
   6
   7         The AES HW is detected 1st thing in
   8                 _aes_encrypt_key (ExpandKeyForEncryption.s)
   9                 _aes_decrypt_key (ExpandKeyForDecryption.s)
  10         and, if AES HW is detected, branch without link (ie, jump) to the functions here.
  11
  12         The implementation here follows the examples in an Intel White Paper
  13         "Intel Advanced Encryption Standard (AES) Instruction Set" Rev.2 01
  14
  15         Note: Rev. 03 Final 2010 01 26 is available. Looks like some code change from Rev.2 01
  16
  17         cclee 3-13-10
  18 */
  19
  20         .text
  21         .align  4,0x90
  22
  23         // hw_aes_encrypt_key(key, klen, hwectx);
  24         // klen = 16, 24, or 32, or (128/192/256)
  25
  26         .globl  _aes_encrypt_key_hw
  27 _aes_encrypt_key_hw:
  28
  29 #ifdef  __i386__
  30         push    %ebp
  31         mov             %esp, %ebp
  32         push    %ebx
  33         push    %edi
  34         mov             8(%ebp), %eax           // pointer to key
  35         mov             12(%ebp), %ebx          // klen
  36         mov             16(%ebp), %edi          // ctx
  37         #define pkey    %eax
  38         #define klen    %ebx
  39         #define ctx             %edi
  40         #define sp              %esp
  41         #define cx              %ecx
  42 #else
  43         #define pkey    %rdi
  44         #define klen    %rsi
  45         #define ctx             %rdx
  46         #define sp              %rsp
  47         #define cx              %rcx
  48         push    %rbp
  49         mov             %rsp, %rbp
  50 #endif
  51
  52 #ifdef  KERNEL
  53         // for xmm registers save and restore
  54         sub             $(16*4), sp
  55 #endif
  56
  57         cmp             $32, klen
  58         jg              0f                                      // klen>32
  59         shl             $3, klen                        // convert 16/24/32 to 128/192/256
  60 0:
  61
  62         cmp             $128, klen                      // AES-128 ?
  63         je              L_AES_128_Encrypt_Key
  64         cmp             $192, klen                      // AES-192 ?
  65         je              L_AES_192_Encrypt_Key
  66         cmp             $256, klen                      // AES-256 ?
  67         je              L_AES_256_Encrypt_Key
  68         mov             $1, %eax                        // return error for wrong klen
  69 L_Encrypt_Key_2_return:
  70 #ifdef  KERNEL
  71         add             $(16*4), sp
  72 #endif
  73 #ifdef  __i386__
  74         pop             %edi
  75         pop             %ebx
  76 #endif
  77         leave
  78         ret
  79
  80 L_AES_128_Encrypt_Key:
  81 #ifdef  KERNEL
  82         // save xmm registers
  83         movaps  %xmm1, (sp)
  84         movaps  %xmm2, 16(sp)
  85         movaps  %xmm3, 32(sp)
  86 #endif  // KERNEL
  87
  88         movl    $160, 240(ctx)          // write expanded key length to ctx
  89         xor             cx, cx
  90
  91         movups  (pkey), %xmm1
  92         movups  %xmm1, (ctx)
  93         aeskeygenassist $1, %xmm1, %xmm2
  94         call    L_key_expansion_128
  95         aeskeygenassist $2, %xmm1, %xmm2
  96         call    L_key_expansion_128
  97         aeskeygenassist $4, %xmm1, %xmm2
  98         call    L_key_expansion_128
  99         aeskeygenassist $8, %xmm1, %xmm2
 100         call    L_key_expansion_128
 101         aeskeygenassist $0x10, %xmm1, %xmm2
 102         call    L_key_expansion_128
 103         aeskeygenassist $0x20, %xmm1, %xmm2
 104         call    L_key_expansion_128
 105         aeskeygenassist $0x40, %xmm1, %xmm2
 106         call    L_key_expansion_128
 107         aeskeygenassist $0x80, %xmm1, %xmm2
 108         call    L_key_expansion_128
 109         aeskeygenassist $0x1b, %xmm1, %xmm2
 110         call    L_key_expansion_128
 111         aeskeygenassist $0x36, %xmm1, %xmm2
 112         call    L_key_expansion_128
 113
 114 #ifdef  KERNEL
 115         // restore xmm registers
 116         movaps  (sp), %xmm1
 117         movaps  16(sp), %xmm2
 118         movaps  32(sp), %xmm3
 119 #endif  // KERNEL
 120         xor             %eax, %eax                      // return 0 for success
 121         jmp             L_Encrypt_Key_2_return
 122
 123         .align  4, 0x90
 124 L_key_expansion_128:
 125         pshufd  $0xff, %xmm2, %xmm2
 126         movaps  %xmm1, %xmm3
 127         pslldq  $4, %xmm3
 128         pxor    %xmm3, %xmm1
 129         movaps  %xmm1, %xmm3
 130         pslldq  $4, %xmm3
 131         pxor    %xmm3, %xmm1
 132         movaps  %xmm1, %xmm3
 133         pslldq  $4, %xmm3
 134         pxor    %xmm3, %xmm1
 135         pxor    %xmm2, %xmm1
 136         add             $16, cx
 137         movups  %xmm1, (ctx, cx)
 138         ret
 139
 140 L_AES_192_Encrypt_Key:
 141 #ifdef  KERNEL
 142         // save xmm registers
 143         movaps  %xmm1, (sp)
 144         movaps  %xmm2, 16(sp)
 145         movaps  %xmm3, 32(sp)
 146         movaps  %xmm4, 48(sp)
 147 #endif  // KERNEL
 148         movl    $192, 240(ctx)          // write expanded key length to ctx
 149
 150         movups  (pkey), %xmm1
 151         movq    16(pkey), %xmm3
 152
 153         movups  %xmm1, (ctx)
 154         movq    %xmm3, 16(ctx)
 155
 156         lea             24(ctx), cx
 157
 158         aeskeygenassist $1, %xmm3, %xmm2
 159         call    L_key_expansion_192
 160         aeskeygenassist $2, %xmm3, %xmm2
 161         call    L_key_expansion_192
 162         aeskeygenassist $4, %xmm3, %xmm2
 163         call    L_key_expansion_192
 164         aeskeygenassist $8, %xmm3, %xmm2
 165         call    L_key_expansion_192
 166         aeskeygenassist $0x10, %xmm3, %xmm2
 167         call    L_key_expansion_192
 168         aeskeygenassist $0x20, %xmm3, %xmm2
 169         call    L_key_expansion_192
 170         aeskeygenassist $0x40, %xmm3, %xmm2
 171         call    L_key_expansion_192
 172         aeskeygenassist $0x80, %xmm3, %xmm2
 173         call    L_key_expansion_192
 174
 175 #ifdef  KERNEL
 176         // restore xmm registers
 177         movaps  (sp), %xmm1
 178         movaps  16(sp), %xmm2
 179         movaps  32(sp), %xmm3
 180         movaps  48(sp), %xmm4
 181 #endif  // KERNEL
 182         xor             %eax, %eax                      // return 0 for success
 183         jmp             L_Encrypt_Key_2_return
 184
 185         .align  4, 0x90
 186 L_key_expansion_192:
 187         pshufd  $0x55, %xmm2, %xmm2
 188
 189         movaps  %xmm1, %xmm4
 190         pslldq  $4, %xmm4
 191
 192         pxor    %xmm4, %xmm1
 193         pslldq  $4, %xmm4
 194
 195         pxor    %xmm4, %xmm1
 196         pslldq  $4, %xmm4
 197
 198         pxor    %xmm4, %xmm1
 199         pxor    %xmm2, %xmm1
 200
 201         pshufd  $0xff, %xmm1, %xmm2
 202
 203         movaps  %xmm3, %xmm4
 204         pslldq  $4, %xmm4
 205
 206         pxor    %xmm4, %xmm3
 207         pxor    %xmm2, %xmm3
 208
 209         movups  %xmm1, (cx)
 210         movq    %xmm3, 16(cx)
 211
 212         add             $24, cx
 213         ret
 214
 215 L_AES_256_Encrypt_Key:
 216 #ifdef  KERNEL
 217         // save xmm registers
 218         movaps  %xmm1, (sp)
 219         movaps  %xmm2, 16(sp)
 220         movaps  %xmm3, 32(sp)
 221         movaps  %xmm4, 48(sp)
 222 #endif  // KERNEL
 223         movl    $224, 240(ctx)          // write expanded key length to ctx
 224
 225         movups  (pkey), %xmm1
 226         movups  16(pkey), %xmm3
 227         movups  %xmm1, (ctx)
 228         movups  %xmm3, 16(ctx)
 229
 230         lea             32(ctx), cx
 231
 232         aeskeygenassist $1, %xmm3, %xmm2
 233         call    L_key_expansion_256
 234         aeskeygenassist $2, %xmm3, %xmm2
 235         call    L_key_expansion_256
 236         aeskeygenassist $4, %xmm3, %xmm2
 237         call    L_key_expansion_256
 238         aeskeygenassist $8, %xmm3, %xmm2
 239         call    L_key_expansion_256
 240         aeskeygenassist $0x10, %xmm3, %xmm2
 241         call    L_key_expansion_256
 242         aeskeygenassist $0x20, %xmm3, %xmm2
 243         call    L_key_expansion_256
 244         aeskeygenassist $0x40, %xmm3, %xmm2
 245         call    L_key_expansion_256_final
 246
 247 #ifdef  KERNEL
 248         // restore xmm registers
 249         movaps  (sp), %xmm1
 250         movaps  16(sp), %xmm2
 251         movaps  32(sp), %xmm3
 252         movaps  48(sp), %xmm4
 253 #endif  // KERNEL
 254         xor             %eax, %eax                      // return 0 for success
 255         jmp             L_Encrypt_Key_2_return
 256
 257         .align  4, 0x90
 258 L_key_expansion_256:
 259
 260         pshufd  $0xff, %xmm2, %xmm2
 261
 262         movaps  %xmm1, %xmm4
 263         pslldq  $4, %xmm4
 264
 265         pxor    %xmm4, %xmm1
 266         pslldq  $4, %xmm4
 267
 268         pxor    %xmm4, %xmm1
 269         pslldq  $4, %xmm4
 270
 271         pxor    %xmm4, %xmm1
 272         pxor    %xmm2, %xmm1
 273
 274         movups  %xmm1, (cx)
 275
 276         aeskeygenassist $0, %xmm1, %xmm4
 277
 278         pshufd  $0xaa, %xmm4, %xmm2
 279
 280         movaps  %xmm3, %xmm4
 281         pslldq  $4, %xmm4
 282
 283         pxor    %xmm4, %xmm3
 284         pslldq  $4, %xmm4
 285
 286         pxor    %xmm4, %xmm3
 287         pslldq  $4, %xmm4
 288
 289         pxor    %xmm4, %xmm3
 290         pxor    %xmm2, %xmm3
 291
 292         movups  %xmm3, 16(cx)
 293
 294         add             $32, cx
 295         ret
 296
 297         .align  4, 0x90
 298 L_key_expansion_256_final:
 299
 300         pshufd  $0xff, %xmm2, %xmm2
 301
 302         movaps  %xmm1, %xmm4
 303         pslldq  $4, %xmm4
 304
 305         pxor    %xmm4, %xmm1
 306         pslldq  $4, %xmm4
 307
 308         pxor    %xmm4, %xmm1
 309         pslldq  $4, %xmm4
 310
 311         pxor    %xmm4, %xmm1
 312         pxor    %xmm2, %xmm1
 313
 314         movups  %xmm1, (cx)
 315         ret
 316
 317 // _aes_decrypt_key_hw is implemented as
 318 //      1. call _aes_encrypt_key_hw
 319 //      2. use aesimc to convert the expanded round keys (except the 1st and last round keys)
 320
 321         .text
 322         .align  4, 0x90
 323         .globl  _aes_decrypt_key_hw
 324 _aes_decrypt_key_hw:
 325
 326 #ifdef  __i386__
 327
 328         push    %ebp
 329         mov             %esp, %ebp
 330         sub             $(8+16), %esp
 331
 332         // copy input arguments for calling aes_decrypt_key_hw
 333
 334         mov             8(%ebp), %eax
 335         mov             %eax, (%esp)
 336         mov             12(%ebp), %eax
 337         mov             %eax, 4(%esp)
 338         mov             16(%ebp), %eax
 339         mov             %eax, 8(%esp)
 340
 341 #else
 342
 343         push    %rbp
 344         mov             %rsp, %rbp
 345         sub             $16, %rsp
 346
 347         // calling arguments %rdi/%rsi/%rdx will be used for encrypt_key
 348         // %rdx (ctx) will return unchanged
 349         // %rsi (klen) will (<<3) if <= 32
 350
 351 #endif
 352         call    _aes_encrypt_key_hw
 353         cmp             $0, %eax
 354         je              L_decrypt_inv
 355 L_decrypt_almost_done:
 356 #ifdef  __i386__
 357         add             $(8+16), %esp
 358 #else
 359         add             $16, %rsp
 360 #endif
 361         leave
 362         ret
 363
 364 L_decrypt_inv:
 365 #ifdef  KERNEL
 366         movaps  %xmm0, (sp)
 367 #endif
 368
 369 #ifdef  __i386__
 370         #undef  klen
 371         #undef  ctx
 372         mov     12(%ebp), %eax      // klen
 373     mov     16(%ebp), %edx      // ctx
 374         #define klen    %eax
 375         #define ctx             %edx
 376         cmp             $32, klen
 377         jg              0f                                      // klen>32
 378         shl             $3, klen                        // convert 16/24/32 to 128/192/256
 379 0:
 380 #endif
 381
 382         mov             $9, cx                          // default is AES-128
 383         cmp             $128, klen
 384         je              L_Decrypt_Key
 385         add             $2, cx
 386         cmp             $192, klen
 387         je              L_Decrypt_Key
 388         add             $2, cx
 389
 390 L_Decrypt_Key:
 391         add             $16, ctx
 392         movups  (ctx), %xmm0
 393         aesimc  %xmm0, %xmm0
 394         movups  %xmm0, (ctx)
 395         sub             $1, cx
 396         jg              L_Decrypt_Key
 397
 398 #ifdef  KERNEL
 399         movaps  (sp), %xmm0
 400 #endif
 401 #ifdef  __i386__
 402         xor             %eax, %eax
 403 #endif
 404         jmp             L_decrypt_almost_done
 405