EXTERNAL_HEADERS/corecrypto/cc_priv.h

   1 /* Copyright (c) (2010,2011,2012,2014,2015,2016,2017,2018,2019,2020) Apple Inc. All rights reserved.
   2  *
   3  * corecrypto is licensed under Apple Inc.’s Internal Use License Agreement (which
   4  * is contained in the License.txt file distributed with corecrypto) and only to
   5  * people who accept that license. IMPORTANT:  Any license rights granted to you by
   6  * Apple Inc. (if any) are limited to internal use within your organization only on
   7  * devices and computers you own or control, for the sole purpose of verifying the
   8  * security characteristics and correct functioning of the Apple Software.  You may
   9  * not, directly or indirectly, redistribute the Apple Software or any portions thereof.
  10  */
  11
  12 #ifndef _CORECRYPTO_CC_PRIV_H_
  13 #define _CORECRYPTO_CC_PRIV_H_
  14
  15 #include <corecrypto/cc.h>
  16 #include <stdbool.h>
  17 #include <stdint.h>
  18
  19 // Fork handlers for the stateful components of corecrypto.
  20 void cc_atfork_prepare(void);
  21 void cc_atfork_parent(void);
  22 void cc_atfork_child(void);
  23
  24 #ifndef __has_builtin
  25 #define __has_builtin(x) 0
  26 #endif
  27
  28 #ifndef __DECONST
  29 #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
  30 #endif
  31
  32 /* defines the following macros :
  33
  34  CC_ARRAY_LEN: returns the number of elements in an array
  35
  36  CC_STORE32_BE : store 32 bit value in big endian in unaligned buffer.
  37  CC_STORE32_LE : store 32 bit value in little endian in unaligned buffer.
  38  CC_STORE64_BE : store 64 bit value in big endian in unaligned buffer.
  39  CC_STORE64_LE : store 64 bit value in little endian in unaligned buffer.
  40
  41  CC_LOAD32_BE : load 32 bit value in big endian from unaligned buffer.
  42  CC_LOAD32_LE : load 32 bit value in little endian from unaligned buffer.
  43  CC_LOAD64_BE : load 64 bit value in big endian from unaligned buffer.
  44  CC_LOAD64_LE : load 64 bit value in little endian from unaligned buffer.
  45
  46  CC_ROR  : Rotate Right 32 bits. Rotate count can be a variable.
  47  CC_ROL  : Rotate Left 32 bits. Rotate count can be a variable.
  48  CC_RORc : Rotate Right 32 bits. Rotate count must be a constant.
  49  CC_ROLc : Rotate Left 32 bits. Rotate count must be a constant.
  50
  51  CC_ROR64  : Rotate Right 64 bits. Rotate count can be a variable.
  52  CC_ROL64  : Rotate Left 64 bits. Rotate count can be a variable.
  53  CC_ROR64c : Rotate Right 64 bits. Rotate count must be a constant.
  54  CC_ROL64c : Rotate Left 64 bits. Rotate count must be a constant.
  55
  56  CC_BSWAP  : byte swap a 32 bits variable.
  57
  58  CC_H2BE32 : convert a 32 bits value between host and big endian order.
  59  CC_H2LE32 : convert a 32 bits value between host and little endian order.
  60
  61  CC_BSWAP64  : byte swap a 64 bits variable
  62
  63  CC_READ_LE32 : read a 32 bits little endian value
  64
  65  CC_WRITE_LE32 : write a 32 bits little endian value
  66  CC_WRITE_LE64 : write a 64 bits little endian value
  67
  68  CC_H2BE64 : convert a 64 bits value between host and big endian order
  69  CC_H2LE64 : convert a 64 bits value between host and little endian order
  70
  71 */
  72
  73 // RTKitOSPlatform should replace CC_MEMCPY with memcpy
  74 #define CC_MEMCPY(D,S,L) cc_memcpy((D),(S),(L))
  75 #define CC_MEMMOVE(D,S,L) cc_memmove((D),(S),(L))
  76 #define CC_MEMSET(D,V,L) cc_memset((D),(V),(L))
  77
  78 #if __has_builtin(__builtin___memcpy_chk) && !defined(_MSC_VER)
  79 #define cc_memcpy(dst, src, len) __builtin___memcpy_chk((dst), (src), (len), __builtin_object_size((dst), 1))
  80 #define cc_memcpy_nochk(dst, src, len) __builtin___memcpy_chk((dst), (src), (len), __builtin_object_size((dst), 0))
  81 #else
  82 #define cc_memcpy(dst, src, len) memcpy((dst), (src), (len))
  83 #define cc_memcpy_nochk(dst, src, len) memcpy((dst), (src), (len))
  84 #endif
  85
  86 #if __has_builtin(__builtin___memmove_chk) && !defined(_MSC_VER)
  87 #define cc_memmove(dst, src, len) __builtin___memmove_chk((dst), (src), (len), __builtin_object_size((dst), 1))
  88 #else
  89 #define cc_memmove(dst, src, len) memmove((dst), (src), (len))
  90 #endif
  91
  92 #if __has_builtin(__builtin___memset_chk) && !defined(_MSC_VER)
  93 #define cc_memset(dst, val, len) __builtin___memset_chk((dst), (val), (len), __builtin_object_size((dst), 1))
  94 #else
  95 #define cc_memset(dst, val, len) memset((dst), (val), (len))
  96 #endif
  97
  98 #define CC_ARRAY_LEN(x) (sizeof((x))/sizeof((x)[0]))
  99
 100 // MARK: - Loads and Store
 101
 102 // MARK: -- 32 bits - little endian
 103
 104 // MARK: --- Default version
 105
 106 #define CC_STORE32_LE(x, y) do {                                    \
 107     ((unsigned char *)(y))[3] = (unsigned char)(((x)>>24)&255);         \
 108     ((unsigned char *)(y))[2] = (unsigned char)(((x)>>16)&255);         \
 109     ((unsigned char *)(y))[1] = (unsigned char)(((x)>>8)&255);          \
 110     ((unsigned char *)(y))[0] = (unsigned char)((x)&255);                       \
 111 } while(0)
 112
 113 #define CC_LOAD32_LE(x, y) do {                                     \
 114 x = ((uint32_t)(((const unsigned char *)(y))[3] & 255)<<24) |                       \
 115     ((uint32_t)(((const unsigned char *)(y))[2] & 255)<<16) |                       \
 116     ((uint32_t)(((const unsigned char *)(y))[1] & 255)<<8)  |                       \
 117     ((uint32_t)(((const unsigned char *)(y))[0] & 255));                                    \
 118 } while(0)
 119
 120 // MARK: -- 64 bits - little endian
 121
 122 #define CC_STORE64_LE(x, y) do {                                    \
 123     ((unsigned char *)(y))[7] = (unsigned char)(((x)>>56)&255);     \
 124     ((unsigned char *)(y))[6] = (unsigned char)(((x)>>48)&255);         \
 125     ((unsigned char *)(y))[5] = (unsigned char)(((x)>>40)&255);         \
 126     ((unsigned char *)(y))[4] = (unsigned char)(((x)>>32)&255);         \
 127     ((unsigned char *)(y))[3] = (unsigned char)(((x)>>24)&255);         \
 128     ((unsigned char *)(y))[2] = (unsigned char)(((x)>>16)&255);         \
 129     ((unsigned char *)(y))[1] = (unsigned char)(((x)>>8)&255);          \
 130     ((unsigned char *)(y))[0] = (unsigned char)((x)&255);                       \
 131 } while(0)
 132
 133 #define CC_LOAD64_LE(x, y) do {                                     \
 134 x = (((uint64_t)(((const unsigned char *)(y))[7] & 255))<<56) |           \
 135     (((uint64_t)(((const unsigned char *)(y))[6] & 255))<<48) |           \
 136     (((uint64_t)(((const unsigned char *)(y))[5] & 255))<<40) |           \
 137     (((uint64_t)(((const unsigned char *)(y))[4] & 255))<<32) |           \
 138     (((uint64_t)(((const unsigned char *)(y))[3] & 255))<<24) |           \
 139     (((uint64_t)(((const unsigned char *)(y))[2] & 255))<<16) |           \
 140     (((uint64_t)(((const unsigned char *)(y))[1] & 255))<<8)  |           \
 141     (((uint64_t)(((const unsigned char *)(y))[0] & 255)));                \
 142 } while(0)
 143
 144 // MARK: -- 32 bits - big endian
 145 // MARK: --- intel version
 146
 147 #if (defined(__i386__) || defined(__x86_64__)) && !defined(_MSC_VER)
 148
 149 #define CC_STORE32_BE(x, y)     \
 150     __asm__ __volatile__ (      \
 151     "bswapl %0     \n\t"        \
 152     "movl   %0,(%1)\n\t"        \
 153     "bswapl %0     \n\t"        \
 154     ::"r"(x), "r"(y))
 155
 156 #define CC_LOAD32_BE(x, y)      \
 157     __asm__ __volatile__ (      \
 158     "movl (%1),%0\n\t"          \
 159     "bswapl %0\n\t"             \
 160     :"=r"(x): "r"(y))
 161
 162 #else
 163 // MARK: --- default version
 164 #define CC_STORE32_BE(x, y) do {                                \
 165     ((unsigned char *)(y))[0] = (unsigned char)(((x)>>24)&255); \
 166     ((unsigned char *)(y))[1] = (unsigned char)(((x)>>16)&255); \
 167     ((unsigned char *)(y))[2] = (unsigned char)(((x)>>8)&255);  \
 168     ((unsigned char *)(y))[3] = (unsigned char)((x)&255);       \
 169 } while(0)
 170
 171 #define CC_LOAD32_BE(x, y) do {                             \
 172 x = ((uint32_t)(((const unsigned char *)(y))[0] & 255)<<24) |       \
 173     ((uint32_t)(((const unsigned char *)(y))[1] & 255)<<16) |           \
 174     ((uint32_t)(((const unsigned char *)(y))[2] & 255)<<8)  |           \
 175     ((uint32_t)(((const unsigned char *)(y))[3] & 255));          \
 176 } while(0)
 177
 178 #endif
 179
 180 // MARK: -- 64 bits - big endian
 181
 182 // MARK: --- intel 64 bits version
 183
 184 #if defined(__x86_64__) && !defined (_MSC_VER)
 185
 186 #define CC_STORE64_BE(x, y)   \
 187 __asm__ __volatile__ (        \
 188 "bswapq %0     \n\t"          \
 189 "movq   %0,(%1)\n\t"          \
 190 "bswapq %0     \n\t"          \
 191 ::"r"(x), "r"(y))
 192
 193 #define CC_LOAD64_BE(x, y)    \
 194 __asm__ __volatile__ (        \
 195 "movq (%1),%0\n\t"            \
 196 "bswapq %0\n\t"               \
 197 :"=r"(x): "r"(y))
 198
 199 #else
 200
 201 // MARK: --- default version
 202
 203 #define CC_STORE64_BE(x, y) do {                                    \
 204     ((unsigned char *)(y))[0] = (unsigned char)(((x)>>56)&255);         \
 205     ((unsigned char *)(y))[1] = (unsigned char)(((x)>>48)&255);         \
 206     ((unsigned char *)(y))[2] = (unsigned char)(((x)>>40)&255);         \
 207     ((unsigned char *)(y))[3] = (unsigned char)(((x)>>32)&255);         \
 208     ((unsigned char *)(y))[4] = (unsigned char)(((x)>>24)&255);         \
 209     ((unsigned char *)(y))[5] = (unsigned char)(((x)>>16)&255);         \
 210     ((unsigned char *)(y))[6] = (unsigned char)(((x)>>8)&255);          \
 211     ((unsigned char *)(y))[7] = (unsigned char)((x)&255);                       \
 212 } while(0)
 213
 214 #define CC_LOAD64_BE(x, y) do {                                     \
 215 x = (((uint64_t)(((const unsigned char *)(y))[0] & 255))<<56) |           \
 216     (((uint64_t)(((const unsigned char *)(y))[1] & 255))<<48) |           \
 217     (((uint64_t)(((const unsigned char *)(y))[2] & 255))<<40) |           \
 218     (((uint64_t)(((const unsigned char *)(y))[3] & 255))<<32) |           \
 219     (((uint64_t)(((const unsigned char *)(y))[4] & 255))<<24) |           \
 220     (((uint64_t)(((const unsigned char *)(y))[5] & 255))<<16) |           \
 221     (((uint64_t)(((const unsigned char *)(y))[6] & 255))<<8)  |                 \
 222     (((uint64_t)(((const unsigned char *)(y))[7] & 255)));                  \
 223 } while(0)
 224
 225 #endif
 226
 227 // MARK: - 32-bit Rotates
 228
 229 #if defined(_MSC_VER)
 230 // MARK: -- MSVC version
 231
 232 #include <stdlib.h>
 233 #if !defined(__clang__)
 234  #pragma intrinsic(_lrotr,_lrotl)
 235 #endif
 236 #define CC_ROR(x,n) _lrotr(x,n)
 237 #define CC_ROL(x,n) _lrotl(x,n)
 238 #define CC_RORc(x,n) _lrotr(x,n)
 239 #define CC_ROLc(x,n) _lrotl(x,n)
 240
 241 #elif (defined(__i386__) || defined(__x86_64__))
 242 // MARK: -- intel asm version
 243
 244 CC_INLINE uint32_t CC_ROL(uint32_t word, int i)
 245 {
 246     __asm__ ("roll %%cl,%0"
 247          :"=r" (word)
 248          :"0" (word),"c" (i));
 249     return word;
 250 }
 251
 252 CC_INLINE uint32_t CC_ROR(uint32_t word, int i)
 253 {
 254     __asm__ ("rorl %%cl,%0"
 255          :"=r" (word)
 256          :"0" (word),"c" (i));
 257     return word;
 258 }
 259
 260 /* Need to be a macro here, because 'i' is an immediate (constant) */
 261 #define CC_ROLc(word, i)                \
 262 ({  uint32_t _word=(word);              \
 263     __asm__ __volatile__ ("roll %2,%0"  \
 264         :"=r" (_word)                   \
 265         :"0" (_word),"I" (i));          \
 266     _word;                              \
 267 })
 268
 269
 270 #define CC_RORc(word, i)                \
 271 ({  uint32_t _word=(word);              \
 272     __asm__ __volatile__ ("rorl %2,%0"  \
 273         :"=r" (_word)                   \
 274         :"0" (_word),"I" (i));          \
 275     _word;                              \
 276 })
 277
 278 #else
 279
 280 // MARK: -- default version
 281
 282 CC_INLINE uint32_t CC_ROL(uint32_t word, int i)
 283 {
 284     return ( (word<<(i&31)) | (word>>(32-(i&31))) );
 285 }
 286
 287 CC_INLINE uint32_t CC_ROR(uint32_t word, int i)
 288 {
 289     return ( (word>>(i&31)) | (word<<(32-(i&31))) );
 290 }
 291
 292 #define CC_ROLc(x, y) CC_ROL(x, y)
 293 #define CC_RORc(x, y) CC_ROR(x, y)
 294
 295 #endif
 296
 297 // MARK: - 64 bits rotates
 298
 299 #if defined(__x86_64__) && !defined(_MSC_VER) //clang _MSVC doesn't support GNU-style inline assembly
 300 // MARK: -- intel 64 asm version
 301
 302 CC_INLINE uint64_t CC_ROL64(uint64_t word, int i)
 303 {
 304     __asm__("rolq %%cl,%0"
 305         :"=r" (word)
 306         :"0" (word),"c" (i));
 307     return word;
 308 }
 309
 310 CC_INLINE uint64_t CC_ROR64(uint64_t word, int i)
 311 {
 312     __asm__("rorq %%cl,%0"
 313         :"=r" (word)
 314         :"0" (word),"c" (i));
 315     return word;
 316 }
 317
 318 /* Need to be a macro here, because 'i' is an immediate (constant) */
 319 #define CC_ROL64c(word, i)      \
 320 ({                              \
 321     uint64_t _word=(word);      \
 322     __asm__("rolq %2,%0"        \
 323         :"=r" (_word)           \
 324         :"0" (_word),"J" (i));  \
 325     _word;                      \
 326 })
 327
 328 #define CC_ROR64c(word, i)      \
 329 ({                              \
 330     uint64_t _word=(word);      \
 331     __asm__("rorq %2,%0"        \
 332         :"=r" (_word)           \
 333         :"0" (_word),"J" (i));  \
 334     _word;                      \
 335 })
 336
 337
 338 #else /* Not x86_64  */
 339
 340 // MARK: -- default C version
 341
 342 CC_INLINE uint64_t CC_ROL64(uint64_t word, int i)
 343 {
 344     return ( (word<<(i&63)) | (word>>(64-(i&63))) );
 345 }
 346
 347 CC_INLINE uint64_t CC_ROR64(uint64_t word, int i)
 348 {
 349     return ( (word>>(i&63)) | (word<<(64-(i&63))) );
 350 }
 351
 352 #define CC_ROL64c(x, y) CC_ROL64(x, y)
 353 #define CC_ROR64c(x, y) CC_ROR64(x, y)
 354
 355 #endif
 356
 357
 358 // MARK: - Byte Swaps
 359
 360 #if __has_builtin(__builtin_bswap32)
 361 #define CC_BSWAP32(x) __builtin_bswap32(x)
 362 #else
 363 CC_INLINE uint32_t CC_BSWAP32(uint32_t x)
 364 {
 365     return
 366         ((x & 0xff000000) >> 24) |
 367         ((x & 0x00ff0000) >>  8) |
 368         ((x & 0x0000ff00) <<  8) |
 369         ((x & 0x000000ff) << 24);
 370 }
 371 #endif
 372
 373 #if __has_builtin(__builtin_bswap64)
 374 #define CC_BSWAP64(x) __builtin_bswap64(x)
 375 #else
 376 CC_INLINE uint64_t CC_BSWAP64(uint64_t x)
 377 {
 378     return
 379         ((x & 0xff00000000000000ULL) >> 56) |
 380         ((x & 0x00ff000000000000ULL) >> 40) |
 381         ((x & 0x0000ff0000000000ULL) >> 24) |
 382         ((x & 0x000000ff00000000ULL) >>  8) |
 383         ((x & 0x00000000ff000000ULL) <<  8) |
 384         ((x & 0x0000000000ff0000ULL) << 24) |
 385         ((x & 0x000000000000ff00ULL) << 40) |
 386         ((x & 0x00000000000000ffULL) << 56);
 387 }
 388 #endif
 389
 390 #ifdef __LITTLE_ENDIAN__
 391 #define CC_H2BE32(x) CC_BSWAP32(x)
 392 #define CC_H2LE32(x) (x)
 393 #define CC_H2BE64(x) CC_BSWAP64(x)
 394 #define CC_H2LE64(x) (x)
 395 #else
 396 #define CC_H2BE32(x) (x)
 397 #define CC_H2LE32(x) CC_BSWAP32(x)
 398 #define CC_H2BE64(x) (x)
 399 #define CC_H2LE64(x) CC_BSWAP64(x)
 400 #endif
 401
 402 #define CC_READ_LE32(ptr) \
 403 ( (uint32_t)( \
 404 ((uint32_t)((const uint8_t *)(ptr))[0]) | \
 405 (((uint32_t)((const uint8_t *)(ptr))[1]) <<  8) | \
 406 (((uint32_t)((const uint8_t *)(ptr))[2]) << 16) | \
 407 (((uint32_t)((const uint8_t *)(ptr))[3]) << 24)))
 408
 409 #define CC_WRITE_LE32(ptr, x) \
 410 do { \
 411 ((uint8_t *)(ptr))[0] = (uint8_t)( (x)        & 0xFF); \
 412 ((uint8_t *)(ptr))[1] = (uint8_t)(((x) >>  8) & 0xFF); \
 413 ((uint8_t *)(ptr))[2] = (uint8_t)(((x) >> 16) & 0xFF); \
 414 ((uint8_t *)(ptr))[3] = (uint8_t)(((x) >> 24) & 0xFF); \
 415 } while(0)
 416
 417 #define CC_WRITE_LE64(ptr, x) \
 418 do { \
 419 ((uint8_t *)(ptr))[0] = (uint8_t)( (x)        & 0xFF); \
 420 ((uint8_t *)(ptr))[1] = (uint8_t)(((x) >>  8) & 0xFF); \
 421 ((uint8_t *)(ptr))[2] = (uint8_t)(((x) >> 16) & 0xFF); \
 422 ((uint8_t *)(ptr))[3] = (uint8_t)(((x) >> 24) & 0xFF); \
 423 ((uint8_t *)(ptr))[4] = (uint8_t)(((x) >> 32) & 0xFF); \
 424 ((uint8_t *)(ptr))[5] = (uint8_t)(((x) >> 40) & 0xFF); \
 425 ((uint8_t *)(ptr))[6] = (uint8_t)(((x) >> 48) & 0xFF); \
 426 ((uint8_t *)(ptr))[7] = (uint8_t)(((x) >> 56) & 0xFF); \
 427 } while(0)
 428
 429 /* extract a byte portably */
 430 #ifdef _MSC_VER
 431 #define cc_byte(x, n) ((unsigned char)((x) >> (8 * (n))))
 432 #else
 433 #define cc_byte(x, n) (((x) >> (8 * (n))) & 255)
 434 #endif
 435
 436 /* Count leading zeros (for nonzero inputs) */
 437
 438 /*
 439  *  On i386 and x86_64, we know clang and GCC will generate BSR for
 440  *  __builtin_clzl.  This instruction IS NOT constant time on all micro-
 441  *  architectures, but it *is* constant time on all micro-architectures that
 442  *  have been used by Apple, and we expect that to continue to be the case.
 443  *
 444  *  When building for x86_64h with clang, this produces LZCNT, which is exactly
 445  *  what we want.
 446  *
 447  *  On arm and arm64, we know that clang and GCC generate the constant-time CLZ
 448  *  instruction from __builtin_clzl( ).
 449  */
 450
 451 #if defined(_WIN32)
 452 /* We use the Windows implementations below. */
 453 #elif defined(__x86_64__) || defined(__i386__) || defined(__arm64__) || defined(__arm__)
 454 /* We use a thought-to-be-good version of __builtin_clz. */
 455 #elif defined __GNUC__
 456 #warning Using __builtin_clz() on an unknown architecture; it may not be constant-time.
 457 /* If you find yourself seeing this warning, file a radar for someone to
 458  * check whether or not __builtin_clz() generates a constant-time
 459  * implementation on the architecture you are targeting.  If it does, append
 460  * the name of that architecture to the list of "safe" architectures above.  */
 461 #endif
 462
 463 CC_INLINE CC_CONST unsigned cc_clz32_fallback(uint32_t data)
 464 {
 465     unsigned int b = 0;
 466     unsigned int bit = 0;
 467     // Work from LSB to MSB
 468     for (int i = 0; i < 32; i++) {
 469         bit = (data >> i) & 1;
 470         // If the bit is 0, update the "leading bits are zero" counter "b".
 471         b += (1 - bit);
 472         /* If the bit is 0, (bit - 1) is 0xffff... therefore b is retained.
 473          * If the bit is 1, (bit - 1) is 0 therefore b is set to 0.
 474          */
 475         b &= (bit - 1);
 476     }
 477     return b;
 478 }
 479
 480 CC_INLINE CC_CONST unsigned cc_clz64_fallback(uint64_t data)
 481 {
 482     unsigned int b = 0;
 483     unsigned int bit = 0;
 484     // Work from LSB to MSB
 485     for (int i = 0; i < 64; i++) {
 486         bit = (data >> i) & 1;
 487         // If the bit is 0, update the "leading bits are zero" counter.
 488         b += (1 - bit);
 489         /* If the bit is 0, (bit - 1) is 0xffff... therefore b is retained.
 490          * If the bit is 1, (bit - 1) is 0 therefore b is set to 0.
 491          */
 492         b &= (bit - 1);
 493     }
 494     return b;
 495 }
 496
 497 CC_INLINE CC_CONST unsigned cc_ctz32_fallback(uint32_t data)
 498 {
 499     unsigned int b = 0;
 500     unsigned int bit = 0;
 501     // Work from MSB to LSB
 502     for (int i = 31; i >= 0; i--) {
 503         bit = (data >> i) & 1;
 504         // If the bit is 0, update the "trailing zero bits" counter.
 505         b += (1 - bit);
 506         /* If the bit is 0, (bit - 1) is 0xffff... therefore b is retained.
 507          * If the bit is 1, (bit - 1) is 0 therefore b is set to 0.
 508          */
 509         b &= (bit - 1);
 510     }
 511     return b;
 512 }
 513
 514 CC_INLINE CC_CONST unsigned cc_ctz64_fallback(uint64_t data)
 515 {
 516     unsigned int b = 0;
 517     unsigned int bit = 0;
 518     // Work from MSB to LSB
 519     for (int i = 63; i >= 0; i--) {
 520         bit = (data >> i) & 1;
 521         // If the bit is 0, update the "trailing zero bits" counter.
 522         b += (1 - bit);
 523         /* If the bit is 0, (bit - 1) is 0xffff... therefore b is retained.
 524          * If the bit is 1, (bit - 1) is 0 therefore b is set to 0.
 525          */
 526         b &= (bit - 1);
 527     }
 528     return b;
 529 }
 530
 531 /*!
 532   @function cc_clz32
 533   @abstract Count leading zeros of a nonzero 32-bit value
 534
 535   @param data A nonzero 32-bit value
 536
 537   @result Count of leading zeros of @p data
 538
 539   @discussion @p data is assumed to be nonzero.
 540 */
 541 CC_INLINE CC_CONST unsigned cc_clz32(uint32_t data) {
 542     cc_assert(data != 0);
 543 #if defined(_WIN32)
 544     return cc_clz32_fallback(data);
 545 #elif defined(__x86_64__) || defined(__i386__) || defined(__arm64__) || defined(__arm__) || defined(__GNUC__)
 546     cc_static_assert(sizeof(unsigned) == 4, "clz relies on an unsigned int being 4 bytes");
 547     return (unsigned)__builtin_clz(data);
 548 #else
 549     return cc_clz32_fallback(data);
 550 #endif
 551 }
 552
 553 /*!
 554   @function cc_clz64
 555   @abstract Count leading zeros of a nonzero 64-bit value
 556
 557   @param data A nonzero 64-bit value
 558
 559   @result Count of leading zeros of @p data
 560
 561   @discussion @p data is assumed to be nonzero.
 562 */
 563 CC_INLINE CC_CONST unsigned cc_clz64(uint64_t data) {
 564     cc_assert(data != 0);
 565 #if defined(_WIN32)
 566     return cc_clz64_fallback(data);
 567 #elif defined(__x86_64__) || defined(__i386__) || defined(__arm64__) || defined(__arm__) || defined(__GNUC__)
 568     return (unsigned)__builtin_clzll(data);
 569 #else
 570     return cc_clz64_fallback(data);
 571 #endif
 572 }
 573
 574 /*!
 575   @function cc_ctz32
 576   @abstract Count trailing zeros of a nonzero 32-bit value
 577
 578   @param data A nonzero 32-bit value
 579
 580   @result Count of trailing zeros of @p data
 581
 582   @discussion @p data is assumed to be nonzero.
 583 */
 584 CC_INLINE CC_CONST unsigned cc_ctz32(uint32_t data) {
 585     cc_assert(data != 0);
 586 #if defined(_WIN32)
 587     return cc_ctz32_fallback(data);
 588 #elif defined(__x86_64__) || defined(__i386__) || defined(__arm64__) || defined(__arm__) || defined(__GNUC__)
 589     cc_static_assert(sizeof(unsigned) == 4, "ctz relies on an unsigned int being 4 bytes");
 590     return (unsigned)__builtin_ctz(data);
 591 #else
 592     return cc_ctz32_fallback(data);
 593 #endif
 594 }
 595
 596 /*!
 597   @function cc_ctz64
 598   @abstract Count trailing zeros of a nonzero 64-bit value
 599
 600   @param data A nonzero 64-bit value
 601
 602   @result Count of trailing zeros of @p data
 603
 604   @discussion @p data is assumed to be nonzero.
 605 */
 606 CC_INLINE CC_CONST unsigned cc_ctz64(uint64_t data) {
 607     cc_assert(data != 0);
 608 #if defined(_WIN32)
 609     return cc_ctz64_fallback(data);
 610 #elif defined(__x86_64__) || defined(__i386__) || defined(__arm64__) || defined(__arm__) || defined(__GNUC__)
 611     return (unsigned)__builtin_ctzll(data);
 612 #else
 613     return cc_ctz64_fallback(data);
 614 #endif
 615 }
 616
 617 /*!
 618   @function cc_ffs32_fallback
 619   @abstract Find first bit set in a 32-bit value
 620
 621   @param data A 32-bit value
 622
 623   @result One plus the index of the least-significant bit set in @p data or, if @p data is zero, zero
 624  */
 625 CC_INLINE CC_CONST unsigned cc_ffs32_fallback(int32_t data)
 626 {
 627     unsigned b = 0;
 628     unsigned bit = 0;
 629     unsigned seen = 0;
 630
 631     // Work from LSB to MSB
 632     for (int i = 0; i < 32; i++) {
 633         bit = ((uint32_t)data >> i) & 1;
 634
 635         // Track whether we've seen a 1 bit.
 636         seen |= bit;
 637
 638         // If the bit is 0 and we haven't seen a 1 yet, increment b.
 639         b += (1 - bit) & (seen - 1);
 640     }
 641
 642     // If we saw a 1, return b + 1, else 0.
 643     return (~(seen - 1)) & (b + 1);
 644 }
 645
 646 /*!
 647   @function cc_ffs64_fallback
 648   @abstract Find first bit set in a 64-bit value
 649
 650   @param data A 64-bit value
 651
 652   @result One plus the index of the least-significant bit set in @p data or, if @p data is zero, zero
 653  */
 654 CC_INLINE CC_CONST unsigned cc_ffs64_fallback(int64_t data)
 655 {
 656     unsigned b = 0;
 657     unsigned bit = 0;
 658     unsigned seen = 0;
 659
 660     // Work from LSB to MSB
 661     for (int i = 0; i < 64; i++) {
 662         bit = ((uint64_t)data >> i) & 1;
 663
 664         // Track whether we've seen a 1 bit.
 665         seen |= bit;
 666
 667         // If the bit is 0 and we haven't seen a 1 yet, increment b.
 668         b += (1 - bit) & (seen - 1);
 669     }
 670
 671     // If we saw a 1, return b + 1, else 0.
 672     return (~(seen - 1)) & (b + 1);
 673 }
 674
 675 /*!
 676   @function cc_ffs32
 677   @abstract Find first bit set in a 32-bit value
 678
 679   @param data A 32-bit value
 680
 681   @result One plus the index of the least-significant bit set in @p data or, if @p data is zero, zero
 682  */
 683 CC_INLINE CC_CONST unsigned cc_ffs32(int32_t data)
 684 {
 685     cc_static_assert(sizeof(int) == 4, "ffs relies on an int being 4 bytes");
 686 #ifdef _WIN32
 687     return cc_ffs32_fallback(data);
 688 #else
 689     return (unsigned)__builtin_ffs(data);
 690 #endif
 691 }
 692
 693 /*!
 694   @function cc_ffs64
 695   @abstract Find first bit set in a 64-bit value
 696
 697   @param data A 64-bit value
 698
 699   @result One plus the index of the least-significant bit set in @p data or, if @p data is zero, zero
 700  */
 701 CC_INLINE CC_CONST unsigned cc_ffs64(int64_t data)
 702 {
 703 #ifdef _WIN32
 704     return cc_ffs64_fallback(data);
 705 #else
 706     return (unsigned)__builtin_ffsll(data);
 707 #endif
 708 }
 709
 710 #define cc_add_overflow __builtin_add_overflow
 711 #define cc_mul_overflow __builtin_mul_overflow
 712
 713 /* HEAVISIDE_STEP (shifted by one)
 714    function f(x): x->0, when x=0
 715                   x->1, when x>0
 716    Can also be seen as a bitwise operation:
 717       f(x): x -> y
 718         y[0]=(OR x[i]) for all i (all bits)
 719         y[i]=0 for all i>0
 720    Run in constant time (log2(<bitsize of x>))
 721    Useful to run constant time checks
 722 */
 723 #define CC_HEAVISIDE_STEP(r, s) {                       \
 724     const uint64_t _s = (uint64_t)s;                    \
 725     const uint64_t _t = (_s & 0xffffffff) | (_s >> 32); \
 726     r = (__typeof__(r))((0xffffffff + _t) >> 32);       \
 727 }
 728
 729 /* Return 1 if x mod 4 =1,2,3, 0 otherwise */
 730 #define CC_CARRY_2BITS(x) (((x>>1) | x) & 0x1)
 731 #define CC_CARRY_3BITS(x) (((x>>2) | (x>>1) | x) & 0x1)
 732
 733 #define cc_ceiling(a,b)  (((a)+((b)-1))/(b))
 734 #define CC_BITLEN_TO_BYTELEN(x) cc_ceiling((x), 8)
 735
 736 /*!
 737  @brief     cc_muxp(s, a, b) is equivalent to z = s ? a : b, but it executes in constant time
 738  @param a       input pointer
 739  @param b       input pointer
 740  @param s       The selection parameter s must be 0 or 1. if s is integer 1 a is returned. If s is integer 0, b is returned. Otherwise, the output is undefined.
 741  @return    Returns a, if s is 1 and b if s is 0
 742  */
 743 void *cc_muxp(int s, const void *a, const void *b);
 744
 745 /*!
 746  @brief     CC_MUXU(r, s, a, b) is equivalent to r = s ? a : b, but executes in constant time
 747  @param a   Input a
 748  @param b   Input b
 749  @param s   Selection parameter s. Must be 0 or 1.
 750  @param r   Output, set to a if s=1, or b if s=0.
 751  */
 752 #define CC_MUXU(r, s, a, b)                           \
 753     {                                                 \
 754         __typeof__(r) _cond = (__typeof__(r))((s)-1); \
 755         r = (~_cond & (a)) | (_cond & (b));           \
 756     }
 757
 758 #define CC_PROVIDES_ABORT (!(CC_USE_SEPROM || CC_USE_S3 || CC_BASEBAND || CC_EFI || CC_IBOOT || CC_RTKITROM))
 759
 760 /*!
 761  @function cc_abort
 762  @abstract Abort execution unconditionally
 763  */
 764 CC_NORETURN
 765 void cc_abort(const char *msg);
 766
 767 /*!
 768   @function cc_try_abort
 769   @abstract Abort execution iff the platform provides a function like @p abort() or @p panic()
 770
 771   @discussion If the platform does not provide a means to abort execution, this function does nothing; therefore, callers should return an error code after calling this function.
 772 */
 773 #if CC_PROVIDES_ABORT
 774
 775 #pragma clang diagnostic push
 776 #pragma clang diagnostic ignored "-Wmissing-noreturn"
 777
 778 CC_INLINE
 779 void cc_try_abort(const char *msg)
 780 {
 781     cc_abort(msg);
 782 }
 783
 784 #pragma clang diagnostic pop
 785
 786 #else
 787
 788 CC_INLINE
 789 void cc_try_abort(CC_UNUSED const char *msg)
 790 {
 791
 792 }
 793
 794 #endif
 795
 796 #if __has_builtin(__builtin_expect)
 797  #define CC_UNLIKELY(cond) __builtin_expect(cond, 0)
 798 #else
 799  #define CC_UNLIKELY(cond) cond
 800 #endif
 801
 802 CC_INLINE
 803 void cc_try_abort_if(bool condition, const char *msg)
 804 {
 805     if (CC_UNLIKELY(condition)) {
 806         cc_try_abort(msg);
 807     }
 808 }
 809
 810 /*
 811   Unfortunately, since we export this symbol, this declaration needs
 812   to be in a public header to satisfy TAPI.
 813
 814   See fipspost_trace_priv.h for more details.
 815 */
 816 extern const void *fipspost_trace_vtable;
 817
 818 #endif /* _CORECRYPTO_CC_PRIV_H_ */