OSX/libsecurity_apple_csp/open_ssl/rsa/rsa_eay.c

   1 /*
   2  * Copyright (c) 2000-2001,2011,2014 Apple Inc. All Rights Reserved.
   3  *
   4  * The contents of this file constitute Original Code as defined in and are
   5  * subject to the Apple Public Source License Version 1.2 (the 'License').
   6  * You may not use this file except in compliance with the License. Please obtain
   7  * a copy of the License at http://www.apple.com/publicsource and read it before
   8  * using this file.
   9  *
  10  * This Original Code and all software distributed under the License are
  11  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS
  12  * OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, INCLUDING WITHOUT
  13  * LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  14  * PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. Please see the License for the
  15  * specific language governing rights and limitations under the License.
  16  */
  17
  18
  19 /* crypto/rsa/rsa_eay.c */
  20 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  21  * All rights reserved.
  22  *
  23  * This package is an SSL implementation written
  24  * by Eric Young (eay@cryptsoft.com).
  25  * The implementation was written so as to conform with Netscapes SSL.
  26  *
  27  * This library is free for commercial and non-commercial use as long as
  28  * the following conditions are aheared to.  The following conditions
  29  * apply to all code found in this distribution, be it the RC4, RSA,
  30  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
  31  * included with this distribution is covered by the same copyright terms
  32  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
  33  *
  34  * Copyright remains Eric Young's, and as such any Copyright notices in
  35  * the code are not to be removed.
  36  * If this package is used in a product, Eric Young should be given attribution
  37  * as the author of the parts of the library used.
  38  * This can be in the form of a textual message at program startup or
  39  * in documentation (online or textual) provided with the package.
  40  *
  41  * Redistribution and use in source and binary forms, with or without
  42  * modification, are permitted provided that the following conditions
  43  * are met:
  44  * 1. Redistributions of source code must retain the copyright
  45  *    notice, this list of conditions and the following disclaimer.
  46  * 2. Redistributions in binary form must reproduce the above copyright
  47  *    notice, this list of conditions and the following disclaimer in the
  48  *    documentation and/or other materials provided with the distribution.
  49  * 3. All advertising materials mentioning features or use of this software
  50  *    must display the following acknowledgement:
  51  *    "This product includes cryptographic software written by
  52  *     Eric Young (eay@cryptsoft.com)"
  53  *    The word 'cryptographic' can be left out if the rouines from the library
  54  *    being used are not cryptographic related :-).
  55  * 4. If you include any Windows specific code (or a derivative thereof) from
  56  *    the apps directory (application code) you must include an acknowledgement:
  57  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
  58  *
  59  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
  60  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  61  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  62  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  63  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  64  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  65  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  66  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  67  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  68  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  69  * SUCH DAMAGE.
  70  *
  71  * The licence and distribution terms for any publically available version or
  72  * derivative of this code cannot be changed.  i.e. this code cannot simply be
  73  * copied and put under another distribution licence
  74  * [including the GNU Public Licence.]
  75  */
  76
  77 #include <stdio.h>
  78 #include "cryptlib.h"
  79 #include <openssl/bn.h>
  80 #include <openssl/rsa.h>
  81 #include <openssl/rand.h>
  82 #include <pthread.h>
  83
  84 #ifndef RSA_NULL
  85
  86 static int RSA_eay_public_encrypt(int flen, unsigned char *from,
  87                 unsigned char *to, RSA *rsa,int padding);
  88 static int RSA_eay_private_encrypt(int flen, unsigned char *from,
  89                 unsigned char *to, RSA *rsa,int padding);
  90 static int RSA_eay_public_decrypt(int flen, unsigned char *from,
  91                 unsigned char *to, RSA *rsa,int padding);
  92 static int RSA_eay_private_decrypt(int flen, unsigned char *from,
  93                 unsigned char *to, RSA *rsa,int padding);
  94 static int RSA_eay_mod_exp(BIGNUM *r0, BIGNUM *i, RSA *rsa);
  95 static int RSA_eay_init(RSA *rsa);
  96 static int RSA_eay_finish(RSA *rsa);
  97 static const RSA_METHOD rsa_pkcs1_eay_meth={
  98         "Eric Young's PKCS#1 RSA",
  99         RSA_eay_public_encrypt,
 100         RSA_eay_public_decrypt,
 101         RSA_eay_private_encrypt,
 102         RSA_eay_private_decrypt,
 103         RSA_eay_mod_exp,
 104         BN_mod_exp_mont,
 105         RSA_eay_init,
 106         RSA_eay_finish,
 107         0,
 108         NULL,
 109         };
 110
 111 const RSA_METHOD *RSA_PKCS1_SSLeay(void)
 112         {
 113         return(&rsa_pkcs1_eay_meth);
 114         }
 115
 116 static int RSA_eay_public_encrypt(int flen, unsigned char *from,
 117              unsigned char *to, RSA *rsa, int padding)
 118         {
 119         BIGNUM f,ret;
 120         int i,j,k,num=0,r= -1;
 121         unsigned char *buf=NULL;
 122         BN_CTX *ctx=NULL;
 123
 124         BN_init(&f);
 125         BN_init(&ret);
 126         if ((ctx=BN_CTX_new()) == NULL) goto err;
 127         num=BN_num_bytes(rsa->n);
 128         if ((buf=(unsigned char *)Malloc(num)) == NULL)
 129                 {
 130                 RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);
 131                 goto err;
 132                 }
 133
 134         switch (padding)
 135                 {
 136         case RSA_PKCS1_PADDING:
 137                 i=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);
 138                 break;
 139 #ifndef _OPENSSL_APPLE_CDSA_
 140 #ifndef NO_SHA
 141         case RSA_PKCS1_OAEP_PADDING:
 142                 i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
 143                 break;
 144 #endif
 145 #endif
 146         case RSA_SSLV23_PADDING:
 147                 i=RSA_padding_add_SSLv23(buf,num,from,flen);
 148                 break;
 149         case RSA_NO_PADDING:
 150                 i=RSA_padding_add_none(buf,num,from,flen);
 151                 break;
 152         default:
 153                 RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
 154                 goto err;
 155                 }
 156         if (i <= 0) goto err;
 157
 158         if (BN_bin2bn(buf,num,&f) == NULL) goto err;
 159
 160         if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
 161                 {
 162                 if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)
 163                         if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))
 164                             goto err;
 165                 }
 166
 167         if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
 168                 rsa->_method_mod_n)) goto err;
 169
 170         /* put in leading 0 bytes if the number is less than the
 171          * length of the modulus */
 172         j=BN_num_bytes(&ret);
 173         i=BN_bn2bin(&ret,&(to[num-j]));
 174         for (k=0; k<(num-i); k++)
 175                 to[k]=0;
 176
 177         r=num;
 178 err:
 179         if (ctx != NULL) BN_CTX_free(ctx);
 180         BN_clear_free(&f);
 181         BN_clear_free(&ret);
 182         if (buf != NULL)
 183                 {
 184                 memset(buf,0,num);
 185                 Free(buf);
 186                 }
 187         return(r);
 188         }
 189
 190 static int RSA_eay_private_encrypt(int flen, unsigned char *from,
 191              unsigned char *to, RSA *rsa, int padding)
 192         {
 193         BIGNUM f,ret;
 194         int i,j,k,num=0,r= -1;
 195         unsigned char *buf=NULL;
 196         BN_CTX *ctx=NULL;
 197
 198         BN_init(&f);
 199         BN_init(&ret);
 200
 201         if ((ctx=BN_CTX_new()) == NULL) goto err;
 202         num=BN_num_bytes(rsa->n);
 203         if ((buf=(unsigned char *)Malloc(num)) == NULL)
 204                 {
 205                 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,ERR_R_MALLOC_FAILURE);
 206                 goto err;
 207                 }
 208
 209         switch (padding)
 210                 {
 211         case RSA_PKCS1_PADDING:
 212                 i=RSA_padding_add_PKCS1_type_1(buf,num,from,flen);
 213                 break;
 214         case RSA_NO_PADDING:
 215                 i=RSA_padding_add_none(buf,num,from,flen);
 216                 break;
 217         case RSA_SSLV23_PADDING:
 218         default:
 219                 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
 220                 goto err;
 221                 }
 222         if (i <= 0) goto err;
 223
 224         if (BN_bin2bn(buf,num,&f) == NULL) goto err;
 225
 226         if ((rsa->flags & RSA_FLAG_BLINDING) && (RSA_get_thread_blinding_ptr(rsa) == NULL))
 227                 RSA_blinding_on(rsa,ctx);
 228         if (rsa->flags & RSA_FLAG_BLINDING)
 229                 if (!BN_BLINDING_convert(&f,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err;
 230
 231         if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
 232                 ((rsa->p != NULL) &&
 233                 (rsa->q != NULL) &&
 234                 (rsa->dmp1 != NULL) &&
 235                 (rsa->dmq1 != NULL) &&
 236                 (rsa->iqmp != NULL)) )
 237                 { if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
 238         else
 239                 {
 240                 if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err;
 241                 }
 242
 243         if (rsa->flags & RSA_FLAG_BLINDING)
 244                 if (!BN_BLINDING_invert(&ret,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err;
 245
 246         /* put in leading 0 bytes if the number is less than the
 247          * length of the modulus */
 248         j=BN_num_bytes(&ret);
 249         i=BN_bn2bin(&ret,&(to[num-j]));
 250         for (k=0; k<(num-i); k++)
 251                 to[k]=0;
 252
 253         r=num;
 254 err:
 255         if (ctx != NULL) BN_CTX_free(ctx);
 256         BN_clear_free(&ret);
 257         BN_clear_free(&f);
 258         if (buf != NULL)
 259                 {
 260                 memset(buf,0,num);
 261                 Free(buf);
 262                 }
 263         return(r);
 264         }
 265
 266 static int RSA_eay_private_decrypt(int flen, unsigned char *from,
 267              unsigned char *to, RSA *rsa, int padding)
 268         {
 269         BIGNUM f,ret;
 270         int j,num=0,r= -1;
 271         unsigned char *p;
 272         unsigned char *buf=NULL;
 273         BN_CTX *ctx=NULL;
 274
 275         BN_init(&f);
 276         BN_init(&ret);
 277         ctx=BN_CTX_new();
 278         if (ctx == NULL) goto err;
 279
 280         num=BN_num_bytes(rsa->n);
 281
 282         if ((buf=(unsigned char *)Malloc(num)) == NULL)
 283                 {
 284                 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);
 285                 goto err;
 286                 }
 287
 288         /* This check was for equality but PGP does evil things
 289          * and chops off the top '0' bytes */
 290         if (flen > num)
 291                 {
 292                 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
 293                 goto err;
 294                 }
 295
 296         /* make data into a big number */
 297         if (BN_bin2bn(from,(int)flen,&f) == NULL) goto err;
 298
 299         if ((rsa->flags & RSA_FLAG_BLINDING) && (RSA_get_thread_blinding_ptr(rsa) == NULL))
 300                 RSA_blinding_on(rsa,ctx);
 301         if (rsa->flags & RSA_FLAG_BLINDING)
 302                 if (!BN_BLINDING_convert(&f,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err;
 303
 304         /* do the decrypt */
 305         if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
 306                 ((rsa->p != NULL) &&
 307                 (rsa->q != NULL) &&
 308                 (rsa->dmp1 != NULL) &&
 309                 (rsa->dmq1 != NULL) &&
 310                 (rsa->iqmp != NULL)) )
 311                 { if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
 312         else
 313                 {
 314                 if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))
 315                         goto err;
 316                 }
 317
 318         if (rsa->flags & RSA_FLAG_BLINDING)
 319                 if (!BN_BLINDING_invert(&ret,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err;
 320
 321         p=buf;
 322         j=BN_bn2bin(&ret,p); /* j is only used with no-padding mode */
 323
 324         switch (padding)
 325                 {
 326         case RSA_PKCS1_PADDING:
 327                 r=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num);
 328                 break;
 329 #ifndef _OPENSSL_APPLE_CDSA_
 330 #ifndef NO_SHA
 331         case RSA_PKCS1_OAEP_PADDING:
 332                 r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0);
 333                 break;
 334 #endif
 335 #endif
 336         case RSA_SSLV23_PADDING:
 337                 r=RSA_padding_check_SSLv23(to,num,buf,j,num);
 338                 break;
 339         case RSA_NO_PADDING:
 340                 r=RSA_padding_check_none(to,num,buf,j,num);
 341                 break;
 342         default:
 343                 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
 344                 goto err;
 345                 }
 346         if (r < 0)
 347                 RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
 348
 349 err:
 350         if (ctx != NULL) BN_CTX_free(ctx);
 351         BN_clear_free(&f);
 352         BN_clear_free(&ret);
 353         if (buf != NULL)
 354                 {
 355                 memset(buf,0,num);
 356                 Free(buf);
 357                 }
 358         return(r);
 359         }
 360
 361 static int RSA_eay_public_decrypt(int flen, unsigned char *from,
 362              unsigned char *to, RSA *rsa, int padding)
 363         {
 364         BIGNUM f,ret;
 365         int i,num=0,r= -1;
 366         unsigned char *p;
 367         unsigned char *buf=NULL;
 368         BN_CTX *ctx=NULL;
 369
 370         BN_init(&f);
 371         BN_init(&ret);
 372         ctx=BN_CTX_new();
 373         if (ctx == NULL) goto err;
 374
 375         num=BN_num_bytes(rsa->n);
 376         buf=(unsigned char *)Malloc(num);
 377         if (buf == NULL)
 378                 {
 379                 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE);
 380                 goto err;
 381                 }
 382
 383         /* This check was for equality but PGP does evil things
 384          * and chops off the top '0' bytes */
 385         if (flen > num)
 386                 {
 387                 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
 388                 goto err;
 389                 }
 390
 391         if (BN_bin2bn(from,flen,&f) == NULL) goto err;
 392         /* do the decrypt */
 393         if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
 394                 {
 395                 if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)
 396                         if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))
 397                             goto err;
 398                 }
 399
 400         if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
 401                 rsa->_method_mod_n)) goto err;
 402
 403         p=buf;
 404         i=BN_bn2bin(&ret,p);
 405
 406         switch (padding)
 407                 {
 408         case RSA_PKCS1_PADDING:
 409                 r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num);
 410                 break;
 411         case RSA_NO_PADDING:
 412                 r=RSA_padding_check_none(to,num,buf,i,num);
 413                 break;
 414         default:
 415                 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
 416                 goto err;
 417                 }
 418         if (r < 0)
 419                 RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
 420
 421 err:
 422         if (ctx != NULL) BN_CTX_free(ctx);
 423         BN_clear_free(&f);
 424         BN_clear_free(&ret);
 425         if (buf != NULL)
 426                 {
 427                 memset(buf,0,num);
 428                 Free(buf);
 429                 }
 430         return(r);
 431         }
 432
 433 static int RSA_eay_mod_exp(BIGNUM *r0, BIGNUM *I, RSA *rsa)
 434         {
 435         BIGNUM r1,m1;
 436         int ret=0;
 437         BN_CTX *ctx;
 438
 439         if ((ctx=BN_CTX_new()) == NULL) goto err;
 440         BN_init(&m1);
 441         BN_init(&r1);
 442
 443         if (rsa->flags & RSA_FLAG_CACHE_PRIVATE)
 444                 {
 445                 if (rsa->_method_mod_p == NULL)
 446                         {
 447                         if ((rsa->_method_mod_p=BN_MONT_CTX_new()) != NULL)
 448                                 if (!BN_MONT_CTX_set(rsa->_method_mod_p,rsa->p,
 449                                                      ctx))
 450                                         goto err;
 451                         }
 452                 if (rsa->_method_mod_q == NULL)
 453                         {
 454                         if ((rsa->_method_mod_q=BN_MONT_CTX_new()) != NULL)
 455                                 if (!BN_MONT_CTX_set(rsa->_method_mod_q,rsa->q,
 456                                                      ctx))
 457                                         goto err;
 458                         }
 459                 }
 460
 461         if (!BN_mod(&r1,I,rsa->q,ctx)) goto err;
 462         if (!rsa->meth->bn_mod_exp(&m1,&r1,rsa->dmq1,rsa->q,ctx,
 463                 rsa->_method_mod_q)) goto err;
 464
 465         if (!BN_mod(&r1,I,rsa->p,ctx)) goto err;
 466         if (!rsa->meth->bn_mod_exp(r0,&r1,rsa->dmp1,rsa->p,ctx,
 467                 rsa->_method_mod_p)) goto err;
 468
 469         if (!BN_sub(r0,r0,&m1)) goto err;
 470         /* This will help stop the size of r0 increasing, which does
 471          * affect the multiply if it optimised for a power of 2 size */
 472         if (r0->neg)
 473                 if (!BN_add(r0,r0,rsa->p)) goto err;
 474
 475         if (!BN_mul(&r1,r0,rsa->iqmp,ctx)) goto err;
 476         if (!BN_mod(r0,&r1,rsa->p,ctx)) goto err;
 477         /* If p < q it is occasionally possible for the correction of
 478          * adding 'p' if r0 is negative above to leave the result still
 479          * negative. This can break the private key operations: the following
 480          * second correction should *always* correct this rare occurrence.
 481          * This will *never* happen with OpenSSL generated keys because
 482          * they ensure p > q [steve]
 483          */
 484         if (r0->neg)
 485                 if (!BN_add(r0,r0,rsa->p)) goto err;
 486         if (!BN_mul(&r1,r0,rsa->q,ctx)) goto err;
 487         if (!BN_add(r0,&r1,&m1)) goto err;
 488
 489         ret=1;
 490 err:
 491         BN_clear_free(&m1);
 492         BN_clear_free(&r1);
 493         BN_CTX_free(ctx);
 494         return(ret);
 495         }
 496
 497 static int RSA_eay_init(RSA *rsa)
 498         {
 499         rsa->flags|=RSA_FLAG_CACHE_PUBLIC|RSA_FLAG_CACHE_PRIVATE;
 500         return(1);
 501         }
 502
 503 static int RSA_eay_finish(RSA *rsa)
 504         {
 505         if (rsa->_method_mod_n != NULL)
 506                 BN_MONT_CTX_free(rsa->_method_mod_n);
 507         if (rsa->_method_mod_p != NULL)
 508                 BN_MONT_CTX_free(rsa->_method_mod_p);
 509         if (rsa->_method_mod_q != NULL)
 510                 BN_MONT_CTX_free(rsa->_method_mod_q);
 511         return(1);
 512         }
 513
 514 #endif