1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 /*
  27  * RSA provider for the Kernel Cryptographic Framework (KCF)
  28  */
  29 
  30 #include <sys/types.h>
  31 #include <sys/systm.h>
  32 #include <sys/modctl.h>
  33 #include <sys/cmn_err.h>
  34 #include <sys/ddi.h>
  35 #include <sys/crypto/spi.h>
  36 #include <sys/sysmacros.h>
  37 #include <sys/strsun.h>
  38 #include <sys/md5.h>
  39 #include <sys/sha1.h>
  40 #define _SHA2_IMPL
  41 #include <sys/sha2.h>
  42 #include <sys/random.h>
  43 #include <sys/crypto/impl.h>
  44 #include <sha1/sha1_impl.h>
  45 #include <sha2/sha2_impl.h>
  46 #include <padding/padding.h>
  47 #include <rsa/rsa_impl.h>
  48 
  49 extern struct mod_ops mod_cryptoops;
  50 
  51 /*
  52  * Module linkage information for the kernel.
  53  */
  54 static struct modlcrypto modlcrypto = {
  55         &mod_cryptoops,
  56         "RSA Kernel SW Provider"
  57 };
  58 
  59 static struct modlinkage modlinkage = {
  60         MODREV_1,
  61         (void *)&modlcrypto,
  62         NULL
  63 };
  64 
  65 /*
  66  * CSPI information (entry points, provider info, etc.)
  67  */
  68 typedef enum rsa_mech_type {
  69         RSA_PKCS_MECH_INFO_TYPE,        /* SUN_CKM_RSA_PKCS */
  70         RSA_X_509_MECH_INFO_TYPE,       /* SUN_CKM_RSA_X_509 */
  71         MD5_RSA_PKCS_MECH_INFO_TYPE,    /* SUN_MD5_RSA_PKCS */
  72         SHA1_RSA_PKCS_MECH_INFO_TYPE,   /* SUN_SHA1_RSA_PKCS */
  73         SHA256_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA256_RSA_PKCS */
  74         SHA384_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA384_RSA_PKCS */
  75         SHA512_RSA_PKCS_MECH_INFO_TYPE  /* SUN_SHA512_RSA_PKCS */
  76 } rsa_mech_type_t;
  77 
  78 /*
  79  * Context for RSA_PKCS and RSA_X_509 mechanisms.
  80  */
  81 typedef struct rsa_ctx {
  82         rsa_mech_type_t mech_type;
  83         crypto_key_t *key;
  84         size_t keychunk_size;
  85 } rsa_ctx_t;
  86 
  87 /*
  88  * Context for MD5_RSA_PKCS and SHA*_RSA_PKCS mechanisms.
  89  */
  90 typedef struct digest_rsa_ctx {
  91         rsa_mech_type_t mech_type;
  92         crypto_key_t *key;
  93         size_t keychunk_size;
  94         union {
  95                 MD5_CTX md5ctx;
  96                 SHA1_CTX sha1ctx;
  97                 SHA2_CTX sha2ctx;
  98         } dctx_u;
  99 } digest_rsa_ctx_t;
 100 
 101 #define md5_ctx         dctx_u.md5ctx
 102 #define sha1_ctx        dctx_u.sha1ctx
 103 #define sha2_ctx        dctx_u.sha2ctx
 104 
 105 /*
 106  * Mechanism info structure passed to KCF during registration.
 107  */
 108 static crypto_mech_info_t rsa_mech_info_tab[] = {
 109         /* RSA_PKCS */
 110         {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
 111             CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
 112             CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
 113             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 114             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
 115             CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
 116             CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
 117             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 118 
 119         /* RSA_X_509 */
 120         {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
 121             CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
 122             CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
 123             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 124             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
 125             CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
 126             CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
 127             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 128 
 129         /* MD5_RSA_PKCS */
 130         {SUN_CKM_MD5_RSA_PKCS, MD5_RSA_PKCS_MECH_INFO_TYPE,
 131             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 132             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 133             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 134 
 135         /* SHA1_RSA_PKCS */
 136         {SUN_CKM_SHA1_RSA_PKCS, SHA1_RSA_PKCS_MECH_INFO_TYPE,
 137             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 138             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 139             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 140 
 141         /* SHA256_RSA_PKCS */
 142         {SUN_CKM_SHA256_RSA_PKCS, SHA256_RSA_PKCS_MECH_INFO_TYPE,
 143             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 144             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 145             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 146 
 147         /* SHA384_RSA_PKCS */
 148         {SUN_CKM_SHA384_RSA_PKCS, SHA384_RSA_PKCS_MECH_INFO_TYPE,
 149             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 150             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 151             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 152 
 153         /* SHA512_RSA_PKCS */
 154         {SUN_CKM_SHA512_RSA_PKCS, SHA512_RSA_PKCS_MECH_INFO_TYPE,
 155             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 156             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 157             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS}
 158 
 159 };
 160 
 161 #define RSA_VALID_MECH(mech)                                    \
 162         (((mech)->cm_type == RSA_PKCS_MECH_INFO_TYPE ||              \
 163         (mech)->cm_type == RSA_X_509_MECH_INFO_TYPE ||               \
 164         (mech)->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||    \
 165         (mech)->cm_type == SHA1_RSA_PKCS_MECH_INFO_TYPE ||   \
 166         (mech)->cm_type == SHA256_RSA_PKCS_MECH_INFO_TYPE || \
 167         (mech)->cm_type == SHA384_RSA_PKCS_MECH_INFO_TYPE || \
 168         (mech)->cm_type == SHA512_RSA_PKCS_MECH_INFO_TYPE) ? 1 : 0)
 169 
 170 /* operations are in-place if the output buffer is NULL */
 171 #define RSA_ARG_INPLACE(input, output)                          \
 172         if ((output) == NULL)                                   \
 173                 (output) = (input);
 174 
 175 static void rsa_provider_status(crypto_provider_handle_t, uint_t *);
 176 
 177 static crypto_control_ops_t rsa_control_ops = {
 178         rsa_provider_status
 179 };
 180 
 181 static int rsa_common_init(crypto_ctx_t *, crypto_mechanism_t *,
 182     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 183 static int rsaprov_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
 184     crypto_req_handle_t);
 185 static int rsa_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
 186     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
 187     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 188 static int rsaprov_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
 189     crypto_req_handle_t);
 190 static int rsa_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
 191     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
 192     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 193 
 194 /*
 195  * The RSA mechanisms do not have multiple-part cipher operations.
 196  * So, the update and final routines are set to NULL.
 197  */
 198 static crypto_cipher_ops_t rsa_cipher_ops = {
 199         rsa_common_init,
 200         rsaprov_encrypt,
 201         NULL,
 202         NULL,
 203         rsa_encrypt_atomic,
 204         rsa_common_init,
 205         rsaprov_decrypt,
 206         NULL,
 207         NULL,
 208         rsa_decrypt_atomic
 209 };
 210 
 211 static int rsa_sign_verify_common_init(crypto_ctx_t *, crypto_mechanism_t *,
 212     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 213 static int rsaprov_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
 214     crypto_req_handle_t);
 215 static int rsa_sign_update(crypto_ctx_t *, crypto_data_t *,
 216     crypto_req_handle_t);
 217 static int rsa_sign_final(crypto_ctx_t *, crypto_data_t *,
 218     crypto_req_handle_t);
 219 static int rsa_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
 220     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
 221     crypto_spi_ctx_template_t, crypto_req_handle_t);
 222 
 223 /*
 224  * We use the same routine for sign_init and sign_recover_init fields
 225  * as they do the same thing. Same holds for sign and sign_recover fields,
 226  * and sign_atomic and sign_recover_atomic fields.
 227  */
 228 static crypto_sign_ops_t rsa_sign_ops = {
 229         rsa_sign_verify_common_init,
 230         rsaprov_sign,
 231         rsa_sign_update,
 232         rsa_sign_final,
 233         rsa_sign_atomic,
 234         rsa_sign_verify_common_init,
 235         rsaprov_sign,
 236         rsa_sign_atomic
 237 };
 238 
 239 static int rsaprov_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
 240     crypto_req_handle_t);
 241 static int rsa_verify_update(crypto_ctx_t *, crypto_data_t *,
 242     crypto_req_handle_t);
 243 static int rsa_verify_final(crypto_ctx_t *, crypto_data_t *,
 244     crypto_req_handle_t);
 245 static int rsa_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
 246     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
 247     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 248 static int rsa_verify_recover(crypto_ctx_t *, crypto_data_t *,
 249     crypto_data_t *, crypto_req_handle_t);
 250 static int rsa_verify_recover_atomic(crypto_provider_handle_t,
 251     crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
 252     crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
 253     crypto_req_handle_t);
 254 
 255 /*
 256  * We use the same routine (rsa_sign_verify_common_init) for verify_init
 257  * and verify_recover_init fields as they do the same thing.
 258  */
 259 static crypto_verify_ops_t rsa_verify_ops = {
 260         rsa_sign_verify_common_init,
 261         rsaprov_verify,
 262         rsa_verify_update,
 263         rsa_verify_final,
 264         rsa_verify_atomic,
 265         rsa_sign_verify_common_init,
 266         rsa_verify_recover,
 267         rsa_verify_recover_atomic
 268 };
 269 
 270 static int rsa_free_context(crypto_ctx_t *);
 271 
 272 static crypto_ctx_ops_t rsa_ctx_ops = {
 273         NULL,
 274         rsa_free_context
 275 };
 276 
 277 static crypto_ops_t rsa_crypto_ops = {
 278         &rsa_control_ops,
 279         NULL,
 280         &rsa_cipher_ops,
 281         NULL,
 282         &rsa_sign_ops,
 283         &rsa_verify_ops,
 284         NULL,
 285         NULL,
 286         NULL,
 287         NULL,
 288         NULL,
 289         NULL,
 290         NULL,
 291         &rsa_ctx_ops,
 292         NULL,
 293         NULL,
 294         NULL,
 295 };
 296 
 297 static crypto_provider_info_t rsa_prov_info = {
 298         CRYPTO_SPI_VERSION_4,
 299         "RSA Software Provider",
 300         CRYPTO_SW_PROVIDER,
 301         {&modlinkage},
 302         NULL,
 303         &rsa_crypto_ops,
 304         sizeof (rsa_mech_info_tab)/sizeof (crypto_mech_info_t),
 305         rsa_mech_info_tab
 306 };
 307 
 308 static int rsa_encrypt_common(rsa_mech_type_t, crypto_key_t *,
 309     crypto_data_t *, crypto_data_t *);
 310 static int rsa_decrypt_common(rsa_mech_type_t, crypto_key_t *,
 311     crypto_data_t *, crypto_data_t *);
 312 static int rsa_sign_common(rsa_mech_type_t, crypto_key_t *,
 313     crypto_data_t *, crypto_data_t *);
 314 static int rsa_verify_common(rsa_mech_type_t, crypto_key_t *,
 315     crypto_data_t *, crypto_data_t *);
 316 static int compare_data(crypto_data_t *, uchar_t *);
 317 
 318 /* EXPORT DELETE START */
 319 
 320 static int core_rsa_encrypt(crypto_key_t *, uchar_t *, int, uchar_t *, int);
 321 static int core_rsa_decrypt(crypto_key_t *, uchar_t *, int, uchar_t *);
 322 
 323 /* EXPORT DELETE END */
 324 
 325 static crypto_kcf_provider_handle_t rsa_prov_handle = NULL;
 326 
 327 int
 328 _init(void)
 329 {
 330         int ret;
 331 
 332         if ((ret = mod_install(&modlinkage)) != 0)
 333                 return (ret);
 334 
 335         /* Register with KCF.  If the registration fails, remove the module. */
 336         if (crypto_register_provider(&rsa_prov_info, &rsa_prov_handle)) {
 337                 (void) mod_remove(&modlinkage);
 338                 return (EACCES);
 339         }
 340 
 341         return (0);
 342 }
 343 
 344 int
 345 _fini(void)
 346 {
 347         /* Unregister from KCF if module is registered */
 348         if (rsa_prov_handle != NULL) {
 349                 if (crypto_unregister_provider(rsa_prov_handle))
 350                         return (EBUSY);
 351 
 352                 rsa_prov_handle = NULL;
 353         }
 354 
 355         return (mod_remove(&modlinkage));
 356 }
 357 
 358 int
 359 _info(struct modinfo *modinfop)
 360 {
 361         return (mod_info(&modlinkage, modinfop));
 362 }
 363 
 364 /* ARGSUSED */
 365 static void
 366 rsa_provider_status(crypto_provider_handle_t provider, uint_t *status)
 367 {
 368         *status = CRYPTO_PROVIDER_READY;
 369 }
 370 
 371 static int
 372 check_mech_and_key(crypto_mechanism_t *mechanism, crypto_key_t *key)
 373 {
 374         int rv = CRYPTO_FAILED;
 375 
 376 /* EXPORT DELETE START */
 377 
 378         uchar_t *modulus;
 379         ssize_t modulus_len; /* In bytes */
 380 
 381         if (!RSA_VALID_MECH(mechanism))
 382                 return (CRYPTO_MECHANISM_INVALID);
 383 
 384         /*
 385          * We only support RSA keys that are passed as a list of
 386          * object attributes.
 387          */
 388         if (key->ck_format != CRYPTO_KEY_ATTR_LIST) {
 389                 return (CRYPTO_KEY_TYPE_INCONSISTENT);
 390         }
 391 
 392         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 393             &modulus_len)) != CRYPTO_SUCCESS) {
 394                 return (rv);
 395         }
 396         if (modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES ||
 397             modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)
 398                 return (CRYPTO_KEY_SIZE_RANGE);
 399 
 400 /* EXPORT DELETE END */
 401 
 402         return (rv);
 403 }
 404 
 405 void
 406 kmemset(uint8_t *buf, char pattern, size_t len)
 407 {
 408         int i = 0;
 409 
 410         while (i < len)
 411                 buf[i++] = pattern;
 412 }
 413 
 414 /*
 415  * This function guarantees to return non-zero random numbers.
 416  * This is needed as the /dev/urandom kernel interface,
 417  * random_get_pseudo_bytes(), may return zeros.
 418  */
 419 int
 420 knzero_random_generator(uint8_t *ran_out, size_t ran_len)
 421 {
 422         int rv;
 423         size_t ebc = 0; /* count of extra bytes in extrarand */
 424         size_t i = 0;
 425         uint8_t extrarand[32];
 426         size_t extrarand_len;
 427 
 428         if ((rv = random_get_pseudo_bytes(ran_out, ran_len)) != 0)
 429                 return (rv);
 430 
 431         /*
 432          * Walk through the returned random numbers pointed by ran_out,
 433          * and look for any random number which is zero.
 434          * If we find zero, call random_get_pseudo_bytes() to generate
 435          * another 32 random numbers pool. Replace any zeros in ran_out[]
 436          * from the random number in pool.
 437          */
 438         while (i < ran_len) {
 439                 if (ran_out[i] != 0) {
 440                         i++;
 441                         continue;
 442                 }
 443 
 444                 /*
 445                  * Note that it is 'while' so we are guaranteed a
 446                  * non-zero value on exit.
 447                  */
 448                 if (ebc == 0) {
 449                         /* refresh extrarand */
 450                         extrarand_len = sizeof (extrarand);
 451                         if ((rv = random_get_pseudo_bytes(extrarand,
 452                             extrarand_len)) != 0) {
 453                                 return (rv);
 454                         }
 455 
 456                         ebc = extrarand_len;
 457                 }
 458                 /* Replace zero with byte from extrarand. */
 459                 -- ebc;
 460 
 461                 /*
 462                  * The new random byte zero/non-zero will be checked in
 463                  * the next pass through the loop.
 464                  */
 465                 ran_out[i] = extrarand[ebc];
 466         }
 467 
 468         return (CRYPTO_SUCCESS);
 469 }
 470 
 471 static int
 472 compare_data(crypto_data_t *data, uchar_t *buf)
 473 {
 474         int len;
 475         uchar_t *dptr;
 476 
 477         len = data->cd_length;
 478         switch (data->cd_format) {
 479         case CRYPTO_DATA_RAW:
 480                 dptr = (uchar_t *)(data->cd_raw.iov_base +
 481                     data->cd_offset);
 482 
 483                 return (bcmp(dptr, buf, len));
 484 
 485         case CRYPTO_DATA_UIO:
 486                 return (crypto_uio_data(data, buf, len,
 487                     COMPARE_TO_DATA, NULL, NULL));
 488 
 489         case CRYPTO_DATA_MBLK:
 490                 return (crypto_mblk_data(data, buf, len,
 491                     COMPARE_TO_DATA, NULL, NULL));
 492         }
 493 
 494         return (CRYPTO_FAILED);
 495 }
 496 
 497 /* ARGSUSED */
 498 static int
 499 rsa_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
 500     crypto_key_t *key, crypto_spi_ctx_template_t template,
 501     crypto_req_handle_t req)
 502 {
 503         int rv;
 504         int kmflag;
 505         rsa_ctx_t *ctxp;
 506 
 507         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
 508                 return (rv);
 509 
 510         /*
 511          * Allocate a RSA context.
 512          */
 513         kmflag = crypto_kmflag(req);
 514         if ((ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag)) == NULL)
 515                 return (CRYPTO_HOST_MEMORY);
 516 
 517         if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
 518             kmflag)) != CRYPTO_SUCCESS) {
 519                 kmem_free(ctxp, sizeof (rsa_ctx_t));
 520                 return (rv);
 521         }
 522         ctxp->mech_type = mechanism->cm_type;
 523 
 524         ctx->cc_provider_private = ctxp;
 525 
 526         return (CRYPTO_SUCCESS);
 527 }
 528 
 529 /* ARGSUSED */
 530 static int
 531 rsaprov_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
 532     crypto_data_t *ciphertext, crypto_req_handle_t req)
 533 {
 534         int rv;
 535         rsa_ctx_t *ctxp;
 536 
 537         ASSERT(ctx->cc_provider_private != NULL);
 538         ctxp = ctx->cc_provider_private;
 539 
 540         RSA_ARG_INPLACE(plaintext, ciphertext);
 541 
 542         /*
 543          * Note on the KM_SLEEP flag passed to the routine below -
 544          * rsaprov_encrypt() is a single-part encryption routine which is
 545          * currently usable only by /dev/crypto. Since /dev/crypto calls are
 546          * always synchronous, we can safely pass KM_SLEEP here.
 547          */
 548         rv = rsa_encrypt_common(ctxp->mech_type, ctxp->key, plaintext,
 549             ciphertext);
 550 
 551         if (rv != CRYPTO_BUFFER_TOO_SMALL)
 552                 (void) rsa_free_context(ctx);
 553 
 554         return (rv);
 555 }
 556 
 557 /* ARGSUSED */
 558 static int
 559 rsa_encrypt_atomic(crypto_provider_handle_t provider,
 560     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
 561     crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
 562     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
 563 {
 564         int rv;
 565 
 566         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
 567                 return (rv);
 568         RSA_ARG_INPLACE(plaintext, ciphertext);
 569 
 570         return (rsa_encrypt_common(mechanism->cm_type, key, plaintext,
 571             ciphertext));
 572 }
 573 
 574 static int
 575 rsa_free_context(crypto_ctx_t *ctx)
 576 {
 577         rsa_ctx_t *ctxp = ctx->cc_provider_private;
 578 
 579         if (ctxp != NULL) {
 580                 bzero(ctxp->key, ctxp->keychunk_size);
 581                 kmem_free(ctxp->key, ctxp->keychunk_size);
 582 
 583                 if (ctxp->mech_type == RSA_PKCS_MECH_INFO_TYPE ||
 584                     ctxp->mech_type == RSA_X_509_MECH_INFO_TYPE)
 585                         kmem_free(ctxp, sizeof (rsa_ctx_t));
 586                 else
 587                         kmem_free(ctxp, sizeof (digest_rsa_ctx_t));
 588 
 589                 ctx->cc_provider_private = NULL;
 590         }
 591 
 592         return (CRYPTO_SUCCESS);
 593 }
 594 
 595 static int
 596 rsa_encrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
 597     crypto_data_t *plaintext, crypto_data_t *ciphertext)
 598 {
 599         int rv = CRYPTO_FAILED;
 600 
 601 /* EXPORT DELETE START */
 602 
 603         int plen;
 604         uchar_t *ptptr;
 605         uchar_t *modulus;
 606         ssize_t modulus_len;
 607         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 608         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 609         uchar_t cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 610 
 611         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 612             &modulus_len)) != CRYPTO_SUCCESS) {
 613                 return (rv);
 614         }
 615 
 616         plen = plaintext->cd_length;
 617         if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
 618                 if (plen > (modulus_len - MIN_PKCS1_PADLEN))
 619                         return (CRYPTO_DATA_LEN_RANGE);
 620         } else {
 621                 if (plen > modulus_len)
 622                         return (CRYPTO_DATA_LEN_RANGE);
 623         }
 624 
 625         /*
 626          * Output buf len must not be less than RSA modulus size.
 627          */
 628         if (ciphertext->cd_length < modulus_len) {
 629                 ciphertext->cd_length = modulus_len;
 630                 return (CRYPTO_BUFFER_TOO_SMALL);
 631         }
 632 
 633         ASSERT(plaintext->cd_length <= sizeof (tmp_data));
 634         if ((rv = crypto_get_input_data(plaintext, &ptptr, tmp_data))
 635             != CRYPTO_SUCCESS)
 636                 return (rv);
 637 
 638         if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
 639                 rv = pkcs1_encode(PKCS1_ENCRYPT, ptptr, plen,
 640                     plain_data, modulus_len);
 641 
 642                 if (rv != CRYPTO_SUCCESS)
 643                         return (rv);
 644         } else {
 645                 bzero(plain_data, modulus_len - plen);
 646                 bcopy(ptptr, &plain_data[modulus_len - plen], plen);
 647         }
 648 
 649         rv = core_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
 650         if (rv == CRYPTO_SUCCESS) {
 651                 /* copy out to ciphertext */
 652                 if ((rv = crypto_put_output_data(cipher_data,
 653                     ciphertext, modulus_len)) != CRYPTO_SUCCESS)
 654                         return (rv);
 655 
 656                 ciphertext->cd_length = modulus_len;
 657         }
 658 
 659 /* EXPORT DELETE END */
 660 
 661         return (rv);
 662 }
 663 
 664 /* EXPORT DELETE START */
 665 
 666 static int
 667 core_rsa_encrypt(crypto_key_t *key, uchar_t *in,
 668     int in_len, uchar_t *out, int is_public)
 669 {
 670         int rv;
 671         uchar_t *expo, *modulus;
 672         ssize_t expo_len;
 673         ssize_t modulus_len;
 674         RSAbytekey k;
 675 
 676         if (is_public) {
 677                 if ((rv = crypto_get_key_attr(key, SUN_CKA_PUBLIC_EXPONENT,
 678                     &expo, &expo_len)) != CRYPTO_SUCCESS)
 679                         return (rv);
 680         } else {
 681                 /*
 682                  * SUN_CKA_PRIVATE_EXPONENT is a required attribute for a
 683                  * RSA secret key. See the comments in core_rsa_decrypt
 684                  * routine which calls this routine with a private key.
 685                  */
 686                 if ((rv = crypto_get_key_attr(key, SUN_CKA_PRIVATE_EXPONENT,
 687                     &expo, &expo_len)) != CRYPTO_SUCCESS)
 688                         return (rv);
 689         }
 690 
 691         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 692             &modulus_len)) != CRYPTO_SUCCESS) {
 693                 return (rv);
 694         }
 695 
 696         k.modulus = modulus;
 697         k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
 698         k.pubexpo = expo;
 699         k.pubexpo_bytes = expo_len;
 700         k.rfunc = NULL;
 701 
 702         rv = rsa_encrypt(&k, in, in_len, out);
 703 
 704         return (rv);
 705 }
 706 
 707 /* EXPORT DELETE END */
 708 
 709 /* ARGSUSED */
 710 static int
 711 rsaprov_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
 712     crypto_data_t *plaintext, crypto_req_handle_t req)
 713 {
 714         int rv;
 715         rsa_ctx_t *ctxp;
 716 
 717         ASSERT(ctx->cc_provider_private != NULL);
 718         ctxp = ctx->cc_provider_private;
 719 
 720         RSA_ARG_INPLACE(ciphertext, plaintext);
 721 
 722         /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
 723         rv = rsa_decrypt_common(ctxp->mech_type, ctxp->key,
 724             ciphertext, plaintext);
 725 
 726         if (rv != CRYPTO_BUFFER_TOO_SMALL)
 727                 (void) rsa_free_context(ctx);
 728 
 729         return (rv);
 730 }
 731 
 732 /* ARGSUSED */
 733 static int
 734 rsa_decrypt_atomic(crypto_provider_handle_t provider,
 735     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
 736     crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
 737     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
 738 {
 739         int rv;
 740 
 741         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
 742                 return (rv);
 743         RSA_ARG_INPLACE(ciphertext, plaintext);
 744 
 745         return (rsa_decrypt_common(mechanism->cm_type, key, ciphertext,
 746             plaintext));
 747 }
 748 
 749 static int
 750 rsa_decrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
 751     crypto_data_t *ciphertext, crypto_data_t *plaintext)
 752 {
 753         int rv = CRYPTO_FAILED;
 754 
 755 /* EXPORT DELETE START */
 756 
 757         size_t plain_len;
 758         uchar_t *ctptr;
 759         uchar_t *modulus;
 760         ssize_t modulus_len;
 761         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 762         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 763 
 764         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 765             &modulus_len)) != CRYPTO_SUCCESS) {
 766                 return (rv);
 767         }
 768 
 769         /*
 770          * Ciphertext length must be equal to RSA modulus size.
 771          */
 772         if (ciphertext->cd_length != modulus_len)
 773                 return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
 774 
 775         ASSERT(ciphertext->cd_length <= sizeof (tmp_data));
 776         if ((rv = crypto_get_input_data(ciphertext, &ctptr, tmp_data))
 777             != CRYPTO_SUCCESS)
 778                 return (rv);
 779 
 780         rv = core_rsa_decrypt(key, ctptr, modulus_len, plain_data);
 781         if (rv == CRYPTO_SUCCESS) {
 782                 plain_len = modulus_len;
 783 
 784                 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
 785                         /* Strip off the PKCS block formatting data. */
 786                         rv = pkcs1_decode(PKCS1_DECRYPT, plain_data,
 787                             &plain_len);
 788                         if (rv != CRYPTO_SUCCESS)
 789                                 return (rv);
 790                 }
 791 
 792                 if (plain_len > plaintext->cd_length) {
 793                         plaintext->cd_length = plain_len;
 794                         return (CRYPTO_BUFFER_TOO_SMALL);
 795                 }
 796 
 797                 if ((rv = crypto_put_output_data(
 798                     plain_data + modulus_len - plain_len,
 799                     plaintext, plain_len)) != CRYPTO_SUCCESS)
 800                         return (rv);
 801 
 802                 plaintext->cd_length = plain_len;
 803         }
 804 
 805 /* EXPORT DELETE END */
 806 
 807         return (rv);
 808 }
 809 
 810 /* EXPORT DELETE START */
 811 
 812 static int
 813 core_rsa_decrypt(crypto_key_t *key, uchar_t *in, int in_len, uchar_t *out)
 814 {
 815         int rv;
 816         uchar_t *modulus, *prime1, *prime2, *expo1, *expo2, *coef;
 817         ssize_t modulus_len;
 818         ssize_t prime1_len, prime2_len;
 819         ssize_t expo1_len, expo2_len, coef_len;
 820         RSAbytekey k;
 821 
 822         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 823             &modulus_len)) != CRYPTO_SUCCESS) {
 824                 return (rv);
 825         }
 826 
 827         /*
 828          * The following attributes are not required to be
 829          * present in a RSA secret key. If any of them is not present
 830          * we call the encrypt routine with a flag indicating use of
 831          * private exponent (d). Note that SUN_CKA_PRIVATE_EXPONENT is
 832          * a required attribute for a RSA secret key.
 833          */
 834         if ((crypto_get_key_attr(key, SUN_CKA_PRIME_1, &prime1, &prime1_len)
 835             != CRYPTO_SUCCESS) ||
 836             (crypto_get_key_attr(key, SUN_CKA_PRIME_2, &prime2, &prime2_len)
 837             != CRYPTO_SUCCESS) ||
 838             (crypto_get_key_attr(key, SUN_CKA_EXPONENT_1, &expo1, &expo1_len)
 839             != CRYPTO_SUCCESS) ||
 840             (crypto_get_key_attr(key, SUN_CKA_EXPONENT_2, &expo2, &expo2_len)
 841             != CRYPTO_SUCCESS) ||
 842             (crypto_get_key_attr(key, SUN_CKA_COEFFICIENT, &coef, &coef_len)
 843             != CRYPTO_SUCCESS)) {
 844                 return (core_rsa_encrypt(key, in, in_len, out, 0));
 845         }
 846 
 847         k.modulus = modulus;
 848         k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
 849         k.prime1 = prime1;
 850         k.prime1_bytes = prime1_len;
 851         k.prime2 = prime2;
 852         k.prime2_bytes = prime2_len;
 853         k.expo1 = expo1;
 854         k.expo1_bytes = expo1_len;
 855         k.expo2 = expo2;
 856         k.expo2_bytes = expo2_len;
 857         k.coeff = coef;
 858         k.coeff_bytes = coef_len;
 859         k.rfunc = NULL;
 860 
 861         rv = rsa_decrypt(&k, in, in_len, out);
 862 
 863         return (rv);
 864 }
 865 
 866 /* EXPORT DELETE END */
 867 
 868 /* ARGSUSED */
 869 static int
 870 rsa_sign_verify_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
 871     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
 872     crypto_req_handle_t req)
 873 {
 874         int rv;
 875         int kmflag;
 876         rsa_ctx_t *ctxp;
 877         digest_rsa_ctx_t *dctxp;
 878 
 879         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
 880                 return (rv);
 881 
 882         /*
 883          * Allocate a RSA context.
 884          */
 885         kmflag = crypto_kmflag(req);
 886         switch (mechanism->cm_type) {
 887         case MD5_RSA_PKCS_MECH_INFO_TYPE:
 888         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
 889         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
 890         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
 891         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
 892                 dctxp = kmem_zalloc(sizeof (digest_rsa_ctx_t), kmflag);
 893                 ctxp = (rsa_ctx_t *)dctxp;
 894                 break;
 895         default:
 896                 ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag);
 897                 break;
 898         }
 899 
 900         if (ctxp == NULL)
 901                 return (CRYPTO_HOST_MEMORY);
 902 
 903         ctxp->mech_type = mechanism->cm_type;
 904         if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
 905             kmflag)) != CRYPTO_SUCCESS) {
 906                 switch (mechanism->cm_type) {
 907                 case MD5_RSA_PKCS_MECH_INFO_TYPE:
 908                 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
 909                 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
 910                 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
 911                 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
 912                         kmem_free(dctxp, sizeof (digest_rsa_ctx_t));
 913                         break;
 914                 default:
 915                         kmem_free(ctxp, sizeof (rsa_ctx_t));
 916                         break;
 917                 }
 918                 return (rv);
 919         }
 920 
 921         switch (mechanism->cm_type) {
 922         case MD5_RSA_PKCS_MECH_INFO_TYPE:
 923                 MD5Init(&(dctxp->md5_ctx));
 924                 break;
 925 
 926         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
 927                 SHA1Init(&(dctxp->sha1_ctx));
 928                 break;
 929 
 930         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
 931                 SHA2Init(SHA256, &(dctxp->sha2_ctx));
 932                 break;
 933 
 934         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
 935                 SHA2Init(SHA384, &(dctxp->sha2_ctx));
 936                 break;
 937 
 938         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
 939                 SHA2Init(SHA512, &(dctxp->sha2_ctx));
 940                 break;
 941         }
 942 
 943         ctx->cc_provider_private = ctxp;
 944 
 945         return (CRYPTO_SUCCESS);
 946 }
 947 
 948 #define SHA1_DIGEST_SIZE 20
 949 #define MD5_DIGEST_SIZE 16
 950 
 951 #define INIT_RAW_CRYPTO_DATA(data, base, len, cd_len)   \
 952         (data).cd_format = CRYPTO_DATA_RAW;             \
 953         (data).cd_offset = 0;                           \
 954         (data).cd_raw.iov_base = (char *)base;          \
 955         (data).cd_raw.iov_len = len;                    \
 956         (data).cd_length = cd_len;
 957 
 958 static int
 959 rsa_digest_svrfy_common(digest_rsa_ctx_t *ctxp, crypto_data_t *data,
 960     crypto_data_t *signature, uchar_t flag)
 961 {
 962         int rv = CRYPTO_FAILED;
 963 
 964 /* EXPORT DELETE START */
 965 
 966         uchar_t digest[SHA512_DIGEST_LENGTH];
 967         /* The der_data size is enough for MD5 also */
 968         uchar_t der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
 969         ulong_t der_data_len;
 970         crypto_data_t der_cd;
 971         rsa_mech_type_t mech_type;
 972 
 973         ASSERT(flag & CRYPTO_DO_SIGN || flag & CRYPTO_DO_VERIFY);
 974         ASSERT(data != NULL || (flag & CRYPTO_DO_FINAL));
 975 
 976         mech_type = ctxp->mech_type;
 977         if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
 978             mech_type == RSA_X_509_MECH_INFO_TYPE)
 979                 return (CRYPTO_MECHANISM_INVALID);
 980 
 981         /*
 982          * We need to do the BUFFER_TOO_SMALL check before digesting
 983          * the data. No check is needed for verify as signature is not
 984          * an output argument for verify.
 985          */
 986         if (flag & CRYPTO_DO_SIGN) {
 987                 uchar_t *modulus;
 988                 ssize_t modulus_len;
 989 
 990                 if ((rv = crypto_get_key_attr(ctxp->key, SUN_CKA_MODULUS,
 991                     &modulus, &modulus_len)) != CRYPTO_SUCCESS) {
 992                         return (rv);
 993                 }
 994 
 995                 if (signature->cd_length < modulus_len) {
 996                         signature->cd_length = modulus_len;
 997                         return (CRYPTO_BUFFER_TOO_SMALL);
 998                 }
 999         }
1000 
1001         if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1002                 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1003                     digest, MD5Update, MD5Final, flag | CRYPTO_DO_MD5);
1004 
1005         else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1006                 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1007                     digest, SHA1Update, SHA1Final,  flag | CRYPTO_DO_SHA1);
1008 
1009         else
1010                 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1011                     digest, SHA2Update, SHA2Final, flag | CRYPTO_DO_SHA2);
1012 
1013         if (rv != CRYPTO_SUCCESS)
1014                 return (rv);
1015 
1016 
1017         /*
1018          * Prepare the DER encoding of the DigestInfo value as follows:
1019          * MD5:         MD5_DER_PREFIX || H
1020          * SHA-1:       SHA1_DER_PREFIX || H
1021          *
1022          * See rsa_impl.c for more details.
1023          */
1024         switch (mech_type) {
1025         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1026                 bcopy(MD5_DER_PREFIX, der_data, MD5_DER_PREFIX_Len);
1027                 bcopy(digest, der_data + MD5_DER_PREFIX_Len, MD5_DIGEST_SIZE);
1028                 der_data_len = MD5_DER_PREFIX_Len + MD5_DIGEST_SIZE;
1029                 break;
1030 
1031         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1032                 bcopy(SHA1_DER_PREFIX, der_data, SHA1_DER_PREFIX_Len);
1033                 bcopy(digest, der_data + SHA1_DER_PREFIX_Len,
1034                     SHA1_DIGEST_SIZE);
1035                 der_data_len = SHA1_DER_PREFIX_Len + SHA1_DIGEST_SIZE;
1036                 break;
1037 
1038         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1039                 bcopy(SHA256_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1040                 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1041                     SHA256_DIGEST_LENGTH);
1042                 der_data_len = SHA2_DER_PREFIX_Len + SHA256_DIGEST_LENGTH;
1043                 break;
1044 
1045         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1046                 bcopy(SHA384_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1047                 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1048                     SHA384_DIGEST_LENGTH);
1049                 der_data_len = SHA2_DER_PREFIX_Len + SHA384_DIGEST_LENGTH;
1050                 break;
1051 
1052         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1053                 bcopy(SHA512_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1054                 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1055                     SHA512_DIGEST_LENGTH);
1056                 der_data_len = SHA2_DER_PREFIX_Len + SHA512_DIGEST_LENGTH;
1057                 break;
1058         }
1059 
1060         INIT_RAW_CRYPTO_DATA(der_cd, der_data, der_data_len, der_data_len);
1061         /*
1062          * Now, we are ready to sign or verify the DER_ENCODED data.
1063          */
1064         if (flag & CRYPTO_DO_SIGN)
1065                 rv = rsa_sign_common(mech_type, ctxp->key, &der_cd,
1066                     signature);
1067         else
1068                 rv = rsa_verify_common(mech_type, ctxp->key, &der_cd,
1069                     signature);
1070 
1071 /* EXPORT DELETE END */
1072 
1073         return (rv);
1074 }
1075 
1076 static int
1077 rsa_sign_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1078     crypto_data_t *data, crypto_data_t *signature)
1079 {
1080         int rv = CRYPTO_FAILED;
1081 
1082 /* EXPORT DELETE START */
1083 
1084         int dlen;
1085         uchar_t *dataptr, *modulus;
1086         ssize_t modulus_len;
1087         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1088         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1089         uchar_t signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1090 
1091         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1092             &modulus_len)) != CRYPTO_SUCCESS) {
1093                 return (rv);
1094         }
1095 
1096         dlen = data->cd_length;
1097         switch (mech_type) {
1098         case RSA_PKCS_MECH_INFO_TYPE:
1099                 if (dlen > (modulus_len - MIN_PKCS1_PADLEN))
1100                         return (CRYPTO_DATA_LEN_RANGE);
1101                 break;
1102         case RSA_X_509_MECH_INFO_TYPE:
1103                 if (dlen > modulus_len)
1104                         return (CRYPTO_DATA_LEN_RANGE);
1105                 break;
1106         }
1107 
1108         if (signature->cd_length < modulus_len) {
1109                 signature->cd_length = modulus_len;
1110                 return (CRYPTO_BUFFER_TOO_SMALL);
1111         }
1112 
1113         ASSERT(data->cd_length <= sizeof (tmp_data));
1114         if ((rv = crypto_get_input_data(data, &dataptr, tmp_data))
1115             != CRYPTO_SUCCESS)
1116                 return (rv);
1117 
1118         switch (mech_type) {
1119         case RSA_PKCS_MECH_INFO_TYPE:
1120         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1121         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1122         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1123         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1124         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1125                 /*
1126                  * Add PKCS padding to the input data to format a block
1127                  * type "01" encryption block.
1128                  */
1129                 rv = pkcs1_encode(PKCS1_SIGN, dataptr, dlen, plain_data,
1130                     modulus_len);
1131                 if (rv != CRYPTO_SUCCESS)
1132                         return (rv);
1133 
1134                 break;
1135 
1136         case RSA_X_509_MECH_INFO_TYPE:
1137                 bzero(plain_data, modulus_len - dlen);
1138                 bcopy(dataptr, &plain_data[modulus_len - dlen], dlen);
1139                 break;
1140         }
1141 
1142         rv = core_rsa_decrypt(key, plain_data, modulus_len, signed_data);
1143         if (rv == CRYPTO_SUCCESS) {
1144                 /* copy out to signature */
1145                 if ((rv = crypto_put_output_data(signed_data,
1146                     signature, modulus_len)) != CRYPTO_SUCCESS)
1147                         return (rv);
1148 
1149                 signature->cd_length = modulus_len;
1150         }
1151 
1152 /* EXPORT DELETE END */
1153 
1154         return (rv);
1155 }
1156 
1157 /* ARGSUSED */
1158 static int
1159 rsaprov_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1160     crypto_req_handle_t req)
1161 {
1162         int rv;
1163         rsa_ctx_t *ctxp;
1164 
1165         ASSERT(ctx->cc_provider_private != NULL);
1166         ctxp = ctx->cc_provider_private;
1167 
1168         /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1169         switch (ctxp->mech_type) {
1170         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1171         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1172         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1173         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1174         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1175                 rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1176                     signature, CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE |
1177                     CRYPTO_DO_FINAL);
1178                 break;
1179         default:
1180                 rv = rsa_sign_common(ctxp->mech_type, ctxp->key, data,
1181                     signature);
1182                 break;
1183         }
1184 
1185         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1186                 (void) rsa_free_context(ctx);
1187 
1188         return (rv);
1189 }
1190 
1191 /* ARGSUSED */
1192 static int
1193 rsa_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
1194 {
1195         int rv;
1196         digest_rsa_ctx_t *ctxp;
1197         rsa_mech_type_t mech_type;
1198 
1199         ASSERT(ctx->cc_provider_private != NULL);
1200         ctxp = ctx->cc_provider_private;
1201         mech_type = ctxp->mech_type;
1202 
1203         if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1204             mech_type == RSA_X_509_MECH_INFO_TYPE)
1205                 return (CRYPTO_MECHANISM_INVALID);
1206 
1207         if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1208                 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1209                     NULL, MD5Update, MD5Final,
1210                     CRYPTO_DO_MD5 | CRYPTO_DO_UPDATE);
1211 
1212         else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1213                 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1214                     NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1215                     CRYPTO_DO_UPDATE);
1216 
1217         else
1218                 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1219                     NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1220                     CRYPTO_DO_UPDATE);
1221 
1222         return (rv);
1223 }
1224 
1225 /* ARGSUSED2 */
1226 static int
1227 rsa_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1228     crypto_req_handle_t req)
1229 {
1230         int rv;
1231         digest_rsa_ctx_t *ctxp;
1232 
1233         ASSERT(ctx->cc_provider_private != NULL);
1234         ctxp = ctx->cc_provider_private;
1235 
1236         rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1237             CRYPTO_DO_SIGN | CRYPTO_DO_FINAL);
1238         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1239                 (void) rsa_free_context(ctx);
1240 
1241         return (rv);
1242 }
1243 
1244 /* ARGSUSED */
1245 static int
1246 rsa_sign_atomic(crypto_provider_handle_t provider,
1247     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1248     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
1249     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1250 {
1251         int rv;
1252         digest_rsa_ctx_t dctx;
1253 
1254         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1255                 return (rv);
1256 
1257         if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1258             mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1259                 rv = rsa_sign_common(mechanism->cm_type, key, data,
1260                     signature);
1261 
1262         else {
1263                 dctx.mech_type = mechanism->cm_type;
1264                 dctx.key = key;
1265                 switch (mechanism->cm_type) {
1266                 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1267                         MD5Init(&(dctx.md5_ctx));
1268                         break;
1269 
1270                 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1271                         SHA1Init(&(dctx.sha1_ctx));
1272                         break;
1273 
1274                 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1275                         SHA2Init(SHA256, &(dctx.sha2_ctx));
1276                         break;
1277 
1278                 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1279                         SHA2Init(SHA384, &(dctx.sha2_ctx));
1280                         break;
1281 
1282                 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1283                         SHA2Init(SHA512, &(dctx.sha2_ctx));
1284                         break;
1285                 }
1286 
1287                 rv = rsa_digest_svrfy_common(&dctx, data, signature,
1288                     CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1289         }
1290 
1291         return (rv);
1292 }
1293 
1294 static int
1295 rsa_verify_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1296     crypto_data_t *data, crypto_data_t *signature)
1297 {
1298         int rv = CRYPTO_FAILED;
1299 
1300 /* EXPORT DELETE START */
1301 
1302         uchar_t *sigptr, *modulus;
1303         ssize_t modulus_len;
1304         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1305         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1306 
1307         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1308             &modulus_len)) != CRYPTO_SUCCESS) {
1309                 return (rv);
1310         }
1311 
1312         if (signature->cd_length != modulus_len)
1313                 return (CRYPTO_SIGNATURE_LEN_RANGE);
1314 
1315         ASSERT(signature->cd_length <= sizeof (tmp_data));
1316         if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1317             != CRYPTO_SUCCESS)
1318                 return (rv);
1319 
1320         rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1321         if (rv != CRYPTO_SUCCESS)
1322                 return (rv);
1323 
1324         if (mech_type == RSA_X_509_MECH_INFO_TYPE) {
1325                 if (compare_data(data, (plain_data + modulus_len
1326                     - data->cd_length)) != 0)
1327                         rv = CRYPTO_SIGNATURE_INVALID;
1328 
1329         } else {
1330                 size_t data_len = modulus_len;
1331 
1332                 /*
1333                  * Strip off the encoded padding bytes in front of the
1334                  * recovered data, then compare the recovered data with
1335                  * the original data.
1336                  */
1337                 rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1338                 if (rv != CRYPTO_SUCCESS)
1339                         return (rv);
1340 
1341                 if (data_len != data->cd_length)
1342                         return (CRYPTO_SIGNATURE_LEN_RANGE);
1343 
1344                 if (compare_data(data, (plain_data + modulus_len
1345                     - data_len)) != 0)
1346                         rv = CRYPTO_SIGNATURE_INVALID;
1347         }
1348 
1349 /* EXPORT DELETE END */
1350 
1351         return (rv);
1352 }
1353 
1354 /* ARGSUSED */
1355 static int
1356 rsaprov_verify(crypto_ctx_t *ctx, crypto_data_t *data,
1357     crypto_data_t *signature, crypto_req_handle_t req)
1358 {
1359         int rv;
1360         rsa_ctx_t *ctxp;
1361 
1362         ASSERT(ctx->cc_provider_private != NULL);
1363         ctxp = ctx->cc_provider_private;
1364 
1365         /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1366         switch (ctxp->mech_type) {
1367         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1368         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1369         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1370         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1371         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1372                 rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1373                     signature, CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE |
1374                     CRYPTO_DO_FINAL);
1375                 break;
1376         default:
1377                 rv = rsa_verify_common(ctxp->mech_type, ctxp->key, data,
1378                     signature);
1379                 break;
1380         }
1381 
1382         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1383                 (void) rsa_free_context(ctx);
1384 
1385         return (rv);
1386 }
1387 
1388 /* ARGSUSED */
1389 static int
1390 rsa_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
1391     crypto_req_handle_t req)
1392 {
1393         int rv;
1394         digest_rsa_ctx_t *ctxp;
1395 
1396         ASSERT(ctx->cc_provider_private != NULL);
1397         ctxp = ctx->cc_provider_private;
1398 
1399         switch (ctxp->mech_type) {
1400 
1401         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1402                 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1403                     NULL, MD5Update, MD5Final, CRYPTO_DO_MD5 |
1404                     CRYPTO_DO_UPDATE);
1405                 break;
1406 
1407         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1408                 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1409                     NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1410                     CRYPTO_DO_UPDATE);
1411                 break;
1412 
1413         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1414         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1415         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1416                 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1417                     NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1418                     CRYPTO_DO_UPDATE);
1419                 break;
1420 
1421         default:
1422                 return (CRYPTO_MECHANISM_INVALID);
1423         }
1424 
1425         return (rv);
1426 }
1427 
1428 /* ARGSUSED2 */
1429 static int
1430 rsa_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1431     crypto_req_handle_t req)
1432 {
1433         int rv;
1434         digest_rsa_ctx_t *ctxp;
1435 
1436         ASSERT(ctx->cc_provider_private != NULL);
1437         ctxp = ctx->cc_provider_private;
1438 
1439         rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1440             CRYPTO_DO_VERIFY | CRYPTO_DO_FINAL);
1441         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1442                 (void) rsa_free_context(ctx);
1443 
1444         return (rv);
1445 }
1446 
1447 
1448 /* ARGSUSED */
1449 static int
1450 rsa_verify_atomic(crypto_provider_handle_t provider,
1451     crypto_session_id_t session_id,
1452     crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data,
1453     crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template,
1454     crypto_req_handle_t req)
1455 {
1456         int rv;
1457         digest_rsa_ctx_t dctx;
1458 
1459         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1460                 return (rv);
1461 
1462         if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1463             mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1464                 rv = rsa_verify_common(mechanism->cm_type, key, data,
1465                     signature);
1466 
1467         else {
1468                 dctx.mech_type = mechanism->cm_type;
1469                 dctx.key = key;
1470 
1471                 switch (mechanism->cm_type) {
1472                 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1473                         MD5Init(&(dctx.md5_ctx));
1474                         break;
1475 
1476                 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1477                         SHA1Init(&(dctx.sha1_ctx));
1478                         break;
1479 
1480                 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1481                         SHA2Init(SHA256, &(dctx.sha2_ctx));
1482                         break;
1483 
1484                 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1485                         SHA2Init(SHA384, &(dctx.sha2_ctx));
1486                         break;
1487 
1488                 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1489                         SHA2Init(SHA512, &(dctx.sha2_ctx));
1490                         break;
1491                 }
1492 
1493                 rv = rsa_digest_svrfy_common(&dctx, data, signature,
1494                     CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1495         }
1496 
1497         return (rv);
1498 }
1499 
1500 static int
1501 rsa_verify_recover_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1502     crypto_data_t *signature, crypto_data_t *data)
1503 {
1504         int rv = CRYPTO_FAILED;
1505 
1506 /* EXPORT DELETE START */
1507 
1508         size_t data_len;
1509         uchar_t *sigptr, *modulus;
1510         ssize_t modulus_len;
1511         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1512         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1513 
1514         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1515             &modulus_len)) != CRYPTO_SUCCESS) {
1516                 return (rv);
1517         }
1518 
1519         if (signature->cd_length != modulus_len)
1520                 return (CRYPTO_SIGNATURE_LEN_RANGE);
1521 
1522         ASSERT(signature->cd_length <= sizeof (tmp_data));
1523         if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1524             != CRYPTO_SUCCESS)
1525                 return (rv);
1526 
1527         rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1528         if (rv != CRYPTO_SUCCESS)
1529                 return (rv);
1530 
1531         data_len = modulus_len;
1532 
1533         if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1534                 /*
1535                  * Strip off the encoded padding bytes in front of the
1536                  * recovered data, then compare the recovered data with
1537                  * the original data.
1538                  */
1539                 rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1540                 if (rv != CRYPTO_SUCCESS)
1541                         return (rv);
1542         }
1543 
1544         if (data->cd_length < data_len) {
1545                 data->cd_length = data_len;
1546                 return (CRYPTO_BUFFER_TOO_SMALL);
1547         }
1548 
1549         if ((rv = crypto_put_output_data(plain_data + modulus_len - data_len,
1550             data, data_len)) != CRYPTO_SUCCESS)
1551                 return (rv);
1552         data->cd_length = data_len;
1553 
1554 /* EXPORT DELETE END */
1555 
1556         return (rv);
1557 }
1558 
1559 /* ARGSUSED */
1560 static int
1561 rsa_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
1562     crypto_data_t *data, crypto_req_handle_t req)
1563 {
1564         int rv;
1565         rsa_ctx_t *ctxp;
1566 
1567         ASSERT(ctx->cc_provider_private != NULL);
1568         ctxp = ctx->cc_provider_private;
1569 
1570         /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1571         rv = rsa_verify_recover_common(ctxp->mech_type, ctxp->key,
1572             signature, data);
1573 
1574         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1575                 (void) rsa_free_context(ctx);
1576 
1577         return (rv);
1578 }
1579 
1580 /* ARGSUSED */
1581 static int
1582 rsa_verify_recover_atomic(crypto_provider_handle_t provider,
1583     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1584     crypto_key_t *key, crypto_data_t *signature, crypto_data_t *data,
1585     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1586 {
1587         int rv;
1588 
1589         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1590                 return (rv);
1591 
1592         return (rsa_verify_recover_common(mechanism->cm_type, key,
1593             signature, data));
1594 }