1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 #include <pthread.h>
  27 #include <stdlib.h>
  28 #include <string.h>
  29 #include <strings.h>
  30 #include <sys/types.h>
  31 #include <security/cryptoki.h>
  32 #include <cryptoutil.h>
  33 #include "softGlobal.h"
  34 #include "softSession.h"
  35 #include "softObject.h"
  36 #include "softOps.h"
  37 #include "softRSA.h"
  38 #include "softMAC.h"
  39 #include "softCrypt.h"
  40 
  41 CK_RV
  42 soft_rsa_encrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
  43     CK_BYTE_PTR out, int realpublic)
  44 {
  45 
  46         CK_RV rv = CKR_OK;
  47 
  48 /* EXPORT DELETE START */
  49 
  50         uchar_t expo[MAX_KEY_ATTR_BUFLEN];
  51         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
  52         uint32_t expo_len = sizeof (expo);
  53         uint32_t modulus_len = sizeof (modulus);
  54         RSAbytekey k;
  55 
  56         if (realpublic) {
  57                 rv = soft_get_public_value(key, CKA_PUBLIC_EXPONENT, expo,
  58                     &expo_len);
  59                 if (rv != CKR_OK) {
  60                         goto clean1;
  61                 }
  62         } else {
  63                 rv = soft_get_private_value(key, CKA_PRIVATE_EXPONENT, expo,
  64                     &expo_len);
  65                 if (rv != CKR_OK) {
  66                         goto clean1;
  67                 }
  68         }
  69 
  70         rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
  71         if (rv != CKR_OK) {
  72                 goto clean1;
  73         }
  74 
  75         k.modulus = modulus;
  76         k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
  77         k.pubexpo = expo;
  78         k.pubexpo_bytes = expo_len;
  79         k.rfunc = NULL;
  80 
  81         rv = rsa_encrypt(&k, in, in_len, out);
  82 
  83 clean1:
  84 
  85 /* EXPORT DELETE END */
  86 
  87         return (rv);
  88 }
  89 
  90 
  91 CK_RV
  92 soft_rsa_decrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
  93     CK_BYTE_PTR out)
  94 {
  95 
  96         CK_RV rv = CKR_OK;
  97 
  98 /* EXPORT DELETE START */
  99 
 100         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
 101         uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
 102         uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
 103         uchar_t expo1[MAX_KEY_ATTR_BUFLEN];
 104         uchar_t expo2[MAX_KEY_ATTR_BUFLEN];
 105         uchar_t coef[MAX_KEY_ATTR_BUFLEN];
 106         uint32_t modulus_len = sizeof (modulus);
 107         uint32_t prime1_len = sizeof (prime1);
 108         uint32_t prime2_len = sizeof (prime2);
 109         uint32_t expo1_len = sizeof (expo1);
 110         uint32_t expo2_len = sizeof (expo2);
 111         uint32_t coef_len = sizeof (coef);
 112         RSAbytekey k;
 113 
 114         rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
 115         if (rv != CKR_OK) {
 116                 goto clean1;
 117         }
 118 
 119         rv = soft_get_private_value(key, CKA_PRIME_1, prime1, &prime1_len);
 120 
 121         if ((prime1_len == 0) && (rv == CKR_OK)) {
 122                 rv = soft_rsa_encrypt(key, in, in_len, out, 0);
 123                 goto clean1;
 124         } else {
 125                 if (rv != CKR_OK)
 126                         goto clean1;
 127         }
 128 
 129         rv = soft_get_private_value(key, CKA_PRIME_2, prime2, &prime2_len);
 130 
 131         if ((prime2_len == 0) && (rv == CKR_OK)) {
 132                 rv = soft_rsa_encrypt(key, in, in_len, out, 0);
 133                 goto clean1;
 134         } else {
 135                 if (rv != CKR_OK)
 136                         goto clean1;
 137         }
 138 
 139         rv = soft_get_private_value(key, CKA_EXPONENT_1, expo1, &expo1_len);
 140 
 141         if ((expo1_len == 0) && (rv == CKR_OK)) {
 142                 rv = soft_rsa_encrypt(key, in, in_len, out, 0);
 143                 goto clean1;
 144         } else {
 145                 if (rv != CKR_OK)
 146                         goto clean1;
 147         }
 148 
 149         rv = soft_get_private_value(key, CKA_EXPONENT_2, expo2, &expo2_len);
 150 
 151         if ((expo2_len == 0) && (rv == CKR_OK)) {
 152                 rv = soft_rsa_encrypt(key, in, in_len, out, 0);
 153                 goto clean1;
 154         } else {
 155                 if (rv != CKR_OK)
 156                         goto clean1;
 157         }
 158 
 159         rv = soft_get_private_value(key, CKA_COEFFICIENT, coef, &coef_len);
 160 
 161         if ((coef_len == 0) && (rv == CKR_OK)) {
 162                 rv = soft_rsa_encrypt(key, in, in_len, out, 0);
 163                 goto clean1;
 164         } else {
 165                 if (rv != CKR_OK)
 166                         goto clean1;
 167         }
 168 
 169         k.modulus = modulus;
 170         k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
 171         k.prime1 = prime1;
 172         k.prime1_bytes = prime1_len;
 173         k.prime2 = prime2;
 174         k.prime2_bytes = prime2_len;
 175         k.expo1 = expo1;
 176         k.expo1_bytes = expo1_len;
 177         k.expo2 = expo2;
 178         k.expo2_bytes = expo2_len;
 179         k.coeff = coef;
 180         k.coeff_bytes = coef_len;
 181         k.rfunc = NULL;
 182 
 183         rv = rsa_decrypt(&k, in, in_len, out);
 184 
 185 clean1:
 186 
 187 /* EXPORT DELETE END */
 188 
 189         return (rv);
 190 }
 191 
 192 /*
 193  * Allocate a RSA context for the active encryption or decryption operation.
 194  * This function is called without the session lock held.
 195  */
 196 CK_RV
 197 soft_rsa_crypt_init_common(soft_session_t *session_p,
 198     CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
 199     boolean_t encrypt)
 200 {
 201 
 202         soft_rsa_ctx_t *rsa_ctx;
 203         soft_object_t *tmp_key = NULL;
 204         CK_RV rv;
 205 
 206         rsa_ctx = calloc(1, sizeof (soft_rsa_ctx_t));
 207         if (rsa_ctx == NULL) {
 208                 return (CKR_HOST_MEMORY);
 209         }
 210 
 211         /*
 212          * Make a copy of the encryption or decryption key, and save it
 213          * in the RSA crypto context since it will be used later for
 214          * encryption/decryption. We don't want to hold any object reference
 215          * on this original key while doing encryption/decryption.
 216          */
 217         (void) pthread_mutex_lock(&key_p->object_mutex);
 218         rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
 219             NULL);
 220 
 221         if ((rv != CKR_OK) || (tmp_key == NULL)) {
 222                 /* Most likely we ran out of space. */
 223                 (void) pthread_mutex_unlock(&key_p->object_mutex);
 224                 free(rsa_ctx);
 225                 return (rv);
 226         }
 227 
 228         /* No need to hold the lock on the old object. */
 229         (void) pthread_mutex_unlock(&key_p->object_mutex);
 230         rsa_ctx->key = tmp_key;
 231 
 232         (void) pthread_mutex_lock(&session_p->session_mutex);
 233         if (encrypt) {
 234                 /* Called by C_EncryptInit. */
 235                 session_p->encrypt.context = rsa_ctx;
 236                 session_p->encrypt.mech.mechanism = pMechanism->mechanism;
 237         } else {
 238                 /* Called by C_DecryptInit. */
 239                 session_p->decrypt.context = rsa_ctx;
 240                 session_p->decrypt.mech.mechanism = pMechanism->mechanism;
 241         }
 242         (void) pthread_mutex_unlock(&session_p->session_mutex);
 243 
 244         return (CKR_OK);
 245 }
 246 
 247 CK_RV
 248 soft_rsa_encrypt_common(soft_session_t *session_p, CK_BYTE_PTR pData,
 249     CK_ULONG ulDataLen, CK_BYTE_PTR pEncrypted,
 250     CK_ULONG_PTR pulEncryptedLen, CK_MECHANISM_TYPE mechanism)
 251 {
 252 
 253         soft_rsa_ctx_t *rsa_ctx = session_p->encrypt.context;
 254         soft_object_t *key = rsa_ctx->key;
 255         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
 256         uint32_t modulus_len = sizeof (modulus);
 257         CK_BYTE plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 258         CK_BYTE cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 259         CK_RV rv = CKR_OK;
 260 
 261         rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
 262         if (rv != CKR_OK) {
 263                 goto clean_exit;
 264         }
 265 
 266         if (pEncrypted == NULL) {
 267                 /*
 268                  * Application asks for the length of the output buffer
 269                  * to hold the ciphertext.
 270                  */
 271                 *pulEncryptedLen = modulus_len;
 272                 rv = CKR_OK;
 273                 goto clean1;
 274         }
 275 
 276         if (mechanism == CKM_RSA_PKCS) {
 277                 /*
 278                  * Input data length needs to be <=
 279                  * modulus length-MIN_PKCS1_PADLEN.
 280                  */
 281                 if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
 282                         *pulEncryptedLen = modulus_len;
 283                         rv = CKR_DATA_LEN_RANGE;
 284                         goto clean_exit;
 285                 }
 286         } else {
 287                 /* Input data length needs to be <= modulus length. */
 288                 if (ulDataLen > (CK_ULONG)modulus_len) {
 289                         *pulEncryptedLen = modulus_len;
 290                         rv = CKR_DATA_LEN_RANGE;
 291                         goto clean_exit;
 292                 }
 293         }
 294 
 295         /* Is the application-supplied buffer large enough? */
 296         if (*pulEncryptedLen < (CK_ULONG)modulus_len) {
 297                 *pulEncryptedLen = modulus_len;
 298                 rv = CKR_BUFFER_TOO_SMALL;
 299                 goto clean1;
 300         }
 301 
 302         if (mechanism == CKM_RSA_PKCS) {
 303                 /*
 304                  * Add PKCS padding to the input data to format a block
 305                  * type "02" encryption block.
 306                  */
 307                 rv = pkcs1_encode(PKCS1_ENCRYPT, pData, ulDataLen, plain_data,
 308                     modulus_len);
 309 
 310                 if (rv != CKR_OK)
 311                         goto clean_exit;
 312         } else {
 313                 /* Pad zeros for the leading bytes of the input data. */
 314                 (void) memset(plain_data, 0x0, modulus_len - ulDataLen);
 315                 (void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
 316                     ulDataLen);
 317         }
 318 
 319         rv = soft_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
 320         if (rv == CKR_OK) {
 321                 (void) memcpy(pEncrypted, cipher_data, modulus_len);
 322                 *pulEncryptedLen = modulus_len;
 323         }
 324 
 325 clean_exit:
 326         (void) pthread_mutex_lock(&session_p->session_mutex);
 327         free(session_p->encrypt.context);
 328         session_p->encrypt.context = NULL;
 329         (void) pthread_mutex_unlock(&session_p->session_mutex);
 330         soft_cleanup_object(key);
 331         free(key);
 332 clean1:
 333         return (rv);
 334 }
 335 
 336 
 337 CK_RV
 338 soft_rsa_decrypt_common(soft_session_t *session_p, CK_BYTE_PTR pEncrypted,
 339     CK_ULONG ulEncryptedLen, CK_BYTE_PTR pData,
 340     CK_ULONG_PTR pulDataLen, CK_MECHANISM_TYPE mechanism)
 341 {
 342 
 343         soft_rsa_ctx_t *rsa_ctx = session_p->decrypt.context;
 344         soft_object_t *key = rsa_ctx->key;
 345         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
 346         uint32_t modulus_len = sizeof (modulus);
 347         CK_BYTE plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 348         CK_RV rv = CKR_OK;
 349 
 350         rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
 351         if (rv != CKR_OK) {
 352                 goto clean_exit;
 353         }
 354 
 355         if (ulEncryptedLen != (CK_ULONG)modulus_len) {
 356                 rv = CKR_ENCRYPTED_DATA_LEN_RANGE;
 357                 goto clean_exit;
 358         }
 359 
 360         if (pData == NULL) {
 361                 /*
 362                  * Application asks for the length of the output buffer
 363                  * to hold the recovered data.
 364                  */
 365                 *pulDataLen = modulus_len;
 366                 rv = CKR_OK;
 367                 goto clean1;
 368         }
 369 
 370         if (mechanism == CKM_RSA_X_509) {
 371                 if (*pulDataLen < (CK_ULONG)modulus_len) {
 372                         *pulDataLen = modulus_len;
 373                         rv = CKR_BUFFER_TOO_SMALL;
 374                         goto clean1;
 375                 }
 376         }
 377 
 378         rv = soft_rsa_decrypt(key, pEncrypted, modulus_len, plain_data);
 379         if (rv != CKR_OK) {
 380                 goto clean_exit;
 381         }
 382 
 383         if (mechanism == CKM_RSA_PKCS) {
 384                 size_t plain_len = modulus_len;
 385                 size_t num_padding;
 386 
 387                 /* Strip off the PKCS block formatting data. */
 388                 rv = pkcs1_decode(PKCS1_DECRYPT, plain_data, &plain_len);
 389                 if (rv != CKR_OK)
 390                         goto clean_exit;
 391 
 392                 num_padding = modulus_len - plain_len;
 393                 if (ulEncryptedLen - num_padding > *pulDataLen) {
 394                         *pulDataLen = plain_len;
 395                         rv = CKR_BUFFER_TOO_SMALL;
 396                         goto clean1;
 397                 }
 398 
 399                 (void) memcpy(pData, &plain_data[num_padding], plain_len);
 400                 *pulDataLen = plain_len;
 401         } else {
 402                 (void) memcpy(pData, plain_data, modulus_len);
 403                 *pulDataLen = modulus_len;
 404         }
 405 
 406 clean_exit:
 407         (void) pthread_mutex_lock(&session_p->session_mutex);
 408         free(session_p->decrypt.context);
 409         session_p->decrypt.context = NULL;
 410         (void) pthread_mutex_unlock(&session_p->session_mutex);
 411         soft_cleanup_object(key);
 412         free(key);
 413 
 414 clean1:
 415         return (rv);
 416 }
 417 
 418 /*
 419  * Allocate a RSA context for the active sign or verify operation.
 420  * This function is called without the session lock held.
 421  */
 422 CK_RV
 423 soft_rsa_sign_verify_init_common(soft_session_t *session_p,
 424     CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
 425     boolean_t sign)
 426 {
 427         CK_RV rv = CKR_OK;
 428         soft_rsa_ctx_t *rsa_ctx;
 429         CK_MECHANISM digest_mech;
 430         soft_object_t *tmp_key = NULL;
 431 
 432         if (sign) {
 433                 if ((key_p->class != CKO_PRIVATE_KEY) ||
 434                     (key_p->key_type != CKK_RSA))
 435                         return (CKR_KEY_TYPE_INCONSISTENT);
 436         } else {
 437                 if ((key_p->class != CKO_PUBLIC_KEY) ||
 438                     (key_p->key_type != CKK_RSA))
 439                         return (CKR_KEY_TYPE_INCONSISTENT);
 440         }
 441 
 442         switch (pMechanism->mechanism) {
 443         case CKM_MD5_RSA_PKCS:
 444                 digest_mech.mechanism = CKM_MD5;
 445                 rv = soft_digest_init_internal(session_p, &digest_mech);
 446                 if (rv != CKR_OK)
 447                         return (rv);
 448                 break;
 449 
 450         case CKM_SHA1_RSA_PKCS:
 451                 digest_mech.mechanism = CKM_SHA_1;
 452                 digest_mech.pParameter = pMechanism->pParameter;
 453                 digest_mech.ulParameterLen = pMechanism->ulParameterLen;
 454                 rv = soft_digest_init_internal(session_p, &digest_mech);
 455                 if (rv != CKR_OK)
 456                         return (rv);
 457                 break;
 458 
 459         case CKM_SHA256_RSA_PKCS:
 460                 digest_mech.mechanism = CKM_SHA256;
 461                 rv = soft_digest_init_internal(session_p, &digest_mech);
 462                 if (rv != CKR_OK)
 463                         return (rv);
 464                 break;
 465 
 466         case CKM_SHA384_RSA_PKCS:
 467                 digest_mech.mechanism = CKM_SHA384;
 468                 rv = soft_digest_init_internal(session_p, &digest_mech);
 469                 if (rv != CKR_OK)
 470                         return (rv);
 471                 break;
 472 
 473         case CKM_SHA512_RSA_PKCS:
 474                 digest_mech.mechanism = CKM_SHA512;
 475                 rv = soft_digest_init_internal(session_p, &digest_mech);
 476                 if (rv != CKR_OK)
 477                         return (rv);
 478                 break;
 479         }
 480 
 481         rsa_ctx = malloc(sizeof (soft_rsa_ctx_t));
 482 
 483         if (rsa_ctx == NULL) {
 484                 rv = CKR_HOST_MEMORY;
 485                 goto clean_exit;
 486         }
 487 
 488         (void) pthread_mutex_lock(&key_p->object_mutex);
 489         rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
 490             NULL);
 491 
 492         if ((rv != CKR_OK) || (tmp_key == NULL)) {
 493                 /* Most likely we ran out of space. */
 494                 (void) pthread_mutex_unlock(&key_p->object_mutex);
 495                 free(rsa_ctx);
 496                 goto clean_exit;
 497         }
 498 
 499         /* No need to hold the lock on the old object. */
 500         (void) pthread_mutex_unlock(&key_p->object_mutex);
 501         rsa_ctx->key = tmp_key;
 502 
 503         (void) pthread_mutex_lock(&session_p->session_mutex);
 504 
 505         if (sign) {
 506                 session_p->sign.context = rsa_ctx;
 507                 session_p->sign.mech.mechanism = pMechanism->mechanism;
 508         } else {
 509                 session_p->verify.context = rsa_ctx;
 510                 session_p->verify.mech.mechanism = pMechanism->mechanism;
 511         }
 512 
 513         (void) pthread_mutex_unlock(&session_p->session_mutex);
 514 
 515         return (CKR_OK);
 516 
 517 clean_exit:
 518         (void) pthread_mutex_lock(&session_p->session_mutex);
 519         if (session_p->digest.context != NULL) {
 520                 free(session_p->digest.context);
 521                 session_p->digest.context = NULL;
 522                 session_p->digest.flags = 0;
 523         }
 524         (void) pthread_mutex_unlock(&session_p->session_mutex);
 525         return (rv);
 526 
 527 }
 528 
 529 
 530 CK_RV
 531 soft_rsa_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
 532     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
 533     CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism)
 534 {
 535 
 536         CK_RV rv = CKR_OK;
 537         soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
 538         soft_object_t *key = rsa_ctx->key;
 539         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
 540         uint32_t modulus_len = sizeof (modulus);
 541         CK_BYTE plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 542         CK_BYTE signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 543 
 544         rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
 545         if (rv != CKR_OK) {
 546                 goto clean_exit;
 547         }
 548 
 549         if (pSigned == NULL) {
 550                 /* Application asks for the length of the output buffer. */
 551                 *pulSignedLen = modulus_len;
 552                 rv = CKR_OK;
 553                 goto clean1;
 554         }
 555 
 556         switch (mechanism) {
 557 
 558         case CKM_RSA_PKCS:
 559 
 560                 /*
 561                  * Input data length needs to be <=
 562                  * modulus length-MIN_PKCS1_PADLEN.
 563                  */
 564                 if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
 565                         *pulSignedLen = modulus_len;
 566                         rv = CKR_DATA_LEN_RANGE;
 567                         goto clean_exit;
 568                 }
 569                 break;
 570 
 571         case CKM_RSA_X_509:
 572 
 573                 /* Input data length needs to be <= modulus length. */
 574                 if (ulDataLen > (CK_ULONG)modulus_len) {
 575                         *pulSignedLen = modulus_len;
 576                         rv = CKR_DATA_LEN_RANGE;
 577                         goto clean_exit;
 578                 }
 579                 break;
 580         }
 581 
 582         /* Is the application-supplied buffer large enough? */
 583         if (*pulSignedLen < (CK_ULONG)modulus_len) {
 584                 *pulSignedLen = modulus_len;
 585                 rv = CKR_BUFFER_TOO_SMALL;
 586                 goto clean1;
 587         }
 588 
 589         switch (mechanism) {
 590 
 591         case CKM_RSA_PKCS:
 592         case CKM_MD5_RSA_PKCS:
 593         case CKM_SHA1_RSA_PKCS:
 594         case CKM_SHA256_RSA_PKCS:
 595         case CKM_SHA384_RSA_PKCS:
 596         case CKM_SHA512_RSA_PKCS:
 597                 /*
 598                  * Add PKCS padding to the input data to format a block
 599                  * type "01" encryption block.
 600                  */
 601                 rv = pkcs1_encode(PKCS1_SIGN, pData, ulDataLen, plain_data,
 602                     modulus_len);
 603 
 604                 if (rv != CKR_OK) {
 605                         goto clean_exit;
 606                 }
 607                 break;
 608 
 609         case CKM_RSA_X_509:
 610 
 611                 /* Pad zeros for the leading bytes of the input data. */
 612                 (void) memset(plain_data, 0x0, modulus_len - ulDataLen);
 613                 (void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
 614                     ulDataLen);
 615                 break;
 616         }
 617 
 618         /*
 619          * Perform RSA encryption with the signer's RSA private key
 620          * for signature process.
 621          */
 622         rv = soft_rsa_decrypt(key, plain_data, modulus_len, signed_data);
 623 
 624         if (rv == CKR_OK) {
 625                 (void) memcpy(pSigned, signed_data, modulus_len);
 626                 *pulSignedLen = modulus_len;
 627         }
 628 
 629 clean_exit:
 630         (void) pthread_mutex_lock(&session_p->session_mutex);
 631         free(session_p->sign.context);
 632         session_p->sign.context = NULL;
 633         if (session_p->digest.context != NULL) {
 634                 free(session_p->digest.context);
 635                 session_p->digest.context = NULL;
 636                 session_p->digest.flags = 0;
 637         }
 638         (void) pthread_mutex_unlock(&session_p->session_mutex);
 639         soft_cleanup_object(key);
 640         free(key);
 641 
 642 clean1:
 643         return (rv);
 644 }
 645 
 646 
 647 CK_RV
 648 soft_rsa_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
 649     CK_ULONG ulDataLen, CK_BYTE_PTR pSignature,
 650     CK_ULONG ulSignatureLen, CK_MECHANISM_TYPE mechanism)
 651 {
 652 
 653         CK_RV rv = CKR_OK;
 654         soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
 655         soft_object_t *key = rsa_ctx->key;
 656         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
 657         uint32_t modulus_len = sizeof (modulus);
 658         CK_BYTE plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 659 
 660         rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
 661         if (rv != CKR_OK) {
 662                 goto clean_exit;
 663         }
 664 
 665         if (ulDataLen == 0) {
 666                 rv = CKR_DATA_LEN_RANGE;
 667                 goto clean_exit;
 668         }
 669 
 670         if (ulSignatureLen != (CK_ULONG)modulus_len) {
 671                 rv = CKR_SIGNATURE_LEN_RANGE;
 672                 goto clean_exit;
 673         }
 674 
 675         /*
 676          * Perform RSA decryption with the signer's RSA public key
 677          * for verification process.
 678          */
 679         rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
 680         if (rv == CKR_OK) {
 681                 switch (mechanism) {
 682 
 683                 case CKM_RSA_PKCS:
 684                 case CKM_MD5_RSA_PKCS:
 685                 case CKM_SHA1_RSA_PKCS:
 686                 case CKM_SHA256_RSA_PKCS:
 687                 case CKM_SHA384_RSA_PKCS:
 688                 case CKM_SHA512_RSA_PKCS:
 689                 {
 690                         /*
 691                          * Strip off the encoded padding bytes in front of the
 692                          * recovered data, then compare the recovered data with
 693                          * the original data.
 694                          */
 695                         size_t data_len = modulus_len;
 696 
 697                         rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
 698                         if (rv != CKR_OK) {
 699                                 goto clean_exit;
 700                         }
 701 
 702                         if ((CK_ULONG)data_len != ulDataLen) {
 703                                 rv = CKR_DATA_LEN_RANGE;
 704                                 goto clean_exit;
 705                         } else if (memcmp(pData,
 706                             &plain_data[modulus_len - data_len],
 707                             ulDataLen) != 0) {
 708                                 rv = CKR_SIGNATURE_INVALID;
 709                                 goto clean_exit;
 710                         }
 711                         break;
 712                 }
 713 
 714                 case CKM_RSA_X_509:
 715                         /*
 716                          * Strip off the encoded padding bytes in front of the
 717                          * recovered plain_data, then compare the input data
 718                          * with the recovered data.
 719                          */
 720                         if (memcmp(pData,
 721                             plain_data + ulSignatureLen - ulDataLen,
 722                             ulDataLen) != 0) {
 723                                 rv = CKR_SIGNATURE_INVALID;
 724                                 goto clean_exit;
 725                         }
 726                         break;
 727                 }
 728         }
 729 
 730         if (rv == CKR_DATA_LEN_RANGE) {
 731                 if ((mechanism == CKM_MD5_RSA_PKCS) ||
 732                     (mechanism == CKM_SHA1_RSA_PKCS) ||
 733                     (mechanism == CKM_SHA256_RSA_PKCS) ||
 734                     (mechanism == CKM_SHA384_RSA_PKCS) ||
 735                     (mechanism == CKM_SHA512_RSA_PKCS))
 736                         rv = CKR_SIGNATURE_INVALID;
 737         }
 738 
 739 clean_exit:
 740         (void) pthread_mutex_lock(&session_p->session_mutex);
 741         free(session_p->verify.context);
 742         session_p->verify.context = NULL;
 743         if (session_p->digest.context != NULL) {
 744                 free(session_p->digest.context);
 745                 session_p->digest.context = NULL;
 746                 session_p->digest.flags = 0;
 747         }
 748         (void) pthread_mutex_unlock(&session_p->session_mutex);
 749         soft_cleanup_object(key);
 750         free(key);
 751         return (rv);
 752 }
 753 
 754 CK_RV
 755 soft_genRSAkey_set_attribute(soft_object_t *key, CK_ATTRIBUTE_TYPE type,
 756     uchar_t *buf, uint32_t buflen, boolean_t public)
 757 {
 758         CK_RV rv = CKR_OK;
 759         biginteger_t *dst = NULL;
 760         biginteger_t src;
 761 
 762         switch (type) {
 763 
 764         case CKA_MODULUS:
 765 
 766                 if (public)
 767                         dst = OBJ_PUB_RSA_MOD(key);
 768                 else
 769                         dst = OBJ_PRI_RSA_MOD(key);
 770                 break;
 771 
 772         case CKA_PUBLIC_EXPONENT:
 773 
 774                 if (public)
 775                         dst = OBJ_PUB_RSA_PUBEXPO(key);
 776                 else
 777                         dst = OBJ_PRI_RSA_PUBEXPO(key);
 778                 break;
 779 
 780         case CKA_PRIVATE_EXPONENT:
 781 
 782                 dst = OBJ_PRI_RSA_PRIEXPO(key);
 783                 break;
 784 
 785         case CKA_PRIME_1:
 786 
 787                 dst = OBJ_PRI_RSA_PRIME1(key);
 788                 break;
 789 
 790         case CKA_PRIME_2:
 791 
 792                 dst = OBJ_PRI_RSA_PRIME2(key);
 793                 break;
 794 
 795         case CKA_EXPONENT_1:
 796 
 797                 dst = OBJ_PRI_RSA_EXPO1(key);
 798                 break;
 799 
 800         case CKA_EXPONENT_2:
 801 
 802                 dst = OBJ_PRI_RSA_EXPO2(key);
 803                 break;
 804 
 805         case CKA_COEFFICIENT:
 806 
 807                 dst = OBJ_PRI_RSA_COEF(key);
 808                 break;
 809         }
 810 
 811         /* Note: no explanation found for why this is needed */
 812         while (buf[0] == 0) {   /* remove proceeding 0x00 */
 813                 buf++;
 814                 buflen--;
 815         }
 816 
 817         if ((rv = dup_bigint_attr(&src, buf, buflen)) != CKR_OK)
 818                 goto cleanexit;
 819 
 820         /* Copy the attribute in the key object. */
 821         copy_bigint_attr(&src, dst);
 822 
 823 cleanexit:
 824         return (rv);
 825 
 826 }
 827 
 828 
 829 CK_RV
 830 soft_rsa_genkey_pair(soft_object_t *pubkey, soft_object_t *prikey)
 831 {
 832         CK_RV rv = CKR_OK;
 833         CK_ATTRIBUTE template;
 834         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
 835         uint32_t modulus_len;
 836         uchar_t pub_expo[MAX_KEY_ATTR_BUFLEN];
 837         uint32_t pub_expo_len = sizeof (pub_expo);
 838         uchar_t private_exponent[MAX_KEY_ATTR_BUFLEN];
 839         uint32_t private_exponent_len = sizeof (private_exponent);
 840         uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
 841         uint32_t prime1_len = sizeof (prime1);
 842         uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
 843         uint32_t prime2_len = sizeof (prime2);
 844         uchar_t exponent1[MAX_KEY_ATTR_BUFLEN];
 845         uint32_t exponent1_len = sizeof (exponent1);
 846         uchar_t exponent2[MAX_KEY_ATTR_BUFLEN];
 847         uint32_t exponent2_len = sizeof (exponent2);
 848         uchar_t coefficient[MAX_KEY_ATTR_BUFLEN];
 849         uint32_t coefficient_len = sizeof (coefficient);
 850         RSAbytekey k;
 851 
 852         if ((pubkey == NULL) || (prikey == NULL)) {
 853                 return (CKR_ARGUMENTS_BAD);
 854         }
 855 
 856         template.pValue = malloc(sizeof (CK_ULONG));
 857         if (template.pValue == NULL) {
 858                 return (CKR_HOST_MEMORY);
 859         }
 860         template.ulValueLen = sizeof (CK_ULONG);
 861 
 862         rv = get_ulong_attr_from_object(OBJ_PUB_RSA_MOD_BITS(pubkey),
 863             &template);
 864         if (rv != CKR_OK) {
 865                 free(template.pValue);
 866                 goto clean0;
 867         }
 868 
 869 #ifdef  __sparcv9
 870         /* LINTED */
 871         modulus_len = (uint32_t)(*((CK_ULONG *)(template.pValue)));
 872 #else   /* !__sparcv9 */
 873         modulus_len = *((CK_ULONG *)(template.pValue));
 874 #endif  /* __sparcv9 */
 875 
 876         free(template.pValue);
 877 
 878         rv = soft_get_public_value(pubkey, CKA_PUBLIC_EXPONENT, pub_expo,
 879             &pub_expo_len);
 880         if (rv != CKR_OK) {
 881                 goto clean0;
 882         }
 883 
 884         /* Inputs to RSA key pair generation */
 885         k.modulus_bits = modulus_len;           /* save modulus len in bits  */
 886         modulus_len = CRYPTO_BITS2BYTES(modulus_len);   /* convert to bytes */
 887         k.modulus = modulus;
 888         k.pubexpo = pub_expo;
 889         k.pubexpo_bytes = pub_expo_len;
 890         k.rfunc = (IS_TOKEN_OBJECT(pubkey) || IS_TOKEN_OBJECT(prikey)) ?
 891             pkcs11_get_random : pkcs11_get_urandom;
 892 
 893         /* Outputs from RSA key pair generation */
 894         k.privexpo = private_exponent;
 895         k.privexpo_bytes = private_exponent_len;
 896         k.prime1 = prime1;
 897         k.prime1_bytes = prime1_len;
 898         k.prime2 = prime2;
 899         k.prime2_bytes = prime2_len;
 900         k.expo1 = exponent1;
 901         k.expo1_bytes = exponent1_len;
 902         k.expo2 = exponent2;
 903         k.expo2_bytes = exponent2_len;
 904         k.coeff = coefficient;
 905         k.coeff_bytes = coefficient_len;
 906 
 907         rv = rsa_genkey_pair(&k);
 908 
 909         if (rv != CKR_OK) {
 910                 goto clean0;
 911         }
 912 
 913         /*
 914          * Add modulus in public template, and add all eight key fields
 915          * in private template.
 916          */
 917         if ((rv = soft_genRSAkey_set_attribute(pubkey, CKA_MODULUS,
 918             modulus, CRYPTO_BITS2BYTES(k.modulus_bits), B_TRUE)) != CKR_OK) {
 919                 goto clean0;
 920         }
 921 
 922         if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_MODULUS,
 923             modulus, CRYPTO_BITS2BYTES(k.modulus_bits), B_FALSE)) != CKR_OK) {
 924                 goto clean0;
 925         }
 926 
 927         if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIVATE_EXPONENT,
 928             private_exponent, k.privexpo_bytes, B_FALSE)) != CKR_OK) {
 929                 goto clean0;
 930         }
 931 
 932         if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PUBLIC_EXPONENT,
 933             pub_expo, k.pubexpo_bytes, B_FALSE)) != CKR_OK) {
 934                 goto clean0;
 935         }
 936 
 937         if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIME_1,
 938             prime1, k.prime1_bytes, B_FALSE)) != CKR_OK) {
 939                 goto clean0;
 940         }
 941 
 942         if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIME_2,
 943             prime2, k.prime2_bytes, B_FALSE)) != CKR_OK) {
 944                 goto clean0;
 945         }
 946 
 947         if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_EXPONENT_1,
 948             exponent1, k.expo1_bytes, B_FALSE)) != CKR_OK) {
 949                 goto clean0;
 950         }
 951 
 952         if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_EXPONENT_2,
 953             exponent2, k.expo2_bytes, B_FALSE)) != CKR_OK) {
 954                 goto clean0;
 955         }
 956 
 957         if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_COEFFICIENT,
 958             coefficient, k.coeff_bytes, B_FALSE)) != CKR_OK) {
 959                 goto clean0;
 960         }
 961 
 962 clean0:
 963         return (rv);
 964 }
 965 
 966 
 967 CK_ULONG
 968 get_rsa_sha1_prefix(CK_MECHANISM_PTR mech, CK_BYTE_PTR *prefix) {
 969         if (mech->pParameter == NULL) {
 970                 *prefix = (CK_BYTE *)SHA1_DER_PREFIX;
 971                 return (SHA1_DER_PREFIX_Len);
 972         }
 973 
 974         *prefix = (CK_BYTE *)SHA1_DER_PREFIX_OID;
 975         return (SHA1_DER_PREFIX_OID_Len);
 976 }
 977 
 978 CK_RV
 979 soft_rsa_digest_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
 980     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
 981     CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
 982 {
 983 
 984         CK_RV rv = CKR_OK;
 985         CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space enough for all mechs */
 986         CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
 987         /* space enough for all mechs */
 988         CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
 989         CK_ULONG der_data_len;
 990         soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
 991         soft_object_t *key = rsa_ctx->key;
 992         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
 993         uint32_t modulus_len = sizeof (modulus);
 994         CK_ULONG der_len;
 995         CK_BYTE_PTR der_prefix;
 996 
 997         rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
 998         if (rv != CKR_OK) {
 999                 (void) pthread_mutex_lock(&session_p->session_mutex);
1000                 free(session_p->digest.context);
1001                 session_p->digest.context = NULL;
1002                 session_p->digest.flags = 0;
1003                 (void) pthread_mutex_unlock(&session_p->session_mutex);
1004                 soft_cleanup_object(key);
1005                 free(key);
1006                 goto clean1;
1007         }
1008 
1009         /* Check arguments before performing message digest. */
1010         if (pSigned == NULL) {
1011                 /* Application asks for the length of the output buffer. */
1012                 *pulSignedLen = modulus_len;
1013                 rv = CKR_OK;
1014                 goto clean1;
1015         }
1016 
1017         /* Is the application-supplied buffer large enough? */
1018         if (*pulSignedLen < (CK_ULONG)modulus_len) {
1019                 *pulSignedLen = modulus_len;
1020                 rv = CKR_BUFFER_TOO_SMALL;
1021                 goto clean1;
1022         }
1023 
1024         if (Final) {
1025                 rv = soft_digest_final(session_p, hash, &hash_len);
1026         } else {
1027                 rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1028         }
1029 
1030         if (rv != CKR_OK) {
1031                 /* free the signature key */
1032                 soft_cleanup_object(key);
1033                 free(key);
1034                 goto clean_exit;
1035         }
1036 
1037         /*
1038          * Prepare the DER encoding of the DigestInfo value by setting it to:
1039          *      <MECH>_DER_PREFIX || H
1040          *
1041          * See rsa_impl.c for more details.
1042          */
1043         switch (session_p->digest.mech.mechanism) {
1044         case CKM_MD5:
1045                 (void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1046                 (void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1047                 der_data_len = MD5_DER_PREFIX_Len + hash_len;
1048                 break;
1049         case CKM_SHA_1:
1050                 der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
1051                     &der_prefix);
1052                 (void) memcpy(der_data, der_prefix, der_len);
1053                 (void) memcpy(der_data + der_len, hash, hash_len);
1054                 der_data_len = der_len + hash_len;
1055                 break;
1056         case CKM_SHA256:
1057                 (void) memcpy(der_data, SHA256_DER_PREFIX,
1058                     SHA2_DER_PREFIX_Len);
1059                 (void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1060                 der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1061                 break;
1062         case CKM_SHA384:
1063                 (void) memcpy(der_data, SHA384_DER_PREFIX,
1064                     SHA2_DER_PREFIX_Len);
1065                 (void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1066                 der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1067                 break;
1068         case CKM_SHA512:
1069                 (void) memcpy(der_data, SHA512_DER_PREFIX,
1070                     SHA2_DER_PREFIX_Len);
1071                 (void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1072                 der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1073                 break;
1074         }
1075 
1076         /*
1077          * Now, we are ready to sign the DER_ENCODED data
1078          * soft_rsa_sign_common() will free the signature key.
1079          */
1080         rv = soft_rsa_sign_common(session_p, der_data, der_data_len,
1081             pSigned, pulSignedLen, mechanism);
1082 
1083 clean_exit:
1084         (void) pthread_mutex_lock(&session_p->session_mutex);
1085         /* soft_digest_common() has freed the digest context */
1086         session_p->digest.flags = 0;
1087         (void) pthread_mutex_unlock(&session_p->session_mutex);
1088 
1089 clean1:
1090         return (rv);
1091 }
1092 
1093 
1094 CK_RV
1095 soft_rsa_digest_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
1096     CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
1097     CK_ULONG ulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
1098 {
1099 
1100         CK_RV rv = CKR_OK;
1101         CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space for all mechs */
1102         CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
1103         CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
1104         CK_ULONG der_data_len;
1105         soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1106         soft_object_t *key = rsa_ctx->key;
1107         CK_ULONG der_len;
1108         CK_BYTE_PTR der_prefix;
1109 
1110         if (Final) {
1111                 rv = soft_digest_final(session_p, hash, &hash_len);
1112         } else {
1113                 rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1114         }
1115 
1116         if (rv != CKR_OK) {
1117                 /* free the verification key */
1118                 soft_cleanup_object(key);
1119                 free(key);
1120                 goto clean_exit;
1121         }
1122 
1123         /*
1124          * Prepare the DER encoding of the DigestInfo value as follows:
1125          * MD5:         MD5_DER_PREFIX || H
1126          * SHA-1:       SHA1_DER_PREFIX || H
1127          * SHA2:        SHA2_DER_PREFIX || H
1128          *
1129          * See rsa_impl.c for more details.
1130          */
1131         switch (session_p->digest.mech.mechanism) {
1132         case CKM_MD5:
1133                 (void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1134                 (void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1135                 der_data_len = MD5_DER_PREFIX_Len + hash_len;
1136                 break;
1137         case CKM_SHA_1:
1138                 der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
1139                     &der_prefix);
1140                 (void) memcpy(der_data, der_prefix, der_len);
1141                 (void) memcpy(der_data + der_len, hash, hash_len);
1142                 der_data_len = der_len + hash_len;
1143                 break;
1144         case CKM_SHA256:
1145                 (void) memcpy(der_data, SHA256_DER_PREFIX,
1146                     SHA2_DER_PREFIX_Len);
1147                 (void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1148                 der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1149                 break;
1150         case CKM_SHA384:
1151                 (void) memcpy(der_data, SHA384_DER_PREFIX,
1152                     SHA2_DER_PREFIX_Len);
1153                 (void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1154                 der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1155                 break;
1156         case CKM_SHA512:
1157                 (void) memcpy(der_data, SHA512_DER_PREFIX,
1158                     SHA2_DER_PREFIX_Len);
1159                 (void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1160                 der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1161                 break;
1162         }
1163 
1164         /*
1165          * Now, we are ready to verify the DER_ENCODED data using signature.
1166          * soft_rsa_verify_common() will free the verification key.
1167          */
1168         rv = soft_rsa_verify_common(session_p, der_data, der_data_len,
1169             pSigned, ulSignedLen, mechanism);
1170 
1171 clean_exit:
1172         (void) pthread_mutex_lock(&session_p->session_mutex);
1173         /* soft_digest_common() has freed the digest context */
1174         session_p->digest.flags = 0;
1175         (void) pthread_mutex_unlock(&session_p->session_mutex);
1176 
1177         return (rv);
1178 
1179 }
1180 
1181 
1182 CK_RV
1183 soft_rsa_verify_recover(soft_session_t *session_p, CK_BYTE_PTR pSignature,
1184     CK_ULONG ulSignatureLen, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
1185 {
1186 
1187         CK_RV rv = CKR_OK;
1188         soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1189         CK_MECHANISM_TYPE mechanism = session_p->verify.mech.mechanism;
1190         soft_object_t *key = rsa_ctx->key;
1191         uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
1192         uint32_t modulus_len = sizeof (modulus);
1193         CK_BYTE plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1194 
1195         rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
1196         if (rv != CKR_OK) {
1197                 goto clean_exit;
1198         }
1199 
1200         if (ulSignatureLen != (CK_ULONG)modulus_len) {
1201                 rv = CKR_SIGNATURE_LEN_RANGE;
1202                 goto clean_exit;
1203         }
1204 
1205         /*
1206          * Perform RSA decryption with the signer's RSA public key
1207          * for verification process.
1208          */
1209         rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
1210         if (rv == CKR_OK) {
1211                 switch (mechanism) {
1212 
1213                 case CKM_RSA_PKCS:
1214                 {
1215                         /*
1216                          * Strip off the encoded padding bytes in front of the
1217                          * recovered data.
1218                          */
1219                         size_t data_len = modulus_len;
1220 
1221                         rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1222                         if (rv != CKR_OK) {
1223                                 goto clean_exit;
1224                         }
1225 
1226                         /*
1227                          * If application asks for the length of the output
1228                          * buffer?
1229                          */
1230                         if (pData == NULL) {
1231                                 *pulDataLen = data_len;
1232                                 rv = CKR_OK;
1233                                 goto clean1;
1234                         }
1235 
1236                         /* Is the application-supplied buffer large enough? */
1237                         if (*pulDataLen < (CK_ULONG)data_len) {
1238                                 *pulDataLen = data_len;
1239                                 rv = CKR_BUFFER_TOO_SMALL;
1240                                 goto clean1;
1241                         }
1242 
1243                         (void) memcpy(pData,
1244                             &plain_data[modulus_len - data_len], data_len);
1245                         *pulDataLen = data_len;
1246 
1247                         break;
1248                 }
1249 
1250                 case CKM_RSA_X_509:
1251                         /*
1252                          * If application asks for the length of the output
1253                          * buffer?
1254                          */
1255                         if (pData == NULL) {
1256                                 *pulDataLen = modulus_len;
1257                                 rv = CKR_OK;
1258                                 goto clean1;
1259                         }
1260 
1261                         /* Is the application-supplied buffer large enough? */
1262                         if (*pulDataLen < (CK_ULONG)modulus_len) {
1263                                 *pulDataLen = modulus_len;
1264                                 rv = CKR_BUFFER_TOO_SMALL;
1265                                 goto clean1;
1266                         }
1267 
1268                         (void) memcpy(pData, plain_data, modulus_len);
1269                         *pulDataLen = modulus_len;
1270 
1271                         break;
1272                 }
1273         }
1274 
1275 clean_exit:
1276         (void) pthread_mutex_lock(&session_p->session_mutex);
1277         free(session_p->verify.context);
1278         session_p->verify.context = NULL;
1279         (void) pthread_mutex_unlock(&session_p->session_mutex);
1280         soft_cleanup_object(key);
1281         free(key);
1282 
1283 clean1:
1284         return (rv);
1285 }