Print this page
3882 remove xmod & friends


  66  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  67  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  68  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  69  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  70  * OF THE POSSIBILITY OF SUCH DAMAGE.
  71  * ====================================================================
  72  */
  73 
  74 /*
  75  * ====================================================================
  76  * OpenSolaris OS modifications
  77  *
  78  * This source originates as files aes-intel.S and eng_aesni_asm.pl, in
  79  * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
  80  * Huang Ying of Intel to the openssl-dev mailing list under the subject
  81  * of "Add support to Intel AES-NI instruction set for x86_64 platform".
  82  *
  83  * This OpenSolaris version has these major changes from the original source:
  84  *
  85  * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
  86  * /usr/include/sys/asm_linkage.h, lint(1B) guards, EXPORT DELETE START
  87  * and EXPORT DELETE END markers, and dummy C function definitions for lint.
  88  *
  89  * 2. Formatted code, added comments, and added #includes and #defines.
  90  *
  91  * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
  92  * calling kpreempt_disable() and kpreempt_enable().
  93  * If the TS bit is not set, Save and restore %xmm registers at the beginning
  94  * and end of function calls (%xmm* registers are not saved and restored by
  95  * during kernel thread preemption).
  96  *
  97  * 4. Renamed functions, reordered parameters, and changed return value
  98  * to match OpenSolaris:
  99  *
 100  * OpenSSL interface:
 101  *      int intel_AES_set_encrypt_key(const unsigned char *userKey,
 102  *              const int bits, AES_KEY *key);
 103  *      int intel_AES_set_decrypt_key(const unsigned char *userKey,
 104  *              const int bits, AES_KEY *key);
 105  *      Return values for above are non-zero on error, 0 on success.
 106  *
 107  *      void intel_AES_encrypt(const unsigned char *in, unsigned char *out,


 288 #define SET_TS_OR_POP_XMM0_XMM1(tmpreg)
 289 #define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg)
 290 #define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg)
 291 #endif  /* _KERNEL */
 292 
 293 
 294 /*
 295  * _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(),
 296  * _key_expansion_256a(), _key_expansion_256b()
 297  *
 298  * Helper functions called by rijndael_key_setup_inc_intel().
 299  * Also used indirectly by rijndael_key_setup_dec_intel().
 300  *
 301  * Input:
 302  * %xmm0        User-provided cipher key
 303  * %xmm1        Round constant
 304  * Output:
 305  * (%rcx)       AES key
 306  */
 307 
 308         /* EXPORT DELETE START */
 309 .align  16
 310 _key_expansion_128:
 311 _key_expansion_256a:
 312         pshufd  $0b11111111, %xmm1, %xmm1
 313         shufps  $0b00010000, %xmm0, %xmm4
 314         pxor    %xmm4, %xmm0
 315         shufps  $0b10001100, %xmm0, %xmm4
 316         pxor    %xmm4, %xmm0
 317         pxor    %xmm1, %xmm0
 318         movaps  %xmm0, (%rcx)
 319         add     $0x10, %rcx
 320         ret
 321         SET_SIZE(_key_expansion_128)
 322         SET_SIZE(_key_expansion_256a)
 323 
 324 .align 16
 325 _key_expansion_192a:
 326         pshufd  $0b01010101, %xmm1, %xmm1
 327         shufps  $0b00010000, %xmm0, %xmm4
 328         pxor    %xmm4, %xmm0


 361         pxor    %xmm3, %xmm2
 362         pxor    %xmm5, %xmm2
 363 
 364         movaps  %xmm0, (%rcx)
 365         add     $0x10, %rcx
 366         ret
 367         SET_SIZE(_key_expansion_192b)
 368 
 369 .align 16
 370 _key_expansion_256b:
 371         pshufd  $0b10101010, %xmm1, %xmm1
 372         shufps  $0b00010000, %xmm2, %xmm4
 373         pxor    %xmm4, %xmm2
 374         shufps  $0b10001100, %xmm2, %xmm4
 375         pxor    %xmm4, %xmm2
 376         pxor    %xmm1, %xmm2
 377         movaps  %xmm2, (%rcx)
 378         add     $0x10, %rcx
 379         ret
 380         SET_SIZE(_key_expansion_256b)
 381         /* EXPORT DELETE END */
 382 
 383 
 384 /*
 385  * rijndael_key_setup_enc_intel()
 386  * Expand the cipher key into the encryption key schedule.
 387  *
 388  * For kernel code, caller is responsible for ensuring kpreempt_disable()
 389  * has been called.  This is because %xmm registers are not saved/restored.
 390  * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 391  * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
 392  * on the stack.
 393  *
 394  * OpenSolaris interface:
 395  * int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
 396  *      uint64_t keyBits);
 397  * Return value is 0 on error, number of rounds on success.
 398  *
 399  * Original Intel OpenSSL interface:
 400  * int intel_AES_set_encrypt_key(const unsigned char *userKey,
 401  *      const int bits, AES_KEY *key);


 407 #define rijndael_key_setup_dec_intel    intel_AES_set_decrypt_key
 408 
 409 #define USERCIPHERKEY           rdi     /* P1, 64 bits */
 410 #define KEYSIZE32               esi     /* P2, 32 bits */
 411 #define KEYSIZE64               rsi     /* P2, 64 bits */
 412 #define AESKEY                  rdx     /* P3, 64 bits */
 413 
 414 #else   /* OpenSolaris Interface */
 415 #define AESKEY                  rdi     /* P1, 64 bits */
 416 #define USERCIPHERKEY           rsi     /* P2, 64 bits */
 417 #define KEYSIZE32               edx     /* P3, 32 bits */
 418 #define KEYSIZE64               rdx     /* P3, 64 bits */
 419 #endif  /* OPENSSL_INTERFACE */
 420 
 421 #define ROUNDS32                KEYSIZE32       /* temp */
 422 #define ROUNDS64                KEYSIZE64       /* temp */
 423 #define ENDAESKEY               USERCIPHERKEY   /* temp */
 424 
 425 
 426 ENTRY_NP(rijndael_key_setup_enc_intel)
 427         /* EXPORT DELETE START */
 428         CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10)
 429 
 430         / NULL pointer sanity check
 431         test    %USERCIPHERKEY, %USERCIPHERKEY
 432         jz      .Lenc_key_invalid_param
 433         test    %AESKEY, %AESKEY
 434         jz      .Lenc_key_invalid_param
 435 
 436         movups  (%USERCIPHERKEY), %xmm0 / user key (first 16 bytes)
 437         movaps  %xmm0, (%AESKEY)
 438         lea     0x10(%AESKEY), %rcx     / key addr
 439         pxor    %xmm4, %xmm4            / xmm4 is assumed 0 in _key_expansion_x
 440 
 441         cmp     $256, %KEYSIZE32
 442         jnz     .Lenc_key192
 443 
 444         / AES 256: 14 rounds in encryption key schedule
 445 #ifdef OPENSSL_INTERFACE
 446         mov     $14, %ROUNDS32
 447         movl    %ROUNDS32, 240(%AESKEY)         / key.rounds = 14


 563 #endif
 564         ret
 565 
 566 .Lenc_key_invalid_param:
 567 #ifdef  OPENSSL_INTERFACE
 568         SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
 569         mov     $-1, %rax       / user key or AES key pointer is NULL
 570         ret
 571 #else
 572         /* FALLTHROUGH */
 573 #endif  /* OPENSSL_INTERFACE */
 574 
 575 .Lenc_key_invalid_key_bits:
 576         SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
 577 #ifdef  OPENSSL_INTERFACE
 578         mov     $-2, %rax       / keysize is invalid
 579 #else   /* Open Solaris Interface */
 580         xor     %rax, %rax      / a key pointer is NULL or invalid keysize
 581 #endif  /* OPENSSL_INTERFACE */
 582 
 583         /* EXPORT DELETE END */
 584         ret
 585         SET_SIZE(rijndael_key_setup_enc_intel)
 586 
 587 
 588 /*
 589  * rijndael_key_setup_dec_intel()
 590  * Expand the cipher key into the decryption key schedule.
 591  *
 592  * For kernel code, caller is responsible for ensuring kpreempt_disable()
 593  * has been called.  This is because %xmm registers are not saved/restored.
 594  * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 595  * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
 596  * on the stack.
 597  *
 598  * OpenSolaris interface:
 599  * int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
 600  *      uint64_t keyBits);
 601  * Return value is 0 on error, number of rounds on success.
 602  * P1->P2, P2->P3, P3->P1
 603  *
 604  * Original Intel OpenSSL interface:
 605  * int intel_AES_set_decrypt_key(const unsigned char *userKey,
 606  *      const int bits, AES_KEY *key);
 607  * Return value is non-zero on error, 0 on success.
 608  */
 609 ENTRY_NP(rijndael_key_setup_dec_intel)
 610         /* EXPORT DELETE START */
 611         / Generate round keys used for encryption
 612         call    rijndael_key_setup_enc_intel
 613         test    %rax, %rax
 614 #ifdef  OPENSSL_INTERFACE
 615         jnz     .Ldec_key_exit  / Failed if returned non-0
 616 #else   /* OpenSolaris Interface */
 617         jz      .Ldec_key_exit  / Failed if returned 0
 618 #endif  /* OPENSSL_INTERFACE */
 619 
 620         CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
 621 
 622         /*
 623          * Convert round keys used for encryption
 624          * to a form usable for decryption
 625          */
 626 #ifndef OPENSSL_INTERFACE               /* OpenSolaris Interface */
 627         mov     %rax, %ROUNDS64         / set # rounds (10, 12, or 14)
 628                                         / (already set for OpenSSL)
 629 #endif
 630 


 643         lea     -0x10(%ROUNDS64), %ROUNDS64
 644         cmp     %AESKEY, %ROUNDS64
 645         ja      .Ldec_key_reorder_loop
 646 
 647 .align 4
 648 .Ldec_key_inv_loop:
 649         movaps  (%rcx), %xmm0
 650         / Convert an encryption round key to a form usable for decryption
 651         / with the "AES Inverse Mix Columns" instruction
 652         aesimc  %xmm0, %xmm1
 653         movaps  %xmm1, (%rcx)
 654         lea     0x10(%rcx), %rcx
 655         cmp     %ENDAESKEY, %rcx
 656         jnz     .Ldec_key_inv_loop
 657 
 658         SET_TS_OR_POP_XMM0_XMM1(%r10)
 659 
 660 .Ldec_key_exit:
 661         / OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error
 662         / OpenSSL: rax = 0 for OK, or non-zero for error
 663         /* EXPORT DELETE END */
 664         ret
 665         SET_SIZE(rijndael_key_setup_dec_intel)
 666 
 667 
 668 /*
 669  * aes_encrypt_intel()
 670  * Encrypt a single block (in and out can overlap).
 671  *
 672  * For kernel code, caller is responsible for ensuring kpreempt_disable()
 673  * has been called.  This is because %xmm registers are not saved/restored.
 674  * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 675  * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
 676  * on the stack.
 677  *
 678  * Temporary register usage:
 679  * %xmm0        State
 680  * %xmm1        Key
 681  *
 682  * Original OpenSolaris Interface:
 683  * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,


 694 
 695 #define INP             rdi     /* P1, 64 bits */
 696 #define OUTP            rsi     /* P2, 64 bits */
 697 #define KEYP            rdx     /* P3, 64 bits */
 698 
 699 /* No NROUNDS parameter--offset 240 from KEYP saved in %ecx:  */
 700 #define NROUNDS32       ecx     /* temporary, 32 bits */
 701 #define NROUNDS         cl      /* temporary,  8 bits */
 702 
 703 #else   /* OpenSolaris Interface */
 704 #define KEYP            rdi     /* P1, 64 bits */
 705 #define NROUNDS         esi     /* P2, 32 bits */
 706 #define INP             rdx     /* P3, 64 bits */
 707 #define OUTP            rcx     /* P4, 64 bits */
 708 #endif  /* OPENSSL_INTERFACE */
 709 
 710 #define STATE           xmm0    /* temporary, 128 bits */
 711 #define KEY             xmm1    /* temporary, 128 bits */
 712 
 713 ENTRY_NP(aes_encrypt_intel)
 714         /* EXPORT DELETE START */
 715         CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
 716 
 717         movups  (%INP), %STATE                  / input
 718         movaps  (%KEYP), %KEY                   / key
 719 #ifdef  OPENSSL_INTERFACE
 720         mov     240(%KEYP), %NROUNDS32          / round count
 721 #else   /* OpenSolaris Interface */
 722         /* Round count is already present as P2 in %rsi/%esi */
 723 #endif  /* OPENSSL_INTERFACE */
 724 
 725         pxor    %KEY, %STATE                    / round 0
 726         lea     0x30(%KEYP), %KEYP
 727         cmp     $12, %NROUNDS
 728         jb      .Lenc128
 729         lea     0x20(%KEYP), %KEYP
 730         je      .Lenc192
 731 
 732         / AES 256
 733         lea     0x20(%KEYP), %KEYP
 734         movaps  -0x60(%KEYP), %KEY


 753         aesenc  %KEY, %STATE
 754         movaps  (%KEYP), %KEY
 755         aesenc  %KEY, %STATE
 756         movaps  0x10(%KEYP), %KEY
 757         aesenc  %KEY, %STATE
 758         movaps  0x20(%KEYP), %KEY
 759         aesenc  %KEY, %STATE
 760         movaps  0x30(%KEYP), %KEY
 761         aesenc  %KEY, %STATE
 762         movaps  0x40(%KEYP), %KEY
 763         aesenc  %KEY, %STATE
 764         movaps  0x50(%KEYP), %KEY
 765         aesenc  %KEY, %STATE
 766         movaps  0x60(%KEYP), %KEY
 767         aesenc  %KEY, %STATE
 768         movaps  0x70(%KEYP), %KEY
 769         aesenclast       %KEY, %STATE           / last round
 770         movups  %STATE, (%OUTP)                 / output
 771 
 772         SET_TS_OR_POP_XMM0_XMM1(%r10)
 773         /* EXPORT DELETE END */
 774         ret
 775         SET_SIZE(aes_encrypt_intel)
 776 
 777 
 778 /*
 779  * aes_decrypt_intel()
 780  * Decrypt a single block (in and out can overlap).
 781  *
 782  * For kernel code, caller is responsible for ensuring kpreempt_disable()
 783  * has been called.  This is because %xmm registers are not saved/restored.
 784  * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 785  * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
 786  * on the stack.
 787  *
 788  * Temporary register usage:
 789  * %xmm0        State
 790  * %xmm1        Key
 791  *
 792  * Original OpenSolaris Interface:
 793  * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
 794  *      const uint32_t pt[4], uint32_t ct[4])/
 795  *
 796  * Original Intel OpenSSL Interface:
 797  * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
 798  *      const AES_KEY *key);
 799  */
 800 ENTRY_NP(aes_decrypt_intel)
 801         /* EXPORT DELETE START */
 802         CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
 803 
 804         movups  (%INP), %STATE                  / input
 805         movaps  (%KEYP), %KEY                   / key
 806 #ifdef  OPENSSL_INTERFACE
 807         mov     240(%KEYP), %NROUNDS32          / round count
 808 #else   /* OpenSolaris Interface */
 809         /* Round count is already present as P2 in %rsi/%esi */
 810 #endif  /* OPENSSL_INTERFACE */
 811 
 812         pxor    %KEY, %STATE                    / round 0
 813         lea     0x30(%KEYP), %KEYP
 814         cmp     $12, %NROUNDS
 815         jb      .Ldec128
 816         lea     0x20(%KEYP), %KEYP
 817         je      .Ldec192
 818 
 819         / AES 256
 820         lea     0x20(%KEYP), %KEYP
 821         movaps  -0x60(%KEYP), %KEY


 841         movaps  (%KEYP), %KEY
 842         aesdec  %KEY, %STATE
 843         movaps  0x10(%KEYP), %KEY
 844         aesdec  %KEY, %STATE
 845         movaps  0x20(%KEYP), %KEY
 846         aesdec  %KEY, %STATE
 847         movaps  0x30(%KEYP), %KEY
 848         aesdec  %KEY, %STATE
 849         movaps  0x40(%KEYP), %KEY
 850         aesdec  %KEY, %STATE
 851         movaps  0x50(%KEYP), %KEY
 852         aesdec  %KEY, %STATE
 853         movaps  0x60(%KEYP), %KEY
 854         aesdec  %KEY, %STATE
 855         movaps  0x70(%KEYP), %KEY
 856         aesdeclast      %KEY, %STATE            / last round
 857         movups  %STATE, (%OUTP)                 / output
 858 
 859         SET_TS_OR_POP_XMM0_XMM1(%r10)
 860         ret
 861         /* EXPORT DELETE END */
 862         SET_SIZE(aes_decrypt_intel)
 863 
 864 #endif  /* lint || __lint */


  66  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  67  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  68  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  69  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  70  * OF THE POSSIBILITY OF SUCH DAMAGE.
  71  * ====================================================================
  72  */
  73 
  74 /*
  75  * ====================================================================
  76  * OpenSolaris OS modifications
  77  *
  78  * This source originates as files aes-intel.S and eng_aesni_asm.pl, in
  79  * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
  80  * Huang Ying of Intel to the openssl-dev mailing list under the subject
  81  * of "Add support to Intel AES-NI instruction set for x86_64 platform".
  82  *
  83  * This OpenSolaris version has these major changes from the original source:
  84  *
  85  * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
  86  * /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function
  87  * definitions for lint.
  88  *
  89  * 2. Formatted code, added comments, and added #includes and #defines.
  90  *
  91  * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
  92  * calling kpreempt_disable() and kpreempt_enable().
  93  * If the TS bit is not set, Save and restore %xmm registers at the beginning
  94  * and end of function calls (%xmm* registers are not saved and restored by
  95  * during kernel thread preemption).
  96  *
  97  * 4. Renamed functions, reordered parameters, and changed return value
  98  * to match OpenSolaris:
  99  *
 100  * OpenSSL interface:
 101  *      int intel_AES_set_encrypt_key(const unsigned char *userKey,
 102  *              const int bits, AES_KEY *key);
 103  *      int intel_AES_set_decrypt_key(const unsigned char *userKey,
 104  *              const int bits, AES_KEY *key);
 105  *      Return values for above are non-zero on error, 0 on success.
 106  *
 107  *      void intel_AES_encrypt(const unsigned char *in, unsigned char *out,


 288 #define SET_TS_OR_POP_XMM0_XMM1(tmpreg)
 289 #define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg)
 290 #define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg)
 291 #endif  /* _KERNEL */
 292 
 293 
 294 /*
 295  * _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(),
 296  * _key_expansion_256a(), _key_expansion_256b()
 297  *
 298  * Helper functions called by rijndael_key_setup_inc_intel().
 299  * Also used indirectly by rijndael_key_setup_dec_intel().
 300  *
 301  * Input:
 302  * %xmm0        User-provided cipher key
 303  * %xmm1        Round constant
 304  * Output:
 305  * (%rcx)       AES key
 306  */
 307 

 308 .align  16
 309 _key_expansion_128:
 310 _key_expansion_256a:
 311         pshufd  $0b11111111, %xmm1, %xmm1
 312         shufps  $0b00010000, %xmm0, %xmm4
 313         pxor    %xmm4, %xmm0
 314         shufps  $0b10001100, %xmm0, %xmm4
 315         pxor    %xmm4, %xmm0
 316         pxor    %xmm1, %xmm0
 317         movaps  %xmm0, (%rcx)
 318         add     $0x10, %rcx
 319         ret
 320         SET_SIZE(_key_expansion_128)
 321         SET_SIZE(_key_expansion_256a)
 322 
 323 .align 16
 324 _key_expansion_192a:
 325         pshufd  $0b01010101, %xmm1, %xmm1
 326         shufps  $0b00010000, %xmm0, %xmm4
 327         pxor    %xmm4, %xmm0


 360         pxor    %xmm3, %xmm2
 361         pxor    %xmm5, %xmm2
 362 
 363         movaps  %xmm0, (%rcx)
 364         add     $0x10, %rcx
 365         ret
 366         SET_SIZE(_key_expansion_192b)
 367 
 368 .align 16
 369 _key_expansion_256b:
 370         pshufd  $0b10101010, %xmm1, %xmm1
 371         shufps  $0b00010000, %xmm2, %xmm4
 372         pxor    %xmm4, %xmm2
 373         shufps  $0b10001100, %xmm2, %xmm4
 374         pxor    %xmm4, %xmm2
 375         pxor    %xmm1, %xmm2
 376         movaps  %xmm2, (%rcx)
 377         add     $0x10, %rcx
 378         ret
 379         SET_SIZE(_key_expansion_256b)

 380 
 381 
 382 /*
 383  * rijndael_key_setup_enc_intel()
 384  * Expand the cipher key into the encryption key schedule.
 385  *
 386  * For kernel code, caller is responsible for ensuring kpreempt_disable()
 387  * has been called.  This is because %xmm registers are not saved/restored.
 388  * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 389  * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
 390  * on the stack.
 391  *
 392  * OpenSolaris interface:
 393  * int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
 394  *      uint64_t keyBits);
 395  * Return value is 0 on error, number of rounds on success.
 396  *
 397  * Original Intel OpenSSL interface:
 398  * int intel_AES_set_encrypt_key(const unsigned char *userKey,
 399  *      const int bits, AES_KEY *key);


 405 #define rijndael_key_setup_dec_intel    intel_AES_set_decrypt_key
 406 
 407 #define USERCIPHERKEY           rdi     /* P1, 64 bits */
 408 #define KEYSIZE32               esi     /* P2, 32 bits */
 409 #define KEYSIZE64               rsi     /* P2, 64 bits */
 410 #define AESKEY                  rdx     /* P3, 64 bits */
 411 
 412 #else   /* OpenSolaris Interface */
 413 #define AESKEY                  rdi     /* P1, 64 bits */
 414 #define USERCIPHERKEY           rsi     /* P2, 64 bits */
 415 #define KEYSIZE32               edx     /* P3, 32 bits */
 416 #define KEYSIZE64               rdx     /* P3, 64 bits */
 417 #endif  /* OPENSSL_INTERFACE */
 418 
 419 #define ROUNDS32                KEYSIZE32       /* temp */
 420 #define ROUNDS64                KEYSIZE64       /* temp */
 421 #define ENDAESKEY               USERCIPHERKEY   /* temp */
 422 
 423 
 424 ENTRY_NP(rijndael_key_setup_enc_intel)

 425         CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10)
 426 
 427         / NULL pointer sanity check
 428         test    %USERCIPHERKEY, %USERCIPHERKEY
 429         jz      .Lenc_key_invalid_param
 430         test    %AESKEY, %AESKEY
 431         jz      .Lenc_key_invalid_param
 432 
 433         movups  (%USERCIPHERKEY), %xmm0 / user key (first 16 bytes)
 434         movaps  %xmm0, (%AESKEY)
 435         lea     0x10(%AESKEY), %rcx     / key addr
 436         pxor    %xmm4, %xmm4            / xmm4 is assumed 0 in _key_expansion_x
 437 
 438         cmp     $256, %KEYSIZE32
 439         jnz     .Lenc_key192
 440 
 441         / AES 256: 14 rounds in encryption key schedule
 442 #ifdef OPENSSL_INTERFACE
 443         mov     $14, %ROUNDS32
 444         movl    %ROUNDS32, 240(%AESKEY)         / key.rounds = 14


 560 #endif
 561         ret
 562 
 563 .Lenc_key_invalid_param:
 564 #ifdef  OPENSSL_INTERFACE
 565         SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
 566         mov     $-1, %rax       / user key or AES key pointer is NULL
 567         ret
 568 #else
 569         /* FALLTHROUGH */
 570 #endif  /* OPENSSL_INTERFACE */
 571 
 572 .Lenc_key_invalid_key_bits:
 573         SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
 574 #ifdef  OPENSSL_INTERFACE
 575         mov     $-2, %rax       / keysize is invalid
 576 #else   /* Open Solaris Interface */
 577         xor     %rax, %rax      / a key pointer is NULL or invalid keysize
 578 #endif  /* OPENSSL_INTERFACE */
 579 

 580         ret
 581         SET_SIZE(rijndael_key_setup_enc_intel)
 582 
 583 
 584 /*
 585  * rijndael_key_setup_dec_intel()
 586  * Expand the cipher key into the decryption key schedule.
 587  *
 588  * For kernel code, caller is responsible for ensuring kpreempt_disable()
 589  * has been called.  This is because %xmm registers are not saved/restored.
 590  * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 591  * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
 592  * on the stack.
 593  *
 594  * OpenSolaris interface:
 595  * int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
 596  *      uint64_t keyBits);
 597  * Return value is 0 on error, number of rounds on success.
 598  * P1->P2, P2->P3, P3->P1
 599  *
 600  * Original Intel OpenSSL interface:
 601  * int intel_AES_set_decrypt_key(const unsigned char *userKey,
 602  *      const int bits, AES_KEY *key);
 603  * Return value is non-zero on error, 0 on success.
 604  */
 605 ENTRY_NP(rijndael_key_setup_dec_intel)

 606         / Generate round keys used for encryption
 607         call    rijndael_key_setup_enc_intel
 608         test    %rax, %rax
 609 #ifdef  OPENSSL_INTERFACE
 610         jnz     .Ldec_key_exit  / Failed if returned non-0
 611 #else   /* OpenSolaris Interface */
 612         jz      .Ldec_key_exit  / Failed if returned 0
 613 #endif  /* OPENSSL_INTERFACE */
 614 
 615         CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
 616 
 617         /*
 618          * Convert round keys used for encryption
 619          * to a form usable for decryption
 620          */
 621 #ifndef OPENSSL_INTERFACE               /* OpenSolaris Interface */
 622         mov     %rax, %ROUNDS64         / set # rounds (10, 12, or 14)
 623                                         / (already set for OpenSSL)
 624 #endif
 625 


 638         lea     -0x10(%ROUNDS64), %ROUNDS64
 639         cmp     %AESKEY, %ROUNDS64
 640         ja      .Ldec_key_reorder_loop
 641 
 642 .align 4
 643 .Ldec_key_inv_loop:
 644         movaps  (%rcx), %xmm0
 645         / Convert an encryption round key to a form usable for decryption
 646         / with the "AES Inverse Mix Columns" instruction
 647         aesimc  %xmm0, %xmm1
 648         movaps  %xmm1, (%rcx)
 649         lea     0x10(%rcx), %rcx
 650         cmp     %ENDAESKEY, %rcx
 651         jnz     .Ldec_key_inv_loop
 652 
 653         SET_TS_OR_POP_XMM0_XMM1(%r10)
 654 
 655 .Ldec_key_exit:
 656         / OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error
 657         / OpenSSL: rax = 0 for OK, or non-zero for error

 658         ret
 659         SET_SIZE(rijndael_key_setup_dec_intel)
 660 
 661 
 662 /*
 663  * aes_encrypt_intel()
 664  * Encrypt a single block (in and out can overlap).
 665  *
 666  * For kernel code, caller is responsible for ensuring kpreempt_disable()
 667  * has been called.  This is because %xmm registers are not saved/restored.
 668  * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 669  * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
 670  * on the stack.
 671  *
 672  * Temporary register usage:
 673  * %xmm0        State
 674  * %xmm1        Key
 675  *
 676  * Original OpenSolaris Interface:
 677  * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,


 688 
 689 #define INP             rdi     /* P1, 64 bits */
 690 #define OUTP            rsi     /* P2, 64 bits */
 691 #define KEYP            rdx     /* P3, 64 bits */
 692 
 693 /* No NROUNDS parameter--offset 240 from KEYP saved in %ecx:  */
 694 #define NROUNDS32       ecx     /* temporary, 32 bits */
 695 #define NROUNDS         cl      /* temporary,  8 bits */
 696 
 697 #else   /* OpenSolaris Interface */
 698 #define KEYP            rdi     /* P1, 64 bits */
 699 #define NROUNDS         esi     /* P2, 32 bits */
 700 #define INP             rdx     /* P3, 64 bits */
 701 #define OUTP            rcx     /* P4, 64 bits */
 702 #endif  /* OPENSSL_INTERFACE */
 703 
 704 #define STATE           xmm0    /* temporary, 128 bits */
 705 #define KEY             xmm1    /* temporary, 128 bits */
 706 
 707 ENTRY_NP(aes_encrypt_intel)

 708         CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
 709 
 710         movups  (%INP), %STATE                  / input
 711         movaps  (%KEYP), %KEY                   / key
 712 #ifdef  OPENSSL_INTERFACE
 713         mov     240(%KEYP), %NROUNDS32          / round count
 714 #else   /* OpenSolaris Interface */
 715         /* Round count is already present as P2 in %rsi/%esi */
 716 #endif  /* OPENSSL_INTERFACE */
 717 
 718         pxor    %KEY, %STATE                    / round 0
 719         lea     0x30(%KEYP), %KEYP
 720         cmp     $12, %NROUNDS
 721         jb      .Lenc128
 722         lea     0x20(%KEYP), %KEYP
 723         je      .Lenc192
 724 
 725         / AES 256
 726         lea     0x20(%KEYP), %KEYP
 727         movaps  -0x60(%KEYP), %KEY


 746         aesenc  %KEY, %STATE
 747         movaps  (%KEYP), %KEY
 748         aesenc  %KEY, %STATE
 749         movaps  0x10(%KEYP), %KEY
 750         aesenc  %KEY, %STATE
 751         movaps  0x20(%KEYP), %KEY
 752         aesenc  %KEY, %STATE
 753         movaps  0x30(%KEYP), %KEY
 754         aesenc  %KEY, %STATE
 755         movaps  0x40(%KEYP), %KEY
 756         aesenc  %KEY, %STATE
 757         movaps  0x50(%KEYP), %KEY
 758         aesenc  %KEY, %STATE
 759         movaps  0x60(%KEYP), %KEY
 760         aesenc  %KEY, %STATE
 761         movaps  0x70(%KEYP), %KEY
 762         aesenclast       %KEY, %STATE           / last round
 763         movups  %STATE, (%OUTP)                 / output
 764 
 765         SET_TS_OR_POP_XMM0_XMM1(%r10)

 766         ret
 767         SET_SIZE(aes_encrypt_intel)
 768 
 769 
 770 /*
 771  * aes_decrypt_intel()
 772  * Decrypt a single block (in and out can overlap).
 773  *
 774  * For kernel code, caller is responsible for ensuring kpreempt_disable()
 775  * has been called.  This is because %xmm registers are not saved/restored.
 776  * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 777  * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
 778  * on the stack.
 779  *
 780  * Temporary register usage:
 781  * %xmm0        State
 782  * %xmm1        Key
 783  *
 784  * Original OpenSolaris Interface:
 785  * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
 786  *      const uint32_t pt[4], uint32_t ct[4])/
 787  *
 788  * Original Intel OpenSSL Interface:
 789  * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
 790  *      const AES_KEY *key);
 791  */
 792 ENTRY_NP(aes_decrypt_intel)

 793         CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
 794 
 795         movups  (%INP), %STATE                  / input
 796         movaps  (%KEYP), %KEY                   / key
 797 #ifdef  OPENSSL_INTERFACE
 798         mov     240(%KEYP), %NROUNDS32          / round count
 799 #else   /* OpenSolaris Interface */
 800         /* Round count is already present as P2 in %rsi/%esi */
 801 #endif  /* OPENSSL_INTERFACE */
 802 
 803         pxor    %KEY, %STATE                    / round 0
 804         lea     0x30(%KEYP), %KEYP
 805         cmp     $12, %NROUNDS
 806         jb      .Ldec128
 807         lea     0x20(%KEYP), %KEYP
 808         je      .Ldec192
 809 
 810         / AES 256
 811         lea     0x20(%KEYP), %KEYP
 812         movaps  -0x60(%KEYP), %KEY


 832         movaps  (%KEYP), %KEY
 833         aesdec  %KEY, %STATE
 834         movaps  0x10(%KEYP), %KEY
 835         aesdec  %KEY, %STATE
 836         movaps  0x20(%KEYP), %KEY
 837         aesdec  %KEY, %STATE
 838         movaps  0x30(%KEYP), %KEY
 839         aesdec  %KEY, %STATE
 840         movaps  0x40(%KEYP), %KEY
 841         aesdec  %KEY, %STATE
 842         movaps  0x50(%KEYP), %KEY
 843         aesdec  %KEY, %STATE
 844         movaps  0x60(%KEYP), %KEY
 845         aesdec  %KEY, %STATE
 846         movaps  0x70(%KEYP), %KEY
 847         aesdeclast      %KEY, %STATE            / last round
 848         movups  %STATE, (%OUTP)                 / output
 849 
 850         SET_TS_OR_POP_XMM0_XMM1(%r10)
 851         ret

 852         SET_SIZE(aes_decrypt_intel)
 853 
 854 #endif  /* lint || __lint */