66 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
67 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
68 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
69 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
70 * OF THE POSSIBILITY OF SUCH DAMAGE.
71 * ====================================================================
72 */
73
74 /*
75 * ====================================================================
76 * OpenSolaris OS modifications
77 *
78 * This source originates as files aes-intel.S and eng_aesni_asm.pl, in
79 * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
80 * Huang Ying of Intel to the openssl-dev mailing list under the subject
81 * of "Add support to Intel AES-NI instruction set for x86_64 platform".
82 *
83 * This OpenSolaris version has these major changes from the original source:
84 *
85 * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
86 * /usr/include/sys/asm_linkage.h, lint(1B) guards, EXPORT DELETE START
87 * and EXPORT DELETE END markers, and dummy C function definitions for lint.
88 *
89 * 2. Formatted code, added comments, and added #includes and #defines.
90 *
91 * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
92 * calling kpreempt_disable() and kpreempt_enable().
93 * If the TS bit is not set, Save and restore %xmm registers at the beginning
94 * and end of function calls (%xmm* registers are not saved and restored by
95 * during kernel thread preemption).
96 *
97 * 4. Renamed functions, reordered parameters, and changed return value
98 * to match OpenSolaris:
99 *
100 * OpenSSL interface:
101 * int intel_AES_set_encrypt_key(const unsigned char *userKey,
102 * const int bits, AES_KEY *key);
103 * int intel_AES_set_decrypt_key(const unsigned char *userKey,
104 * const int bits, AES_KEY *key);
105 * Return values for above are non-zero on error, 0 on success.
106 *
107 * void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
288 #define SET_TS_OR_POP_XMM0_XMM1(tmpreg)
289 #define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg)
290 #define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg)
291 #endif /* _KERNEL */
292
293
294 /*
295 * _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(),
296 * _key_expansion_256a(), _key_expansion_256b()
297 *
298 * Helper functions called by rijndael_key_setup_inc_intel().
299 * Also used indirectly by rijndael_key_setup_dec_intel().
300 *
301 * Input:
302 * %xmm0 User-provided cipher key
303 * %xmm1 Round constant
304 * Output:
305 * (%rcx) AES key
306 */
307
308 /* EXPORT DELETE START */
309 .align 16
310 _key_expansion_128:
311 _key_expansion_256a:
312 pshufd $0b11111111, %xmm1, %xmm1
313 shufps $0b00010000, %xmm0, %xmm4
314 pxor %xmm4, %xmm0
315 shufps $0b10001100, %xmm0, %xmm4
316 pxor %xmm4, %xmm0
317 pxor %xmm1, %xmm0
318 movaps %xmm0, (%rcx)
319 add $0x10, %rcx
320 ret
321 SET_SIZE(_key_expansion_128)
322 SET_SIZE(_key_expansion_256a)
323
324 .align 16
325 _key_expansion_192a:
326 pshufd $0b01010101, %xmm1, %xmm1
327 shufps $0b00010000, %xmm0, %xmm4
328 pxor %xmm4, %xmm0
361 pxor %xmm3, %xmm2
362 pxor %xmm5, %xmm2
363
364 movaps %xmm0, (%rcx)
365 add $0x10, %rcx
366 ret
367 SET_SIZE(_key_expansion_192b)
368
369 .align 16
370 _key_expansion_256b:
371 pshufd $0b10101010, %xmm1, %xmm1
372 shufps $0b00010000, %xmm2, %xmm4
373 pxor %xmm4, %xmm2
374 shufps $0b10001100, %xmm2, %xmm4
375 pxor %xmm4, %xmm2
376 pxor %xmm1, %xmm2
377 movaps %xmm2, (%rcx)
378 add $0x10, %rcx
379 ret
380 SET_SIZE(_key_expansion_256b)
381 /* EXPORT DELETE END */
382
383
384 /*
385 * rijndael_key_setup_enc_intel()
386 * Expand the cipher key into the encryption key schedule.
387 *
388 * For kernel code, caller is responsible for ensuring kpreempt_disable()
389 * has been called. This is because %xmm registers are not saved/restored.
390 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
391 * on entry. Otherwise, if TS is not set, save and restore %xmm registers
392 * on the stack.
393 *
394 * OpenSolaris interface:
395 * int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
396 * uint64_t keyBits);
397 * Return value is 0 on error, number of rounds on success.
398 *
399 * Original Intel OpenSSL interface:
400 * int intel_AES_set_encrypt_key(const unsigned char *userKey,
401 * const int bits, AES_KEY *key);
407 #define rijndael_key_setup_dec_intel intel_AES_set_decrypt_key
408
409 #define USERCIPHERKEY rdi /* P1, 64 bits */
410 #define KEYSIZE32 esi /* P2, 32 bits */
411 #define KEYSIZE64 rsi /* P2, 64 bits */
412 #define AESKEY rdx /* P3, 64 bits */
413
414 #else /* OpenSolaris Interface */
415 #define AESKEY rdi /* P1, 64 bits */
416 #define USERCIPHERKEY rsi /* P2, 64 bits */
417 #define KEYSIZE32 edx /* P3, 32 bits */
418 #define KEYSIZE64 rdx /* P3, 64 bits */
419 #endif /* OPENSSL_INTERFACE */
420
421 #define ROUNDS32 KEYSIZE32 /* temp */
422 #define ROUNDS64 KEYSIZE64 /* temp */
423 #define ENDAESKEY USERCIPHERKEY /* temp */
424
425
426 ENTRY_NP(rijndael_key_setup_enc_intel)
427 /* EXPORT DELETE START */
428 CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10)
429
430 / NULL pointer sanity check
431 test %USERCIPHERKEY, %USERCIPHERKEY
432 jz .Lenc_key_invalid_param
433 test %AESKEY, %AESKEY
434 jz .Lenc_key_invalid_param
435
436 movups (%USERCIPHERKEY), %xmm0 / user key (first 16 bytes)
437 movaps %xmm0, (%AESKEY)
438 lea 0x10(%AESKEY), %rcx / key addr
439 pxor %xmm4, %xmm4 / xmm4 is assumed 0 in _key_expansion_x
440
441 cmp $256, %KEYSIZE32
442 jnz .Lenc_key192
443
444 / AES 256: 14 rounds in encryption key schedule
445 #ifdef OPENSSL_INTERFACE
446 mov $14, %ROUNDS32
447 movl %ROUNDS32, 240(%AESKEY) / key.rounds = 14
563 #endif
564 ret
565
566 .Lenc_key_invalid_param:
567 #ifdef OPENSSL_INTERFACE
568 SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
569 mov $-1, %rax / user key or AES key pointer is NULL
570 ret
571 #else
572 /* FALLTHROUGH */
573 #endif /* OPENSSL_INTERFACE */
574
575 .Lenc_key_invalid_key_bits:
576 SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
577 #ifdef OPENSSL_INTERFACE
578 mov $-2, %rax / keysize is invalid
579 #else /* Open Solaris Interface */
580 xor %rax, %rax / a key pointer is NULL or invalid keysize
581 #endif /* OPENSSL_INTERFACE */
582
583 /* EXPORT DELETE END */
584 ret
585 SET_SIZE(rijndael_key_setup_enc_intel)
586
587
588 /*
589 * rijndael_key_setup_dec_intel()
590 * Expand the cipher key into the decryption key schedule.
591 *
592 * For kernel code, caller is responsible for ensuring kpreempt_disable()
593 * has been called. This is because %xmm registers are not saved/restored.
594 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
595 * on entry. Otherwise, if TS is not set, save and restore %xmm registers
596 * on the stack.
597 *
598 * OpenSolaris interface:
599 * int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
600 * uint64_t keyBits);
601 * Return value is 0 on error, number of rounds on success.
602 * P1->P2, P2->P3, P3->P1
603 *
604 * Original Intel OpenSSL interface:
605 * int intel_AES_set_decrypt_key(const unsigned char *userKey,
606 * const int bits, AES_KEY *key);
607 * Return value is non-zero on error, 0 on success.
608 */
609 ENTRY_NP(rijndael_key_setup_dec_intel)
610 /* EXPORT DELETE START */
611 / Generate round keys used for encryption
612 call rijndael_key_setup_enc_intel
613 test %rax, %rax
614 #ifdef OPENSSL_INTERFACE
615 jnz .Ldec_key_exit / Failed if returned non-0
616 #else /* OpenSolaris Interface */
617 jz .Ldec_key_exit / Failed if returned 0
618 #endif /* OPENSSL_INTERFACE */
619
620 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
621
622 /*
623 * Convert round keys used for encryption
624 * to a form usable for decryption
625 */
626 #ifndef OPENSSL_INTERFACE /* OpenSolaris Interface */
627 mov %rax, %ROUNDS64 / set # rounds (10, 12, or 14)
628 / (already set for OpenSSL)
629 #endif
630
643 lea -0x10(%ROUNDS64), %ROUNDS64
644 cmp %AESKEY, %ROUNDS64
645 ja .Ldec_key_reorder_loop
646
647 .align 4
648 .Ldec_key_inv_loop:
649 movaps (%rcx), %xmm0
650 / Convert an encryption round key to a form usable for decryption
651 / with the "AES Inverse Mix Columns" instruction
652 aesimc %xmm0, %xmm1
653 movaps %xmm1, (%rcx)
654 lea 0x10(%rcx), %rcx
655 cmp %ENDAESKEY, %rcx
656 jnz .Ldec_key_inv_loop
657
658 SET_TS_OR_POP_XMM0_XMM1(%r10)
659
660 .Ldec_key_exit:
661 / OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error
662 / OpenSSL: rax = 0 for OK, or non-zero for error
663 /* EXPORT DELETE END */
664 ret
665 SET_SIZE(rijndael_key_setup_dec_intel)
666
667
668 /*
669 * aes_encrypt_intel()
670 * Encrypt a single block (in and out can overlap).
671 *
672 * For kernel code, caller is responsible for ensuring kpreempt_disable()
673 * has been called. This is because %xmm registers are not saved/restored.
674 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
675 * on entry. Otherwise, if TS is not set, save and restore %xmm registers
676 * on the stack.
677 *
678 * Temporary register usage:
679 * %xmm0 State
680 * %xmm1 Key
681 *
682 * Original OpenSolaris Interface:
683 * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
694
695 #define INP rdi /* P1, 64 bits */
696 #define OUTP rsi /* P2, 64 bits */
697 #define KEYP rdx /* P3, 64 bits */
698
699 /* No NROUNDS parameter--offset 240 from KEYP saved in %ecx: */
700 #define NROUNDS32 ecx /* temporary, 32 bits */
701 #define NROUNDS cl /* temporary, 8 bits */
702
703 #else /* OpenSolaris Interface */
704 #define KEYP rdi /* P1, 64 bits */
705 #define NROUNDS esi /* P2, 32 bits */
706 #define INP rdx /* P3, 64 bits */
707 #define OUTP rcx /* P4, 64 bits */
708 #endif /* OPENSSL_INTERFACE */
709
710 #define STATE xmm0 /* temporary, 128 bits */
711 #define KEY xmm1 /* temporary, 128 bits */
712
713 ENTRY_NP(aes_encrypt_intel)
714 /* EXPORT DELETE START */
715 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
716
717 movups (%INP), %STATE / input
718 movaps (%KEYP), %KEY / key
719 #ifdef OPENSSL_INTERFACE
720 mov 240(%KEYP), %NROUNDS32 / round count
721 #else /* OpenSolaris Interface */
722 /* Round count is already present as P2 in %rsi/%esi */
723 #endif /* OPENSSL_INTERFACE */
724
725 pxor %KEY, %STATE / round 0
726 lea 0x30(%KEYP), %KEYP
727 cmp $12, %NROUNDS
728 jb .Lenc128
729 lea 0x20(%KEYP), %KEYP
730 je .Lenc192
731
732 / AES 256
733 lea 0x20(%KEYP), %KEYP
734 movaps -0x60(%KEYP), %KEY
753 aesenc %KEY, %STATE
754 movaps (%KEYP), %KEY
755 aesenc %KEY, %STATE
756 movaps 0x10(%KEYP), %KEY
757 aesenc %KEY, %STATE
758 movaps 0x20(%KEYP), %KEY
759 aesenc %KEY, %STATE
760 movaps 0x30(%KEYP), %KEY
761 aesenc %KEY, %STATE
762 movaps 0x40(%KEYP), %KEY
763 aesenc %KEY, %STATE
764 movaps 0x50(%KEYP), %KEY
765 aesenc %KEY, %STATE
766 movaps 0x60(%KEYP), %KEY
767 aesenc %KEY, %STATE
768 movaps 0x70(%KEYP), %KEY
769 aesenclast %KEY, %STATE / last round
770 movups %STATE, (%OUTP) / output
771
772 SET_TS_OR_POP_XMM0_XMM1(%r10)
773 /* EXPORT DELETE END */
774 ret
775 SET_SIZE(aes_encrypt_intel)
776
777
778 /*
779 * aes_decrypt_intel()
780 * Decrypt a single block (in and out can overlap).
781 *
782 * For kernel code, caller is responsible for ensuring kpreempt_disable()
783 * has been called. This is because %xmm registers are not saved/restored.
784 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
785 * on entry. Otherwise, if TS is not set, save and restore %xmm registers
786 * on the stack.
787 *
788 * Temporary register usage:
789 * %xmm0 State
790 * %xmm1 Key
791 *
792 * Original OpenSolaris Interface:
793 * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
794 * const uint32_t pt[4], uint32_t ct[4])/
795 *
796 * Original Intel OpenSSL Interface:
797 * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
798 * const AES_KEY *key);
799 */
800 ENTRY_NP(aes_decrypt_intel)
801 /* EXPORT DELETE START */
802 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
803
804 movups (%INP), %STATE / input
805 movaps (%KEYP), %KEY / key
806 #ifdef OPENSSL_INTERFACE
807 mov 240(%KEYP), %NROUNDS32 / round count
808 #else /* OpenSolaris Interface */
809 /* Round count is already present as P2 in %rsi/%esi */
810 #endif /* OPENSSL_INTERFACE */
811
812 pxor %KEY, %STATE / round 0
813 lea 0x30(%KEYP), %KEYP
814 cmp $12, %NROUNDS
815 jb .Ldec128
816 lea 0x20(%KEYP), %KEYP
817 je .Ldec192
818
819 / AES 256
820 lea 0x20(%KEYP), %KEYP
821 movaps -0x60(%KEYP), %KEY
841 movaps (%KEYP), %KEY
842 aesdec %KEY, %STATE
843 movaps 0x10(%KEYP), %KEY
844 aesdec %KEY, %STATE
845 movaps 0x20(%KEYP), %KEY
846 aesdec %KEY, %STATE
847 movaps 0x30(%KEYP), %KEY
848 aesdec %KEY, %STATE
849 movaps 0x40(%KEYP), %KEY
850 aesdec %KEY, %STATE
851 movaps 0x50(%KEYP), %KEY
852 aesdec %KEY, %STATE
853 movaps 0x60(%KEYP), %KEY
854 aesdec %KEY, %STATE
855 movaps 0x70(%KEYP), %KEY
856 aesdeclast %KEY, %STATE / last round
857 movups %STATE, (%OUTP) / output
858
859 SET_TS_OR_POP_XMM0_XMM1(%r10)
860 ret
861 /* EXPORT DELETE END */
862 SET_SIZE(aes_decrypt_intel)
863
864 #endif /* lint || __lint */
|
66 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
67 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
68 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
69 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
70 * OF THE POSSIBILITY OF SUCH DAMAGE.
71 * ====================================================================
72 */
73
74 /*
75 * ====================================================================
76 * OpenSolaris OS modifications
77 *
78 * This source originates as files aes-intel.S and eng_aesni_asm.pl, in
79 * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
80 * Huang Ying of Intel to the openssl-dev mailing list under the subject
81 * of "Add support to Intel AES-NI instruction set for x86_64 platform".
82 *
83 * This OpenSolaris version has these major changes from the original source:
84 *
85 * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
86 * /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function
87 * definitions for lint.
88 *
89 * 2. Formatted code, added comments, and added #includes and #defines.
90 *
91 * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
92 * calling kpreempt_disable() and kpreempt_enable().
93 * If the TS bit is not set, Save and restore %xmm registers at the beginning
94 * and end of function calls (%xmm* registers are not saved and restored by
95 * during kernel thread preemption).
96 *
97 * 4. Renamed functions, reordered parameters, and changed return value
98 * to match OpenSolaris:
99 *
100 * OpenSSL interface:
101 * int intel_AES_set_encrypt_key(const unsigned char *userKey,
102 * const int bits, AES_KEY *key);
103 * int intel_AES_set_decrypt_key(const unsigned char *userKey,
104 * const int bits, AES_KEY *key);
105 * Return values for above are non-zero on error, 0 on success.
106 *
107 * void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
288 #define SET_TS_OR_POP_XMM0_XMM1(tmpreg)
289 #define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg)
290 #define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg)
291 #endif /* _KERNEL */
292
293
294 /*
295 * _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(),
296 * _key_expansion_256a(), _key_expansion_256b()
297 *
298 * Helper functions called by rijndael_key_setup_inc_intel().
299 * Also used indirectly by rijndael_key_setup_dec_intel().
300 *
301 * Input:
302 * %xmm0 User-provided cipher key
303 * %xmm1 Round constant
304 * Output:
305 * (%rcx) AES key
306 */
307
308 .align 16
309 _key_expansion_128:
310 _key_expansion_256a:
311 pshufd $0b11111111, %xmm1, %xmm1
312 shufps $0b00010000, %xmm0, %xmm4
313 pxor %xmm4, %xmm0
314 shufps $0b10001100, %xmm0, %xmm4
315 pxor %xmm4, %xmm0
316 pxor %xmm1, %xmm0
317 movaps %xmm0, (%rcx)
318 add $0x10, %rcx
319 ret
320 SET_SIZE(_key_expansion_128)
321 SET_SIZE(_key_expansion_256a)
322
323 .align 16
324 _key_expansion_192a:
325 pshufd $0b01010101, %xmm1, %xmm1
326 shufps $0b00010000, %xmm0, %xmm4
327 pxor %xmm4, %xmm0
360 pxor %xmm3, %xmm2
361 pxor %xmm5, %xmm2
362
363 movaps %xmm0, (%rcx)
364 add $0x10, %rcx
365 ret
366 SET_SIZE(_key_expansion_192b)
367
368 .align 16
369 _key_expansion_256b:
370 pshufd $0b10101010, %xmm1, %xmm1
371 shufps $0b00010000, %xmm2, %xmm4
372 pxor %xmm4, %xmm2
373 shufps $0b10001100, %xmm2, %xmm4
374 pxor %xmm4, %xmm2
375 pxor %xmm1, %xmm2
376 movaps %xmm2, (%rcx)
377 add $0x10, %rcx
378 ret
379 SET_SIZE(_key_expansion_256b)
380
381
382 /*
383 * rijndael_key_setup_enc_intel()
384 * Expand the cipher key into the encryption key schedule.
385 *
386 * For kernel code, caller is responsible for ensuring kpreempt_disable()
387 * has been called. This is because %xmm registers are not saved/restored.
388 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
389 * on entry. Otherwise, if TS is not set, save and restore %xmm registers
390 * on the stack.
391 *
392 * OpenSolaris interface:
393 * int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
394 * uint64_t keyBits);
395 * Return value is 0 on error, number of rounds on success.
396 *
397 * Original Intel OpenSSL interface:
398 * int intel_AES_set_encrypt_key(const unsigned char *userKey,
399 * const int bits, AES_KEY *key);
405 #define rijndael_key_setup_dec_intel intel_AES_set_decrypt_key
406
407 #define USERCIPHERKEY rdi /* P1, 64 bits */
408 #define KEYSIZE32 esi /* P2, 32 bits */
409 #define KEYSIZE64 rsi /* P2, 64 bits */
410 #define AESKEY rdx /* P3, 64 bits */
411
412 #else /* OpenSolaris Interface */
413 #define AESKEY rdi /* P1, 64 bits */
414 #define USERCIPHERKEY rsi /* P2, 64 bits */
415 #define KEYSIZE32 edx /* P3, 32 bits */
416 #define KEYSIZE64 rdx /* P3, 64 bits */
417 #endif /* OPENSSL_INTERFACE */
418
419 #define ROUNDS32 KEYSIZE32 /* temp */
420 #define ROUNDS64 KEYSIZE64 /* temp */
421 #define ENDAESKEY USERCIPHERKEY /* temp */
422
423
424 ENTRY_NP(rijndael_key_setup_enc_intel)
425 CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10)
426
427 / NULL pointer sanity check
428 test %USERCIPHERKEY, %USERCIPHERKEY
429 jz .Lenc_key_invalid_param
430 test %AESKEY, %AESKEY
431 jz .Lenc_key_invalid_param
432
433 movups (%USERCIPHERKEY), %xmm0 / user key (first 16 bytes)
434 movaps %xmm0, (%AESKEY)
435 lea 0x10(%AESKEY), %rcx / key addr
436 pxor %xmm4, %xmm4 / xmm4 is assumed 0 in _key_expansion_x
437
438 cmp $256, %KEYSIZE32
439 jnz .Lenc_key192
440
441 / AES 256: 14 rounds in encryption key schedule
442 #ifdef OPENSSL_INTERFACE
443 mov $14, %ROUNDS32
444 movl %ROUNDS32, 240(%AESKEY) / key.rounds = 14
560 #endif
561 ret
562
563 .Lenc_key_invalid_param:
564 #ifdef OPENSSL_INTERFACE
565 SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
566 mov $-1, %rax / user key or AES key pointer is NULL
567 ret
568 #else
569 /* FALLTHROUGH */
570 #endif /* OPENSSL_INTERFACE */
571
572 .Lenc_key_invalid_key_bits:
573 SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
574 #ifdef OPENSSL_INTERFACE
575 mov $-2, %rax / keysize is invalid
576 #else /* Open Solaris Interface */
577 xor %rax, %rax / a key pointer is NULL or invalid keysize
578 #endif /* OPENSSL_INTERFACE */
579
580 ret
581 SET_SIZE(rijndael_key_setup_enc_intel)
582
583
584 /*
585 * rijndael_key_setup_dec_intel()
586 * Expand the cipher key into the decryption key schedule.
587 *
588 * For kernel code, caller is responsible for ensuring kpreempt_disable()
589 * has been called. This is because %xmm registers are not saved/restored.
590 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
591 * on entry. Otherwise, if TS is not set, save and restore %xmm registers
592 * on the stack.
593 *
594 * OpenSolaris interface:
595 * int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
596 * uint64_t keyBits);
597 * Return value is 0 on error, number of rounds on success.
598 * P1->P2, P2->P3, P3->P1
599 *
600 * Original Intel OpenSSL interface:
601 * int intel_AES_set_decrypt_key(const unsigned char *userKey,
602 * const int bits, AES_KEY *key);
603 * Return value is non-zero on error, 0 on success.
604 */
605 ENTRY_NP(rijndael_key_setup_dec_intel)
606 / Generate round keys used for encryption
607 call rijndael_key_setup_enc_intel
608 test %rax, %rax
609 #ifdef OPENSSL_INTERFACE
610 jnz .Ldec_key_exit / Failed if returned non-0
611 #else /* OpenSolaris Interface */
612 jz .Ldec_key_exit / Failed if returned 0
613 #endif /* OPENSSL_INTERFACE */
614
615 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
616
617 /*
618 * Convert round keys used for encryption
619 * to a form usable for decryption
620 */
621 #ifndef OPENSSL_INTERFACE /* OpenSolaris Interface */
622 mov %rax, %ROUNDS64 / set # rounds (10, 12, or 14)
623 / (already set for OpenSSL)
624 #endif
625
638 lea -0x10(%ROUNDS64), %ROUNDS64
639 cmp %AESKEY, %ROUNDS64
640 ja .Ldec_key_reorder_loop
641
642 .align 4
643 .Ldec_key_inv_loop:
644 movaps (%rcx), %xmm0
645 / Convert an encryption round key to a form usable for decryption
646 / with the "AES Inverse Mix Columns" instruction
647 aesimc %xmm0, %xmm1
648 movaps %xmm1, (%rcx)
649 lea 0x10(%rcx), %rcx
650 cmp %ENDAESKEY, %rcx
651 jnz .Ldec_key_inv_loop
652
653 SET_TS_OR_POP_XMM0_XMM1(%r10)
654
655 .Ldec_key_exit:
656 / OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error
657 / OpenSSL: rax = 0 for OK, or non-zero for error
658 ret
659 SET_SIZE(rijndael_key_setup_dec_intel)
660
661
662 /*
663 * aes_encrypt_intel()
664 * Encrypt a single block (in and out can overlap).
665 *
666 * For kernel code, caller is responsible for ensuring kpreempt_disable()
667 * has been called. This is because %xmm registers are not saved/restored.
668 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
669 * on entry. Otherwise, if TS is not set, save and restore %xmm registers
670 * on the stack.
671 *
672 * Temporary register usage:
673 * %xmm0 State
674 * %xmm1 Key
675 *
676 * Original OpenSolaris Interface:
677 * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
688
689 #define INP rdi /* P1, 64 bits */
690 #define OUTP rsi /* P2, 64 bits */
691 #define KEYP rdx /* P3, 64 bits */
692
693 /* No NROUNDS parameter--offset 240 from KEYP saved in %ecx: */
694 #define NROUNDS32 ecx /* temporary, 32 bits */
695 #define NROUNDS cl /* temporary, 8 bits */
696
697 #else /* OpenSolaris Interface */
698 #define KEYP rdi /* P1, 64 bits */
699 #define NROUNDS esi /* P2, 32 bits */
700 #define INP rdx /* P3, 64 bits */
701 #define OUTP rcx /* P4, 64 bits */
702 #endif /* OPENSSL_INTERFACE */
703
704 #define STATE xmm0 /* temporary, 128 bits */
705 #define KEY xmm1 /* temporary, 128 bits */
706
707 ENTRY_NP(aes_encrypt_intel)
708 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
709
710 movups (%INP), %STATE / input
711 movaps (%KEYP), %KEY / key
712 #ifdef OPENSSL_INTERFACE
713 mov 240(%KEYP), %NROUNDS32 / round count
714 #else /* OpenSolaris Interface */
715 /* Round count is already present as P2 in %rsi/%esi */
716 #endif /* OPENSSL_INTERFACE */
717
718 pxor %KEY, %STATE / round 0
719 lea 0x30(%KEYP), %KEYP
720 cmp $12, %NROUNDS
721 jb .Lenc128
722 lea 0x20(%KEYP), %KEYP
723 je .Lenc192
724
725 / AES 256
726 lea 0x20(%KEYP), %KEYP
727 movaps -0x60(%KEYP), %KEY
746 aesenc %KEY, %STATE
747 movaps (%KEYP), %KEY
748 aesenc %KEY, %STATE
749 movaps 0x10(%KEYP), %KEY
750 aesenc %KEY, %STATE
751 movaps 0x20(%KEYP), %KEY
752 aesenc %KEY, %STATE
753 movaps 0x30(%KEYP), %KEY
754 aesenc %KEY, %STATE
755 movaps 0x40(%KEYP), %KEY
756 aesenc %KEY, %STATE
757 movaps 0x50(%KEYP), %KEY
758 aesenc %KEY, %STATE
759 movaps 0x60(%KEYP), %KEY
760 aesenc %KEY, %STATE
761 movaps 0x70(%KEYP), %KEY
762 aesenclast %KEY, %STATE / last round
763 movups %STATE, (%OUTP) / output
764
765 SET_TS_OR_POP_XMM0_XMM1(%r10)
766 ret
767 SET_SIZE(aes_encrypt_intel)
768
769
770 /*
771 * aes_decrypt_intel()
772 * Decrypt a single block (in and out can overlap).
773 *
774 * For kernel code, caller is responsible for ensuring kpreempt_disable()
775 * has been called. This is because %xmm registers are not saved/restored.
776 * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
777 * on entry. Otherwise, if TS is not set, save and restore %xmm registers
778 * on the stack.
779 *
780 * Temporary register usage:
781 * %xmm0 State
782 * %xmm1 Key
783 *
784 * Original OpenSolaris Interface:
785 * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
786 * const uint32_t pt[4], uint32_t ct[4])/
787 *
788 * Original Intel OpenSSL Interface:
789 * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
790 * const AES_KEY *key);
791 */
792 ENTRY_NP(aes_decrypt_intel)
793 CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
794
795 movups (%INP), %STATE / input
796 movaps (%KEYP), %KEY / key
797 #ifdef OPENSSL_INTERFACE
798 mov 240(%KEYP), %NROUNDS32 / round count
799 #else /* OpenSolaris Interface */
800 /* Round count is already present as P2 in %rsi/%esi */
801 #endif /* OPENSSL_INTERFACE */
802
803 pxor %KEY, %STATE / round 0
804 lea 0x30(%KEYP), %KEYP
805 cmp $12, %NROUNDS
806 jb .Ldec128
807 lea 0x20(%KEYP), %KEYP
808 je .Ldec192
809
810 / AES 256
811 lea 0x20(%KEYP), %KEYP
812 movaps -0x60(%KEYP), %KEY
832 movaps (%KEYP), %KEY
833 aesdec %KEY, %STATE
834 movaps 0x10(%KEYP), %KEY
835 aesdec %KEY, %STATE
836 movaps 0x20(%KEYP), %KEY
837 aesdec %KEY, %STATE
838 movaps 0x30(%KEYP), %KEY
839 aesdec %KEY, %STATE
840 movaps 0x40(%KEYP), %KEY
841 aesdec %KEY, %STATE
842 movaps 0x50(%KEYP), %KEY
843 aesdec %KEY, %STATE
844 movaps 0x60(%KEYP), %KEY
845 aesdec %KEY, %STATE
846 movaps 0x70(%KEYP), %KEY
847 aesdeclast %KEY, %STATE / last round
848 movups %STATE, (%OUTP) / output
849
850 SET_TS_OR_POP_XMM0_XMM1(%r10)
851 ret
852 SET_SIZE(aes_decrypt_intel)
853
854 #endif /* lint || __lint */
|