1 /* ====================================================================
2 * Copyright (c) 1999-2016 The OpenSSL Project. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
16 * 3. All advertising materials mentioning features or use of this
17 * software must display the following acknowledgment:
18 * "This product includes software developed by the OpenSSL Project
19 * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
21 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
22 * endorse or promote products derived from this software without
23 * prior written permission. For written permission, please contact
24 * openssl-core@OpenSSL.org.
26 * 5. Products derived from this software may not be called "OpenSSL"
27 * nor may "OpenSSL" appear in their names without prior written
28 * permission of the OpenSSL Project.
30 * 6. Redistributions of any form whatsoever must retain the following
32 * "This product includes software developed by the OpenSSL Project
33 * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
35 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
36 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
37 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
38 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
39 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
41 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
42 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
43 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
44 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
45 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
46 * OF THE POSSIBILITY OF SUCH DAMAGE.
47 * ====================================================================
49 * This product includes cryptographic software written by Eric Young
50 * (eay@cryptsoft.com). This product includes software written by Tim
51 * Hudson (tjh@cryptsoft.com).
55 /* Required for vmsplice */
61 #include <openssl/engine.h>
62 #include <openssl/async.h>
64 #include <linux/version.h>
68 #if LINUX_VERSION_CODE <= KERNEL_VERSION(K_MAJ, K_MIN1, K_MIN2)
69 # warning "AFALG ENGINE requires Kernel Headers >= 4.1.0"
70 # warning "Skipping Compilation of AFALG engine"
73 # include <linux/if_alg.h>
74 # include <sys/socket.h>
76 # include <sys/utsname.h>
78 # include <linux/aio_abi.h>
79 # include <sys/syscall.h>
84 # define AFALG_LIB_NAME "AFALG"
85 # include "e_afalg_err.h"
92 # ifndef SPLICE_F_GIFT
93 # define SPLICE_F_GIFT (0x08)
97 # define ALG_AES_IV_LEN 16
98 # define ALG_IV_LEN(len) (sizeof(struct af_alg_iv) + (len))
99 # define ALG_OP_TYPE unsigned int
100 # define ALG_OP_LEN (sizeof(ALG_OP_TYPE))
102 /* Local Linkage Functions */
103 static int afalg_init_aio(afalg_aio *aio);
104 static int afalg_fin_cipher_aio(afalg_aio *ptr, int sfd,
105 unsigned char *buf, size_t len);
106 static int afalg_create_bind_sk(void);
107 static int afalg_destroy(ENGINE *e);
108 static int afalg_init(ENGINE *e);
109 static int afalg_finish(ENGINE *e);
110 const EVP_CIPHER *afalg_aes_128_cbc(void);
111 static int afalg_ciphers(ENGINE *e, const EVP_CIPHER **cipher,
112 const int **nids, int nid);
113 static int afalg_cipher_init(EVP_CIPHER_CTX *ctx, const unsigned char *key,
114 const unsigned char *iv, int enc);
115 static int afalg_do_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
116 const unsigned char *in, size_t inl);
117 static int afalg_cipher_cleanup(EVP_CIPHER_CTX *ctx);
118 static int afalg_chk_platform(void);
120 /* Engine Id and Name */
121 static const char *engine_afalg_id = "afalg";
122 static const char *engine_afalg_name = "AFLAG engine support";
124 int afalg_cipher_nids[] = {
128 static EVP_CIPHER *_hidden_aes_128_cbc = NULL;
130 static inline int io_setup(unsigned n, aio_context_t *ctx)
132 return syscall(__NR_io_setup, n, ctx);
135 static inline int eventfd(int n)
137 return syscall(__NR_eventfd, n);
140 static inline int io_destroy(aio_context_t ctx)
142 return syscall(__NR_io_destroy, ctx);
145 static inline int io_read(aio_context_t ctx, long n, struct iocb **iocb)
147 return syscall(__NR_io_submit, ctx, n, iocb);
150 static inline int io_getevents(aio_context_t ctx, long min, long max,
151 struct io_event *events,
152 struct timespec *timeout)
154 return syscall(__NR_io_getevents, ctx, min, max, events, timeout);
157 static void afalg_waitfd_cleanup(ASYNC_WAIT_CTX *ctx, const void *key,
158 OSSL_ASYNC_FD waitfd, void *custom)
163 static int afalg_setup_async_event_notification(afalg_aio *aio)
166 ASYNC_WAIT_CTX *waitctx;
169 if ((job = ASYNC_get_current_job()) != NULL) {
171 waitctx = ASYNC_get_wait_ctx(job);
172 if (waitctx == NULL) {
173 ALG_WARN("%s: ASYNC_get_wait_ctx error", __func__);
176 /* Get waitfd from ASYNC_WAIT_CTX if it is alreday set */
177 if (0 == ASYNC_WAIT_CTX_get_fd(waitctx, engine_afalg_id,
178 &aio->efd_async, &custom)) {
179 /* waitfd is not set in ASYNC_WAIT_CTX so we set it */
180 /* efd_async will be signaled when AIO operation completes */
181 aio->efd_async = eventfd(0);
182 if (aio->efd_async == -1) {
183 ALG_PERR("%s: Failed to get eventfd : ", __func__);
184 AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION,
185 AFALG_R_EVENTFD_FAILED);
189 ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_afalg_id,
190 aio->efd_async, custom,
191 afalg_waitfd_cleanup)) {
192 ALG_WARN("%s: Failed to set wait fd", __func__);
193 close(aio->efd_async);
196 /* make fd non-blocking in async mode */
197 if (fcntl(aio->efd_async, F_SETFL, O_NONBLOCK) != 0) {
198 ALG_WARN("%s: Failed to set event fd as NONBLOCKING",
202 /* efd_async is the active fd to be used */
203 aio->efd = aio->efd_async;
206 /* efd_sync will be signaled when AIO operation completes */
207 aio->efd_sync = eventfd(0);
208 if (aio->efd_sync == -1) {
209 ALG_PERR("%s: Failed to get eventfd : ", __func__);
210 AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION,
211 AFALG_R_EVENTFD_FAILED);
214 /* efd_async is the active fd to be used */
215 aio->efd = aio->efd_sync;
220 int afalg_init_aio(afalg_aio *aio)
224 /* Initialise for AIO */
226 r = io_setup(MAX_INFLIGHTS, &aio->aio_ctx);
228 ALG_PERR("%s: io_setup error : ", __func__);
229 AFALGerr(AFALG_F_AFALG_INIT_AIO, AFALG_R_IO_SETUP_FAILED);
233 memset(aio->cbt, 0, sizeof(aio->cbt));
241 int afalg_fin_cipher_aio(afalg_aio *aio, int sfd, unsigned char *buf,
246 unsigned int done = 0;
248 struct timespec timeout;
249 struct io_event events[MAX_INFLIGHTS];
255 /* if efd has not been initialised yet do it here */
256 if (aio->efd == -1) {
257 r = afalg_setup_async_event_notification(aio);
262 cb = &(aio->cbt[0 % MAX_INFLIGHTS]);
263 memset(cb, '\0', sizeof(*cb));
264 cb->aio_fildes = sfd;
265 cb->aio_lio_opcode = IOCB_CMD_PREAD;
266 cb->aio_buf = (unsigned long)buf;
269 cb->aio_nbytes = len;
270 cb->aio_flags = IOCB_FLAG_RESFD;
271 cb->aio_resfd = aio->efd;
274 * Perform AIO read on AFALG socket, this in turn performs an async
275 * crypto operation in kernel space
277 r = io_read(aio->aio_ctx, 1, &cb);
279 ALG_PERR("%s: io_read failed : ", __func__);
284 /* While AIO read is being performed pause job */
287 /* Check for completion of AIO read */
288 r = read(aio->efd, &eval, sizeof(eval));
290 if (errno == EAGAIN || errno == EWOULDBLOCK)
292 ALG_PERR("%s: read failed for event fd : ", __func__);
294 } else if (r == 0 || eval <= 0) {
295 ALG_WARN("%s: eventfd read %d bytes, eval = %lu\n", __func__, r,
300 /* Get results of AIO read */
301 r = io_getevents(aio->aio_ctx, 1, 1, events, &timeout);
304 * events.res indicates the actual status of the operation.
305 * Handle the error condition first.
307 if (events[0].res < 0) {
309 * Underlying operation cannot be completed at the time
310 * of previous submission. Resubmit for the operation.
312 if (events[0].res == -EBUSY && retry++ < 3) {
313 r = io_read(aio->aio_ctx, 1, &cb);
315 ALG_PERR("%s: retry %d for io_read failed : ",
322 * Retries exceed for -EBUSY or unrecoverable error
323 * condition for this instance of operation.
326 ("%s: Crypto Operation failed with code %lld\n",
327 __func__, events[0].res);
331 /* Operation successful. */
334 ALG_PERR("%s: io_getevents failed : ", __func__);
337 ALG_WARN("%s: io_geteventd read 0 bytes\n", __func__);
345 static int afalg_create_bind_sk(void)
347 struct sockaddr_alg sa = {
348 .salg_family = AF_ALG,
349 .salg_type = "skcipher",
350 .salg_name = "cbc(aes)"
356 /* Create AFALG socket for crypto processing */
357 sfd = socket(AF_ALG, SOCK_SEQPACKET, 0);
359 ALG_PERR("%s: Failed to open socket : ", __func__);
360 AFALGerr(AFALG_F_AFALG_CREATE_BIND_SK, AFALG_R_SOCKET_CREATE_FAILED);
364 /* Set cipher algorithm for aes-cbc */
365 r = bind(sfd, (struct sockaddr *)&sa, sizeof(sa));
367 ALG_PERR("%s: Failed to bind socket : ", __func__);
368 AFALGerr(AFALG_F_AFALG_CREATE_BIND_SK, AFALG_R_SOCKET_BIND_FAILED);
380 static inline void afalg_set_op_sk(struct cmsghdr *cmsg,
381 const unsigned int op)
383 cmsg->cmsg_level = SOL_ALG;
384 cmsg->cmsg_type = ALG_SET_OP;
385 cmsg->cmsg_len = CMSG_LEN(ALG_OP_LEN);
386 *CMSG_DATA(cmsg) = (char)op;
389 static void afalg_set_iv_sk(struct cmsghdr *cmsg, const unsigned char *iv,
390 const unsigned int len)
392 struct af_alg_iv *aiv;
394 cmsg->cmsg_level = SOL_ALG;
395 cmsg->cmsg_type = ALG_SET_IV;
396 cmsg->cmsg_len = CMSG_LEN(ALG_IV_LEN(len));
397 aiv = (struct af_alg_iv *)CMSG_DATA(cmsg);
399 memcpy(aiv->iv, iv, len);
402 static void afalg_socket(afalg_ctx *actx, const unsigned char *key,
407 actx->bfd = actx->sfd = -1;
409 actx->bfd = afalg_create_bind_sk();
415 * Send cipher key to the kernel in preparation for future crypto
418 ret = setsockopt(actx->bfd, SOL_ALG, ALG_SET_KEY, key, klen);
420 ALG_PERR("%s: Failed to set socket option : ", __func__);
421 AFALGerr(AFALG_F_AFALG_SOCKET, AFALG_R_SOCKET_SET_KEY_FAILED);
425 /* Get fd to perform future aes-cbc operations with given key */
426 actx->sfd = accept(actx->bfd, NULL, 0);
428 ALG_PERR("%s: Socket Accept Failed : ", __func__);
429 AFALGerr(AFALG_F_AFALG_SOCKET, AFALG_R_SOCKET_BIND_FAILED);
440 actx->bfd = actx->sfd = -1;
444 static int afalg_start_cipher_sk(afalg_ctx *actx, const unsigned char *in,
445 size_t inl, const unsigned char *iv,
448 struct msghdr msg = { 0 };
449 struct cmsghdr *cmsg;
452 # ifdef ALG_ZERO_COPY
456 const ssize_t cbuf_sz = CMSG_SPACE(ALG_IV_LEN(ALG_AES_IV_LEN))
457 + CMSG_SPACE(ALG_OP_LEN);
460 memset(cbuf, 0, cbuf_sz);
461 msg.msg_control = cbuf;
462 msg.msg_controllen = cbuf_sz;
465 * cipher direction (i.e. encrypt or decrypt) and iv are sent to the
466 * kernel as part of sendmsg()'s ancillary data
468 cmsg = CMSG_FIRSTHDR(&msg);
469 afalg_set_op_sk(cmsg, enc);
470 cmsg = CMSG_NXTHDR(&msg, cmsg);
471 afalg_set_iv_sk(cmsg, iv, ALG_AES_IV_LEN);
473 /* iov that describes input data */
474 iov.iov_base = (unsigned char *)in;
477 msg.msg_flags = MSG_MORE;
479 # ifdef ALG_ZERO_COPY
482 * Works best when buffer is 4k aligned
483 * OPENS: out of place processing (i.e. out != in)
486 /* Input data is not sent as part of call to sendmsg() */
490 /* Sendmsg() sends iv and cipher direction to the kernel */
491 sbytes = sendmsg(actx->sfd, &msg, 0);
493 ALG_PERR("%s: sendmsg failed for zero copy cipher operation : ",
499 * vmsplice and splice are used to pin the user space input buffer for
500 * kernel space processing avoiding copys from user to kernel space
502 ret = vmsplice(actx->zc_pipe[1], &iov, 1, SPLICE_F_GIFT);
504 ALG_PERR("%s: vmsplice failed : ", __func__);
508 ret = splice(actx->zc_pipe[0], NULL, actx->sfd, NULL, inl, 0);
510 ALG_PERR("%s: splice failed : ", __func__);
517 /* Sendmsg() sends iv, cipher direction and input data to the kernel */
518 sbytes = sendmsg(actx->sfd, &msg, 0);
520 ALG_PERR("%s: sendmsg failed for cipher operation : ", __func__);
524 if (sbytes != (ssize_t) inl) {
525 ALG_ERR("Cipher operation send bytes %zd != inlen %zd\n", sbytes,
534 static int afalg_cipher_init(EVP_CIPHER_CTX *ctx, const unsigned char *key,
535 const unsigned char *iv, int enc)
540 if (ctx == NULL || key == NULL) {
541 ALG_WARN("%s: Null Parameter\n", __func__);
545 if (EVP_CIPHER_CTX_cipher(ctx) == NULL) {
546 ALG_WARN("%s: Cipher object NULL\n", __func__);
550 actx = EVP_CIPHER_CTX_cipher_data(ctx);
552 ALG_WARN("%s: Cipher data NULL\n", __func__);
556 ciphertype = EVP_CIPHER_CTX_nid(ctx);
557 switch (ciphertype) {
558 case NID_aes_128_cbc:
561 ALG_WARN("%s: Unsupported Cipher type %d\n", __func__, ciphertype);
565 if (ALG_AES_IV_LEN != EVP_CIPHER_CTX_iv_length(ctx)) {
566 ALG_ERR("%s: Unsupported IV length :%d\n", __func__,
567 EVP_CIPHER_CTX_iv_length(ctx));
571 /* Setup AFALG socket for crypto processing */
572 afalg_socket(actx, key, EVP_CIPHER_CTX_key_length(ctx));
577 /* Setup AIO ctx to allow async AFALG crypto processing */
578 if (afalg_init_aio(&actx->aio) == 0) {
583 # ifdef ALG_ZERO_COPY
587 actx->init_done = MAGIC_INIT_NUM;
592 static int afalg_do_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
593 const unsigned char *in, size_t inl)
597 char nxtiv[ALG_AES_IV_LEN] = { 0 };
599 if (ctx == NULL || out == NULL || in == NULL) {
600 ALG_WARN("NULL parameter passed to function %s\n", __func__);
604 actx = (afalg_ctx *) EVP_CIPHER_CTX_cipher_data(ctx);
605 if (actx == NULL || actx->init_done != MAGIC_INIT_NUM) {
606 ALG_WARN("%s afalg ctx passed\n",
607 ctx == NULL ? "NULL" : "Uninitialised");
612 * set iv now for decrypt operation as the input buffer can be
613 * overwritten for inplace operation where in = out.
615 if (EVP_CIPHER_CTX_encrypting(ctx) == 0) {
616 memcpy(nxtiv, in + (inl - ALG_AES_IV_LEN), ALG_AES_IV_LEN);
619 /* Send input data to kernel space */
620 ret = afalg_start_cipher_sk(actx, (unsigned char *)in, inl,
621 EVP_CIPHER_CTX_iv(ctx),
622 EVP_CIPHER_CTX_encrypting(ctx));
627 /* Perform async crypto operation in kernel space */
628 ret = afalg_fin_cipher_aio(&actx->aio, actx->sfd, out, inl);
630 ALG_WARN("%s: Socket cipher operation failed\n", __func__);
634 if (EVP_CIPHER_CTX_encrypting(ctx)) {
635 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), out + (inl - ALG_AES_IV_LEN),
638 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), nxtiv, ALG_AES_IV_LEN);
644 static int afalg_cipher_cleanup(EVP_CIPHER_CTX *ctx)
649 ALG_WARN("NULL parameter passed to function %s\n", __func__);
653 actx = (afalg_ctx *) EVP_CIPHER_CTX_cipher_data(ctx);
654 if (actx == NULL || actx->init_done != MAGIC_INIT_NUM) {
655 ALG_WARN("%s afalg ctx passed\n",
656 ctx == NULL ? "NULL" : "Uninitialised");
662 # ifdef ALG_ZERO_COPY
663 close(actx->zc_pipe[0]);
664 close(actx->zc_pipe[1]);
666 /* close sync mode efd, async mode is closed in afalg_waitfd_cleanup() */
667 if (actx->aio.efd_sync >= 0)
668 close(actx->aio.efd_sync);
669 io_destroy(actx->aio.aio_ctx);
674 const EVP_CIPHER *afalg_aes_128_cbc(void)
676 if (_hidden_aes_128_cbc == NULL
677 && ((_hidden_aes_128_cbc =
678 EVP_CIPHER_meth_new(NID_aes_128_cbc,
680 AES_KEY_SIZE_128)) == NULL
681 || !EVP_CIPHER_meth_set_iv_length(_hidden_aes_128_cbc, AES_IV_LEN)
682 || !EVP_CIPHER_meth_set_flags(_hidden_aes_128_cbc,
684 EVP_CIPH_FLAG_DEFAULT_ASN1)
685 || !EVP_CIPHER_meth_set_init(_hidden_aes_128_cbc,
687 || !EVP_CIPHER_meth_set_do_cipher(_hidden_aes_128_cbc,
689 || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_128_cbc,
690 afalg_cipher_cleanup)
691 || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_128_cbc,
692 sizeof(afalg_ctx)))) {
693 EVP_CIPHER_meth_free(_hidden_aes_128_cbc);
694 _hidden_aes_128_cbc = NULL;
696 return _hidden_aes_128_cbc;
699 static int afalg_ciphers(ENGINE *e, const EVP_CIPHER **cipher,
700 const int **nids, int nid)
704 if (cipher == NULL) {
705 *nids = afalg_cipher_nids;
706 return (sizeof(afalg_cipher_nids) / sizeof(afalg_cipher_nids[0]));
710 case NID_aes_128_cbc:
711 *cipher = afalg_aes_128_cbc();
721 static int bind_afalg(ENGINE *e)
723 /* Ensure the afalg error handling is set up */
724 ERR_load_AFALG_strings();
726 if (!ENGINE_set_id(e, engine_afalg_id)
727 || !ENGINE_set_name(e, engine_afalg_name)
728 || !ENGINE_set_destroy_function(e, afalg_destroy)
729 || !ENGINE_set_init_function(e, afalg_init)
730 || !ENGINE_set_finish_function(e, afalg_finish)) {
731 AFALGerr(AFALG_F_BIND_AFALG, AFALG_R_INIT_FAILED);
735 if (!ENGINE_set_ciphers(e, afalg_ciphers)) {
736 AFALGerr(AFALG_F_BIND_AFALG, AFALG_R_INIT_FAILED);
743 # ifndef OPENSSL_NO_DYNAMIC_ENGINE
744 static int bind_helper(ENGINE *e, const char *id)
746 if (id && (strcmp(id, engine_afalg_id) != 0))
749 if (!afalg_chk_platform())
757 IMPLEMENT_DYNAMIC_CHECK_FN()
758 IMPLEMENT_DYNAMIC_BIND_FN(bind_helper)
760 static int afalg_chk_platform(void)
764 int kver[3] = { -1, -1, -1 };
770 ALG_ERR("%s: Failed to get system information\n", __func__);
774 str = strtok(ut.release, ".");
775 for (i = 0; i < 3 && str != NULL; i++) {
777 str = strtok(NULL, ".");
780 if (KERNEL_VERSION(kver[0], kver[1], kver[2])
781 < KERNEL_VERSION(K_MAJ, K_MIN1, K_MIN2)) {
782 ALG_WARN("ASYNC AFALG not supported this kernel(%d.%d.%d)\n",
783 kver[0], kver[1], kver[2]);
784 ALG_WARN("ASYNC AFALG requires kernel version %d.%d.%d or later\n",
785 K_MAJ, K_MIN1, K_MIN2);
786 AFALGerr(AFALG_F_AFALG_CHK_PLATFORM,
787 AFALG_R_KERNEL_DOES_NOT_SUPPORT_ASYNC_AFALG);
794 # ifdef OPENSSL_NO_DYNAMIC_ENGINE
795 static ENGINE *engine_afalg(void)
797 ENGINE *ret = ENGINE_new();
800 if (!bind_afalg(ret)) {
807 static void ENGINE_load_afalg(void)
811 if (!afalg_chk_platform())
814 toadd = engine_afalg();
823 static int afalg_init(ENGINE *e)
828 static int afalg_finish(ENGINE *e)
833 static int afalg_destroy(ENGINE *e)
835 ERR_unload_AFALG_strings();
839 #endif /* KERNEL VERSION */