Commit d829e9c4 authored by Daniel Borkmann's avatar Daniel Borkmann Committed by Alexei Starovoitov

tls: convert to generic sk_msg interface

Convert kTLS over to make use of sk_msg interface for plaintext and
encrypted scattergather data, so it reuses all the sk_msg helpers
and data structure which later on in a second step enables to glue
this to BPF.

This also allows to remove quite a bit of open coded helpers which
are covered by the sk_msg API. Recent changes in kTLs 80ece6a0
("tls: Remove redundant vars from tls record structure") and
4e6d4720 ("tls: Add support for inplace records encryption")
changed the data path handling a bit; while we've kept the latter
optimization intact, we had to undo the former change to better
fit the sk_msg model, hence the sg_aead_in and sg_aead_out have
been brought back and are linked into the sk_msg sgs. Now the kTLS
record contains a msg_plaintext and msg_encrypted sk_msg each.

In the original code, the zerocopy_from_iter() has been used out
of TX but also RX path. For the strparser skb-based RX path,
we've left the zerocopy_from_iter() in decrypt_internal() mostly
untouched, meaning it has been moved into tls_setup_from_iter()
with charging logic removed (as not used from RX). Given RX path
is not based on sk_msg objects, we haven't pursued setting up a
dummy sk_msg to call into sk_msg_zerocopy_from_iter(), but it
could be an option to prusue in a later step.

Joint work with John.
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Signed-off-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 604326b4
......@@ -102,6 +102,8 @@ struct sk_psock {
int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
int elem_first_coalesce);
int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
u32 off, u32 len);
void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
int sk_msg_free(struct sock *sk, struct sk_msg *msg);
int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
......
......@@ -2214,10 +2214,6 @@ static inline struct page_frag *sk_page_frag(struct sock *sk)
bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
int sg_start, int *sg_curr, unsigned int *sg_size,
int first_coalesce);
/*
* Default write policy as shown to user space via poll/select/SIGIO
*/
......
......@@ -39,6 +39,8 @@
#include <linux/crypto.h>
#include <linux/socket.h>
#include <linux/tcp.h>
#include <linux/skmsg.h>
#include <net/tcp.h>
#include <net/strparser.h>
#include <crypto/aead.h>
......@@ -103,15 +105,13 @@ struct tls_rec {
int tx_flags;
int inplace_crypto;
/* AAD | sg_plaintext_data | sg_tag */
struct scatterlist sg_plaintext_data[MAX_SKB_FRAGS + 1];
/* AAD | sg_encrypted_data (data contain overhead for hdr&iv&tag) */
struct scatterlist sg_encrypted_data[MAX_SKB_FRAGS + 1];
struct sk_msg msg_plaintext;
struct sk_msg msg_encrypted;
unsigned int sg_plaintext_size;
unsigned int sg_encrypted_size;
int sg_plaintext_num_elem;
int sg_encrypted_num_elem;
/* AAD | msg_plaintext.sg.data | sg_tag */
struct scatterlist sg_aead_in[2];
/* AAD | msg_encrypted.sg.data (data contains overhead for hdr & iv & tag) */
struct scatterlist sg_aead_out[2];
char aad_space[TLS_AAD_SPACE_SIZE];
struct aead_request aead_req;
......@@ -223,8 +223,8 @@ struct tls_context {
unsigned long flags;
bool in_tcp_sendpages;
bool pending_open_record_frags;
u16 pending_open_record_frags;
int (*push_pending_record)(struct sock *sk, int flags);
void (*sk_write_space)(struct sock *sk);
......
......@@ -73,6 +73,45 @@ int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
}
EXPORT_SYMBOL_GPL(sk_msg_alloc);
int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
u32 off, u32 len)
{
int i = src->sg.start;
struct scatterlist *sge = sk_msg_elem(src, i);
u32 sge_len, sge_off;
if (sk_msg_full(dst))
return -ENOSPC;
while (off) {
if (sge->length > off)
break;
off -= sge->length;
sk_msg_iter_var_next(i);
if (i == src->sg.end && off)
return -ENOSPC;
sge = sk_msg_elem(src, i);
}
while (len) {
sge_len = sge->length - off;
sge_off = sge->offset + off;
if (sge_len > len)
sge_len = len;
off = 0;
len -= sge_len;
sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off);
sk_mem_charge(sk, sge_len);
sk_msg_iter_var_next(i);
if (i == src->sg.end && len)
return -ENOSPC;
sge = sk_msg_elem(src, i);
}
return 0;
}
EXPORT_SYMBOL_GPL(sk_msg_clone);
void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes)
{
int i = msg->sg.start;
......
......@@ -2238,67 +2238,6 @@ bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
}
EXPORT_SYMBOL(sk_page_frag_refill);
int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
int sg_start, int *sg_curr_index, unsigned int *sg_curr_size,
int first_coalesce)
{
int sg_curr = *sg_curr_index, use = 0, rc = 0;
unsigned int size = *sg_curr_size;
struct page_frag *pfrag;
struct scatterlist *sge;
len -= size;
pfrag = sk_page_frag(sk);
while (len > 0) {
unsigned int orig_offset;
if (!sk_page_frag_refill(sk, pfrag)) {
rc = -ENOMEM;
goto out;
}
use = min_t(int, len, pfrag->size - pfrag->offset);
if (!sk_wmem_schedule(sk, use)) {
rc = -ENOMEM;
goto out;
}
sk_mem_charge(sk, use);
size += use;
orig_offset = pfrag->offset;
pfrag->offset += use;
sge = sg + sg_curr - 1;
if (sg_curr > first_coalesce && sg_page(sge) == pfrag->page &&
sge->offset + sge->length == orig_offset) {
sge->length += use;
} else {
sge = sg + sg_curr;
sg_unmark_end(sge);
sg_set_page(sge, pfrag->page, use, orig_offset);
get_page(pfrag->page);
sg_curr++;
if (sg_curr == MAX_SKB_FRAGS)
sg_curr = 0;
if (sg_curr == sg_start) {
rc = -ENOSPC;
break;
}
}
len -= use;
}
out:
*sg_curr_size = size;
*sg_curr_index = sg_curr;
return rc;
}
EXPORT_SYMBOL(sk_alloc_sg);
static void __lock_sock(struct sock *sk)
__releases(&sk->sk_lock.slock)
__acquires(&sk->sk_lock.slock)
......
......@@ -8,6 +8,7 @@ config TLS
select CRYPTO_AES
select CRYPTO_GCM
select STREAM_PARSER
select NET_SOCK_MSG
default n
---help---
Enable kernel support for TLS protocol. This allows symmetric
......
......@@ -421,7 +421,7 @@ static int tls_push_data(struct sock *sk,
tls_push_record_flags = flags;
if (more) {
tls_ctx->pending_open_record_frags =
record->num_frags;
!!record->num_frags;
break;
}
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment