Merge pull request #1264 from tempesta-tech/avb-sendfile2

tls: encrypting in new sk_buff pages, if `sendfile()` is used or response from cache

tempesta_fw/cache.c

@@ -1538,6 +1538,13 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce)
 	if (tfw_http_msg_setup((TfwHttpMsg *)resp, &it, ce->hdr_len + 2))
 		goto free;
 
+	/*
+	 * Mark skb as SKBTX_SHARED_FRAG so that when sending it to the client
+	 * via https, encryption routines do not change data in-place.
+	 */
+	if (TFW_CONN_TLS(req->conn))
+		skb_shinfo(it.skb)->tx_flags |= SKBTX_SHARED_FRAG;
+
 	/*
 	 * Allocate HTTP headers table of proper size.
 	 * There were no other allocations since the table is allocated,

tempesta_fw/msg.c

@@ -74,5 +74,7 @@ tfw_msg_iter_append_skb(TfwMsgIter *it)
 	it->skb = ss_skb_peek_tail(&it->skb_head);
 	it->frag = 0;
 
+	skb_shinfo(it->skb)->tx_flags = skb_shinfo(it->skb->prev)->tx_flags;
+
 	return 0;
 }

tempesta_fw/ss_skb.c

@@ -1217,6 +1217,7 @@ __coalesce_frag(struct sk_buff **skb_head, skb_frag_t *frag,
 		skb = ss_skb_alloc(0);
 		if (!skb)
 			return -ENOMEM;
+		skb_shinfo(skb)->tx_flags = skb_shinfo(orig_skb)->tx_flags;
 		ss_skb_queue_tail(skb_head, skb);
 		skb->mark = orig_skb->mark;
 	}
@@ -1396,3 +1397,63 @@ ss_skb_dump(struct sk_buff *skb)
 		ss_skb_dump(f_skb);
 }
 EXPORT_SYMBOL(ss_skb_dump);
+
+/*
+ * Replace the skb fragments with new pages and add them to the scatter list.
+ */
+int
+ss_skb_to_sgvec_with_new_pages(struct sk_buff *skb, struct scatterlist *sgl,
+                               struct page ***old_pages)
+{
+	unsigned int head_data_len = skb_headlen(skb);
+	unsigned int out_frags = 0;
+	int remain = 0, offset = 0;
+	int i;
+
+	/* TODO: process of SKBTX_ZEROCOPY_FRAG for MSG_ZEROCOPY */
+	if (skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG) {
+		if (head_data_len) {
+			sg_set_buf(sgl + out_frags, skb->data, head_data_len);
+			out_frags++;
+		}
+
+		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+			skb_frag_t *f = &skb_shinfo(skb)->frags[i];
+			unsigned int size;
+			struct page *p;
+
+			size = skb_frag_size(f);
+			if (remain < size) {
+				int order = get_order(size);
+				p = alloc_pages(GFP_ATOMIC, order);
+				if (!p)
+					return -ENOMEM;
+				remain = (1 << order) * PAGE_SIZE;
+				offset = 0;
+			} else {
+				skb_frag_t *prev_f;
+
+				prev_f = &skb_shinfo(skb)->frags[i - 1];
+				p = skb_frag_page(prev_f);
+				get_page(p);
+			}
+			**old_pages = skb_frag_page(f);
+			(*old_pages)++;
+			sg_set_page(sgl + out_frags, p, size, offset);
+			__skb_fill_page_desc(skb, i, p, offset, size);
+			remain -= size;
+			offset += size;
+			out_frags++;
+		}
+		if (out_frags > 0)
+			sg_mark_end(&sgl[out_frags - 1]);
+		skb_shinfo(skb)->tx_flags &= ~SKBTX_SHARED_FRAG;
+	} else {
+		int r = skb_to_sgvec(skb, sgl + out_frags, 0, skb->len);
+		if (r <= 0)
+			return r;
+		out_frags += r;
+	}
+
+	return out_frags;
+}

tempesta_fw/ss_skb.h

@@ -187,5 +187,7 @@ int ss_skb_process(struct sk_buff *skb, unsigned int off, unsigned int trail,
 int ss_skb_unroll(struct sk_buff **skb_head, struct sk_buff *skb);
 void ss_skb_init_for_xmit(struct sk_buff *skb);
 void ss_skb_dump(struct sk_buff *skb);
+int ss_skb_to_sgvec_with_new_pages(struct sk_buff *skb, struct scatterlist *sgl,
+                                   struct page ***old_pages);
 
 #endif /* __TFW_SS_SKB_H__ */

tempesta_fw/tls.c

@@ -230,7 +230,7 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 #define AUTO_SEGS_N	8
 
 	int r = -ENOMEM;
-	unsigned int head_sz, tag_sz, len, frags, t_sz;
+	unsigned int head_sz, tag_sz, len, frags, t_sz, out_frags;
 	unsigned char type;
 	struct sk_buff *next = skb, *skb_tail = skb;
 	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
@@ -239,8 +239,12 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 	TlsXfrm *xfrm;
 	struct sg_table sgt = {
 		.nents = skb_shinfo(skb)->nr_frags + !!skb_headlen(skb),
+	}, out_sgt = {
+		.nents = skb_shinfo(skb)->nr_frags + !!skb_headlen(skb),
 	};
-	struct scatterlist sg[AUTO_SEGS_N];
+	struct scatterlist sg[AUTO_SEGS_N], out_sg[AUTO_SEGS_N];
+	struct page **pages = NULL, **pages_end, **p;
+	struct page *auto_pages[AUTO_SEGS_N];
 
 	if (unlikely(sk->sk_user_data == NULL))
 		return -EINVAL;
@@ -297,6 +301,7 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 
 		len += next->len;
 		sgt.nents += skb_shinfo(next)->nr_frags + !!skb_headlen(next);
+		out_sgt.nents += skb_shinfo(next)->nr_frags + !!skb_headlen(next);
 		skb_tail = next;
 	}
 
@@ -317,13 +322,15 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 		if (r < 0)
 			goto out;
 		sgt.nents += r;
+		out_sgt.nents += r;
 
 		r = ss_skb_expand_head_tail(skb_tail->next, skb_tail, 0,
 					    tag_sz);
 		if (r < 0)
 			goto out;
 	}
 	sgt.nents += r;
+	out_sgt.nents += r;
 
 	/*
 	 * The last skb in our list will bring TLS tag - add it to end_seqno.
@@ -379,17 +386,29 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 
 	if (likely(sgt.nents <= AUTO_SEGS_N)) {
 		sgt.sgl = sg;
+		out_sgt.sgl = out_sg;
+		pages = auto_pages;
 	} else {
-		sgt.sgl = kmalloc(sizeof(struct scatterlist) * sgt.nents,
-				  GFP_ATOMIC);
+		char *ptr = kmalloc(sizeof(struct scatterlist) * sgt.nents +
+			            sizeof(struct scatterlist) * out_sgt.nents +
+			            sizeof(struct page *) * out_sgt.nents,
+				    GFP_ATOMIC);
+		sgt.sgl = ptr;
 		if (!sgt.sgl) {
-			T_WARN("cannot alloc TLS encryption scatter list.\n");
+			T_WARN("cannot alloc memory for TLS encryption.\n");
 			return -ENOMEM;
 		}
+
+		ptr += sizeof(struct scatterlist) * sgt.nents;
+		out_sgt.sgl = ptr;
+
+		ptr += sizeof(struct scatterlist) * out_sgt.nents;
+		pages = pages_end = ptr;
 	}
 	sg_init_table(sgt.sgl, sgt.nents);
+	sg_init_table(out_sgt.sgl, out_sgt.nents);
 
-	for (next = skb, frags = 0; ; ) {
+	for (next = skb, frags = 0, out_frags = 0; ; ) {
 		/*
 		 * skb data and tails are already adjusted above,
 		 * so use zero offset and skb->len.
@@ -405,13 +424,22 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 		if (r <= 0)
 			goto out;
 		frags += r;
+
+		r = ss_skb_to_sgvec_with_new_pages(next,
+		                                   out_sgt.sgl + out_frags,
+		                                   &pages_end);
+		if (r <= 0)
+			goto out;
+		out_frags += r;
+
 		tempesta_tls_skb_clear(next);
 		if (next == skb_tail)
 			break;
 		if (WARN_ON_ONCE(frags >= sgt.nents))
 			break;
 		next = tcp_write_queue_next(sk, next);
 		sg_unmark_end(&sgt.sgl[frags - 1]);
+		sg_unmark_end(&out_sgt.sgl[out_frags - 1]);
 	}
 	WARN_ON_ONCE(sgt.nents != frags);
 
@@ -420,11 +448,14 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 	/* Set IO context under the lock before encryption. */
 	io->msglen = len - TLS_HEADER_SIZE;
 	io->msgtype = type;
-	if (!(r = ttls_encrypt(tls, &sgt)))
+	if (!(r = ttls_encrypt(tls, &sgt, &out_sgt)))
 		ttls_aad2hdriv(xfrm, skb->data);
 
 	spin_unlock(&tls->lock);
 
+	for (p = pages; p < pages_end; ++p)
+		put_page(*p);
+
 out:
 	if (unlikely(sgt.nents > AUTO_SEGS_N))
 		kfree(sgt.sgl);

tls/ttls.c

@@ -770,7 +770,7 @@ ttls_aead_req_free(struct crypto_aead *tfm, struct aead_request *req)
  * different sessions, always have different ciphertexts.
  */
 int
-ttls_encrypt(TlsCtx *tls, struct sg_table *sgt)
+ttls_encrypt(TlsCtx *tls, struct sg_table *sgt, struct sg_table *out_sgt)
 {
 	int r, elen;
 	TlsXfrm *xfrm = &tls->xfrm;
@@ -791,10 +791,10 @@ ttls_encrypt(TlsCtx *tls, struct sg_table *sgt)
 	T_DBG3_BUF("IV used", xfrm->iv_enc, xfrm->ivlen);
 
 	elen = ttls_msg2crypt_len(io, xfrm);
-	ttls_make_aad(tls, io, sg_virt(sgt->sgl));
+	ttls_make_aad(tls, io, sg_virt(out_sgt->sgl));
 	aead_request_set_tfm(req, c_ctx->cipher_ctx);
 	aead_request_set_ad(req, TLS_AAD_SPACE_SIZE);
-	aead_request_set_crypt(req, sgt->sgl, sgt->sgl, elen, xfrm->iv_enc);
+	aead_request_set_crypt(req, sgt->sgl, out_sgt->sgl, elen, xfrm->iv_enc);
 
 	T_DBG3("%s encryption: tfm=%pK(req->tfm=%pK req=%pK) reqsize=%u"
 		" key_len=%u data_len=%d\n",
@@ -1760,7 +1760,7 @@ ttls_write_finished(TlsCtx *tls, struct sg_table *sgt, unsigned char **in_buf)
 
 	sg_init_table(&sg, 1);
 	sg_set_buf(&sg, p, TLS_HEADER_SIZE + TTLS_HS_FINISHED_BODY_LEN);
-	if ((r = ttls_encrypt(tls, &enc_sgt)))
+	if ((r = ttls_encrypt(tls, &enc_sgt, &enc_sgt)))
 		return r;
 
 	ttls_aad2hdriv(xfrm, p);

tls/ttls.h

@@ -948,7 +948,7 @@ int ttls_get_session(const ttls_context *ssl, TlsSess *session);
 
 int ttls_recv(void *tls_data, unsigned char *buf, size_t len,
 	      unsigned int *read);
-int ttls_encrypt(TlsCtx *tls, struct sg_table *sgt);
+int ttls_encrypt(TlsCtx *tls, struct sg_table *sgt, struct sg_table *out_sgt);
 
 int ttls_send_alert(TlsCtx *tls, unsigned char lvl, unsigned char msg);