encrypt in new sk_buff pages, if sendfile () or MSG_ZEROCOPY is u…

…sed or resp from cache

tempesta_fw/cache.c

@@ -1579,6 +1579,9 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce)
 	if (tfw_cache_build_resp_body(db, resp, trec, &it, p))
 		goto err;
 
+	if (TFW_CONN_TLS(req->conn))
+		skb_shinfo(it.skb)->tx_flags |= SKBTX_SHARED_FRAG;
+
 	resp->version = ce->version;
 	tfw_http_copy_flags(resp->flags, ce->hmflags);
 

tempesta_fw/ss_skb.c

@@ -1217,6 +1217,7 @@ __coalesce_frag(struct sk_buff **skb_head, skb_frag_t *frag,
 		skb = ss_skb_alloc(0);
 		if (!skb)
 			return -ENOMEM;
+		skb_shinfo(skb)->tx_flags = skb_shinfo(orig_skb)->tx_flags;
 		ss_skb_queue_tail(skb_head, skb);
 		skb->mark = orig_skb->mark;
 	}

tempesta_fw/tls.c

@@ -230,7 +230,7 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 #define AUTO_SEGS_N	8
 
 	int r = -ENOMEM;
-	unsigned int head_sz, tag_sz, len, frags, t_sz;
+	unsigned int head_sz, tag_sz, len, frags, t_sz, out_frags, elt;
 	unsigned char type;
 	struct sk_buff *next = skb, *skb_tail = skb;
 	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
@@ -239,8 +239,14 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 	TlsXfrm *xfrm;
 	struct sg_table sgt = {
 		.nents = skb_shinfo(skb)->nr_frags + !!skb_headlen(skb),
+	}, out_sgt = {
+		.nents = skb_shinfo(skb)->nr_frags + !!skb_headlen(skb),
 	};
-	struct scatterlist sg[AUTO_SEGS_N];
+	struct scatterlist sg[AUTO_SEGS_N], out_sg[AUTO_SEGS_N];
+	struct page **pages = NULL;
+	struct page *auto_pages[AUTO_SEGS_N];
+	int i;
+	int remain, offset;
 
 	if (unlikely(sk->sk_user_data == NULL))
 		return -EINVAL;
@@ -297,6 +303,7 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 
 		len += next->len;
 		sgt.nents += skb_shinfo(next)->nr_frags + !!skb_headlen(next);
+		out_sgt.nents += skb_shinfo(next)->nr_frags + !!skb_headlen(next);
 		skb_tail = next;
 	}
 
@@ -317,13 +324,15 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 		if (r < 0)
 			goto out;
 		sgt.nents += r;
+		out_sgt.nents += r;
 
 		r = ss_skb_expand_head_tail(skb_tail->next, skb_tail, 0,
 					    tag_sz);
 		if (r < 0)
 			goto out;
 	}
 	sgt.nents += r;
+	out_sgt.nents += r;
 
 	/*
 	 * The last skb in our list will bring TLS tag - add it to end_seqno.
@@ -379,23 +388,91 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 
 	if (likely(sgt.nents <= AUTO_SEGS_N)) {
 		sgt.sgl = sg;
+		out_sgt.sgl = out_sg;
+		pages = auto_pages;
 	} else {
 		sgt.sgl = kmalloc(sizeof(struct scatterlist) * sgt.nents,
 				  GFP_ATOMIC);
 		if (!sgt.sgl) {
 			T_WARN("cannot alloc TLS encryption scatter list.\n");
-			return -ENOMEM;
+			r = -ENOMEM;
+			goto err_sgt_alloc;
+		}
+		out_sgt.sgl = kmalloc(sizeof(struct scatterlist) * out_sgt.nents,
+				      GFP_ATOMIC);
+		if (!out_sgt.sgl) {
+			T_WARN("cannot alloc TLS encryption scatter list.\n");
+			r = -ENOMEM;
+			goto err_out_sgt_alloc;
+		}
+		pages = kmalloc(sizeof(struct page *) * out_sgt.nents,
+				GFP_ATOMIC);
+		if (!pages) {
+			T_WARN("cannot alloc memory.\n");
+			r = -ENOMEM;
+			goto err_pages_alloc;
 		}
 	}
 	sg_init_table(sgt.sgl, sgt.nents);
+	sg_init_table(out_sgt.sgl, out_sgt.nents);
 
-	for (next = skb, frags = 0; ; ) {
+	for (next = skb, frags = 0, out_frags = 0, elt = 0; ; ) {
+		unsigned int head_data_len = skb_headlen(next);
 		/*
 		 * skb data and tails are already adjusted above,
 		 * so use zero offset and skb->len.
 		 */
 		r = skb_to_sgvec(next, sgt.sgl + frags, 0, next->len);
 
+		if (skb_shinfo(next)->tx_flags & SKBTX_ZEROCOPY_FRAG) {
+			if (head_data_len) {
+				sg_set_buf(out_sgt.sgl + out_frags, next->data,
+				           head_data_len);
+				out_frags++;
+			}
+
+			remain = offset = 0;
+			for (i = 0; i < skb_shinfo(next)->nr_frags; i++) {
+				skb_frag_t *f = &skb_shinfo(next)->frags[i];
+				unsigned int size;
+				struct page *p;
+
+				size = skb_frag_size(f);
+				if (remain < size) {
+					int order = get_order(size);
+					p = alloc_pages(GFP_ATOMIC, order);
+					if (!p) {
+						r = -ENOMEM;
+						goto out;
+					}
+					remain = (1 << order) * PAGE_SIZE;
+					offset = 0;
+				} else {
+					skb_frag_t *prev_f;
+
+					prev_f = &skb_shinfo(next)->frags[i - 1];
+					p = skb_frag_page(prev_f);
+					get_page(p);
+				}
+				pages[elt++] = skb_frag_page(f);
+				sg_set_page(out_sgt.sgl + out_frags, p, size,
+					    offset);
+				__skb_fill_page_desc(next, i, p, offset, size);
+				remain -= size;
+				offset += size;
+				out_frags++;
+			}
+			if (head_data_len || i)
+				sg_mark_end(&out_sgt.sgl[out_frags - 1]);
+			skb_shinfo(next)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG;
+		} else {
+			r = skb_to_sgvec(next, out_sgt.sgl + out_frags, 0,
+				         next->len);
+			if (r <= 0)
+				goto out;
+			out_frags += r;
+		}
+
 		T_DBG3("skb_to_sgvec (%u segs) from skb %pK"
 		       " (%u bytes, %u segs), done_frags=%u ret=%d\n",
 		       sgt.nents, next, next->len,
@@ -412,6 +489,7 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 			break;
 		next = tcp_write_queue_next(sk, next);
 		sg_unmark_end(&sgt.sgl[frags - 1]);
+		sg_unmark_end(&out_sgt.sgl[out_frags - 1]);
 	}
 	WARN_ON_ONCE(sgt.nents != frags);
 
@@ -420,15 +498,25 @@ tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit)
 	/* Set IO context under the lock before encryption. */
 	io->msglen = len - TLS_HEADER_SIZE;
 	io->msgtype = type;
-	if (!(r = ttls_encrypt(tls, &sgt)))
+	if (!(r = ttls_encrypt(tls, &sgt, &out_sgt)))
 		ttls_aad2hdriv(xfrm, skb->data);
 
 	spin_unlock(&tls->lock);
 
+	for (i = 0; i < elt; ++i) {
+		put_page(pages[i]);
+	}
+
 out:
-	if (unlikely(sgt.nents > AUTO_SEGS_N))
+	if (unlikely(sgt.nents > AUTO_SEGS_N)) {
+		kfree(pages);
+err_pages_alloc:
+		kfree(out_sgt.sgl);
+err_out_sgt_alloc:
 		kfree(sgt.sgl);
+	}
 
+err_sgt_alloc:
 	return r;
 #undef AUTO_SEGS_N
 }

tls/ttls.c

@@ -770,7 +770,7 @@ ttls_aead_req_free(struct crypto_aead *tfm, struct aead_request *req)
  * different sessions, always have different ciphertexts.
  */
 int
-ttls_encrypt(TlsCtx *tls, struct sg_table *sgt)
+ttls_encrypt(TlsCtx *tls, struct sg_table *sgt, struct sg_table *out_sgt)
 {
 	int r, elen;
 	TlsXfrm *xfrm = &tls->xfrm;
@@ -791,10 +791,10 @@ ttls_encrypt(TlsCtx *tls, struct sg_table *sgt)
 	T_DBG3_BUF("IV used", xfrm->iv_enc, xfrm->ivlen);
 
 	elen = ttls_msg2crypt_len(io, xfrm);
-	ttls_make_aad(tls, io, sg_virt(sgt->sgl));
+	ttls_make_aad(tls, io, sg_virt(out_sgt->sgl));
 	aead_request_set_tfm(req, c_ctx->cipher_ctx);
 	aead_request_set_ad(req, TLS_AAD_SPACE_SIZE);
-	aead_request_set_crypt(req, sgt->sgl, sgt->sgl, elen, xfrm->iv_enc);
+	aead_request_set_crypt(req, sgt->sgl, out_sgt->sgl, elen, xfrm->iv_enc);
 
 	T_DBG3("%s encryption: tfm=%pK(req->tfm=%pK req=%pK) reqsize=%u"
 		" key_len=%u data_len=%d\n",
@@ -1760,7 +1760,7 @@ ttls_write_finished(TlsCtx *tls, struct sg_table *sgt, unsigned char **in_buf)
 
 	sg_init_table(&sg, 1);
 	sg_set_buf(&sg, p, TLS_HEADER_SIZE + TTLS_HS_FINISHED_BODY_LEN);
-	if ((r = ttls_encrypt(tls, &enc_sgt)))
+	if ((r = ttls_encrypt(tls, &enc_sgt, &enc_sgt)))
 		return r;
 
 	ttls_aad2hdriv(xfrm, p);

tls/ttls.h

@@ -948,7 +948,7 @@ int ttls_get_session(const ttls_context *ssl, TlsSess *session);
 
 int ttls_recv(void *tls_data, unsigned char *buf, size_t len,
 	      unsigned int *read);
-int ttls_encrypt(TlsCtx *tls, struct sg_table *sgt);
+int ttls_encrypt(TlsCtx *tls, struct sg_table *sgt, struct sg_table *out_sgt);
 
 int ttls_send_alert(TlsCtx *tls, unsigned char lvl, unsigned char msg);