fix(1902): correct trailer headers placement

close #1902

fw/hpack.c

@@ -3489,6 +3489,59 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr,
 	if (!hdr)
 		return -EINVAL;
 
+	// compact str with sparse chunks in place
+	// e.g. changes
+	// "f" "o" ":" " " "b" "a"
+	// into
+	// "fo" ": " "ba"
+	// it's different from `tfw_http_hdr_split()`, which only splits
+	// name chunks and value chunks into two strs, no compact.
+	if (hdr->flags & TFW_STR_TRAILER) {
+		int i;
+		unsigned long name_len = 0;
+		bool compact_chunks = false;
+
+		for (i = 0; i < hdr->nchunks; ++i) {
+			c = TFW_STR_CHUNK(hdr, i);
+			if (c->len == 1 && *c->data == S_COLON) {
+				c = TFW_STR_CHUNK(hdr, i + 1);
+				if (WARN_ON_ONCE(!c))
+					return -EINVAL;
+				if (c->len == 1 && *c->data == S_SP) {
+					c = TFW_STR_CHUNK(hdr, i + 2);
+					if (WARN_ON_ONCE(!c))
+						return -EINVAL;
+					compact_chunks = true;
+					break;
+				}
+				return -EINVAL;
+			} else {
+				name_len += c->len;
+			}
+		}
+
+		if (compact_chunks) {
+			int j;
+			int nchunks = hdr->nchunks;
+			unsigned long value_len = hdr->len - name_len - 2;
+			char *value_data;
+
+			c = TFW_STR_CHUNK(hdr, i);
+			value_data = c->data + 2;
+
+			for (j = 0; j < nchunks - 2; ++j) {
+				tfw_str_del_chunk(hdr, 2);
+			}
+
+			c = TFW_STR_CHUNK(hdr, 0);
+			c->len = name_len;
+
+			c = TFW_STR_CHUNK(hdr, 1);
+			c->len = value_len;
+			c->data = value_data;
+		}
+	}
+
 	ret = tfw_hpack_write_idx(resp, idx, false);
 
 	if (unlikely(ret))
@@ -3497,6 +3550,9 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr,
 	mit->acc_len += idx->sz;
 
 	if (unlikely(!name_indexed)) {
+		if ((hdr->flags & TFW_STR_TRAILER) && (c = TFW_STR_CHUNK(hdr, 0))) {
+			tfw_cstrtolower(c->data, c->data, c->len);
+		}
 		ret = tfw_hpack_str_expand(mit, iter, skb_head,
 					   TFW_STR_CHUNK(hdr, 0), NULL);
 		if (unlikely(ret))
@@ -3648,6 +3704,12 @@ tfw_hpack_transform(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr)
 	return __tfw_hpack_encode(resp, hdr, true, true, true);
 }
 
+int
+tfw_hpack_transform_trailer(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr)
+{
+	return __tfw_hpack_encode(resp, hdr, false, false, true);
+}
+
 void
 tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl, unsigned short new_size)
 {

fw/hpack.h

@@ -304,6 +304,7 @@ void write_int(unsigned long index, unsigned short max, unsigned short mask,
 int tfw_hpack_init(TfwHPack *__restrict hp, unsigned int htbl_sz);
 void tfw_hpack_clean(TfwHPack *__restrict hp);
 int tfw_hpack_transform(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr);
+int tfw_hpack_transform_trailer(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr);
 int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr,
 		     bool use_pool, bool dyn_indexing);
 void tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl,

fw/http.c

@@ -4774,6 +4774,9 @@ tfw_h2_hpack_encode_headers(TfwHttpResp *resp, const TfwHdrMods *h_mods)
 		       __func__, hid, d_num, ht->tbl[hid].nchunks,
 		       h_mods ? h_mods->sz : 0);
 
+		if (tgt->flags & TFW_STR_TRAILER)
+			continue;
+
 		/* Don't encode header if it must be substituted from config */
 		if (tfw_h2_hdr_sub(hid, tgt, h_mods))
 			continue;
@@ -4804,6 +4807,42 @@ tfw_h2_hpack_encode_headers(TfwHttpResp *resp, const TfwHdrMods *h_mods)
 	return 0;
 }
 
+int
+tfw_h2_hpack_encode_trailer_headers(TfwHttpResp *resp)
+{
+	int r;
+	unsigned int i;
+	TfwHttpTransIter *mit = &resp->mit;
+	TfwHttpHdrMap *map = mit->map;
+	TfwHttpHdrTbl *ht = resp->h_tbl;
+
+	for (i = 0; i < map->count; ++i) {
+		unsigned short hid = map->index[i].idx;
+		unsigned short d_num = map->index[i].d_idx;
+		TfwStr *tgt = &ht->tbl[hid];
+
+		if (TFW_STR_DUP(tgt))
+			tgt = TFW_STR_CHUNK(tgt, d_num);
+
+		if (WARN_ON_ONCE(!tgt
+				 || TFW_STR_EMPTY(tgt)
+				 || TFW_STR_DUP(tgt)))
+			return -EINVAL;
+
+		T_DBG3("%s: hid=%hu, d_num=%hu, nchunks=%u\n",
+		       __func__, hid, d_num, ht->tbl[hid].nchunks);
+
+		if (!(tgt->flags & TFW_STR_TRAILER))
+			continue;
+
+		r = tfw_hpack_transform_trailer(resp, tgt);
+		if (unlikely(r))
+			return r;
+	}
+
+	return 0;
+}
+
 /**
  * Split response body stored locally. Allocate a new skb and put body there
  * by fragments. Every skb fragment has size of single page and has frame

fw/http.h

@@ -748,6 +748,7 @@ int tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods,
 int tfw_h2_resp_status_write(TfwHttpResp *resp, unsigned short status,
 			     bool use_pool, bool cache);
 int tfw_h2_resp_encode_headers(TfwHttpResp *resp);
+int tfw_h2_hpack_encode_trailer_headers(TfwHttpResp *resp);
 /*
  * Functions to send an HTTP error response to a client.
  */

fw/http2.c

@@ -493,6 +493,39 @@ tfw_h2_stream_xmit_prepare_resp(TfwStream *stream)
 	    && !test_bit(TFW_HTTP_B_VOID_BODY, resp->flags))
 		r = tfw_http_msg_cutoff_body_chunks(resp);
 
+	if (resp->trailers_len > 0) {
+		// MUST _NOT_ use resp pool to encode trailer headers
+		// because it assumes no headers after body in many code.
+		// Instead, encode trailers header in new skbs
+		// and append them to the resp->msg.skb_head.
+		struct sk_buff *skb_head, *skb, *nskb_head, *nskb;
+		unsigned long acc;
+
+		skb_head = resp->msg.skb_head;
+		skb = resp->mit.iter.skb;
+		resp->msg.skb_head = resp->mit.iter.skb = NULL;
+		acc = resp->mit.acc_len;
+
+		r = tfw_h2_hpack_encode_trailer_headers(resp);
+
+		stream->xmit.t_len = resp->mit.acc_len - acc;
+		nskb_head = resp->msg.skb_head;
+		resp->msg.skb_head = skb_head;
+		resp->mit.iter.skb = skb;
+
+		if (unlikely(r))
+			goto finish;
+
+		nskb = nskb_head;
+		do {
+			skb = nskb->next;
+			nskb->next = nskb->prev = NULL;
+			ss_skb_queue_tail(&resp->msg.skb_head, nskb);
+			nskb = skb;
+			printk("---> acc=%d, nskb=%p, nskb->len=%d\n", stream->xmit.t_len, nskb, nskb->len);
+		} while (nskb != nskb_head);
+	}
+
 finish:
 	swap(stream->xmit.skb_head, resp->msg.skb_head);
 	ss_skb_setup_head_of_list(stream->xmit.skb_head, mark, tls_type);
@@ -513,6 +546,7 @@ tfw_h2_entail_stream_skb(struct sock *sk, TfwH2Ctx *ctx, TfwStream *stream,
 	BUG_ON(!TFW_SKB_CB(stream->xmit.skb_head)->is_head);
 	while (*len) {
 		skb = ss_skb_dequeue(&stream->xmit.skb_head);
+		printk("---> tfw_h2_entail_stream_skb, stream->xmit.skb_head=%p, skb=%p, *len=%d\n", stream->xmit.skb_head, skb, *len);
 		BUG_ON(!skb);
 
 		if (unlikely(!skb->len)) {

fw/http_frame.c

@@ -1897,13 +1897,19 @@ tf2_h2_calc_frame_flags(TfwStream *stream, TfwFrameType type)
 {
 	switch (type) {
 	case HTTP2_HEADERS:
+		if (!stream->xmit.h_len && !stream->xmit.b_len)
+			return HTTP2_F_END_STREAM | HTTP2_F_END_HEADERS;
 		return stream->xmit.h_len ?
 			(stream->xmit.b_len ? 0 : HTTP2_F_END_STREAM) :
 			(stream->xmit.b_len ? HTTP2_F_END_HEADERS :
 			 HTTP2_F_END_HEADERS | HTTP2_F_END_STREAM);
 	case HTTP2_CONTINUATION:
+		if (!stream->xmit.h_len && !stream->xmit.b_len)
+			return stream->xmit.t_len ? 0 : (HTTP2_F_END_HEADERS | HTTP2_F_END_STREAM);
 		return stream->xmit.h_len ? 0 : HTTP2_F_END_HEADERS;
 	case HTTP2_DATA:
+		if (stream->xmit.t_len)
+			return 0;
 		return stream->xmit.b_len ? 0 : HTTP2_F_END_STREAM;
 	default:
 		BUG();
@@ -1932,6 +1938,7 @@ tfw_h2_insert_frame_header(struct sock *sk, TfwH2Ctx *ctx, TfwStream *stream,
 	unsigned int length;
 	char *data;
 	int r;
+	TfwStreamFsmRes rr;
 
 
 	/*
@@ -1952,27 +1959,35 @@ tfw_h2_insert_frame_header(struct sock *sk, TfwH2Ctx *ctx, TfwStream *stream,
 					 stream->xmit.frame_length);
 	BUG_ON(!data);
 
-	if (type == HTTP2_CONTINUATION || type == HTTP2_DATA) {
+	if ((type == HTTP2_HEADERS && !stream->xmit.h_len && stream->xmit.t_len) || type == HTTP2_CONTINUATION || type == HTTP2_DATA) {
 		it.skb = it.skb_head = stream->xmit.skb_head;
 		if ((r = tfw_http_msg_insert(&it, &data, &frame_hdr_str)))
 			return r;
 		stream->xmit.skb_head = it.skb_head;
+		printk("---> insert frame hdr\n");
 	}
 
 	/*
 	 * Set tls_type and mark, because skb_head could be changed
 	 * during previous operations.
 	 */
 	ss_skb_setup_head_of_list(stream->xmit.skb_head, mark, tls_type);
+	printk("---> stream->xmit.skb_head=%p\n", stream->xmit.skb_head);
 
 	length = tfw_h2_calc_frame_length(ctx, stream, type, len,
 					  max_len - FRAME_HEADER_SIZE);
 	if (type == HTTP2_DATA) {
-		ctx->rem_wnd -= length;
-		stream->rem_wnd -= length;
+		if (!stream->xmit.t_len) {
+			ctx->rem_wnd -= length;
+			stream->rem_wnd -= length;
+		}
 		stream->xmit.b_len -= length;
-	} else {
+	} else if (stream->xmit.h_len) {
 		stream->xmit.h_len -= length;
+	} else if (stream->xmit.t_len) {
+		ctx->rem_wnd -= length;
+		stream->rem_wnd -= length;
+		stream->xmit.t_len -= length;
 	}
 
 	*snd_wnd -= length;
@@ -1984,7 +1999,10 @@ tfw_h2_insert_frame_header(struct sock *sk, TfwH2Ctx *ctx, TfwStream *stream,
 	tfw_h2_pack_frame_header(data, &frame_hdr);
 
 	stream->xmit.frame_length += length + FRAME_HEADER_SIZE;
-	switch (tfw_h2_stream_send_process(ctx, stream, type)) {
+	//switch (tfw_h2_stream_send_process(ctx, stream, type)) {
+	rr = tfw_h2_stream_send_process(ctx, stream, type);
+	printk("---> rr=%d, stream->xmit.skb_head=%p\n", rr, stream->xmit.skb_head);
+	switch (rr) {
 	case STREAM_FSM_RES_OK:
 	case STREAM_FSM_RES_IGNORE:
 		break;
@@ -2064,6 +2082,7 @@ do {									\
 			return r;
 		}
 
+		printk("---> header->send\n");
 		T_FSM_JMP(HTTP2_SEND_FRAMES);
 	}
 
@@ -2076,6 +2095,7 @@ do {									\
 			return r;
 		}
 
+		printk("---> cont->send\n");
 		T_FSM_JMP(HTTP2_SEND_FRAMES);
 	}
 
@@ -2096,7 +2116,34 @@ do {									\
 			return r;
 		}
 
-		fallthrough;
+		printk("---> data->send\n");
+		T_FSM_JMP(HTTP2_SEND_FRAMES);
+	}
+
+	T_FSM_STATE(HTTP2_MAKE_TRAILER_FRAMES) {
+		CALC_SND_WND_AND_SET_FRAME_TYPE(HTTP2_HEADERS);
+		r = tfw_h2_insert_frame_header(sk, ctx, stream, frame_type,
+					       snd_wnd, stream->xmit.t_len);
+		if (unlikely(r)) {
+			T_WARN("Failed to make trail headers frame %d", r);
+			return r;
+		}
+
+		printk("---> trailer->send, stream->xmit.skb_head=%p\n", stream->xmit.skb_head);
+		T_FSM_JMP(HTTP2_SEND_FRAMES);
+	}
+
+	T_FSM_STATE(HTTP2_MAKE_TRAILER_CONTINUATION_FRAMES) {
+		CALC_SND_WND_AND_SET_FRAME_TYPE(HTTP2_CONTINUATION);
+		r = tfw_h2_insert_frame_header(sk, ctx, stream, frame_type,
+					       snd_wnd, stream->xmit.t_len);
+		if (unlikely(r)) {
+			T_WARN("Failed to make trail continuation frame %d", r);
+			return r;
+		}
+
+		printk("---> trailer cont->send\n");
+		T_FSM_JMP(HTTP2_SEND_FRAMES);
 	}
 
 	T_FSM_STATE(HTTP2_SEND_FRAMES) {
@@ -2117,6 +2164,9 @@ do {									\
 						       &stream->xmit.postponed);
 			if (stream->xmit.b_len) {
 				T_FSM_JMP(HTTP2_MAKE_DATA_FRAMES);
+			} else if (stream->xmit.t_len) {
+				//T_FSM_JMP(HTTP2_MAKE_TRAILER_CONTINUATION_FRAMES);
+				T_FSM_JMP(HTTP2_MAKE_TRAILER_FRAMES);
 			} else {
 				fallthrough;
 			}

fw/http_msg.h

@@ -2,7 +2,7 @@
  *		Tempesta FW
  *
  * Copyright (C) 2014 NatSys Lab. ([email protected]).
- * Copyright (C) 2015-2023 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -25,6 +25,8 @@
 
 #define S_CRLF			"\r\n"
 #define S_DLM			": "
+#define S_COLON			':'
+#define S_SP			' '
 #define S_SET_COOKIE		"set-cookie"
 #define S_F_SET_COOKIE		S_SET_COOKIE S_DLM
 #define S_LOCATION		"location"

fw/http_stream.c

@@ -881,7 +881,10 @@ tfw_h2_stream_send_process(TfwH2Ctx *ctx, TfwStream *stream, unsigned char type)
 	if (!stream->xmit.h_len && type != HTTP2_DATA)
 		flags |= HTTP2_F_END_HEADERS;
 
-	if (!stream->xmit.h_len && !stream->xmit.b_len
+	if (!stream->xmit.t_len && type != HTTP2_DATA)
+		flags |= HTTP2_F_END_HEADERS;
+
+	if (!stream->xmit.h_len && !stream->xmit.b_len && !stream->xmit.t_len
 	    && !tfw_h2_stream_is_eos_sent(stream))
 		flags |= HTTP2_F_END_STREAM;