From 71d7b7a76d3c4bb5aa8825cad9bd708c03c44656 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Wed, 16 Oct 2019 16:52:54 +0300 Subject: [PATCH 01/64] HTTP/2 Parser implementation (#309). --- tempesta_fw/cache.c | 15 +- tempesta_fw/hpack.c | 1270 ++++++-- tempesta_fw/hpack.h | 69 +- tempesta_fw/http.c | 899 +++++- tempesta_fw/http.h | 22 +- tempesta_fw/http_frame.c | 40 +- tempesta_fw/http_frame.h | 23 + tempesta_fw/http_limits.c | 18 +- tempesta_fw/http_match.c | 39 +- tempesta_fw/http_msg.c | 411 ++- tempesta_fw/http_msg.h | 40 +- tempesta_fw/http_parser.c | 4051 +++++++++++++++++++++++-- tempesta_fw/http_parser.h | 4 +- tempesta_fw/http_sess.c | 28 +- tempesta_fw/msg.h | 29 +- tempesta_fw/ss_skb.c | 186 +- tempesta_fw/ss_skb.h | 5 + tempesta_fw/str.h | 25 +- tempesta_fw/t/unit/test_hpack.c | 50 +- tempesta_fw/t/unit/test_http_parser.c | 31 +- tempesta_fw/t/unit/test_http_sticky.c | 1 - tempesta_fw/t/unit/test_tfw_str.c | 2 +- 22 files changed, 6466 insertions(+), 792 deletions(-) diff --git a/tempesta_fw/cache.c b/tempesta_fw/cache.c index 0a6349a2d5..dfa8c3a44b 100644 --- a/tempesta_fw/cache.c +++ b/tempesta_fw/cache.c @@ -1153,6 +1153,7 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) long n, etag_off = 0; char *p; TdbVRec *trec = &ce->trec, *etag_trec = NULL; + TfwStr *s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; TDB *db = node_db(); TfwStr *field, *h, *end1, *end2, empty = {}; int r, i; @@ -1176,7 +1177,7 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) ce->method = req->method; ce->status = TDB_OFF(db->hdr, p); - if ((n = tfw_cache_strcpy_eol(&p, &trec, &resp->s_line, &tot_len, 1)) < 0) { + if ((n = tfw_cache_strcpy_eol(&p, &trec, s_line, &tot_len, 1)) < 0) { T_ERR("Cache: cannot copy HTTP status line\n"); return -ENOMEM; } @@ -1185,7 +1186,7 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) ce->hdrs = TDB_OFF(db->hdr, p); ce->hdr_len = 0; ce->hdr_num = resp->h_tbl->off; - FOR_EACH_HDR_FIELD(field, end1, resp) { + FOR_EACH_HDR_FIELD_FROM(field, end1, resp, TFW_HTTP_HDR_REGULAR) { bool hdr_304 = false; /* Skip hop-by-hop headers. */ @@ -1282,7 +1283,7 @@ __cache_entry_size(TfwHttpResp *resp) size += req->h_tbl->tbl[TFW_HTTP_HDR_HOST].len; /* Add all the headers size */ - FOR_EACH_HDR_FIELD(hdr, hdr_end, resp) { + FOR_EACH_HDR_FIELD_FROM(hdr, hdr_end, resp, TFW_HTTP_HDR_REGULAR) { /* Skip hop-by-hop headers. */ if (!(hdr->flags & TFW_STR_HBH_HDR)) h = hdr; @@ -1304,7 +1305,7 @@ __cache_entry_size(TfwHttpResp *resp) } /* Add status line length + CRLF */ - size += resp->s_line.len + SLEN(S_CRLF); + size += resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE].len + SLEN(S_CRLF); /* Add body size accounting CRLF after the last chunk */ size += resp->body.len; @@ -1532,6 +1533,7 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce) { int h; char *p; + TfwStr *s_line; TfwHttpResp *resp; TdbVRec *trec = &ce->trec; TDB *db = node_db(); @@ -1577,12 +1579,13 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce) goto err; } + s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; if (tfw_cache_write_field(db, &trec, resp, &it, &p, - ce->status_len, &resp->s_line)) + ce->status_len, s_line)) goto err; resp->h_tbl->off = ce->hdr_num; - for (h = 0; h < ce->hdr_num; ++h) { + for (h = TFW_HTTP_HDR_REGULAR; h < ce->hdr_num; ++h) { TFW_STR_INIT(resp->h_tbl->tbl + h); if (tfw_cache_build_resp_hdr(db, resp, resp->h_tbl->tbl + h, &trec, &it, &p)) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index 0124a66875..b7aff15695 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -950,67 +950,67 @@ static const HTState ht_decode[] ____cacheline_aligned = { }) static const TfwHPackEntry static_table[] ____cacheline_aligned = { - { HP_STR(":authority"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR(":method"), HP_STR("GET"), TFW_HTTP_HDR_RAW }, - { HP_STR(":method"), HP_STR("POST"), TFW_HTTP_HDR_RAW }, - { HP_STR(":path"), HP_STR("/"), TFW_HTTP_HDR_RAW }, - { HP_STR(":path"), HP_STR("/index.html"), TFW_HTTP_HDR_RAW }, - { HP_STR(":scheme"), HP_STR("http"), TFW_HTTP_HDR_RAW }, - { HP_STR(":scheme"), HP_STR("https"), TFW_HTTP_HDR_RAW }, - { HP_STR(":status"), HP_STR("200"), TFW_HTTP_HDR_RAW }, - { HP_STR(":status"), HP_STR("204"), TFW_HTTP_HDR_RAW }, - { HP_STR(":status"), HP_STR("206"), TFW_HTTP_HDR_RAW }, - { HP_STR(":status"), HP_STR("304"), TFW_HTTP_HDR_RAW }, - { HP_STR(":status"), HP_STR("400"), TFW_HTTP_HDR_RAW }, - { HP_STR(":status"), HP_STR("404"), TFW_HTTP_HDR_RAW }, - { HP_STR(":status"), HP_STR("500"), TFW_HTTP_HDR_RAW }, - { HP_STR("accept-charset"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("accept-encoding"), HP_STR("gzip, deflate"), TFW_HTTP_HDR_RAW }, - { HP_STR("accept-language"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("accept-ranges"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("accept"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("access-control-allow-origin"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("age"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("allow"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("authorization"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("cache-control"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("content-disposition"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("content-encoding"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("content-language"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("content-length"), NULL, TFW_HTTP_HDR_CONTENT_LENGTH }, - { HP_STR("content-location"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("content-range"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("content-type"), NULL, TFW_HTTP_HDR_CONTENT_TYPE }, - { HP_STR("cookie"), NULL, TFW_HTTP_HDR_COOKIE }, - { HP_STR("date"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("etag"), NULL, TFW_HTTP_HDR_ETAG }, - { HP_STR("expect"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("expires"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("from"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("host"), NULL, TFW_HTTP_HDR_HOST }, - { HP_STR("if-match"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("if-modified-since"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("if-none-match"), NULL, TFW_HTTP_HDR_IF_NONE_MATCH }, - { HP_STR("if-range"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("if-unmodified-since"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("last-modified"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("link"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("location"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("max-forwards"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("proxy-authenticate"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("proxy-authorization"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("range"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("referer"), NULL, TFW_HTTP_HDR_REFERER }, - { HP_STR("refresh"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("retry-after"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("server"), NULL, TFW_HTTP_HDR_SERVER }, - { HP_STR("set-cookie"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("strict-transport-security"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("transfer-encoding"), NULL, TFW_HTTP_HDR_TRANSFER_ENCODING }, - { HP_STR("user-agent"), NULL, TFW_HTTP_HDR_USER_AGENT }, - { HP_STR("vary"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("via"), NULL, TFW_HTTP_HDR_RAW }, - { HP_STR("www-authenticate"), NULL, TFW_HTTP_HDR_RAW } + { HP_STR(":authority"), NULL, TFW_TAG_HDR_H2_AUTHORITY }, + { HP_STR(":method"), HP_STR("GET"), TFW_TAG_HDR_H2_METHOD }, + { HP_STR(":method"), HP_STR("POST"), TFW_TAG_HDR_H2_METHOD }, + { HP_STR(":path"), HP_STR("/"), TFW_TAG_HDR_H2_PATH }, + { HP_STR(":path"), HP_STR("/index.html"), TFW_TAG_HDR_H2_PATH }, + { HP_STR(":scheme"), HP_STR("http"), TFW_TAG_HDR_H2_SCHEME }, + { HP_STR(":scheme"), HP_STR("https"),TFW_TAG_HDR_H2_SCHEME }, + { HP_STR(":status"), HP_STR("200"), TFW_TAG_HDR_H2_STATUS }, + { HP_STR(":status"), HP_STR("204"), TFW_TAG_HDR_H2_STATUS }, + { HP_STR(":status"), HP_STR("206"), TFW_TAG_HDR_H2_STATUS }, + { HP_STR(":status"), HP_STR("304"), TFW_TAG_HDR_H2_STATUS }, + { HP_STR(":status"), HP_STR("400"), TFW_TAG_HDR_H2_STATUS }, + { HP_STR(":status"), HP_STR("404"), TFW_TAG_HDR_H2_STATUS }, + { HP_STR(":status"), HP_STR("500"), TFW_TAG_HDR_H2_STATUS }, + { HP_STR("accept-charset"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("accept-encoding"), HP_STR("gzip, deflate"), TFW_TAG_HDR_RAW }, + { HP_STR("accept-language"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("accept-ranges"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("accept"), NULL, TFW_TAG_HDR_ACCEPT }, + { HP_STR("access-control-allow-origin"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("age"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("allow"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("authorization"), NULL, TFW_TAG_HDR_AUTHORIZATION }, + { HP_STR("cache-control"), NULL, TFW_TAG_HDR_CACHE_CONTROL }, + { HP_STR("content-disposition"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("content-encoding"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("content-language"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("content-length"), NULL, TFW_TAG_HDR_CONTENT_LENGTH }, + { HP_STR("content-location"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("content-range"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("content-type"), NULL, TFW_TAG_HDR_CONTENT_TYPE }, + { HP_STR("cookie"), NULL, TFW_TAG_HDR_COOKIE }, + { HP_STR("date"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("etag"), NULL, TFW_TAG_HDR_ETAG }, + { HP_STR("expect"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("expires"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("from"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("host"), NULL, TFW_TAG_HDR_HOST }, + { HP_STR("if-match"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("if-modified-since"), NULL, TFW_TAG_HDR_IF_MODIFIED_SINCE }, + { HP_STR("if-none-match"), NULL, TFW_TAG_HDR_IF_NONE_MATCH }, + { HP_STR("if-range"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("if-unmodified-since"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("last-modified"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("link"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("location"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("max-forwards"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("proxy-authenticate"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("proxy-authorization"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("range"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("referer"), NULL, TFW_TAG_HDR_REFERER }, + { HP_STR("refresh"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("retry-after"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("server"), NULL, TFW_TAG_HDR_SERVER }, + { HP_STR("set-cookie"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("strict-transport-security"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("transfer-encoding"), NULL, TFW_TAG_HDR_TRANSFER_ENCODING }, + { HP_STR("user-agent"), NULL, TFW_TAG_HDR_USER_AGENT }, + { HP_STR("vary"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("via"), NULL, TFW_TAG_HDR_RAW }, + { HP_STR("www-authenticate"), NULL, TFW_TAG_HDR_RAW } }; #define HPACK_STATIC_ENTRIES (sizeof(static_table) / sizeof(TfwHPackEntry)) @@ -1131,111 +1131,78 @@ do { \ last - src); \ } while (0) -#define BUFFER_NAME_OPEN(length) \ +#define BUFFER_HDR_INIT(length, it) \ do { \ - WARN_ON_ONCE(!TFW_STR_EMPTY(it->hdr)); \ - BUFFER_GET(length, it); \ - if (!it->pos) { \ - r = T_DROP; \ - goto out; \ - } \ - if (!it->start_pos) \ - it->start_pos = it->pos; \ - it->hdr->data = it->pos; \ - it->hdr->len = length; \ - it->next = it->hdr; \ + if (!(it)->start_pos) \ + (it)->start_pos = (it)->pos; \ + (it)->hdr.data = (it)->pos; \ + (it)->hdr.len = length; \ + (it)->next = &(it)->hdr; \ } while (0) -#define BUFFER_NEXT(len) \ -({ \ - TfwStr *new = NULL; \ - TfwPool *rpool = req->pool; \ - BUFFER_GET(len, it); \ - if (it->pos) { \ - new = tfw_hpack_exp_hdr(rpool, len, it); \ - it->next = new; \ - } \ - new; \ -}) - -#define BUFFER_VAL_OPEN(length) \ -do { \ - WARN_ON_ONCE(TFW_STR_EMPTY(it->hdr)); \ - it->nm_len = it->hdr->len - 1; \ - if (!BUFFER_NEXT(length)) { \ - r = T_DROP; \ - goto out; \ - } \ -} while (0) - -#define BUFFER_WRITE(data, len, it) \ -do { \ - BUG_ON(!(len)); \ - WARN_ON_ONCE((len) > (it)->rspace); \ - memcpy_fast((it)->pos, data, len); \ - (it)->rspace -= len; \ - (it)->pos += len; \ -} while (0) - -#define BUFFER_WRITE_COLON() \ +#define BUFFER_NAME_OPEN(length) \ do { \ - WARN_ON_ONCE(it->rspace); \ - it->pos = tfw_pool_alloc_not_align(it->pool, 1); \ - T_DBG3("%s: write colon, it->pos=[%p], it->pos=%lu\n", \ - __func__, it->pos, (unsigned long)it->pos); \ - if (!it->pos) { \ - r = T_DROP; \ - goto out; \ - } \ - if (!tfw_hpack_exp_hdr(req->pool, 1, it)) { \ - r = T_DROP; \ - goto out; \ + WARN_ON_ONCE(!TFW_STR_EMPTY(&it->hdr)); \ + if (state & HPACK_FLAGS_HUFFMAN_NAME) { \ + BUFFER_GET(length, it); \ + if (!it->pos) { \ + r = T_DROP; \ + goto out; \ + } \ + BUFFER_HDR_INIT(length, it); \ } \ - memcpy_fast(it->pos, ":", 1); \ } while (0) -#define BUFFER_WRITE_CRLF() \ +#define BUFFER_VAL_OPEN(length) \ do { \ - WARN_ON_ONCE(it->rspace); \ - it->pos = tfw_pool_alloc_not_align(it->pool, 2); \ - T_DBG3("%s: write crlf, it->pos=[%p], it->pos=%lu\n", \ - __func__, it->pos, (unsigned long)it->pos); \ - if (!it->pos) { \ - r = T_DROP; \ - goto out; \ + WARN_ON_ONCE(TFW_STR_EMPTY(it->parsed_hdr)); \ + it->nm_len = it->parsed_hdr->len; \ + it->nm_num = it->parsed_hdr->nchunks; \ + if (state & HPACK_FLAGS_HUFFMAN_NAME) { \ + BUFFER_GET(length, it); \ + if (!it->pos) { \ + r = T_DROP; \ + goto out; \ + } \ + if (!TFW_STR_EMPTY(&it->hdr)) \ + it->next = tfw_hpack_exp_hdr(req->pool, \ + length, it); \ + else \ + BUFFER_HDR_INIT(length, it); \ } \ - memcpy_fast(it->pos, "\r\n", 2); \ } while (0) -#define HPACK_DECODE_STRING(len) \ +#define HPACK_DECODE_PROCESS_STRING(field, len) \ do { \ - T_DBG3("%s: decoding, len=%lu, n=%lu, to_parse=%lu\n", \ + T_DBG3("%s: decoding, len=%lu, n=%lu, tail=%lu\n", \ __func__, len, n, last - src); \ r = tfw_huffman_decode(hp, req, src, len); \ src += len; \ if (r) \ goto out; \ - T_DBG3("%s: decoded, to_parse=%lu\n", __func__, \ - last - src); \ WARN_ON_ONCE(hp->length); \ hp->hctx = 0; \ tfw_huffman_init(hp); \ + if ((r = tfw_hpack_process_hdr_##field(req))) \ + goto out; \ + T_DBG3("%s: processed decoded, tail=%lu\n", __func__, \ + last - src); \ } while (0) -#define HPACK_COPY_STRING(len) \ +#define HPACK_PROCESS_STRING(len, value_stage) \ do { \ - BUFFER_WRITE(src, len, it); \ hp->length -= len; \ + r = tfw_h2_parse_req_hdr(src, len, req, !hp->length, \ + value_stage); \ src += len; \ - T_DBG3("%s: copy plain, len=%lu, n=%lu, to_parse=%lu," \ + T_DBG3("%s: processed plain, len=%lu, n=%lu, tail=%lu," \ " hp->length=%lu\n", __func__, len, n, \ last - src, hp->length); \ - if (hp->length) \ + if (r) \ goto out; \ + WARN_ON_ONCE(hp->length); \ } while (0) -#define STATIC_INDEXED(ent) ((ent)->tag >= 0) - static unsigned long act_hp_str_n; static inline TfwStr * @@ -1243,14 +1210,13 @@ tfw_hpack_exp_hdr(TfwPool *__restrict pool, unsigned long len, TfwMsgParseIter *__restrict it) { TfwStr *new; - TfwStr *hdr = it->hdr; - if (!(new = tfw_str_add_compound(pool, hdr))) + if (!(new = tfw_str_add_compound(pool, &it->hdr))) return NULL; new->data = it->pos; new->len = len; - hdr->len += len; + it->hdr.len += len; return new; } @@ -1276,7 +1242,7 @@ tfw_hpack_huffman_write(char sym, TfwHttpReq *__restrict req) it->rspace, sym, np); if (!np) { - TfwStr *hdr = it->hdr; + TfwStr *hdr = &it->hdr; TfwStr *last = TFW_STR_LAST(hdr); T_DBG3("%s: add to hdr, hdr->len=%lu, last->len=%lu," @@ -1550,65 +1516,50 @@ tfw_hpack_str_put(TfwHPackStr *hp_str) } } -static int -tfw_hpack_create_entry(const TfwHPackEntry *__restrict existent, - TfwMsgParseIter *__restrict it, - TfwHPackEntry *__restrict entry_out) +static TfwHPackEntry * +tfw_hpack_create_entry(TfwPool *__restrict pool, TfwMsgParseIter *__restrict it) { - char *pos = NULL; - const TfwStr *c, *end, *hdr = it->hdr; - unsigned long nm_len = it->nm_len; - unsigned long val_len = hdr->len - nm_len - 1; - - BUG_ON(TFW_STR_EMPTY(hdr)); - WARN_ON_ONCE(!nm_len || !val_len); + char *data; + TfwHPackEntry *entry; + const TfwStr *d, *s, *end, *d_hdr, *s_hdr = it->parsed_hdr; + unsigned long size = sizeof(TfwHPackEntry); - T_DBG3("%s: nm_len=%lu, val_len=%lu, entry_out->tag=%ld\n", __func__, - nm_len, val_len, entry_out->tag); + if (WARN_ON_ONCE(TFW_STR_EMPTY(s_hdr))) + return NULL; - if (!existent) { - if (!(entry_out->name = tfw_hpack_create_str(nm_len))) - return -ENOMEM; - entry_out->tag = -1; - pos = entry_out->name->ptr; - } - else { - tfw_hpack_str_get(existent->name); - entry_out->name = existent->name; - entry_out->tag = existent->tag; - } + size += (s_hdr->nchunks + 1) * sizeof(TfwStr) + s_hdr->len; + T_DBG3("%s: size=%lu, s_hdr->nchunks=%u, s_hdr->len=%lu\n", __func__, + size, s_hdr->nchunks, s_hdr->len); + if (!(entry = tfw_pool_alloc(pool, size))) + return NULL; - TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { - if (!existent) { - memcpy_fast(pos, c->data, c->len); - T_DBG3("%s: name cycle, nm_len=%lu, len=%lu, pos='%.*s'" - "\n", __func__, nm_len, c->len, (int)c->len, pos); - pos += c->len; - } - nm_len -= c->len; - if (unlikely(!nm_len)) - break; + d_hdr = entry->hdr; + *d_hdr = *s_hdr; + d_hdr->chunks = d_hdr + 1; + data = (char *)(TFW_STR_LAST(d_hdr) + 1); + + d = TFW_STR_CHUNK(d_hdr, 0); + TFW_STR_FOR_EACH_CHUNK(s, s_hdr, end) { + *d = *s; + d->data = data; + memcpy_fast(data, s->data, s->len); + T_DBG3("%s: copy cycle, d->len=%lu, d->data='%.*s'," + " d->flags=%hu\n", __func__, d->len, (int)d->len, + d->data, d->flags); + data += s->len; + ++d; } - ++c; - WARN_ON_ONCE(c >= end || *c->data != ':'); + T_DBG3("%s: entry created, d_hdr->nchunks=%u, d_hdr->len=%lu," + " d_hdr->flags=%hu, it->nm_len=%lu, it->nm_num=%u, it->tag=%u\n", + __func__, d_hdr->nchunks, d_hdr->len, d_hdr->flags, it->nm_len, + it->nm_num, it->tag); - if (!(entry_out->value = tfw_hpack_create_str(val_len))) { - tfw_hpack_str_put(entry_out->name); - return -ENOMEM; - } + entry->name_len = it->nm_len; + entry->name_num = it->nm_num; + entry->tag = it->tag; - pos = entry_out->value->ptr; - while (++c < end) { - memcpy_fast(pos, c->data, c->len); - T_DBG3("%s: value cycle, val_len=%lu, len=%lu, pos='%.*s'" - "\n", __func__, val_len, c->len, (int)c->len, pos); - pos += c->len; - val_len -= c->len; - } - WARN_ON_ONCE(val_len); - - return 0; + return entry; } static inline void @@ -1620,12 +1571,11 @@ tfw_hpack_free_entry(TfwHPackEntry *entry) static int tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, - const TfwHPackEntry *__restrict existent, TfwMsgParseIter *__restrict it) { unsigned int delta; unsigned long name_len; - TfwHPackEntry entry; + TfwHPackEntry *entry; const TfwHPackStr *name, *value; unsigned int window, size, new_size; unsigned int count = tbl->n; @@ -1634,8 +1584,8 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, TfwHPackEntry *entries = tbl->entries; int r = 0; - if ((r = tfw_hpack_create_entry(existent, it, &entry))) - return r; + if (!(entry = tfw_hpack_create_entry(tbl->pool, it))) + return -ENOMEM; name = entry.name; value = entry.value; @@ -1806,10 +1756,10 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, return r; } -static const TfwHPackEntry * +static TfwHPackEntry * tfw_hpack_find_index(TfwHPackDTbl *__restrict tbl, unsigned long index) { - const TfwHPackEntry *entry = NULL; + TfwHPackEntry *entry = NULL; if (index <= HPACK_STATIC_ENTRIES) { entry = static_table + index - 1; @@ -1946,37 +1896,214 @@ tfw_hpack_clean(TfwHPack *__restrict hp) static inline void tfw_hpack_reinit(TfwHPack *__restrict hp, TfwMsgParseIter *__restrict it) { + WARN_ON_ONCE(!TFW_STR_EMPTY(it->parsed_hdr)); + bzero_fast(it->__off, + sizeof(*it) - offsetof(TfwMsgParseIter, __off)); bzero_fast(hp->__off, sizeof(*hp) - offsetof(TfwHPack, __off)); - bzero_fast(it->__off, - sizeof(*it) - offsetof(TfwHPack, __off)); - TFW_STR_INIT(it->hdr); } -static int -tfw_hpack_hdr_process(TfwHttpReq *req) +static inline int +tfw_hpack_process_hdr_name(TfwHttpReq *req) { - int r = T_OK; + int ret = T_BAD; TfwMsgParseIter *it = &req->pit; - const TfwStr *chunk, *end, *hdr = it->hdr; + const TfwStr *c, *end, *next = it->next; + + WARN_ON_ONCE(next != &it->hdr); + TFW_STR_FOR_EACH_CHUNK(c, next, end) { + bool last = c + 1 == end; - BUG_ON(TFW_STR_DUP(hdr)); - if (TFW_STR_PLAIN(hdr)) { - chunk = hdr; - end = hdr + 1; - WARN_ON_ONCE(chunk != it->next); - } else { - chunk = it->next; - end = hdr->chunks + hdr->nchunks; - WARN_ON_ONCE(hdr == it->next); + WARN_ON_ONCE(ret == T_OK); + ret = tfw_h2_parse_req_hdr(c->data, c->len, req, last, false); + if (unlikely(ret < T_POSTPONE)) + return ret; } + return ret ? T_DROP : T_OK; +} +static inline int +tfw_hpack_process_hdr_value(TfwHttpReq *req) +{ + int ret = T_BAD; + TfwMsgParseIter *it = &req->pit; + const TfwStr *hdr = &it->hdr; + const TfwStr *chunk = it->next; + const TfwStr *end = hdr->chunks + hdr->nchunks; + + WARN_ON_ONCE(chunk == hdr); while (chunk < end) { - r = tfw_h2_parse_hdr(chunk->data, chunk->len, req); - if (unlikely(r < T_POSTPONE)) - return r; + bool last = chunk + 1 == end; + + WARN_ON_ONCE(ret == T_OK); + ret = tfw_h2_parse_req_hdr(chunk->data, chunk->len, + req, last, true); + if (unlikely(ret < T_POSTPONE)) + return ret; ++chunk; } + return ret ? T_DROP : T_OK; +} + +static int +tfw_hpack_hdr_name_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, + TfwHPackEntry *__restrict entry) +{ + char *data; + TfwStr *d; + const TfwStr *s, *end; + TfwHttpParser *parser = &req->stream->parser; + TfwStr *d_hdr = &parser->hdr; + TfwStr *s_hdr = entry->hdr; + TfwMsgParseIter *it = &req->pit; + unsigned int num = entry->name_num; + unsigned long sz = entry->name_len; + + WARN_ON_ONCE(!TFW_STR_EMPTY(d_hdr)); + + if (!(data = tfw_pool_alloc_not_align(it->pool, sz))) + return T_BAD; + + d_hdr->len = sz; + d_hdr->flags = s_hdr->flags; + + if (TFW_STR_PLAIN(s_hdr)) { + d_hdr->data = data; + goto done; + } + + d_hdr->nchunks = num; + if (!(d_hdr->chunks = tfw_pool_alloc(req->pool, num * sizeof(TfwStr)))) + return T_BAD; + + /* + * Since headers in static table cannot be changed, we need to copy only + * descriptors (because they will grow during further processing). + */ + d = __TFW_STR_CH(d_hdr, 0); + if (hp->index <= HPACK_STATIC_ENTRIES) { + WARN_ON_ONCE(!s_hdr->nchunks); + *d = *s_hdr->chunks; + goto done; + } + + for (s = s_hdr->chunks, end = s_hdr->chunks + num; s < end; ++s) { + *d = *s; + d->data = data; + memcpy_fast(data, s->data, s->len); + data += s->len; + ++d; + } + +done: + it->tag = entry->tag; + + return T_OK; +} + +static int +tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, + TfwHPackEntry *__restrict entry) +{ + char *data; + TfwStr *d; + const TfwStr *s, *end; + unsigned long d_size; + TfwMsgParseIter *it = &req->pit; + TfwHttpParser *parser = &req->stream->parser; + TfwStr *d_hdr = &parser->hdr; + TfwStr *s_hdr = entry->hdr; + + WARN_ON_ONCE(TFW_STR_PLAIN(s_hdr)); + WARN_ON_ONCE(!TFW_STR_EMPTY(d_hdr)); + + /* + * The header in static table should not be supplanted and full header + * descriptor (with name and value) should not grow during subsequent + * processing. Thus, we can avoid the descriptor deep copying from the + * table and take only its high-level part. + */ + if (hp->index <= HPACK_STATIC_ENTRIES) { + WARN_ON_ONCE(s_hdr->nchunks > 2); + if (s_hdr->nchunks != 2) + return T_DROP; + *d_hdr = *s_hdr; + goto done; + } + + /* + * We must do a full copy of dynamically indexed headers (in-depth + * descriptor and data), since next header can supplant the processed + * header or change it by adding a new header into dynamic table, and + * any type of reference for header (index or high-level/full + * descriptor) will become invalid. Note, that for static table this + * problem does not exist, since statically indexed headers cannot be + * supplanted or changed - therefore, for subsequent work we keep + * (without full copying) only references for statically indexed + * headers (also, see comment above). + */ + d_size = s_hdr->nchunks * sizeof(TfwStr); + if (!(data = tfw_pool_alloc_not_align(it->pool, s_hdr->len))) + return T_BAD; + + if (!(d_hdr->chunks = tfw_pool_alloc(req->pool, d_size))) + return T_BAD; + + d_hdr->len = s_hdr->len; + d_hdr->flags = s_hdr->flags; + d_hdr->nchunks = s_hdr->nchunks; + + d = TFW_STR_CHUNK(d_hdr, 0); + TFW_STR_FOR_EACH_CHUNK(s, s_hdr, end) { + *d = *s; + d->data = data; + memcpy_fast(data, s->data, s->len); + data += s->len; + ++d; + } + +done: + switch (entry->tag) { + case TFW_TAG_HDR_H2_METHOD: + parser->_hdr_tag = TFW_HTTP_HDR_H2_METHOD; + case TFW_TAG_HDR_H2_SCHEME: + parser->_hdr_tag = TFW_HTTP_HDR_H2_SCHEME; + case TFW_TAG_HDR_H2_AUTHORITY: + parser->_hdr_tag = TFW_HTTP_HDR_H2_AUTHORITY; + case TFW_TAG_HDR_H2_PATH: + parser->_hdr_tag = TFW_HTTP_HDR_H2_PATH; + case TFW_TAG_HDR_ACCEPT: + parser->_hdr_tag = TFW_HTTP_HDR_RAW; + case TFW_TAG_HDR_AUTHORIZATION: + parser->_hdr_tag = TFW_HTTP_HDR_RAW; + case TFW_TAG_HDR_CACHE_CONTROL: + parser->_hdr_tag = TFW_HTTP_HDR_RAW; + case TFW_TAG_HDR_CONTENT_LENGTH: + parser->_hdr_tag = TFW_HTTP_HDR_CONTENT_LENGTH; + case TFW_TAG_HDR_CONTENT_TYPE: + parser->_hdr_tag = TFW_HTTP_HDR_CONTENT_TYPE; + case TFW_TAG_HDR_COOKIE: + parser->_hdr_tag = TFW_HTTP_HDR_COOKIE; + case TFW_TAG_HDR_HOST: + parser->_hdr_tag = TFW_HTTP_HDR_HOST; + case TFW_TAG_HDR_IF_MODIFIED_SINCE: + parser->_hdr_tag = TFW_HTTP_HDR_RAW; + case TFW_TAG_HDR_IF_NONE_MATCH: + parser->_hdr_tag = TFW_HTTP_HDR_IF_NONE_MATCH; + case TFW_TAG_HDR_PRAGMA: + parser->_hdr_tag = TFW_HTTP_HDR_RAW; + case TFW_TAG_HDR_REFERER: + parser->_hdr_tag = TFW_HTTP_HDR_REFERER; + case TFW_TAG_HDR_X_FORWARDED_FOR: + parser->_hdr_tag = TFW_HTTP_HDR_X_FORWARDED_FOR; + case TFW_TAG_HDR_USER_AGENT: + parser->_hdr_tag = TFW_HTTP_HDR_USER_AGENT; + case TFW_TAG_HDR_RAW: + parser->_hdr_tag = TFW_HTTP_HDR_RAW; + default: + WARN_ON_ONCE(1); + return T_DROP; + } return T_OK; } @@ -1985,9 +2112,8 @@ tfw_hpack_hdr_process(TfwHttpReq *req) * HPACK decoder FSM for HTTP/2 message processing. */ int -tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, - unsigned long n, TfwHttpReq *__restrict req, - unsigned int *__restrict parsed) +tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *src, unsigned long n, + TfwHttpReq *__restrict req, unsigned int *__restrict parsed) { int r = T_POSTPONE; unsigned int state = hp->state; @@ -1995,9 +2121,9 @@ tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, const unsigned char *last = src + n; BUILD_BUG_ON(HPACK_STATE_MASK < _HPACK_STATE_NUM - 1); - BUG_ON(!it->hdr); + BUG_ON(!it->parsed_hdr); WARN_ON_ONCE(!n); - *parsed = n; + *parsed += n; do { T_DBG3("%s: header processing, n=%lu, to_parse=%lu, state=%d\n", __func__, n, last - src, state); @@ -2101,6 +2227,7 @@ tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, unsigned char c = *src++; T_DBG3("%s: decode header name length...\n", __func__); + WARN_ON_ONCE(hp->length); hp->length = c & 0x7F; if (c & 0x80) { T_DBG3("%s: Huffman encoding used for name...\n", @@ -2133,16 +2260,9 @@ tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, T_DBG3("%s: decode header name...\n", __func__); m_len = min((unsigned long)(last - src), hp->length); if (state & HPACK_FLAGS_HUFFMAN_NAME) - HPACK_DECODE_STRING(m_len); + HPACK_DECODE_PROCESS_STRING(name, m_len); else - HPACK_COPY_STRING(m_len); - - if (tfw_hpack_hdr_process(req)) { - r = T_DROP; - goto out; - } - - BUFFER_WRITE_COLON(); + HPACK_PROCESS_STRING(m_len, false); NEXT_STATE(HPACK_STATE_VALUE); @@ -2153,33 +2273,17 @@ tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, } case HPACK_STATE_INDEXED_NAME_TEXT: { + TfwHPackEntry *entry; get_indexed_name: T_DBG3("%s: decode indexed (%lu) header name...\n", __func__, hp->index); WARN_ON_ONCE(!hp->index); - hp->entry = tfw_hpack_find_index(&hp->dec_tbl, - hp->index); - if (!hp->entry) { + entry = tfw_hpack_find_index(&hp->dec_tbl, hp->index); + if (!entry || tfw_hpack_hdr_name_set(hp, req, entry)) { r = T_DROP; goto out; } - BUFFER_NAME_OPEN(hp->entry->name->len); - BUFFER_WRITE(hp->entry->name->ptr, - hp->entry->name->len, it); - - if (!STATIC_INDEXED(hp->entry)) { - if (tfw_hpack_hdr_process(req)) { - r = T_DROP; - goto out; - } - } - else { - it->hdr_tag = hp->entry->tag; - } - - BUFFER_WRITE_COLON(); - NEXT_STATE(HPACK_STATE_VALUE); /* Fall through. */ @@ -2190,6 +2294,7 @@ tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, get_value: T_DBG3("%s: decode header value length...\n", __func__); c = *src++; + WARN_ON_ONCE(hp->length); hp->length = c & 0x7F; if (c & 0x80) { T_DBG3("%s: Huffman encoding used for value\n", @@ -2225,22 +2330,27 @@ tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, T_DBG3("%s: decode header value...\n", __func__); m_len = min((unsigned long)(last - src), hp->length); if (state & HPACK_FLAGS_HUFFMAN_VALUE) - HPACK_DECODE_STRING(m_len); + HPACK_DECODE_PROCESS_STRING(value, m_len); else - HPACK_COPY_STRING(m_len); - - BUFFER_WRITE_CRLF(); + HPACK_PROCESS_STRING(m_len, true); - if (tfw_hpack_hdr_process(req)) { + if (state & HPACK_FLAGS_ADD + && tfw_hpack_add_index(&hp->dec_tbl, it)) + { r = T_DROP; goto out; } - tfw_str_set_eolen(it->hdr, 2); + it->hdrs_len += it->parsed_hdr->len; + ++it->hdrs_cnt; - if (state & HPACK_FLAGS_ADD - && tfw_hpack_add_index(&hp->dec_tbl, hp->entry, it)) - { + /* + * Finish parsed header and reinitialize parsing + * context. Note, @parser->hdr and @parser->_hdr_tag + * must be determined during headers' field processing + * above. + */ + if (tfw_http_msg_hdr_close((TfwHttpMsg *)req)) { r = T_DROP; goto out; } @@ -2249,7 +2359,7 @@ tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, } case HPACK_STATE_ALL_INDEXED: { - const TfwHPackEntry *entry; + TfwHPackEntry *entry; get_all_indexed: T_DBG3("%s: get entire header by index: %lu\n", __func__, hp->index); @@ -2262,27 +2372,24 @@ tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, goto out; } - BUFFER_NAME_OPEN(entry->name->len); - BUFFER_WRITE(entry->name->ptr, entry->name->len, it); - - if (!STATIC_INDEXED(entry)) { - if (tfw_hpack_hdr_process(req)) { - r = T_DROP; - goto out; - } - } - else { - it->hdr_tag = entry->tag; + if (tfw_hpack_hdr_set(hp, req, entry)) { + r = T_DROP; + goto out; } - BUFFER_WRITE_COLON(); - - BUFFER_VAL_OPEN(entry->value->len); - BUFFER_WRITE(entry->value->ptr, entry->value->len, it); - BUFFER_WRITE_CRLF(); + it->hdrs_len += it->parsed_hdr->len; + ++it->hdrs_cnt; - tfw_str_set_eolen(it->hdr, 2); - tfw_http_msg_hdr_close((TfwHttpMsg *)req, it->hdr_tag); + /* + * Finish parsed header and reinitialize parsing + * context. Note, in case of indexed header @parser->hdr + * and @parser->_hdr_tag must be determined from the + * decoder static/dynamic tables above. + */ + if (tfw_http_msg_hdr_close((TfwHttpMsg *)req)) { + r = T_DROP; + goto out; + } break; } @@ -2460,6 +2567,9 @@ typedef enum { #define HPACK_IDX_ST_MASK 0x0F #define HPACK_IDX_FLAG_ADD 0x010 +#define HPACK_IDX_RES(res) \ + ((res) & HPACK_IDX_ST_MASK) + #define HPACK_MAX_ENC_EVICTION 5 #define HPACK_RB_IS_BLACK(node) ((int)(node)->color) @@ -3284,11 +3394,11 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, unsigned long node_size, hdr_len; unsigned short new_size, node_len; unsigned short cur_size = tbl->size, window = tbl->window; - unsigned long nm_len = 0, val_off = 0, val_len = 0; + unsigned long nm_len, val_off, val_len; TfwHPackNode *del_list[HPACK_MAX_ENC_EVICTION] = {}; TfwHPackETblIter it = {}; - hdr_len = tfw_http_msg_hdr_length(hdr, &nm_len, &val_off, &val_len); + hdr_len = tfw_h2_msg_hdr_length(hdr, &nm_len, &val_off, &val_len); WARN_ON_ONCE(cur_size > window || window > HPACK_ENC_TABLE_MAX_SIZE); if ((node_size = hdr_len + HPACK_ENTRY_OVERHEAD) > window) { @@ -3383,7 +3493,7 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, it.last->hdr_len = hdr_len; it.last->rindex = ++tbl->idx_acc; - tfw_http_msg_hdr_write(hdr, nm_len, val_off, val_len, it.last->hdr); + tfw_h2_msg_hdr_write(hdr, nm_len, val_off, val_len, it.last->hdr); tfw_hpack_rbuf_commit(tbl, del_list, place, &it); WARN_ON_ONCE(tbl->rb_len > tbl->size); @@ -3396,41 +3506,30 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, * encoder dynamic table with potentially concurrent access from different * threads, so lock is used to protect the find/add/erase operations inside * this procedure. - * - * TODO #309: this function must be called from response sending procedures: - * upstream response headers cycle, upstream response adjusting procedure, - * internal response creation; in all these three cases the call to this - * function must be combined with the sending of fully prepared response, - * in which the @entered variable must be checked; callers of this procedure - * must decide (taking into account information from static indexing: the - * index itself and the special flag indicating whether the entire header - * is indexed or only its name) - whether to call this function and which - * index to send - static or dynamic; note, that dynamic index is returned - * from this function with its own attributes to be taken into account: return - * header/name values and special addition flag, signaling that the header - * must be added into decoder index table on the client side; also note, - * that if only header name has been found (in static or dynamic table), we - * must save in dynamic table (under new index) the entire header nevertheless. */ static TfwHPackETblRes tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, - bool *entered, unsigned short *__restrict out_index) + unsigned short *__restrict out_index, + unsigned long *__restrict flags) { - TfwHPackETblRes res; - const TfwHPackNode *node = NULL; TfwHPackNodeIter place = {}; + const TfwHPackNode *node = NULL; + TfwHPackETblRes res = HPACK_IDX_ST_NOT_FOUND; BUILD_BUG_ON(HPACK_IDX_ST_MASK < _HPACK_IDX_ST_NUM - 1); + if (WARN_ON_ONCE(!hdr)) + return -EINVAL; spin_lock(&tbl->lock); - if (!*entered && atomic64_read(&tbl->guard) < 0) + if (!test_bit(TFW_HTTP_B_H2_TRANS_ENTERED, flags) + && atomic64_read(&tbl->guard) < 0) goto out; res = tfw_hpack_rbtree_find(tbl, hdr, &node, &place); - WARN_ON_ONCE(res != HPACK_IDX_ST_NOT_FOUND && !node); + WARN_ON_ONCE(!node && res != HPACK_IDX_ST_NOT_FOUND); *out_index = HPACK_NODE_GET_INDEX(tbl, node); @@ -3438,25 +3537,25 @@ tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, * Encoder dynamic index can be in three states: initial state (@guard * is zero), read state (@guard is 1 or greater), and write state * (@guard is -1); in read state any thread can search in index, but - * nobody can add or evict entries in index; if index in write state + * nobody can add or evict entries in index; if index in the write state * only one thread (current writer) can add/evict entries in index and * nobody can search in index; index can be switched to write state * only from initial state (in general case) or from read state (if * current reader is the sole read owner of the index). */ - if (!*entered) { + if (!test_bit(TFW_HTTP_B_H2_TRANS_ENTERED, flags)) { if(res != HPACK_IDX_ST_FOUND && !atomic64_read(&tbl->guard) && !tfw_hpack_add_node(tbl, hdr, &place)) { res |= HPACK_IDX_FLAG_ADD; atomic64_set(&tbl->guard, -1); - *entered = true; + __set_bit(TFW_HTTP_B_H2_TRANS_ENTERED, flags); } else if (res != HPACK_IDX_ST_NOT_FOUND) { atomic64_inc(&tbl->guard); - *entered = true; + __set_bit(TFW_HTTP_B_H2_TRANS_ENTERED, flags); } } else { @@ -3464,8 +3563,9 @@ tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, * If value of guard is 1, we are the sole owner of the encoder * dynamic index with read rights, thus we can write to it. * Note, that @guard cannot be zero here, since we are already - * owning encoder index with read or write rights (the value of - * @entered is 'true'), thus we have already set the @guard + * owning encoder index with read or write rights (i.e. the flag + * @TFW_HTTP_B_H2_TRANS_ENTERED is set for the corrently + * processed message), thus we have already set the @guard * equal to 1 (or greater) or to -1 before. */ WARN_ON_ONCE(!atomic64_read(&tbl->guard)); @@ -3484,29 +3584,559 @@ tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, return res; } +void +tfw_hpack_enc_release(TfwHPack *__restrict hp, unsigned long *flags) +{ + TfwHPackETbl *tbl = &hp->enc_tbl; + + if (!test_bit(TFW_HTTP_B_H2_TRANS_ENTERED, flags)) + return; + + if (atomic64_read(&tbl->guard) < 0) { + atomic64_set(&tbl->guard, 0); + } + else { + WARN_ON_ONCE(!atomic64_read(&tbl->guard)); + atomic64_dec(&tbl->guard); + } +} + +static void +write_int(unsigned long index, unsigned short max, unsigned short mask, + TfwHPackInt *__restrict res_idx) +{ + unsigned int size = 0; + unsigned char *dst = res_idx->buf; + + if (likely(index < max)) { + index |= mask; + } + else { + index -= max; + *dst++ = max | mask; + while (index > 0x7F) { + ++size; + *dst++ = (index & 0x7F) | 0x80; + index >>= 7; + } + } + ++size; + *dst = index; + + res_idx->sz = size; +} + +static int +tfw_hpack_idx_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, + TfwHPackInt *__restrict idx) +{ + int ret; + TfwStr it = {}; + TfwMsgTransIter *mit = &resp->mit; + TfwStr idx_str = { + .data = idx->buf, + .len = idx->sz, + }; + + ret = ss_skb_get_room(resp->msg.skb_head, first->skb, first->data, + idx_str.len, &it); + if (ret) + return ret; + + if ((ret = tfw_strcpy(&it, &idx_str))) + return ret; + + mit->acc_len += idx_str.len; + + T_DBG3("%s: idx, acc_len=%lu, idx_str.len=%lu, idx_str.data='%.*s'\n", + __func__, mit->acc_len, idx_str.len, (int)idx_str.len, + idx_str.data); + + return 0; +} + +static inline int +tfw_hpack_idx_expand(TfwHttpResp *__restrict resp, TfwHPackInt *__restrict idx) +{ + int ret; + TfwMsgTransIter *mit = &resp->mit; + TfwStr idx_str = { + .data = idx->buf, + .len = idx->sz + }; + + ret = tfw_http_msg_expand_data(&mit->iter, &resp->msg.skb_head, + &idx_str); + if (ret) + return ret; + + mit->acc_len += idx_str.len; + + T_DBG3("%s: idx, acc_len=%lu, idx_str.len=%lu, idx_str.data='%.*s'\n", + __func__, mit->acc_len, idx_str.len, (int)idx_str.len, + idx_str.data); + + return 0; +} + +/* + * Replace @str in the HTTP message by @idx as HPACK representation, with + * deletion of EOL at the end of @str. + */ +static int +tfw_hpack_idx_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict str, + TfwHPackInt *__restrict idx) +{ + int ret; + struct sk_buff *skb_head = resp->msg.skb_head; + TfwMsgTransIter *mit = &resp->mit; + TfwStr *last, it = {}; + TfwStr idx_str = { + .data = idx->buf, + .len = idx->sz, + }; + + if (WARN_ON_ONCE(!str)) + return -EINVAL; + + if (str->len == idx_str.len) + goto copy; + + if (str->len > idx_str.len) { + ret = ss_skb_cutoff_data(skb_head, str, idx_str.len, + tfw_str_eolen(str)); + if (ret) + return ret; + goto copy; + } + + last = TFW_STR_LAST(str); + ret = ss_skb_get_room(skb_head, last->skb, last->data + last->len, + idx_str.len - str->len, &it); + if (ret) + return ret; + + if ((ret = tfw_strcat(resp->pool, str, &it))) { + T_WARN("Cannot concatenate '%.*s' with '%.*s' (err=%d)\n", + PR_TFW_STR(str), PR_TFW_STR(&idx_str), ret); + return ret; + } + +copy: + if ((ret = tfw_strcpy(str, &idx_str))) + return ret; + + mit->acc_len += idx_str.len; + + T_DBG3("%s: idx, acc_len=%lu, idx_str.len=%lu, idx_str.data='%.*s'\n", + __func__, mit->acc_len, idx_str.len, (int)idx_str.len, + idx_str.data); + + return 0; +} + +//comment is needed +static int +tfw_hpack_lit_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, + TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, + bool name_indexed) +{ + int ret; + TfwHPackInt vlen; + TfwStr *chunk, *end; + TfwMsgTransIter *mit = &resp->mit; + struct sk_buff *skb_head = resp->msg.skb_head; + TfwStr s_val, s_vlen = {}; + TfwStr it = {}; + TfwStr res_hdr = { + .data = idx->buf, + .len = idx->sz, + }; + + if (WARN_ON_ONCE(!hdr || TFW_STR_EMPTY(hdr))) + return -EINVAL; + + if (!name_indexed) { + TfwHPackInt nlen; + TfwStr *s_n = TFW_STR_CHUNK(hdr, 0); + TfwStr s_name = { + .chunks = (TfwStr []){ {}, {} }, + .nchunks = 2 + }; + + write_int(s_n->len, 0x7F, 0, &nlen); + __TFW_STR_CH(&s_name, 0)->data = nlen.buf; + __TFW_STR_CH(&s_name, 0)->len = nlen.sz; + __TFW_STR_CH(&s_name, 1)->data = s_n->data; + __TFW_STR_CH(&s_name, 1)->len = s_n->len; + + if ((ret = tfw_strcat(resp->pool, &res_hdr, &s_name))) + return ret; + } + + /* + * The @hdr must have the HTTP/2-format chunk structure (without the + * colon and OWS), i.e.: { name, value1, value2, ... }. + */ + chunk = TFW_STR_CHUNK(hdr, 1); + end = hdr->chunks + hdr->nchunks; + tfw_str_collect_cmp(chunk, end, &s_val, NULL); + + write_int(s_val.len, 0x7F, 0, &vlen); + s_vlen.data = vlen.buf; + s_vlen.len = vlen.sz; + + if ((ret = tfw_strcat(resp->pool, &res_hdr, &s_vlen))) + return ret; + + if ((ret = tfw_strcat(resp->pool, &res_hdr, &s_val))) + return ret; + + ret = ss_skb_get_room(skb_head, first->skb, first->data, + res_hdr.len, &it); + if (ret) + return ret; + + if ((ret = tfw_strcpy(&it, &res_hdr))) + return ret; + + mit->acc_len += res_hdr.len; + + T_DBG3("%s: idx, acc_len=%lu, it.len=%lu, it.data='%.*s'\n", __func__, + mit->acc_len, it.len, (int)it.len, it.data); + + return 0; +} + +//comment is needed +static int +tfw_hpack_lit_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, + TfwHPackInt *__restrict idx, bool name_indexed) +{ + int ret; + TfwHPackInt nlen, vlen; + TfwStr *chunk, *end, *s_name; + TfwMsgTransIter *mit = &resp->mit; + TfwMsgIter *iter = &mit->iter; + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwStr s_val, s_nlen = {}, s_vlen = {}; + TfwStr idx_str = { + .data = idx->buf, + .len = idx->sz, + }; + + if (WARN_ON_ONCE(!hdr || TFW_STR_EMPTY(hdr))) + return -EINVAL; + + if ((ret = tfw_http_msg_expand_data(iter, skb_head, &idx_str))) + return ret; + + mit->acc_len += idx_str.len; + + T_DBG3("%s: idx, acc_len=%lu, idx_str.len=%lu, idx_str.data='%.*s'\n", + __func__, mit->acc_len, idx_str.len, (int)idx_str.len, + idx_str.data); + + if (!name_indexed) { + s_name = TFW_STR_CHUNK(hdr, 0); + write_int(s_name->len, 0x7F, 0, &nlen); + s_nlen.data = nlen.buf; + s_nlen.len = nlen.sz; + + if ((ret = tfw_http_msg_expand_data(iter, skb_head, &s_nlen))) + return ret; + + mit->acc_len += s_nlen.len; + + T_DBG3("%s: name len, acc_len=%lu, s_nlen.len=%lu, s_nlen.data" + "='%.*s'\n", __func__, mit->acc_len, s_nlen.len, + (int)s_nlen.len, s_nlen.data); + + if ((ret = tfw_http_msg_expand_data(iter, skb_head, s_name))) + return ret; + + mit->acc_len += s_name->len; + + T_DBG3("%s: name str, acc_len=%lu, s_name.len=%lu, s_name.data" + "='%.*s'\n", __func__, mit->acc_len, s_name.len, + (int)s_name.len, s_name.data); + } + /* + * The @hdr must have the HTTP/2-format chunk structure (without the + * colon and OWS), i.e.: { name, value1, value2, ... }. + */ + chunk = TFW_STR_CHUNK(hdr, 1); + end = hdr->chunks + hdr->nchunks; + tfw_str_collect_cmp(chunk, end, &s_val, NULL); + + write_int(s_val.len, 0x7F, 0, &vlen); + s_vlen.data = vlen.buf; + s_vlen.len = vlen.sz; + + if ((ret = tfw_http_msg_expand_data(iter, skb_head, &s_vlen))) + return ret; + + mit->acc_len += s_vlen.len; + + T_DBG3("%s: val len, acc_len=%lu, s_vlen.len=%lu, s_vlen.data='%.*s'\n", + __func__, mit->acc_len, s_vlen.len, (int)s_vlen.len, s_vlen.data); + + if ((ret = tfw_http_msg_expand_data(iter, skb_head, &s_val))) + return ret; + + mit->acc_len += s_val.len; + + T_DBG3("%s: val str, acc_len=%lu, s_val.len=%lu, s_val.data='%.*s'\n", + __func__, mit->acc_len, s_val.len, (int)s_val.len, s_val.data); + + return 0; +} + +//comment is needed (about @fc too) +static inline int +tfw_hpack_lit_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, + TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, + bool name_indexed) +{ + int ret; + TfwStr *fc; + struct sk_buff *skb_head = resp->msg.skb_head; + + if (WARN_ON_ONCE(!tgt || !hdr)) + return -EINVAL; + + fc = TFW_STR_CHUNK(tgt, 0); + + if ((ret = tfw_hpack_lit_add(resp, fc, hdr, idx, name_indexed))) + return ret; + + if ((ret = ss_skb_cutoff_data(skb_head, tgt, 0, tfw_str_eolen(tgt)))) + return ret; + + return 0; +} + +static int +tfw_hpack_lit_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, + TfwHPackInt *__restrict idx, bool name_indexed) +{ + int r; + TfwHPackInt nlen, vlen; + const TfwStr *c, *end, *h; + TfwStr it = {}; + bool val_found = false; + TfwMsgTransIter *mit = &resp->mit; + unsigned long hdr_len, nm_len, val_off, val_len; + struct sk_buff *skb_head = resp->msg.skb_head; + TfwStr s_nlen = {}, s_vlen = {}; + TfwStr s_idx = { + .data = idx->buf, + .len = idx->sz, + }; + + if (WARN_ON_ONCE(!hdr)) + return -EINVAL; + + if ((r = tfw_http_msg_del_eol(skb_head, hdr))) + return r; + + h = TFW_STR_CHUNK(hdr, 0); + if ((r = ss_skb_get_room(skb_head, h->skb, h->data, s_idx.len, &it))) + return r; + + if ((r = tfw_strcpy(&it, &s_idx))) + return r; + + mit->acc_len += s_idx.len; + + T_DBG3("%s: idx, acc_len=%lu, it.len=%lu, it.data='%.*s'\n", __func__, + mit->acc_len, it.len, (int)it.len, it.data); + + hdr_len = tfw_h2_msg_hdr_length(hdr, &nm_len, &val_off, &val_len); + + if (!name_indexed) { + write_int(nm_len, 0x7F, 0, &nlen); + s_nlen.data = nlen.buf; + s_nlen.len = nlen.sz; + r = ss_skb_get_room(skb_head, h->skb, h->data, s_nlen.len, &it); + if (r) + return r; + if ((r = tfw_strcpy(&it, &s_nlen))) + return r; + + mit->acc_len += s_nlen.len + nm_len; + + T_DBG3("%s: name len, acc_len=%lu, it.len=%lu, it.data='%.*s'\n", + __func__, mit->acc_len, it.len, (int)it.len, it.data); + } + + write_int(val_len, 0x7F, 0, &vlen); + s_vlen.data = vlen.buf; + s_vlen.len = vlen.sz; + + mit->acc_len += s_vlen.len + val_len; + + TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { + unsigned long len; + + if (!c->len) + continue; + + len = 0; + if (nm_len) { + WARN_ON_ONCE(nm_len == c->len); + len = min(nm_len, c->len); + nm_len -= len; + if (name_indexed) { + r = skb_fragment(skb_head, c->skb, c->data, + -len, &it); + if (r < 0) + return r; + } + + if (len == c->len) + continue; + + WARN_ON_ONCE(!val_off); + } + + if (val_off) { + WARN_ON_ONCE(val_off < c->len - len); + val_off -= c->len - len; + r = skb_fragment(skb_head, c->skb, c->data + len, + -(c->len - len), &it); + if (r < 0) + return r; + + continue; + } + + if (val_len) { + len = min(val_len, c->len); + val_len -= len; + if (!val_found) { + val_found = true; + r = ss_skb_get_room(skb_head, c->skb, c->data, + s_vlen.len, &it); + if (r) + return r; + if ((r = tfw_strcpy(&it, &s_vlen))) + return r; + T_DBG3("%s: value len, acc_len=%lu, it.len=%lu," + " it.data='%.*s'\n", __func__, + mit->acc_len, it.len, + (int)it.len, it.data); + } + } + + if (len < c->len) { + r = skb_fragment(skb_head, c->skb, c->data + len, + -(c->len - len), &it); + if (r < 0) + return r; + } + } + + return 0; +} + +//comment is needed (including the @tgt description) int -tfw_hpack_hdrs_transform(TfwHttpResp *__restrict resp, bool *entered) +tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, + TfwStr *__restrict hdr, TfwH2TransOp op) { - TfwHPackETblRes r; - const TfwStr *field, *hdrs_end, *hdr, *dup_end; + TfwHPackInt idx; TfwH2Ctx *ctx = tfw_h2_context(resp->req->conn); - unsigned short index = 0; + TfwHPackETbl *tbl = &ctx->hpack.enc_tbl; + TfwStr *h2i = op == TFW_H2_TRANS_INPLACE ? tgt : hdr; + TfwStr *fc = TFW_STR_CHUNK(&resp->crlf, 0); + unsigned short index = 0, st_index = TFW_STR_INDEX(h2i); + bool st_full_index = h2i->flags & TFW_STR_FULL_INDEX; + int r = 0; - FOR_EACH_HDR_FIELD(field, hdrs_end, resp) { - TFW_STR_FOR_EACH_DUP(hdr, field, dup_end) { - r = tfw_hpack_encoder_index(&ctx->hpack.enc_tbl, hdr, - entered, &index); - /* - * TODO #309: replace response headers with HTTP/2 - * indexes. - */ + BUG_ON(tgt == hdr); + + if (!st_full_index) { + r = tfw_hpack_encoder_index(tbl, h2i, &index, resp->flags); + if (r < 0) + return r; + } + + if (st_full_index || HPACK_IDX_RES(r) == HPACK_IDX_ST_FOUND) { + /* + * The full index (whether static or dynamic) always takes + * precedence over partial index (when only the header name is + * indexed). + */ + if (!index) + index = st_index; + + WARN_ON_ONCE(!index); + + write_int(index, 0x7F, 0x80, &idx); + switch (op) { + case TFW_H2_TRANS_ADD: + return tfw_hpack_idx_add(resp, fc, &idx); + case TFW_H2_TRANS_EXPAND: + return tfw_hpack_idx_expand(resp, &idx); + case TFW_H2_TRANS_SUB: + case TFW_H2_TRANS_INPLACE: + return tfw_hpack_idx_sub(resp, tgt, &idx); + default: + BUG(); } } - return 0; + if (st_index || HPACK_IDX_RES(r) == HPACK_IDX_ST_NM_FOUND) { + /* + * If we have only partial indexes (static and/or dynamic), the + * static index, if it had been found, always takes precedence + * over dynamic one. + */ + if (st_index) + index = st_index; + + WARN_ON_ONCE(!index); + + if (r & HPACK_IDX_FLAG_ADD) + write_int(index, 0x3F, 0x40, &idx); + else + write_int(index, 0xF, 0, &idx); + + switch (op) { + case TFW_H2_TRANS_ADD: + return tfw_hpack_lit_add(resp, fc, hdr, &idx, true); + case TFW_H2_TRANS_EXPAND: + return tfw_hpack_lit_expand(resp, hdr, &idx, true); + case TFW_H2_TRANS_SUB: + return tfw_hpack_lit_sub(resp, tgt, hdr, &idx, true); + case TFW_H2_TRANS_INPLACE: + return tfw_hpack_lit_inplace(resp, tgt, &idx, true); + default: + BUG(); + } + } + + WARN_ON_ONCE(index || st_index); + + idx.sz = 1; + idx.buf[0] = (r & HPACK_IDX_FLAG_ADD) ? 0x40 : 0; + + switch (op) { + case TFW_H2_TRANS_ADD: + return tfw_hpack_lit_add(resp, fc, hdr, &idx, false); + case TFW_H2_TRANS_EXPAND: + return tfw_hpack_lit_expand(resp, hdr, &idx, false); + case TFW_H2_TRANS_SUB: + return tfw_hpack_lit_sub(resp, tgt, hdr, &idx, false); + case TFW_H2_TRANS_INPLACE: + return tfw_hpack_lit_inplace(resp, tgt, &idx, false); + default: + BUG(); + } } -static inline void +void tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl, unsigned short new_size) { WARN_ON_ONCE(new_size > HPACK_ENC_TABLE_MAX_SIZE); diff --git a/tempesta_fw/hpack.h b/tempesta_fw/hpack.h index 4213f89f6c..879860ffe6 100644 --- a/tempesta_fw/hpack.h +++ b/tempesta_fw/hpack.h @@ -118,17 +118,51 @@ typedef struct { int count; } TfwHPackStr; +typedef enum { + TFW_H2_TRANS_ADD = 0, + TFW_H2_TRANS_EXPAND, + TFW_H2_TRANS_SUB, + TFW_H2_TRANS_INPLACE +} TfwH2TransOp; + +typedef enum { + TFW_TAG_HDR_H2_STATUS, + TFW_TAG_HDR_H2_METHOD = TFW_TAG_HDR_H2_STATUS, + TFW_TAG_HDR_H2_SCHEME, + TFW_TAG_HDR_H2_AUTHORITY, + TFW_TAG_HDR_H2_PATH, + TFW_TAG_HDR_ACCEPT, + TFW_TAG_HDR_AUTHORIZATION, + TFW_TAG_HDR_CACHE_CONTROL, + TFW_TAG_HDR_CONTENT_LENGTH, + TFW_TAG_HDR_CONTENT_TYPE, + TFW_TAG_HDR_COOKIE, + TFW_TAG_HDR_IF_NONE_MATCH, + TFW_TAG_HDR_ETAG = TFW_TAG_HDR_IF_NONE_MATCH, + TFW_TAG_HDR_HOST, + TFW_TAG_HDR_IF_MODIFIED_SINCE, + TFW_TAG_HDR_PRAGMA, + TFW_TAG_HDR_REFERER, + TFW_TAG_HDR_X_FORWARDED_FOR, + TFW_TAG_HDR_USER_AGENT, + TFW_TAG_HDR_SERVER = TFW_TAG_HDR_USER_AGENT, + TFW_TAG_HDR_TRANSFER_ENCODING, + TFW_TAG_HDR_RAW +} TfwHPackTag; + /** * Representation of the entry in HPack decoder index. * - * @name - name of the indexed header; - * @value - value of the indexed header; - * @tag - ID of the indexed header. + * @name_len - length of the header's name part; + * @name_num - chunks count of the header's name part; + * @tag - tag of the indexed header; + * @hdr - descriptor of the header data. */ typedef struct { - TfwHPackStr *name; - TfwHPackStr *value; - long tag; + unsigned long name_len; + unsigned int name_num; + unsigned int tag; + TfwStr hdr[0]; } TfwHPackEntry; /** @@ -168,7 +202,6 @@ typedef struct { * to absence of the next fragment; * @index - saved index value, used when decoding is interrupted due to * absence of the next fragment; - * @entry - index entry found in context of currently parsed header. */ typedef struct { TfwHPackETbl enc_tbl; @@ -181,9 +214,21 @@ typedef struct { unsigned int state; unsigned int shift; unsigned long index; - const TfwHPackEntry *entry; } TfwHPack; +/** + * Maximum length (in bytes) of HPACK variable-length integer representation, + * encoded from 64-bit unsigned long integer: one byte for each 7-bit part of + * source long integer plus on byte for initial prefix. + */ +#define HPACK_MAX_INT \ + (DIV_ROUND_UP(sizeof(unsigned long), 7) + 1) + +typedef struct { + unsigned int sz; + unsigned char buf[HPACK_MAX_INT]; +} TfwHPackInt; + #define BUFFER_GET(len, it) \ do { \ BUG_ON(!(len)); \ @@ -197,9 +242,13 @@ do { \ int tfw_hpack_init(TfwHPack *__restrict hp, unsigned int htbl_sz); void tfw_hpack_clean(TfwHPack *__restrict hp); -int tfw_hpack_hdrs_transform(TfwHttpResp *__restrict resp, bool *entered); -int tfw_hpack_decode(TfwHPack *__restrict hp, const unsigned char *src, +int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, + TfwStr *__restrict hdr, TfwH2TransOp op); +void tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl, + unsigned short new_size); +int tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *src, unsigned long n, TfwHttpReq *__restrict req, unsigned int *__restrict parsed); +void tfw_hpack_enc_release(TfwHPack *__restrict hp, unsigned long *flags); #endif /* __TFW_HPACK_H__ */ diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 0c467688f9..ede3538f55 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -109,6 +109,13 @@ #include "sync_socket.h" #include "lib/common.h" +#define S_H2_METHOD ":method" +#define S_H2_SCHEME ":scheme" +#define S_H2_AUTH ":authority" +#define S_H2_PATH ":path" +#define S_H2_STAT ":status" +#define H2_STAT_VAL_LEN 3 + #define T_WARN_ADDR_STATUS(msg, addr_ptr, print_port, status) \ TFW_WITH_ADDR_FMT(addr_ptr, print_port, addr_str, \ T_WARN("%s, status %d: %s\n", \ @@ -130,8 +137,9 @@ static struct { #define S_CRLFCRLF "\r\n\r\n" #define S_HTTP "http://" #define S_HTTPS "https://" +#define S_VERSION11 "HTTP/1.1" -#define S_0 "HTTP/1.1 " +#define S_0 S_VERSION11 " " #define S_200 "HTTP/1.1 200 OK" #define S_302 "HTTP/1.1 302 Found" #define S_304 "HTTP/1.1 304 Not Modified" @@ -144,14 +152,18 @@ static struct { #define S_503 "HTTP/1.1 503 Service Unavailable" #define S_504 "HTTP/1.1 504 Gateway Timeout" -#define S_F_HOST "Host: " -#define S_F_DATE "Date: " -#define S_F_CONTENT_LENGTH "Content-Length: " -#define S_F_LOCATION "Location: " -#define S_F_CONNECTION "Connection: " -#define S_F_ETAG "ETag: " -#define S_F_RETRY_AFTER "Retry-After: " -#define S_F_SERVER "Server: " +#define S_CONT_TYPE "content-type" +#define S_XFF "x-forwarded-for" + +#define S_F_HOST "host: " +#define S_F_DATE "date: " +#define S_F_CONTENT_LENGTH "content-length: " +#define S_F_LOCATION "location: " +#define S_F_CONNECTION "connection: " +#define S_F_ETAG "etag: " +#define S_F_RETRY_AFTER "retry-after: " +#define S_F_SERVER "server: " +#define S_F_VIA "via: " #define S_V_DATE "Sun, 06 Nov 1994 08:49:37 GMT" #define S_V_CONTENT_LENGTH "9999" @@ -483,7 +495,8 @@ tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, TfwStr *rmark, if (req->host.len) host = req->host; else - tfw_http_msg_clnthdr_val(&req->h_tbl->tbl[TFW_HTTP_HDR_HOST], + tfw_http_msg_clnthdr_val(req, + &req->h_tbl->tbl[TFW_HTTP_HDR_HOST], TFW_HTTP_HDR_HOST, &host); /* Set "Connection:" header field if needed. */ @@ -665,13 +678,138 @@ tfw_http_resp_build_error(TfwHttpReq *req) } static void -tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code) +tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) { + TfwHttpResp *resp; + struct sk_buff **skb_head; + TfwFrameHdr frame_hdr; + unsigned int stream_id; + TfwStr *start *date, *clen, *srv, *body; + char *pos_dt_nm, *pos_cl_nm, *pos_srv_nm; + char *pos_dt_val, *pos_cl_val, *pos_srv_val; + unsigned long len_dt_nm, len_cl_nm, len_srv_nm; + unsigned long len_dt_val, len_cl_val, len_srv_val; + unsigned long len = SLEN(S_H2_STAT) + H2_STAT_VAL_LEN; + unsigned char buf[FRAME_HEADER_SIZE]; + TfwStr *msg = &http_predef_resps[code]; + TfwMsgIter it = {}; + TfwStr hdr = { + .chunks = (TfwStr []){ {}, {} }, + .len = 0, + .nchunks = 2 + }; + TfwStr f_hdr = { + .data = buf, + .len = FRAME_HEADER_SIZE + }; + + if (!(stream_id = tfw_h2_stream_id_close(req))) + goto err; + + if (!(resp = tfw_http_msg_alloc_resp_light(req))) + goto err; + + skb_head = &resp->msg.skb_head; + /* - * TODO #309: add separate flow for HTTP/2 response preparing - * and sending (HPACK index, encode in HTTP/2 format, add frame - * headers and send via @tfw_h2_resp_fwd()). + * Form HTTP/2 response headers excluding "\r\n", ':' separators + * and OWS. */ + start = TFW_STR_START_CH(msg); + date = TFW_STR_DATE_CH(msg); + pos_dt_nm = start->data + start->len - SLEN(S_F_DATE); + len_dt_nm = SLEN(S_F_DATE) - 2; + pos_dt_val = *this_cpu_ptr(&g_buf); + len_dt_val = date->len; + len += len_dt_nm + len_dt_val; + + clen = TFW_STR_CLEN_CH(msg); + pos_cl_nm = clen->data + SLEN(S_CRLF); + len_cl_nm = SLEN(S_F_CONTENT_LENGTH) - 2; + pos_cl_val = pos_cl_nm + len_cl_nm + 2; + len_cl_val = clen->data + clen->len - pos_cl_val - SLEN(S_CRLF); + len += len_cl_nm + len_cl_val; + + srv = TFW_STR_SRV_CH(msg); + pos_srv_nm = srv->data; + len_srv_nm = SLEN(S_F_SERVER) - 2; + pos_srv_val = pos_srv_nm + len_srv_nm + 2; + len_srv_val = srv->data + srv->len - pos_srv_val - SLEN(S_CRLF); + len += len_srv_nm + len_srv_val; + + body = TFW_STR_BODY_CH(msg); + + /* Create frame header for HEADERS. */ + frame_hdr.stream_id = stream_id; + frame_hdr.length = len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + if (!body->data) + frame_hd.flags |= HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, frame_hdr); + + /* Set frame header and payload for HEADERS. */ + if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr)) + goto err_setup; + + resp->status = status; + if (tfw_h2_pseudo_write(resp, TFW_H2_TRANS_EXPAND)) + goto err_setup; + + __TFW_STR_CH(&hdr, 0)->data = pos_dt_nm; + __TFW_STR_CH(&hdr, 0)->len = len_dt_nm; + tfw_http_prep_date(pos_dt_val); + __TFW_STR_CH(&hdr, 1)->data = pos_dt_val; + __TFW_STR_CH(&hdr, 1)->len = len_dt_val; + hdr->len = len_dt_nm + len_dt_val; + TFW_STR_INDEX_SET(&hdr, 33); + if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) + goto err_setup; + + __TFW_STR_CH(&hdr, 0)->data = pos_cl_nm; + __TFW_STR_CH(&hdr, 0)->len = len_cl_nm; + __TFW_STR_CH(&hdr, 1)->data = pos_cl_val; + __TFW_STR_CH(&hdr, 1)->len = len_cl_val; + hdr->len = len_cl_nm + len_cl_val; + TFW_STR_INDEX_SET(&hdr, 28); + if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) + goto err_setup; + + __TFW_STR_CH(&hdr, 0)->data = pos_srv_nm; + __TFW_STR_CH(&hdr, 0)->len = len_srv_nm; + __TFW_STR_CH(&hdr, 1)->data = pos_srv_val; + __TFW_STR_CH(&hdr, 1)->len = len_srv_val; + hdr->len = len_srv_nm + len_srv_val; + TFW_STR_INDEX_SET(&hdr, 54); + if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) + goto err_setup; + + if (body->data) { + /* Create and set frame header and set payload for DATA. */ + frame_hdr.length = body->len; + frame_hdr.type = HTTP2_DATA; + frame_hdr.flags = HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, frame_hdr); + + if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr) + || tfw_http_msg_expand_data(&resp->mit.iter, skb_head, body)) + goto err_setup; + } + + /* Send resulting HTTP/2 response and release HPACK encoder index. */ + tfw_h2_resp_fwd(resp); + + return; + +err_setup: + TFW_DBG("%s: HTTP/2 response message transformation error:" + " conn=[%p]\n", __func__, req->conn); + + tfw_hpack_enc_release(&ctx->hpack, resp->flags); + + tfw_http_msg_free((TfwHttpMsg *)resp); +err: + tfw_http_resp_build_error(req); } /* @@ -1007,7 +1145,7 @@ tfw_http_error_resp_switch(TfwHttpReq *req, int status) } if (TFW_MSG_H2(req)) - tfw_h2_send_resp(req, code); + tfw_h2_send_resp(req, code, (unsigned short)status); else tfw_h1_send_resp(req, code); } @@ -1992,7 +2130,7 @@ tfw_http_conn_msg_alloc(TfwConn *conn, TfwStream *stream) if(!(req->pit.pool = __tfw_pool_new(0))) goto clean; - req->pit.hdr = &req->stream->parser.hdr; + req->pit.parsed_hdr = &req->stream->parser.hdr; __set_bit(TFW_HTTP_B_H2, req->flags); } @@ -2255,7 +2393,7 @@ tfw_http_set_hdr_date(TfwHttpMsg *hm) char *s_date = *this_cpu_ptr(&g_buf); tfw_http_prep_date_from(s_date, ((TfwHttpResp *)hm)->date); - r = tfw_http_msg_hdr_xfrm(hm, "Date", sizeof("Date") - 1, + r = tfw_http_msg_hdr_xfrm(hm, "date", sizeof("date") - 1, s_date, SLEN(S_V_DATE), TFW_HTTP_HDR_RAW, 0); if (r) @@ -2364,18 +2502,16 @@ tfw_http_add_hdr_via(TfwHttpMsg *hm) }; TfwGlobal *g_vhost = tfw_vhost_get_global(); const TfwStr rh = { -#define S_VIA "Via: " .chunks = (TfwStr []) { - { .data = S_VIA, .len = SLEN(S_VIA) }, + { .data = S_F_VIA, .len = SLEN(S_F_VIA) }, { .data = (void *)s_http_version[hm->version], .len = 4 }, { .data = *this_cpu_ptr(&g_buf), .len = g_vhost->hdr_via_len }, }, - .len = SLEN(S_VIA) + 4 + g_vhost->hdr_via_len, + .len = SLEN(S_F_VIA) + 4 + g_vhost->hdr_via_len, .eolen = 2, .nchunks = 3 -#undef S_VIA }; memcpy_fast(__TFW_STR_CH(&rh, 2)->data, g_vhost->hdr_via, @@ -2383,9 +2519,9 @@ tfw_http_add_hdr_via(TfwHttpMsg *hm) r = tfw_http_msg_hdr_add(hm, &rh); if (r) - T_ERR("Unable to add Via: header to msg [%p]\n", hm); + T_ERR("Unable to add via: header to msg [%p]\n", hm); else - T_DBG2("Added Via: header to msg [%p]\n", hm); + T_DBG2("Added via: header to msg [%p]\n", hm); return r; } @@ -2399,7 +2535,7 @@ tfw_http_add_x_forwarded_for(TfwHttpMsg *hm) r = tfw_http_msg_hdr_xfrm(hm, "X-Forwarded-For", sizeof("X-Forwarded-For") - 1, buf, p - buf, - TFW_HTTP_HDR_X_FORWARDED_FOR, true); + TFW_HTTP_HDR_X_FORWARDED_FOR, 0); if (r) T_ERR("can't add X-Forwarded-For header for %.*s to msg %p", (int)(p - buf), buf, hm); @@ -2491,7 +2627,7 @@ tfw_http_should_validate_post_req(TfwHttpReq *req) * Adjust the request before proxying it to real server. */ static int -tfw_http_adjust_req(TfwHttpReq *req) +tfw_h1_adjust_req(TfwHttpReq *req) { int r; TfwHttpMsg *hm = (TfwHttpMsg *)req; @@ -2528,6 +2664,382 @@ tfw_http_adjust_req(TfwHttpReq *req) return tfw_http_set_hdr_connection(hm, BIT(TFW_HTTP_B_CONN_KA)); } +static inline void +__h2_hdrs_dup_decrease(TfwHttpReq *req, const TfwStr *hdr) +{ + const TfwStr *dup, *dup_end; + TfwMsgParseIter *it = &req->pit; + + TFW_STR_FOR_EACH_DUP(dup, hdr, dup_end) { + --it->hdrs_cnt; + it->hdrs_len -= dup->len; + } +} + +static int +__h2_set_req_loc_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, + bool append) +{ + TfwStr *orig_hdr; + TfwHttpHdrTbl *ht = req->h_tbl; + TfwMsgParseIter *it = &req->pit; + const TfwStr *s_val = TFW_STR_CHUNK(hdr, 2); + + BUG_ON(TFW_STR_CHUNK(hdr, 1)->len != SLEN(S_DLM)); + if (WARN_ON_ONCE(!ht)) + return -EINVAL; + + if (unlikely(append && hid < TFW_HTTP_HDR_NONSINGULAR)) { + T_WARN("Appending to singular header %d\n", hid); + return -ENOENT; + } + + if (hid < TFW_HTTP_HDR_RAW) { + orig_hdr = &ht->tbl[hid]; + /* + * Insert special header if empty and exit we have nothing + * to insert. + */ + if (TFW_STR_EMPTY(orig_hdr)) { + if (unlikely(!s_val)) + return 0; + ++it->hdrs_cnt; + it->hdrs_len += hdr->len - SLEN(S_DLM); + *orig_hdr = *hdr; + return 0; + } + } + else { + hid = __h2_hdr_lookup(req, TFW_STR_CHUNK(hdr, 0)); + if (hid == ht->off && !s_val) + /* + * The raw header not found, and there is nothing + * to delete. + */ + return 0; + if (unlikely(hid == ht->size)) + if (tfw_http_msg_grow_hdr_tbl(req)) + return -ENOMEM; + if (hid == ht->off) { + /* + * The raw header not found, but we have the new + * header to insert. + */ + ++ht->off; + ++it->hdrs_cnt; + it->hdrs_len += hdr->len - SLEN(S_DLM); + ht->tbl[hid] = *hdr; + return 0; + } + orig_hdr = &ht->tbl[hid]; + } + + BUG_ON(TFW_STR_EMPTY(orig_hdr)); + /* + * The original header exists, but we have nothing to insert, thus, + * the original header should be evicted. + */ + if (!s_val) { + __h2_hdrs_dup_decrease(req, orig_hdr); + TFW_STR_INIT(orig_hdr); + return 0; + } + + if (append) { + TfwStr h_app = { + .chunks = (TfwStr []){ + { .data = ", ", .len = 2 }, + { .data = s_val->data, .len = s_val->len } + }, + .len = s_val->len + 2, + .nchunks = 2 + }; + /* + * Concatenate only the first duplicate header, there is no need + * to produce more duplicates. + */ + if (TFW_STR_DUP(orig_hdr)) + orig_hdr = __TFW_STR_CH(orig_hdr, 0); + + it->hdrs_len += h_app->len; + return tfw_strcat(req->pool, orig_hdr, &h_app); + } + /* + * The remaining case is the substitution, since we have both: existing + * original header and the new header to insert. + */ + __h2_hdrs_dup_decrease(req, orig_hdr); + ++it->hdrs_cnt; + it->hdrs_len += hdr->len - SLEN(S_DLM); + *orig_hdr = *hdr; + + return 0; +} +static int +tfw_h2_set_req_loc_hdrs(TfwHttpReq *req) +{ + int i; + TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, + TFW_VHOST_HDRMOD_REQ); + if (!h_mods) + return 0; + + for (i = 0; i < h_mods->sz; ++i) { + TfwHdrModsDesc *d = &h_mods->hdrs[i]; + r = __h2_set_req_loc_hdrs(req, d->hdr, d->hid, d->append); + if (r) { + T_ERR("HTTP/2: can't update location-specific header in" + " request [%p]\n", req); + return r; + } + T_DBG3("%s: hdr updated, req=[%p]\n", __func__, req); + } + + return 0; +} + +/** + * Generation and adjusting HTTP/1.1 request (applicable in + * HTTP/2=>HTTP/1.1 transformation). + */ +static int +tfw_h2_adjust_req(TfwHttpReq *req) +{ + int r; + char *dst; + struct page *p; + unsigned int hid; + TfwStr method; + const TfwStr *fld, *fld_end, *hdr, *dup_end; + TfwHttpHdrTbl *ht = req->h_tbl; + TfwMsgParseIter *it = &req->pit; + bool auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); + bool host = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_HOST]); + TfwGlobal *g_vhost = tfw_vhost_get_global(); + char *xff_buf = *this_cpu_ptr(&g_buf); + char *xff_ptr = ss_skb_fmt_src_addr(req->msg.skb_head, xff_buf); + TfwStr h_xff = { + .chunks = (TfwStr []){ + { .data = S_XFF, .len = SLEN(S_XFF) }, + { .data = xff_buf, .len = xff_ptr - xff_buf } + }, + .len = SLEN(S_XFF) + xff_ptr - xff_buf, + .nchunks = 2 + }; + TfwStr h_ct_new = { + .chunks = (TfwStr []) { + { .data = S_CONT_TYPE, .len = SLEN(S_CONT_TYPE) }, + TFW_STR_F_STRING("multipart/form-data; boundary=", + TFW_STR_HDR_VALUE), + req->multipart_boundary_raw + }, + .nchunks = 3 + }; + const TfwStr h_via = { + .chunks = (TfwStr []) { + { .data = S_F_VIA, .len = SLEN(S_F_VIA) }, + { .data = "1.1 ", .len = 4 }, + { .data = *this_cpu_ptr(&g_buf), + .len = g_vhost->hdr_via_len }, + { .data = S_CRLF, .len = SLEN(S_CRLF) } + }, + .len = SLEN(S_F_VIA) + 4 + g_vhost->hdr_via_len + SLEN(S_CRLF), + .nchunks = 4 + }; + TfwStr h_conn = { + .chunks = (TfwStr []){ + { .data = S_F_CONNECTION, .len = SLEN(S_F_CONNECTION) }, + { .data = S_V_CONN_KA, .len = SLEN(S_V_CONN_KA) }, + { .data = S_CRLF, .len = SLEN(S_CRLF) } + }, + .len = SLEN(S_F_CONNECTION) + SLEN(S_V_CONN_KA) + SLEN(S_CRLF), + .nchunks = 3 + }; + +#define WRITE_LIT(buf, lit) \ +do { \ + memcpy_fast(buf, lit, SLEN(lit)); \ + buf += SLEN(lit); \ +} while (0) + +#define WRITE_STR(buf, str) \ +do { \ + TFW_STR_FOR_EACH_CHUNK(chunk, str, end) { \ + memcpy_fast(buf, chunk->data, chunk->len); \ + buf += chunk->len; \ + } \ +} while (0) + +#define WRITE_HDR(buf, hdr) \ +({ \ + TfwStr *c, *end; \ + bool val_found = false; \ + TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { \ + if (!val_found) { \ + if (c->flags & TFW_STR_HDR_VALUE) { \ + WRITE_LIT(buf, S_DLM); \ + val_found = true; \ + } else if (c->data == ':') { \ + /* For statically configured adjustments. */ \ + WARN_ON_ONCE(c->len != SLEN(S_DLM)); \ + WRITE_LIT(buf, S_DLM); \ + val_found = true; \ + continue; \ + } \ + } \ + memcpy_fast(buf, c->data, c->len); \ + buf += c->len; \ + } \ + WRITE_LIT(buf, S_CRLF); \ +}) + + T_DBG3("%s: it->hdrs_len=%lu, it->hdrs_cnt=%u\n", __func__, + it->hdrs_len, it->hdrs_cnt); + + /* Substitution/addition of 'x-forwarded-for' header. */ + hid = __h2_hdr_lookup(req, TFW_STR_CHUNK(&h_xff, 0)); + if (unlikely(hid == ht->size)) + if (tfw_http_msg_grow_hdr_tbl(req)) + return -ENOMEM; + if (hid < ht->off) + __h2_hdrs_dup_decrease(req, &ht->tbl[hid]); + else + ++ht->off; + + ++it->hdrs_cnt; + it->hdrs_len += h_xff.len; + ht->tbl[hid] = h_xff; + + /* + * Conditional substitution/addition/deletion or appending of statically + * configured headers. + */ + if ((r = tfw_h2_set_req_loc_hdrs(req))) + return r; + + /* + * Conditional substitution/additions of 'content-type' header. This is + * singular header, so we can avoid duplicates processing. + */ + if (req->method == TFW_HTTP_METH_POST && + test_bit(TFW_HTTP_B_CT_MULTIPART, req->flags) && + tfw_http_should_validate_post_req(req)) + { + TfwStr *h_ct = &ht->tbl[TFW_HTTP_HDR_CONTENT_TYPE]; + unsigned long ct_len = TFW_STR_EMPTY(h_ct) ? 0 : h_ct->len; + TfwStr *c = h_ct_new.chunks; + + BUG_ON(!TFW_STR_PLAIN(&req->multipart_boundary_raw)); + h_ct_new.len = c[0].len + c[1].len + c[2].len; + + if (TFW_STR_EMPTY(h_ct)) + ++it->hdrs_cnt; + + it->hdrs_len += h_ct_new.len - ct_len; + *h_ct = h_ct_new; + } + + /* + * Calculation of the entire buffer size needed for headers. Request-line + * should be written at first and separately, thus, the total headers + * count and length is correcting by three (or four) pseudo-headers + * count and length. + */ + it->hdrs_cnt -= 3; + it->hdrs_len += 2 + SLEN(S_VERSION11) + SLEN(S_CRLF) + - ht->tbl[TFW_HTTP_HDR_H2_SCHEME].len + - SLEN(S_H2_METHOD) + - SLEN(S_H2_PATH); + + if (auth) { + BUG_ON(TFW_STR_EMPTY(&req->host)); + it->hdrs_len += SLEN(S_HTTP) + - SLEN(S_H2_AUTH) + + SLEN(S_F_HOST) + + req->host.len; + if (host) { + --it->hdrs_cnt; + it->hdrs_len -= ht->tbl[TFW_HTTP_HDR_HOST].len; + } + } + + it->hdrs_len += it->hdrs_cnt * (SLEN(S_DLM) + SLEN(S_CRLF)); + it->hdrs_len += SLEN(S_CRLF); + it->hdrs_len += h_conn.len + h_via.len; + + /* + * Create special buffer to write headers block into the target HTTP/1.1 + * representation. + */ + if (!(pg = alloc_pages(GFP_ATOMIC, get_order(it->hdrs_len)))) + return -ENOMEM; + + dst = (char *)page_address(pg); + + /* Add request-line into the HTTP/1.1 request. */ + BUG_ON(TFW_STR_EMPTY(&req->uri_path)); + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_METHOD], &method); + + WRITE_STR(dst, &method); + WRITE_LIT(dst, " "); + if (auth) { + WRITE_LIT(dst, S_HTTP); + WRITE_STR(dst, &req->host); + } + WRITE_STR(dst, &req->uri_path); + WRITE_LIT(dst, " "); + WRITE_LIT(dst, S_VERSION11); + WRITE_LIT(dst, S_CRLF); + + /* Add 'host' as the first header in request (RFC 7230 section 5.4). */ + if (auth) { + WRITE_LIT(dst, S_F_HOST); + WRITE_STR(dst, &req->host); + } else if (host) { + WRITE_HDR(dst, &ht->tbl[TFW_HTTP_HDR_HOST]); + } + WRITE_LIT(dst, S_CRLF); + + /* + * We have already added 'host' header, thus, it won't be needed in + * the writing cycle for all headers below. + */ + TFW_STR_INIT(&ht->tbl[TFW_HTTP_HDR_HOST]); + + FOR_EACH_HDR_FIELD_FROM(fld, fld_end, req, TFW_HTTP_HDR_REGULAR) { + if (TFW_STR_EMPTY(fld)) + continue; + TFW_STR_FOR_EACH_DUP(hdr, fld, dup_end) + WRITE_HDR(dst, hdr); + } + + /* + * Headers, which should be added unconditionally, inserted in + * the end. + */ + WRITE_STR(dst, &h_via); + WRITE_STR(dst, &h_conn); + WRITE_LIT(dst, S_CRLF); + + T_DBG3("%s: req adjusted, it->hdrs_len=%lu, it->hdrs_cnt=%u, dst=[%p]," + " p=[%p]\n", __func__, it->hdrs_len, it->hdrs_cnt, dst, + (char *)page_address(p)); + WARN_ON_ONCE(dst - (char *)page_address(p) != it->hdrs_len); + + r = ss_skb_replace_page(&req->msg.skb_head, pg, it->hdrs_len, + it->hb_len); + if (r) { + __free_pages(pg, get_order(it->hdrs_len)); + return r; + } + + return 0; + +#undef WRITE_HDR +#undef WRITE_STR +#undef WRITE_LIT +} + /** * Adjust the response before proxying it to real client. */ @@ -2598,7 +3110,7 @@ tfw_http_adjust_resp(TfwHttpResp *resp) } if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { - r = tfw_http_set_hdr_date(hm); + r = tfw_http_set_hdr_date(resp); if (r < 0) return r; } @@ -2637,7 +3149,7 @@ tfw_h2_resp_fwd(TfwHttpResp *resp) { TfwHttpReq *req = resp->req; - if (tfw_connection_send(req->conn, (TfwMsg *)resp)) { + if (tfw_cli_conn_send(req->conn, (TfwMsg *)resp)) { T_DBG("%s: cannot send data to client via HTTP/2\n", __func__); TFW_INC_STAT_BH(serv.msgs_otherr); tfw_connection_close(req->conn, true); @@ -2646,6 +3158,8 @@ tfw_h2_resp_fwd(TfwHttpResp *resp) TFW_INC_STAT_BH(serv.msgs_forwarded); } + tfw_hpack_enc_release(&ctx->hpack, resp->flags); + tfw_http_resp_pair_free(req); } @@ -2756,30 +3270,321 @@ tfw_http_resp_fwd(TfwHttpResp *resp) } } -void +/* + * Static index determination for response ':status' pseudo-header (see RFC + * 7541 Appendix A for details). + */ +static inline unsigned short +tfw_h2_pseudo_index(unsigned short status) +{ + switch (status) { + case 200: + return 8; + case 204: + return 9; + case 206: + return 10; + case 304: + return 11; + case 400: + return 12; + case 404: + return 13; + case 500: + return 14; + default: + return 0; + } +} + +static int +tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) +{ + int ret; + unsigned short index = tfw_h2_pseudo_index(resp->status); + char buf[H2_STAT_VAL_LEN]; + TfwStr *s_line = NULL; + TfwStr s_hdr = { + .chunks = (TfwStr []){ + { .data = S_H2_STAT, .len = SLEN(S_H2_STAT) }, + { .data = buf, .len = H2_STAT_VAL_LEN } + }, + .len = SLEN(S_H2_STAT) + H2_STAT_VAL_LEN, + .nchunks = 2 + }; + + WARN_ON_ONCE(op != TFW_H2_TRANS_EXPAND && op != TFW_H2_TRANS_SUB); + + if (index) { + TFW_STR_INDEX_SET(&s_hdr, index); + s_hdr.flags |= TFW_STR_FULL_INDEX; + } + + if (op == TFW_H2_TRANS_SUB) + s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; + + if (!tfw_ultoa(resp->status, __TFW_STR_CH(&s_hdr, 1)->data, + H2_STAT_VAL_LEN)) + return -E2BIG; + + if ((ret = tfw_hpack_encode(resp, s_line, &s_hdr, op))) + return ret; + + return 0; +} + +static int +tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, + unsigned long h_len) +{ + int ret; + TfwFrameHdr frame_hdr; + unsigned char buf[FRAME_HEADER_SIZE]; + TfwStr *s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; + unsigned long b_len = resp->body.len; + TfwStr *first, it = {}; + TfwStr s_hdr = { + .data = buf, + .len = FRAME_HEADER_SIZE + }; + + frame_hdr.stream_id = stream_id; + + if (b_len) { + if (b_len > FRAME_MAX_LENGTH) { + T_WARN("Unable to make HTTP/2 DATA frame: too big" + " message body (%lu)\n", b_len); + return -E2BIG; + } + + /* + * Set frame header for DATA, if body part of HTTP/1.1 + * response exists. + */ + frame_hdr.length = b_len; + frame_hdr.type = HTTP2_DATA; + frame_hdr.flags = HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, frame_hdr); + + first = TFW_STR_CHUNK(&resp->crlf, 0); + ret = ss_skb_get_room(resp->msg.skb_head, first->skb, + first->data, s_hdr.len, &it); + if (ret) + return ret; + + if ((ret = tfw_strcpy(&it, &s_hdr))) + return ret; + } + + if (h_len > FRAME_MAX_LENGTH) { + T_WARN("Unable to make HTTP/2 HEADERS frame: too big header block" + " fragment (%lu)\n", h_len); + return -E2BIG; + } + + /* Set frame header for HEADERS. */ + frame_hdr.length = h_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + if (!b_len) + frame_hd.flags |= HTTP2_F_END_STREAM; + + tfw_h2_pack_frame_header(buf, frame_hdr); + first = TFW_STR_CHUNK(s_line, 0); + ret = ss_skb_get_room(resp->msg.skb_head, first->skb, first->data, + s_hdr.len, &it); + if (ret) + return ret; + + if ((ret = tfw_strcpy(&it, &s_hdr))) + return ret; + + return 0; +} + +/* + * Same as @tfw_http_add_hdr_via(), but intended for usage in HTTP/1.1=>HTTP/2 + * transformation. + */ +static int +tfw_h2_add_hdr_via(TfwHttpResp *resp) +{ +#define NM_VIA "via" +#define V_PROTO "2.0 " + int r; + TfwGlobal *g_vhost = tfw_vhost_get_global(); + TfwStr via = { + .chunks = (TfwStr []) { + { .data = NM_VIA, .len = SLEN(NM_VIA) }, + { .data = V_PROTO, .len = SLEN(V_PROTO) }, + { .data = *this_cpu_ptr(&g_buf), + .len = g_vhost->hdr_via_len }, + }, + .len = SLEN(NM_VIA) + SLEN(V_PROTO) + g_vhost->hdr_via_len, + .nchunks = 3 + }; + + memcpy_fast(__TFW_STR_CH(&via, 2)->data, g_vhost->hdr_via, + g_vhost->hdr_via_len); + + TFW_STR_INDEX_SET(&via, 60); + r = __hdr_h2_add(resp, &via); + if (unlikely(r)) + TFW_ERR("HTTP/2: unable to add 'via' header (resp=[%p])\n", resp); + else + TFW_DBG3("%s: added 'via' header, resp=[%p]\n", resp); + return r; +#undef NM_VIA +#undef V_PROTO +} + +/* + * Same as @tfw_http_set_hdr_date(), but intended for usage in HTTP/1.1=>HTTP/2 + * transformation. + */ +static int +tfw_h2_set_hdr_date(TfwHttpResp *resp) +{ + int r; + char *s_date = *this_cpu_ptr(&g_buf); + + tfw_http_prep_date_from(s_date, resp->date); + r = tfw_h2_msg_hdr_sub(resp, "date", SLEN("date"), s_date, + SLEN(S_V_DATE), TFW_HTTP_HDR_RAW, 33); + if (unlikely(r)) + TFW_ERR("HTTP/2: unable to add 'date' header to response" + " [%p]\n", resp); + else + TFW_DBG3("%s: added 'date' header, resp=[%p]\n", resp); + + return r; +} + +static inline int +tfw_h2_set_stale_warn(TfwHttpResp *resp) +{ +#define WARN_NM "warning" +#define WARN_VAL "110 - Response is stale" + if (test_bit(TFW_HTTP_B_RESP_STALE, resp->flags)) { + TfwStr wh = { + .chunks = (TfwStr []){ + { .data = WARN_NM, .len = SLEN(WARN_NM) }, + { .data = WARN_VAL, .len = SLEN(WARN_VAL) } + }, + .len = SLEN(WARN_NM) + SLEN(WARN_VAL), + .nchunks = 2 + }; + + return __hdr_h2_add(resp, &wh); + } + + return 0; +#undef WARN_NM +#undef WARN_VAL +} + +static void tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) { - TfwHttpReq *req = resp->req; + int r; unsigned int stream_id; - bool entered = false; + const TfwStr *field, *hdrs_end, *hdr, *dup_end; + TfwHttpReq *req = resp->req; + TfwH2Ctx *ctx = tfw_h2_context(req->conn); + TfwStr *s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; + TfwMsgTransIter *mit = &resp->mit; - if (!(stream_id = tfw_h2_stream_id_close(req)) - || tfw_hpack_hdrs_transform(resp, &entered)) - { - tfw_http_resp_pair_free(req); - TFW_INC_STAT_BH(serv.msgs_otherr); - return; + WARN_ON_ONCE(mit->acc_len); + + /* + * Get ID of corresponding stream and unlink request from the + * stream. + */ + if (!(stream_id = tfw_h2_stream_id_close(req))) + goto out; + + /* Adjust HTTP/1.1 headers with transformation to HTTP/2 form. */ + r = tfw_http_sess_resp_process(resp) + if (unlikely(r)) + goto clean; + + r = tfw_http_msg_del_hbh_hdrs(resp); + if (unlikely(r)) + goto clean; + + r = TFW_H2_MSG_HDR_DEL(resp, "keep-alive", TFW_HTTP_HDR_KEEP_ALIVE); + if (unlikely(r)) + goto clean; + + r = TFW_H2_MSG_HDR_DEL(resp, "connection", TFW_HTTP_HDR_CONNECTION); + if (unlikely(r)) + goto clean; + + r = tfw_h2_add_hdr_via(resp); + if (unlikely(r)) + goto clean; + + r = tfw_h2_set_stale_warn(resp); + if (unlikely(r)) + goto clean; + + if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { + r = tfw_h2_set_hdr_date(resp); + if (unlikely(r)) + goto clean; } + r = TFW_H2_MSG_HDR_SUB(resp, "server", TFW_NAME "/" TFW_VERSION, + TFW_HTTP_HDR_SERVER, 54); + if (unlikely(r)) + goto clean; + + /* Transform HTTP/1.1 headers to HTTP/2 form. */ + FOR_EACH_HDR_FIELD_FROM(field, hdrs_end, resp, TFW_HTTP_HDR_REGULAR) { + if (TFW_STR_EMPTY(field)) + continue; + TFW_STR_FOR_EACH_DUP(hdr, field, dup_end) { + r = tfw_hpack_encode(resp, hdr, NULL, + TFW_H2_TRANS_INPLACE); + if (unlikely(r)) + goto clean; + } + TFW_STR_INIT(field); + } + resp->h_tbl->off = TFW_HTTP_HDR_RAW; + /* - * TODO #309: transforming response into HTTP/2 format and sending - * (HPACK index, replace headers/names with indexes in HTTP/2 format, - * add lengths in HTTP/2 format for not replaced headers/values, apply - * @tfw_http_adjust_resp() optimized for HTTP/2 HPACK, insert frame - * headers for HEADERS and DATA frames and send via @tfw_h2_resp_fwd()). + * Transform the whole response from HTTP/1.1 to HTTP/2 form and + * forward it to the client. */ + r = tfw_h2_pseudo_write(resp, TFW_H2_TRANS_SUB); + if (unlikely(r)) + goto clean; + + r = tfw_h2_make_frames(resp, stream_id, mit->acc_len); + if (unlikely(r)) + goto clean; + + r = tfw_http_msg_del_str(resp, &resp->crlf); + if (unlikely(r)) + goto clean; + + TFW_STR_INIT(s_line); + TFW_STR_INIT(&resp->body); + + tfw_h2_resp_fwd(resp); + + return; +clean: + tfw_http_conn_msg_free((TfwHttpMsg *)resp); + tfw_http_send_resp(req, 500, + "response dropped: processing error"); + tfw_hpack_enc_release(&ctx->hpack, resp->flags); + TFW_INC_STAT_BH(serv.msgs_otherr); + + return; +out: tfw_http_resp_pair_free(req); - TFW_INC_STAT_BH(serv.msgs_forwarded); } static void @@ -3092,6 +3897,7 @@ tfw_http_req_cache_service(TfwHttpResp *resp) static void tfw_http_req_cache_cb(TfwHttpMsg *msg) { + int r; TfwHttpReq *req = (TfwHttpReq *)msg; TfwSrvConn *srv_conn = NULL; LIST_HEAD(eq); @@ -3119,7 +3925,10 @@ tfw_http_req_cache_cb(TfwHttpMsg *msg) goto send_502; } - if (tfw_http_adjust_req(req)) + r = TFW_MSG_H2(req) + ? tfw_h2_adjust_req(req) + : tfw_h1_adjust_req(req); + if (r) goto send_500; /* Account current request in APM health monitoring statistics */ @@ -3288,7 +4097,7 @@ tfw_http_req_client_link(TfwConn *conn, TfwHttpReq *req) conn_cli = (TfwClient *)conn->peer; ua = &req->h_tbl->tbl[TFW_HTTP_HDR_USER_AGENT]; - tfw_http_msg_clnthdr_val(ua, TFW_HTTP_HDR_USER_AGENT, + tfw_http_msg_clnthdr_val(req, ua, TFW_HTTP_HDR_USER_AGENT, &s_user_agent); cli = tfw_client_obtain(conn_cli->addr, &addr, &s_user_agent, NULL); @@ -4297,7 +5106,7 @@ tfw_http_req_key_calc(TfwHttpReq *req) if (test_bit(TFW_HTTP_B_HMONITOR, req->flags)) return req->hash; - tfw_http_msg_clnthdr_val(&req->h_tbl->tbl[TFW_HTTP_HDR_HOST], + tfw_http_msg_clnthdr_val(req, &req->h_tbl->tbl[TFW_HTTP_HDR_HOST], TFW_HTTP_HDR_HOST, &host); if (!TFW_STR_EMPTY(&host)) req->hash ^= tfw_hash_str(&host); diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index e7fb452e3e..60a195a68f 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -180,7 +180,14 @@ typedef struct { * wasting of headers array slots. */ typedef enum { - TFW_HTTP_HDR_HOST, + TFW_HTTP_STATUS_LINE, + TFW_HTTP_HDR_H2_STATUS = TFW_HTTP_STATUS_LINE, + TFW_HTTP_HDR_H2_METHOD = TFW_HTTP_HDR_H2_STATUS, + TFW_HTTP_HDR_H2_SCHEME, + TFW_HTTP_HDR_H2_AUTHORITY, + TFW_HTTP_HDR_H2_PATH, + TFW_HTTP_HDR_REGULAR, + TFW_HTTP_HDR_HOST = TFW_HTTP_HDR_REGULAR, TFW_HTTP_HDR_CONTENT_LENGTH, TFW_HTTP_HDR_CONTENT_TYPE, TFW_HTTP_HDR_USER_AGENT, @@ -244,6 +251,10 @@ enum { TFW_HTTP_B_FULLY_PARSED, /* Message has HTTP/2 format. */ TFW_HTTP_B_H2, + /* Message has all mandatory pseudo-headers (applicable for HTTP/2 mode only) */ + TFW_HTTP_B_H2_HDRS_FULL, + /* Message in HTTP/2 transformation (applicable for HTTP/2 mode only). */ + TFW_HTTP_B_H2_TRANS_ENTERED, /* Request flags. */ TFW_HTTP_FLAGS_REQ, @@ -284,6 +295,10 @@ enum { #define TFW_MSG_H2(hmmsg) \ test_bit(TFW_HTTP_B_H2, ((TfwHttpMsg *)hmmsg)->flags) +#define TFW_RESP_TO_H2(hmmsg) \ + ((!hmmsg->conn || TFW_CONN_TYPE(hmmsg->conn) & Conn_Srv) && \ + hmmsg->pair && TFW_MSG_H2(hmmsg->pair)) + /** * The structure to hold data for an HTTP error response. * An error response is sent later in an unlocked queue context. @@ -443,14 +458,16 @@ struct tfw_http_req_t { * TfwStr members must be the first for efficient scanning. * * @jrxtstamp - time the message has been received, in jiffies; + * @mit - iterator for controlling HTTP/1.1 => HTTP/2 message + * transformation process (applicable for HTTP/2 mode only). */ struct tfw_http_resp_t { TFW_HTTP_MSG_COMMON; - TfwStr s_line; unsigned short status; time_t date; time_t last_modified; unsigned long jrxtstamp; + TfwMsgTransIter mit; }; #define TFW_HTTP_RESP_STR_START(r) __MSG_STR_START(r) @@ -526,7 +543,6 @@ int tfw_http_msg_process_generic(TfwConn *conn, TfwStream *stream, unsigned long tfw_http_req_key_calc(TfwHttpReq *req); void tfw_http_req_destruct(void *msg); void tfw_http_resp_fwd(TfwHttpResp *resp); -void tfw_h2_resp_adjust_fwd(TfwHttpResp *resp); void tfw_http_resp_build_error(TfwHttpReq *req); int tfw_cfgop_parse_http_status(const char *status, int *out); void tfw_http_hm_srv_send(TfwServer *srv, char *data, unsigned long len); diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index a730cb1f74..c787ec678e 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -35,8 +35,6 @@ #define FRAME_SETTINGS_ENTRY_SIZE 6 #define FRAME_PING_SIZE 8 #define FRAME_GOAWAY_SIZE 8 -#define FRAME_STREAM_ID_MASK ((1U << 31) - 1) -#define FRAME_RESERVED_BIT_MASK (~FRAME_STREAM_ID_MASK) #define WND_INCREMENT_SIZE 4 #define SETTINGS_KEY_SIZE 2 @@ -246,22 +244,6 @@ tfw_h2_unpack_frame_header(TfwFrameHdr *hdr, const unsigned char *buf) __func__, hdr->length, hdr->stream_id, hdr->type, hdr->flags); } -static inline void -tfw_h2_pack_frame_header(unsigned char *p, const TfwFrameHdr *hdr) -{ - *(unsigned int *)p = htonl((unsigned int)(hdr->length << 8)); - p += 3; - *p++ = hdr->type; - *p++ = hdr->flags; - /* - * Stream id must occupy not more than 31 bit and reserved bit - * must be 0. - */ - WARN_ON_ONCE((unsigned int)(hdr->stream_id & FRAME_RESERVED_BIT_MASK)); - - *(unsigned int *)p = htonl(hdr->stream_id); -} - static inline void tfw_h2_unpack_priority(TfwFramePri *pri, const unsigned char *buf) { @@ -313,10 +295,10 @@ __tfw_h2_send_frame(TfwH2Ctx *ctx, TfwFrameHdr *hdr, TfwStr *data, bool close) if ((r = tfw_connection_send((TfwConn *)conn, &msg))) goto err; /* - * For HTTP/2 we do not close client connection automatically in case - * of failed sending (unlike the HTTP/1.1 processing); thus, we should - * set Conn_Stop flag only if sending procedure was successful - to - * avoid hanged unclosed client connection. + * We do not close client connection automatically here in case + * of failed sending, the caller must make such decision instead; + * thus, we should set Conn_Stop flag only if sending procedure + * was successful - to avoid hanged unclosed client connection. */ if (close) TFW_CONN_TYPE((TfwConn *)conn) |= Conn_Stop; @@ -520,6 +502,7 @@ tfw_h2_recv_priority(TfwH2Ctx *ctx) { ctx->to_read = FRAME_PRIORITY_SIZE; ctx->hdr.length -= ctx->to_read; + ctx->plen -= ctx->to_read; VERIFY_FRAME_SIZE(ctx); ctx->state = HTTP2_RECV_HEADER_PRI; return T_OK; @@ -530,6 +513,7 @@ tfw_h2_recv_padded(TfwH2Ctx *ctx) { ctx->to_read = 1; ctx->hdr.length -= ctx->to_read; + ctx->plen -= ctx->to_read; VERIFY_FRAME_SIZE(ctx); ctx->state = HTTP2_RECV_FRAME_PADDED; return T_OK; @@ -899,12 +883,15 @@ tfw_h2_rst_stream_process(TfwH2Ctx *ctx) } static int -tfw_h2_apply_settings_entry(TfwSettings *dest, unsigned short id, +tfw_h2_apply_settings_entry(TfwH2Ctx *ctx, unsigned short id, unsigned int val) { + TfwSettings *dest = &ctx->rsettings; + switch (id) { case HTTP2_SETTINGS_TABLE_SIZE: dest->hdr_tbl_sz = val; + tfw_hpack_set_rbuf_size(&ctx->hpack.enc_tbl, val); break; case HTTP2_SETTINGS_ENABLE_PUSH: @@ -936,7 +923,7 @@ tfw_h2_apply_settings_entry(TfwSettings *dest, unsigned short id, } /* - * TODO: apply settings entry. + * TODO #309: apply settings entry. */ return T_OK; } @@ -962,7 +949,7 @@ tfw_h2_settings_process(TfwH2Ctx *ctx) T_DBG3("%s: entry parsed, id=%hu, val=%u\n", __func__, id, val); - if (tfw_h2_apply_settings_entry(&ctx->rsettings, id, val)) + if (tfw_h2_apply_settings_entry(ctx, id, val)) return T_BAD; ctx->to_read = hdr->length ? FRAME_SETTINGS_ENTRY_SIZE : 0; @@ -1201,6 +1188,7 @@ tfw_h2_frame_type_process(TfwH2Ctx *ctx) STREAM_RECV_PROCESS(ctx, hdr); ctx->data_off = FRAME_HEADER_SIZE; + ctx->plen = ctx->hdr.length; if (hdr->flags & HTTP2_F_PADDED) return tfw_h2_recv_padded(ctx); @@ -1241,6 +1229,7 @@ tfw_h2_frame_type_process(TfwH2Ctx *ctx) } ctx->data_off = FRAME_HEADER_SIZE; + ctx->plen = ctx->hdr.length; if (hdr->flags & HTTP2_F_PADDED) return tfw_h2_recv_padded(ctx); @@ -1413,6 +1402,7 @@ tfw_h2_frame_type_process(TfwH2Ctx *ctx) STREAM_RECV_PROCESS(ctx, hdr); ctx->data_off = FRAME_HEADER_SIZE; + ctx->plen = ctx->hdr.length; SET_TO_READ_VERIFY(ctx, HTTP2_RECV_CONT); return T_OK; diff --git a/tempesta_fw/http_frame.h b/tempesta_fw/http_frame.h index 527e709ba7..d4c054e8f2 100644 --- a/tempesta_fw/http_frame.h +++ b/tempesta_fw/http_frame.h @@ -25,6 +25,9 @@ #include "hpack.h" #define FRAME_HEADER_SIZE 9 +#define FRAME_STREAM_ID_MASK ((1U << 31) - 1) +#define FRAME_RESERVED_BIT_MASK (~FRAME_STREAM_ID_MASK) +#define FRAME_MAX_LENGTH ((1U << 24) - 1) /** * HTTP/2 frame types (RFC 7540 section 6). @@ -152,6 +155,8 @@ typedef struct { * processed HEADERS or PRIORITY frames; * @hdr - unpacked data from header of currently processed * frame; + * @plen - payload length of currently processed frame + * (HEADERS/CONTINUATION/DATA frames); * @state - current FSM state of HTTP/2 processing context; * @to_read - indicates how much data of HTTP/2 frame should * be read on next FSM @state; @@ -183,6 +188,7 @@ typedef struct { TfwStream *cur_stream; TfwFramePri priority; TfwFrameHdr hdr; + unsigned int plen; int state; int to_read; int rlen; @@ -201,4 +207,21 @@ unsigned int tfw_h2_stream_id_close(TfwHttpReq *req); void tfw_h2_conn_terminate_close(TfwH2Ctx *ctx, TfwH2Err err_code, bool close); int tfw_h2_send_rst_stream(TfwH2Ctx *ctx, unsigned int id, TfwH2Err err_code); +static inline void +tfw_h2_pack_frame_header(unsigned char *p, const TfwFrameHdr *hdr) +{ + *(unsigned int *)p = htonl((unsigned int)(hdr->length << 8)); + p += 3; + *p++ = hdr->type; + *p++ = hdr->flags; + /* + * Stream id must occupy not more than 31 bit and reserved bit + * must be 0. + */ + WARN_ON_ONCE((unsigned int)(hdr->stream_id & FRAME_RESERVED_BIT_MASK)); + + *(unsigned int *)p = htonl(hdr->stream_id); +} + + #endif /* __HTTP_FRAME__ */ diff --git a/tempesta_fw/http_limits.c b/tempesta_fw/http_limits.c index 7fc0c8d334..915aebbccf 100644 --- a/tempesta_fw/http_limits.c +++ b/tempesta_fw/http_limits.c @@ -553,7 +553,8 @@ frang_http_ct_check(const TfwHttpReq *req, FrangAcc *ra, FrangCtVals *ct_vals) return TFW_BLOCK; } - tfw_http_msg_clnthdr_val(&req->h_tbl->tbl[TFW_HTTP_HDR_CONTENT_TYPE], + tfw_http_msg_clnthdr_val(req, + &req->h_tbl->tbl[TFW_HTTP_HDR_CONTENT_TYPE], TFW_HTTP_HDR_CONTENT_TYPE, &field); /* @@ -619,7 +620,7 @@ frang_http_host_check(const TfwHttpReq *req, FrangAcc *ra) return req->version > TFW_HTTP_VER_10 ? TFW_BLOCK : TFW_PASS; } - tfw_http_msg_clnthdr_val(&req->h_tbl->tbl[TFW_HTTP_HDR_HOST], + tfw_http_msg_clnthdr_val(req, &req->h_tbl->tbl[TFW_HTTP_HDR_HOST], TFW_HTTP_HDR_HOST, &field); if (!TFW_STR_EMPTY(&field)) { /* Check that host header is not a IP address. */ @@ -874,14 +875,19 @@ frang_http_req_trailer_check(FrangAcc *ra, TfwFsmData *data, if (!tfw_http_parse_is_done((TfwHttpMsg *)req)) return TFW_POSTPONE; /* - * Block request if the same header appear in both main and - * trailer headers part. Some intermediates doesn't read trailers, so - * request processing may depend on implementation. + * Block request if the same header appear in both main and trailer + * headers part. Some intermediates doesn't read trailers, so request + * processing may depend on implementation. + * + * NOTE: we check only regular headers (without HTTP/2-specific + * pseudo-header fields), since pseudo-headers must not appear in + * trailers (RFC 7540 section 8.1.2.1), and during parsing stage, in + * @H2_MSG_VERIFY(), we have already verified that. */ if (!f_cfg->http_trailer_split) return TFW_PASS; - FOR_EACH_HDR_FIELD(field, end, req) { + FOR_EACH_HDR_FIELD_FROM(field, end, req, TFW_HTTP_HDR_REGULAR) { int trailers = 0, dups = 0; TFW_STR_FOR_EACH_DUP(dup, field, dup_end) { trailers += !!(dup->flags & TFW_STR_TRAILER); diff --git a/tempesta_fw/http_match.c b/tempesta_fw/http_match.c index 247dc822b5..a43729006b 100644 --- a/tempesta_fw/http_match.c +++ b/tempesta_fw/http_match.c @@ -125,7 +125,7 @@ hdr_val_eq(const TfwHttpReq *req, const TfwHttpMatchRule *rule, if (op == TFW_HTTP_MATCH_O_WILDCARD) return true; - tfw_http_msg_clnthdr_val(hdr, id, &hdr_val); + tfw_http_msg_clnthdr_val(req, hdr, id, &hdr_val); flags = map_op_to_str_eq_flags(rule->op); /* @@ -221,9 +221,11 @@ static bool match_hdr_raw(const TfwHttpReq *req, const TfwHttpMatchRule *rule) { int i; + bool h2_mode = TFW_MSG_H2(req); tfw_str_eq_flags_t flags = map_op_to_str_eq_flags(rule->op); for (i = TFW_HTTP_HDR_RAW; i < req->h_tbl->off; ++i) { + bool col_found; const TfwStr *hdr, *dup, *end, *chunk; const char *c, *cend, *p, *pend; char prev; @@ -236,6 +238,7 @@ match_hdr_raw(const TfwHttpReq *req, const TfwHttpMatchRule *rule) TFW_STR_FOR_EACH_DUP(dup, hdr, end) { /* Initialize the state - get the first chunk. */ + col_found = false; p = rule->arg.str; pend = rule->arg.str + rule->arg.len; cnum = 0; @@ -259,13 +262,47 @@ match_hdr_raw(const TfwHttpReq *req, const TfwHttpMatchRule *rule) } \ } +#define __H2_VERIFY() \ + if (h2_mode && !col_found \ + && chunk->flags & TFW_STR_HDR_VALUE \ + && c != chunk->data) \ + break; + +#define __H2_VERIFY_NON_MATCH() \ + if (h2_mode) { \ + if (!col_found && *p == ':' \ + && chunk->flags & TFW_STR_HDR_VALUE) \ + { \ + p++; \ + col_found = true; \ + goto state_rule_sp; \ + } \ + break; \ + } + prev = *p; state_common: while (p != pend && c != cend) { + + /* If we reached the value chunk of HTTP/2 + * header, but colon is not found in the rule + * yet, the header does not fit the rule. + */ + __H2_VERIFY(); + /* The rule convert to lower case on the step of * handling the configuration. */ if (*p != tolower(*c)) { + + /* If the characters from the rule and + * HTTP/2 header does not match, this + * could be due to colon and subsequent + * OWS in the rule, so, we should skip + * them and check the next character. + */ + __H2_VERIFY_NON_MATCH(); + /* If the same position of the header * field and rule have a different * number of whitespace characters, diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 2142aad8ba..8c670207ad 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -131,7 +131,8 @@ tfw_http_msg_resp_spec_hid(const TfwStr *hdr) TfwStrDefV("x-forwarded-for:", TFW_HTTP_HDR_X_FORWARDED_FOR), }; - BUILD_BUG_ON(ARRAY_SIZE(resp_hdrs) != TFW_HTTP_HDR_RAW); + BUILD_BUG_ON(ARRAY_SIZE(resp_hdrs) != + TFW_HTTP_HDR_RAW - TFW_HTTP_HDR_REGULAR); return __tfw_http_msg_spec_hid(hdr, resp_hdrs); } @@ -156,7 +157,8 @@ tfw_http_msg_req_spec_hid(const TfwStr *hdr) TfwStrDefV("x-forwarded-for:", TFW_HTTP_HDR_X_FORWARDED_FOR), }; - BUILD_BUG_ON(ARRAY_SIZE(req_hdrs) != TFW_HTTP_HDR_RAW); + BUILD_BUG_ON(ARRAY_SIZE(req_hdrs) != + TFW_HTTP_HDR_RAW - TFW_HTTP_HDR_REGULAR); return __tfw_http_msg_spec_hid(hdr, req_hdrs); } @@ -249,6 +251,33 @@ __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client) } EXPORT_SYMBOL(__http_msg_hdr_val); +void +__h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val) +{ + TfwStr *c, *end; + + if (unlikely(TFW_STR_PLAIN(hdr))) { + TFW_STR_INIT(out_val); + return; + } + + BUG_ON(TFW_STR_DUP(hdr)); + + *out_val = *hdr; + + TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { + if (c->flags & TFW_STR_HDR_VALUE) { + out_val->chunks = c; + return; + } + out_val->len -= c->len; + out_val->nchunks--; + } + + /* Empty header value part. */ + TFW_STR_INIT(out_val); +} + /** * Slow check of generic (raw) header for singularity. * Some of the header should be special and moved to tfw_http_hdr_t enum, @@ -311,6 +340,104 @@ __hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) return id; } +/** + * Special procedure comparing specified name against the header in HTTP/2 + * or HTTP/1.1 format. + */ +static int +__hdr_name_cmp(const TfwStr *hdr, const TfwStr *name) +{ + long n; + int i1, i2, off1, off2; + const TfwStr *c1, *c2; + + BUG_ON(hdr->flags & TFW_STR_DUPLICATE); + BUG_ON(!name->len); + + if (unlikely(!hdr->len)) + return -name->len; + + i1 = i2 = 0; + off1 = off2 = 0; + n = min(hdr->len, name->len); + c1 = TFW_STR_CHUNK(hdr, 0); + c2 = TFW_STR_CHUNK(name, 0); + while (n) { + int cn = min(c1->len - off1, c2->len - off2); + int r = tfw_cstricmp(c1->data + off1, + c2->data + off2, cn); + if (r) + return r; + + n -= cn; + if (cn == c1->len - off1) { + off1 = 0; + ++i1; + c1 = TFW_STR_CHUNK(hdr, i1); + } else { + off1 += cn; + } + if (cn == c2->len - off2) { + off2 = 0; + ++i2; + c2 = TFW_STR_CHUNK(name, i2); + } else { + off2 += cn; + } + BUG_ON(n && (!c1 || !c2)); + } + + /* Only name is contained in the header. */ + if (hdr->len == name->len) + return 0; + + if (hdr->len > name->len) { + /* + * If the header is of HTTP/2 format, the end of name must match + * the end of the chunk, and the following value must have + * appropriate flag. + */ + if (!off1 + && (c1->flags & TFW_STR_HDR_VALUE) + && !(TFW_STR_CHUNK(hdr, i1 - 1)->flags & TFW_STR_HDR_VALUE)) + return 0; + /* + * If this is the HTTP/1.1-format header, the value must begin + * after the colon. + */ + if (*(c1->data + off1) == ':') + return 0; + } + + return (long)hdr->len - (long)name->len; +} + +/** + * As @__hdr_lookup(), but intended for search during HTTP/1.1=>HTTP/2 + * and HTTP/2=>HTTP/1.1 transformations, comparing the specified name + * headers in HTTP/2 or HTTP/1.1 format. + */ +int +__h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name) +{ + unsigned int id; + TfwHttpHdrTbl *ht = hm->h_tbl; + + for (id = TFW_HTTP_HDR_RAW; id < ht->off; ++id) { + TfwStr *h = &ht->tbl[id]; + /* + * We are looking only for the header's name matching, + * thus, there is no sense to compare against all duplicates. + */ + if (h->flags & TFW_STR_DUPLICATE) + h = TFW_STR_CHUNK(h, 0); + if (!__hdr_name_cmp(h, h_name)) + break; + } + + return id; +} + /** * Open currently parsed header. */ @@ -335,11 +462,12 @@ tfw_http_msg_hdr_open(TfwHttpMsg *hm, unsigned char *hdr_start) * HTTP message headers list. */ int -tfw_http_msg_hdr_close(TfwHttpMsg *hm, unsigned int id) +tfw_http_msg_hdr_close(TfwHttpMsg *hm) { TfwStr *h; TfwHttpHdrTbl *ht = hm->h_tbl; TfwHttpParser *parser = &hm->stream->parser; + unsigned int id = parser->_hdr_tag; BUG_ON(parser->hdr.flags & TFW_STR_DUPLICATE); BUG_ON(id > TFW_HTTP_HDR_RAW); @@ -506,6 +634,12 @@ __hdr_add(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid) return 0; } +int +__hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr) +{ + return tfw_hpack_encode(resp, NULL, hdr, TFW_H2_TRANS_ADD); +} + /** * Expand @orig_hdr by appending or replacing with the @hdr. * (CRLF is not accounted in TfwStr representation of HTTP headers). @@ -614,6 +748,35 @@ __hdr_sub(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid) return TFW_PASS; } +static int +__hdr_h2_sub(TfwHttpResp *resp, TfwStr *hdr, unsigned int hid) +{ + int ret; + TfwHttpHdrTbl *ht = resp->h_tbl; + TfwStr *tgt, *dup, *end, *orig_hdr = &ht->tbl[hid]; + + tgt = TFW_STR_DUP(orig_hdr) ? __TFW_STR_CH(orig_hdr, 0) : orig_hdr; + + TFW_STR_FOR_EACH_DUP(dup, orig_hdr, end) { + if (dup == tgt) + continue; + if ((ret = ss_skb_cutoff_data(resp->msg.skb_head, dup, 0, + tfw_str_eolen(dup)))) + return ret; + } + + ret = tfw_hpack_encode(resp, tgt, hdr, TFW_H2_TRANS_SUB); + if (ret) + return ret; + /* + * Exclude header from the subsequent transformation + * processing. + */ + TFW_STR_INIT(orig_hdr); + + return 0; +} + /** * Transform HTTP message @hm header with identifier @hid. * @hdr must be compound string and contain two or three parts: @@ -629,6 +792,17 @@ __hdr_sub(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid) * string @orig_hdr may have a single LF as EOL. We may want to follow * the EOL pattern of the original. For that, the EOL of @hdr needs * to be made the same as in the original header field string. + * + * Note: In case of response transformation from HTTP/1.1 to HTTP/2, for + * optimization purposes, we use special add/replace procedures to adjust + * headers and create HTTP/2 representation at once; for headers deletion + * procedure there is no sense to use special HTTP/2 handling (the header + * must not exist in the resulting response); in case of headers appending + * we at first create the usual HTTP/1.1 representation of the final header + * and then transform it into HTTP/2 form at the common stage of response + * HTTP/2 transformation - we have no other choice, since we need a full + * header for going through the HTTP/2 transformation (i.e. search in the + * HPACK encoder dynamic index). */ int tfw_http_msg_hdr_xfrm_str(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid, @@ -759,6 +933,34 @@ tfw_http_msg_del_hbh_hdrs(TfwHttpMsg *hm) return 0; } +int +tfw_http_msg_del_eol(struct sk_buff *skb_head, TfwStr *hdr) +{ + int r; + TfwStr it = {}; + int eolen = tfw_str_eolen(hdr); + TfwStr *last = TFW_STR_LAST(hdr); + + if ((r = skb_next_data(last->skb, last->data + last->len - 1, &it))) + return r; + + while (eolen) { + char *ptr = it.data; + struct sk_buff *skb = it.skb; + + bzero_fast(&it, sizeof(TfwStr)); + if ((r = skb_fragment(skb_head, skb, ptr, -eolen, &it))) + return r; + + if (WARN_ON_ONCE(r > eolen)) + return -EINVAL; + + eolen -= r; + } + + return 0; +} + /** * Modify message body, add string @data to the end of the body (if @append) or * to its beginning. @@ -842,13 +1044,17 @@ tfw_http_msg_to_chunked(TfwHttpMsg *hm) /** * Add a header, probably duplicated, without any checking of current headers. + * In case of response transformation from HTTP/1.1 to HTTP/2, for optimization + * purposes, we use special handling for headers adding (see note for + * @tfw_http_msg_hdr_xfrm_str() for details). */ int tfw_http_msg_hdr_add(TfwHttpMsg *hm, const TfwStr *hdr) { unsigned int hid; - TfwHttpHdrTbl *ht = hm->h_tbl; + TfwHttpHdrTbl *ht; + ht = hm->h_tbl; hid = ht->off; if (hid == ht->size) if (tfw_http_msg_grow_hdr_tbl(hm)) @@ -1050,54 +1256,39 @@ __tfw_http_msg_alloc(int type, bool full) /** * Determination length of the header's real part (for details see comment - * for @tfw_http_msg_hdr_write() below) to store it in the encoder dynamic + * for @tfw_h2_msg_hdr_write() below) to store it in the encoder dynamic * index. */ unsigned long -tfw_http_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, - unsigned long *val_off, unsigned long *val_len) +tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, + unsigned long *val_off, unsigned long *val_len) { const TfwStr *chunk, *end; unsigned long tail, hdr_tail = 0, hdr_len = 0; bool name_found = false, val_found = false; + *name_len = *val_off = *val_len = 0; + TFW_STR_FOR_EACH_CHUNK(chunk, hdr, end) { unsigned long idx; if (!chunk->len) continue; - idx = chunk->len - 1; hdr_len += chunk->len; if (!name_found) { - for (; (chunk->data[idx] == ' ' - || chunk->data[idx] == '\t') - && idx; - --idx); - - *name_len += idx; - - if (chunk->data[idx] == ':') { - *val_off += hdr_len - *name_len; + *name_len += chunk->len; + if (chunk->data[chunk->len - 1] == ':') { + --*name_len; name_found = true; } - else { - WARN_ON_ONCE(idx != chunk->len - 1 - || chunk->data[idx] == ' ' - || chunk->data[idx] == '\t'); - ++*name_len; - } - continue; } /* - * Skip OWS before header value during header's real length - * calculation. OWS is always on the same chunk with the header - * name, or on the separate chunk; thus header value always - * begins at new chunk, and we can skip length of the entire - * (OWS) chunk. If this is WS of the header value, accumulate - * length in @tail for end OWS cutting off (if this is not the - * end chunk, @tail will be reset). + * Skip OWS before the header value (LWS) during HTTP/2 header's + * real length calculation. LWS is always in the separate chunks + * between the name and value; thus, we can skip length of the + * entire (LWS) chunks. */ if (!val_found) { if (unlikely(chunk->data[0] == ' ' @@ -1106,11 +1297,20 @@ tfw_http_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, *val_off += chunk->len; continue; } + /* + * The colon must not be included into HTTP/2 header, + * thus, it should be counted in the value offset. + */ + ++*val_off; val_found = true; } - + /* + * Skip OWS after the header value (RWS); accumulate the length + * in @tail for RWS cutting off (if this is not the end chunk, + * @tail will be reset). + */ tail = 0; - + idx = chunk->len - 1; while (chunk->data[idx] == ' ' || chunk->data[idx] == '\t') { @@ -1138,18 +1338,18 @@ tfw_http_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, } /** - * Copy the real part (i.e. without name colon and OWS) of header into @out_buf - * from @hdr; @nm_len is the real length of header name, @val_len - the real - * length of header value, and @val_off - the offset between header name and - * value (i.e. the part occupied by OWS); OWS in the end of header's value are - * also skipped and will not be included into header's copied part. Note that - * the size of prepared @out_buf must be not less than sum of @nm_len and - * @val_len. + * Copy the real part of header (i.e. the header in HTTP/2 form - without name + * colon and OWS) into @out_buf from @hdr; @nm_len is the real length of header + * name, @val_len - the real length of header value, and @val_off - the offset + * between header name and value (i.e. the part occupied by colon and OWS); OWS + * in the end of header's value are also skipped and will not be included into + * header's copied part. Note that the size of prepared @out_buf must be not + * less than sum of @nm_len and @val_len. */ void -tfw_http_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, - unsigned long val_off, unsigned long val_len, - char *out_buf) +tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, + unsigned long val_off, unsigned long val_len, + char *out_buf) { const TfwStr *c, *end; @@ -1191,3 +1391,128 @@ tfw_http_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, out_buf += len; } } + +int +tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, + const TfwStr *src) +{ + const TfwStr *c, *end; + + TFW_STR_FOR_EACH_CHUNK(c, src, end) { + char *p; + unsigned long off = 0, cur_len, f_room, min_len; +this_chunk: + if (!it->skb) { + if (!(it->skb = ss_skb_alloc(SKB_MAX_HEADER))) + return -ENOMEM; + ss_skb_queue_tail(skb_head, it->skb); + it->frag = -1; + if (!it->skb_head) + it->skb_head = *skb_head; + T_DBG3("message expanded by new skb [%p]\n", it->skb); + } + + cur_len = c->len - off; + if (it->frag >= 0) { + unsigned int f_size; + skb_frag_t *frag = &skb_shinfo(it->skb)->frags[it->frag]; + + f_size = skb_frag_size(frag); + f_room = PAGE_SIZE - frag->page_offset - f_size; + p = (char *)skb_frag_address(frag) + f_size; + min_len = min(cur_len, f_room); + skb_frag_size_add(frag, min_len); + ss_skb_adjust_data_len(it->skb, min_len); + } else { + f_room = skb_tailroom(it->skb); + min_len = min(cur_len, f_room); + p = skb_put(it->skb, min_len); + } + + memcpy_fast(p, c->data + off, min_len); + + if (cur_len >= f_room) { + /* + * If the amount of skb frags is exhausted, allocate new + * skb on next iteration (if it will be). + */ + if (MAX_SKB_FRAGS <= it->frag + 1) { + it->skb = NULL; + } + else if (cur_len != f_room || c + 1 < end) { + struct page *page = alloc_page(GFP_ATOMIC); + if (!page) + return -ENOMEM; + ++it->frag; + skb_fill_page_desc(it->skb, it->frag, page, + 0, 0); + T_DBG3("message expanded by new frag %d," + " page=[%p], skb=[%p]\n", i, + page_address(page), it->skb); + } + + if (cur_len != f_room) { + off += min_len; + goto this_chunk; + } + } + } + + return 0; +} + +int +tfw_h2_msg_hdr_sub(TfwHttpMsg *hm, char *name, size_t nlen, char *val, + size_t vlen, unsigned int hid, unsigned short idx) +{ + TfwHttpHdrTbl *ht = hm->h_tbl; + TfwStr *orig_hdr = NULL; + TfwStr hdr = { + .chunks = (TfwStr []){ + { .data = name, .len = nlen }, + { .data = val, .len = vlen }, + }, + .len = nlen + vlen, + .nchunks = 2 + }; + + WARN_ON_ONCE(!ht); + + if (hid < TFW_HTTP_HDR_RAW) { + orig_hdr = &ht->tbl[hid]; + } + else { + hid = __h2_hdr_lookup(hm, TFW_STR_CHUNK(&hdr, 0)); + if (hid < ht->off) + orig_hdr = &ht->tbl[hid]; + } + + TFW_STR_INDEX_SET(&hdr, idx); + + if (!orig_hdr || TFW_STR_EMPTY(orig_hdr)) + return __hdr_h2_add((TfwHttpResp *)hm, &hdr); + + return __hdr_h2_sub((TfwHttpResp *)hm, &hdr, hid); +} + +int +tfw_h2_msg_hdr_del(TfwHttpMsg *hm, char *name, size_t nlen, unsigned int hid) +{ + TfwHttpHdrTbl *ht = hm->h_tbl; + TfwStr h_name = { + .data = name, + .len = nlen + }; + + WARN_ON_ONCE(!ht); + + if (hid < TFW_HTTP_HDR_RAW) { + if (TFW_STR_EMPTY(&ht->tbl[hid])) + return 0; + } else { + if ((hid = __h2_hdr_lookup(hm, &h_name)) == ht->off) + return 0; + } + + return __hdr_del(hm, hid); +} diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index cb55f78bec..09ba9a0fb0 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -23,9 +23,10 @@ #include "http.h" -#define S_F_SET_COOKIE "Set-Cookie: " #define S_CRLF "\r\n" #define S_DLM ": " +#define S_SET_COOKIE "set-cookie" +#define S_F_SET_COOKIE S_SET_COOKIE S_DLM #define SLEN(s) (sizeof(s) - 1) @@ -49,12 +50,17 @@ __tfw_http_msg_set_str_data(TfwStr *str, void *data, struct sk_buff *skb) __tfw_http_msg_set_str_data(str, data, \ ss_skb_peek_tail(&hm->msg.skb_head)) +void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val); void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client); static inline void -tfw_http_msg_clnthdr_val(TfwStr *hdr, unsigned id, TfwStr *val) +tfw_http_msg_clnthdr_val(const TfwHttpReq *req, TfwStr *hdr, unsigned id, + TfwStr *val) { - __http_msg_hdr_val(hdr, id, val, true); + if (TFW_MSG_H2(req)) + __h2_msg_hdr_val(hdr, val); + else + __http_msg_hdr_val(hdr, id, val, true); } static inline void @@ -115,6 +121,7 @@ int tfw_http_msg_hdr_xfrm(TfwHttpMsg *hm, char *name, size_t n_len, int tfw_http_msg_del_str(TfwHttpMsg *hm, TfwStr *str); int tfw_http_msg_del_hbh_hdrs(TfwHttpMsg *hm); +int tfw_http_msg_del_eol(struct sk_buff *skb_head, TfwStr *hdr); int tfw_http_msg_to_chunked(TfwHttpMsg *hm); int tfw_http_msg_setup(TfwHttpMsg *hm, TfwMsgIter *it, size_t data_len, @@ -122,14 +129,27 @@ int tfw_http_msg_setup(TfwHttpMsg *hm, TfwMsgIter *it, size_t data_len, int tfw_http_msg_add_data(TfwMsgIter *it, TfwHttpMsg *hm, TfwStr *field, const TfwStr *data); void tfw_http_msg_hdr_open(TfwHttpMsg *hm, unsigned char *hdr_start); -int tfw_http_msg_hdr_close(TfwHttpMsg *hm, unsigned int id); +int tfw_http_msg_hdr_close(TfwHttpMsg *hm); int tfw_http_msg_grow_hdr_tbl(TfwHttpMsg *hm); void tfw_http_msg_free(TfwHttpMsg *m); -unsigned long tfw_http_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, - unsigned long *val_off, - unsigned long *val_len); -void tfw_http_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, - unsigned long val_off, unsigned long val_len, - char *out_buf); +int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, + const TfwStr *src); +int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name); +unsigned long tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, + unsigned long *val_off, + unsigned long *val_len); +void tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, + unsigned long val_off, unsigned long val_len, + char *out_buf); +int __hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr); +int tfw_h2_msg_hdr_sub(TfwHttpMsg *hm, char *name, size_t nlen, char *val, + size_t vlen, unsigned int hid, unsigned short idx); +int tfw_h2_msg_hdr_del(TfwHttpMsg *hm, char *name, size_t nlen, unsigned int hid); + +#define TFW_H2_MSG_HDR_SUB(hm, name, val, hid, idx) \ + tfw_h2_msg_hdr_sub(hm, name, sizeof(name) - 1, val, \ + sizeof(val) - 1, hid, idx) +#define TFW_H2_MSG_HDR_DEL(hm, name, hid) \ + tfw_h2_msg_hdr_del(hm, name, sizeof(name) - 1, hid) #endif /* __TFW_HTTP_MSG_H__ */ diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 74f7d0b124..cc0c07cbab 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -60,7 +60,7 @@ * must be raised. The behavior of macros with @_pos suffixes differ from * the ones specified above in the sense that they fixate the field chunk * with respect to an explicitly defined pointer (instead of only relative - * start of the data).n + * start of the data). */ #define __msg_field_open(field, pos) \ tfw_http_msg_set_str_data(msg, field, pos) @@ -86,6 +86,9 @@ do { \ #define __msg_hdr_chunk_fixup(data, len) \ tfw_http_msg_add_str_data(msg, &msg->stream->parser.hdr, data, len) +#define __msg_hdr_set_hpack_index(idx) \ + TFW_STR_INDEX_SET(&parser->hdr, idx); + /** * GCC 4.8 (CentOS 7) does a poor work on memory reusage of automatic local * variables in nested blocks, so we declare all required temporal variables @@ -222,7 +225,6 @@ do { \ * header values. That is done with separate, nested, or interior * FSMs, and so _I_ in the name means "interior" FSM. */ - #define __FSM_I_field_chunk_flags(field, flag) \ do { \ T_DBG3("parser: add chunk flags: %u\n", flag); \ @@ -317,15 +319,15 @@ __FSM_STATE(st) { \ /* Automaton transition from state @st to @st_next on character @ch. */ #define __FSM_TX(st, ch, st_next) \ - __FSM_TX_COND(st, c == (ch), st_next, NULL) + __FSM_TX_COND(st, c == (ch), st_next, &parser->hdr) #define __FSM_TX_f(st, ch, st_next, field) \ __FSM_TX_COND(st, c == (ch), st_next, field) #define __FSM_TX_nofixup(st, ch, st_next) \ __FSM_TX_COND_nofixup(st, c == (ch), st_next) /* Case-insensitive version of __FSM_TX(). */ -#define __FSM_TX_LC(st, ch, st_next) \ - __FSM_TX_COND(st, TFW_LC(c) == (ch), st_next) +#define __FSM_TX_LC(st, ch, st_next, field) \ + __FSM_TX_COND(st, TFW_LC(c) == (ch), st_next, field) #define __FSM_TX_LC_nofixup(st, ch, st_next) \ __FSM_TX_COND_nofixup(st, TFW_LC(c) == (ch), st_next) @@ -346,7 +348,19 @@ __FSM_STATE(st, cold) { \ __FSM_STATE(st, cold) { \ if (likely(c == ':')) { \ parser->_i_st = &&st_next; \ - __FSM_MOVE(RGen_OWS); \ + __FSM_MOVE(RGen_LWS); \ + } \ + /* It should be checked in st_fallback if `c` is allowed */ \ + __FSM_JMP(RGen_HdrOther); \ +} + +/* As above, but with HPACK static index setting. */ +#define __FSM_TX_AF_OWS_HP(st, st_next, hp_idx) \ +__FSM_STATE(st, cold) { \ + if (likely(c == ':')) { \ + parser->_i_st = &&st_next; \ + __msg_hdr_set_hpack_index(hp_idx); \ + __FSM_MOVE(RGen_LWS); \ } \ /* It should be checked in st_fallback if `c` is allowed */ \ __FSM_JMP(RGen_HdrOther); \ @@ -400,6 +414,14 @@ __FSM_STATE(st, cold) { \ !((*(unsigned long *)(p) | TFW_LC_LONG7) \ ^ TFW_CHAR8_INT(a, b, c, d, e, f, g, h)) +/* + * Matching 4 to 8 characters without conversion to lower case (applicable + * for HTTP/2 headers name comparison). + */ +#define C4_INT(p, a, b, c, d) \ + !(PI(p) ^ TFW_CHAR4_INT(a, b, c, d)) +#define C8_INT(p, a, b, c, d, e, f, g, h) \ + !(*(unsigned long *)(p) ^ TFW_CHAR8_INT(a, b, c, d, e, f, g, h)) #define IN_ALPHABET(c, a) (a[c >> 6] & (1UL << (c & 0x3f))) @@ -550,6 +572,24 @@ parse_int_hex(unsigned char *data, size_t len, unsigned long *acc, unsigned shor return CSTR_POSTPONE; } +/** + * Parse OWS, i.e. the space or horizontal tab characters which + * can exist before or after the header's value. + * @return number of parsed bytes or CSTR_POSTPONE if all @len bytes + * are parsed. + */ +static __always_inline int +parse_ows(unsigned char *__restrict data, size_t len) +{ + unsigned char *p; + + for (p = data; p - data < len; ++p) { + if (!IS_WS(*p)) + return p - data; + } + return CSTR_POSTPONE; +} + /** * Mark existing spec headers of http message @hm as hop-by-hop if they were * listed in Connection header or in @tfw_http_init_parser_* function. @@ -792,7 +832,7 @@ __FSM_STATE(RGen_EoL, hot) { \ if (c == '\n') { \ if (parser->hdr.data) { \ tfw_str_set_eolen(&parser->hdr, 1); \ - if (tfw_http_msg_hdr_close(msg, parser->_hdr_tag)) \ + if (tfw_http_msg_hdr_close(msg)) \ TFW_PARSER_BLOCK(RGen_EoL); \ } \ __FSM_MOVE_nofixup(RGen_Hdr); \ @@ -804,7 +844,7 @@ __FSM_STATE(RGen_CR, hot) { \ TFW_PARSER_BLOCK(RGen_CR); \ if (parser->hdr.data) { \ tfw_str_set_eolen(&parser->hdr, 2); \ - if (tfw_http_msg_hdr_close(msg, parser->_hdr_tag)) \ + if (tfw_http_msg_hdr_close(msg)) \ TFW_PARSER_BLOCK(RGen_CR); \ } \ /* Process next header if any. */ \ @@ -879,8 +919,6 @@ __FSM_STATE(st_curr) { \ if (id < TFW_HTTP_HDR_NONSINGULAR \ && unlikely(!TFW_STR_EMPTY(&(msg)->h_tbl->tbl[id]))) \ TFW_PARSER_BLOCK(st_curr); \ - /* Store header name and field in different chunks. */ \ - __msg_hdr_chunk_fixup(data, p - data); \ __fsm_n = func(hm, p, __fsm_sz); \ T_DBG3("parse special header " #func ": ret=%d data_len=%lu" \ " id=%d\n", __fsm_n, __fsm_sz, id); \ @@ -901,7 +939,9 @@ __FSM_STATE(st_curr) { \ mark_trailer_hdr(msg, &parser->hdr); \ parser->_i_st = &&RGen_EoL; \ parser->_hdr_tag = id; \ - __FSM_MOVE_n(RGen_OWS, __fsm_n); /* skip OWS */ \ + p += __fsm_n; \ + BUG_ON(unlikely(__data_off(p) >= len)); \ + __FSM_JMP(RGen_RWS); /* skip RWS */ \ } \ } @@ -914,12 +954,6 @@ __FSM_STATE(st_curr) { \ __fsm_sz = __data_remain(p); \ if (!parser->_i_st) \ TRY_STR_INIT(); \ - /* In 'func' the pointer at the beginning of this piece of the request - * is not available to us. If the request ends in 'func', we can not - * correctly create a new chunk, which includes part of the request - * before the header-value, and we lose this part. It should be forced - * to save it.*/ \ - __msg_hdr_chunk_fixup(data, p - data); \ __fsm_n = func(hm, p, __fsm_sz); \ T_DBG3("parse raw header " #func ": ret=%d data_len=%lu\n", \ __fsm_n, __fsm_sz); \ @@ -941,7 +975,9 @@ __FSM_STATE(st_curr) { \ mark_trailer_hdr(msg, &parser->hdr); \ parser->_i_st = &&RGen_EoL; \ parser->_hdr_tag = TFW_HTTP_HDR_RAW; \ - __FSM_MOVE_n(RGen_OWS, __fsm_n); /* skip OWS */ \ + p += __fsm_n; \ + BUG_ON(unlikely(__data_off(p) >= len)); \ + __FSM_JMP(RGen_RWS); /* skip RWS */ \ } \ } @@ -961,13 +997,11 @@ __FSM_STATE(RGen_HdrOtherN) { \ __FSM_MATCH_MOVE(token, RGen_HdrOtherN); \ if (likely(*(p + __fsm_sz) == ':')) { \ parser->_i_st = &&RGen_HdrOtherV; \ - __FSM_MOVE_n(RGen_OWS, __fsm_sz + 1); \ + __FSM_MOVE_n(RGen_LWS, __fsm_sz + 1); \ } \ TFW_PARSER_BLOCK(RGen_HdrOtherN); \ } \ __FSM_STATE(RGen_HdrOtherV) { \ - /* Store header name and value in different chunks. */ \ - __msg_hdr_chunk_fixup(data, p - data); \ /* \ * The header content is opaque for us, \ * so pass ctext and VCHAR. \ @@ -979,7 +1013,7 @@ __FSM_STATE(RGen_HdrOtherV) { \ __msg_hdr_chunk_fixup(p, __fsm_sz); \ mark_raw_hbh(msg, &parser->hdr); \ mark_trailer_hdr(msg, &parser->hdr); \ - __FSM_MOVE_n(RGen_EoL, __fsm_sz); \ + __FSM_MOVE_nofixup_n(RGen_EoL, __fsm_sz); \ } /* Process according RFC 7230 3.3.3 */ @@ -1156,14 +1190,36 @@ __FSM_STATE(RGen_BodyCR, __VA_ARGS__) { \ } /* - * Read OWS at arbitrary position and move to stashed state. - * This is bit complicated (however you can think about this as - * a plain pushdown automaton), but reduces FSM code size. + * Read OWS and move to stashed state. This is bit complicated (however + * you can think about this as a plain pushdown automaton), but reduces + * FSM code size. */ #define RGEN_OWS() \ -__FSM_STATE(RGen_OWS, hot) { \ +__FSM_STATE(RGen_LWS, hot) { \ + /* Store header name, LWS and value in different chunks. */ \ + __msg_hdr_chunk_fixup(data, p - data); \ + __fsm_sz = __data_remain(p); \ + __fsm_n = parse_ows(p, __fsm_sz); \ + T_DBG3("parse LWS: __fsm_n=%d, __fsm_sz=%lu, len=%lu," \ + " off=%lu\n", __fsm_n, __fsm_sz, len, __data_off(p)); \ + if (__fsm_n == CSTR_POSTPONE) { \ + p += __fsm_sz; \ + parser->state = &&RGen_LWS; \ + __FSM_EXIT(TFW_POSTPONE); \ + } \ + BUG_ON(__fsm_n < 0); \ + if (__fsm_n) \ + __msg_hdr_chunk_fixup(p, __fsm_n); \ + parser->state = parser->_i_st; \ + parser->_i_st = NULL; \ + p += __fsm_n; \ + BUG_ON(unlikely(__data_off(p) >= len)); \ + goto *parser->state; \ +} \ +__FSM_STATE(RGen_RWS, hot) { \ if (likely(IS_WS(c))) \ - __FSM_MOVE(RGen_OWS); \ + __FSM_MOVE_nofixup(RGen_RWS); \ + T_DBG3("parse RWS: len=%lu, off=%lu\n", len, __data_off(p)); \ parser->state = parser->_i_st; \ parser->_i_st = NULL; \ BUG_ON(unlikely(__data_off(p) >= len)); \ @@ -3316,7 +3372,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 13) == ':')) { parser->_i_st = &&Req_HdrAcceptV; - __FSM_MOVE_n(RGen_OWS, 7); + __FSM_MOVE_n(RGen_LWS, 7); } if (likely(__data_available(p, 14) && C8_INT_LCM(p + 1, 'u', 't', 'h', 'o', @@ -3325,7 +3381,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 13) == ':')) { parser->_i_st = &&Req_HdrAuthorizationV; - __FSM_MOVE_n(RGen_OWS, 14); + __FSM_MOVE_n(RGen_LWS, 14); } __FSM_MOVE(Req_HdrA); case 'c': @@ -3341,7 +3397,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, ':'))) { parser->_i_st = &&Req_HdrCache_ControlV; - __FSM_MOVE_n(RGen_OWS, 14); + __FSM_MOVE_n(RGen_LWS, 14); } __FSM_MOVE_n(RGen_HdrOther, 5); case TFW_CHAR4_INT('o', 'n', 'n', 'e'): @@ -3350,7 +3406,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 10) == ':')) { parser->_i_st = &&Req_HdrConnectionV; - __FSM_MOVE_n(RGen_OWS, 11); + __FSM_MOVE_n(RGen_LWS, 11); } __FSM_MOVE_n(RGen_HdrOther, 5); case TFW_CHAR4_INT('o', 'n', 't', 'e'): @@ -3366,7 +3422,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 6) == ':')) { parser->_i_st = &&Req_HdrCookieV; - __FSM_MOVE_n(RGen_OWS, 7); + __FSM_MOVE_n(RGen_LWS, 7); } __FSM_MOVE_n(RGen_HdrOther, 5); default: @@ -3378,7 +3434,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, { parser->_i_st = &&Req_HdrHostV; parser->_hdr_tag = TFW_HTTP_HDR_HOST; - __FSM_MOVE_n(RGen_OWS, 5); + __FSM_MOVE_n(RGen_LWS, 5); } __FSM_MOVE(Req_HdrH); case 'i': @@ -3393,7 +3449,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 17) == ':')) { parser->_i_st = &&Req_HdrIf_Modified_SinceV; - __FSM_MOVE_n(RGen_OWS, 18); + __FSM_MOVE_n(RGen_LWS, 18); } if (likely(__data_available(p, 14) && TFW_LC(*(p + 1)) == 'f' @@ -3405,7 +3461,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 13) == ':')) { parser->_i_st = &&Req_HdrIf_None_MatchV; - __FSM_MOVE_n(RGen_OWS, 14); + __FSM_MOVE_n(RGen_LWS, 14); } __FSM_MOVE(Req_HdrI); case 'k': @@ -3417,7 +3473,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 10) == ':')) { parser->_i_st = &&Req_HdrKeep_AliveV; - __FSM_MOVE_n(RGen_OWS, 11); + __FSM_MOVE_n(RGen_LWS, 11); } __FSM_MOVE(Req_HdrK); case 'p': @@ -3427,7 +3483,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 6) == ':')) { parser->_i_st = &&Req_HdrPragmaV; - __FSM_MOVE_n(RGen_OWS, 7); + __FSM_MOVE_n(RGen_LWS, 7); } __FSM_MOVE(Req_HdrP); case 'r': @@ -3438,7 +3494,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 7) == ':')) { parser->_i_st = &&Req_HdrRefererV; - __FSM_MOVE_n(RGen_OWS, 8); + __FSM_MOVE_n(RGen_LWS, 8); } __FSM_MOVE(Req_HdrR); case 't': @@ -3451,7 +3507,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 17) == ':')) { parser->_i_st = &&Req_HdrTransfer_EncodingV; - __FSM_MOVE_n(RGen_OWS, 18); + __FSM_MOVE_n(RGen_LWS, 18); } __FSM_MOVE(Req_HdrT); case 'x': @@ -3465,7 +3521,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, 'f', 'o', 'r', ':'))) { parser->_i_st = &&Req_HdrX_Forwarded_ForV; - __FSM_MOVE_n(RGen_OWS, 16); + __FSM_MOVE_n(RGen_LWS, 16); } __FSM_MOVE(Req_HdrX); case 'u': @@ -3477,7 +3533,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 10) == ':')) { parser->_i_st = &&Req_HdrUser_AgentV; - __FSM_MOVE_n(RGen_OWS, 11); + __FSM_MOVE_n(RGen_LWS, 11); } __FSM_MOVE(Req_HdrU); default: @@ -3495,7 +3551,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 6) == ':')) { parser->_i_st = &&Req_HdrContent_LengthV; - __FSM_MOVE_n(RGen_OWS, 7); + __FSM_MOVE_n(RGen_LWS, 7); } __FSM_MOVE(Req_HdrContent_L); case 't': @@ -3503,7 +3559,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C4_INT3_LCM(p + 1, 'y', 'p', 'e', ':'))) { parser->_i_st = &&Req_HdrContent_TypeV; - __FSM_MOVE_n(RGen_OWS, 5); + __FSM_MOVE_n(RGen_LWS, 5); } __FSM_MOVE(Req_HdrContent_T); default: @@ -3982,7 +4038,7 @@ Req_Method_1CharStep: __attribute__((cold)) __FSM_STATE(Req_HdrHost, cold) { if (likely(c == ':')) { parser->_i_st = &&Req_HdrHostV; - __FSM_MOVE(RGen_OWS); + __FSM_MOVE(RGen_LWS); } __FSM_JMP(RGen_HdrOther); } @@ -4118,201 +4174,3191 @@ Req_Method_1CharStep: __attribute__((cold)) } STACK_FRAME_NON_STANDARD(tfw_http_parse_req); -int -tfw_h2_parse_hdr(const char *data, unsigned long len, TfwHttpReq *req) -{ - /* - * TODO #309: implementation of separate parsing procedure for message - * headers in HTTP/2 context. - */ - return T_OK; -} +/* + * ------------------------------------------------------------------------ + * Special stuff for HTTP/2 parsing + * ------------------------------------------------------------------------ + */ +#define __FSM_H2_OK(st_next) \ +do { \ + T_DBG3("%s: parsed, st_next=" #st_next ", input=%#x('%.*s')," \ + " len=%lu, off=%lu\n", __func__, (char)c, \ + min(16U, (unsigned int)(data + len - p)), p, len, \ + p - data); \ + parser->state = &&st_next; \ + goto out; \ +} while (0) -static int -tfw_h2_parse_body(const char *data, unsigned long len, TfwHttpReq *req, - unsigned int *parsed) -{ - /* - * TODO #309: implementation of message body parsing (i.e. payload of - * HTTP/2 DATA frame). - */ - return T_OK; -} +#define __FSM_H2_POSTPONE(st_next) \ +do { \ + T_DBG3("%s: postponed, state=" #st_next ", input=%#x('%.*s')," \ + " len=%lu, off=%lu\n", __func__, (char)c, \ + min(16U, (unsigned int)(data + len - p)), p, len, \ + p - data); \ + parser->state = &&st_next; \ + ret = T_POSTPONE; \ + goto out; \ +} while (0) -static inline int -tfw_h2_parse_fin(TfwHttpReq *req) -{ - TfwStr *crlf = &req->crlf; - TfwMsgParseIter *it = &req->pit; +#define __FSM_H2_DROP(st) \ +do { \ + T_WARN("HTTP/2 request dropped: state=" #st " input=%#x('%.*s')," \ + " len=%lu, off=%lu\n", (char)c, \ + min(16U, (unsigned int)(data + len - p)), p, \ + len, p - data); \ + ret = T_DROP; \ + goto out; \ +} while (0) - WARN_ON_ONCE(!TFW_STR_EMPTY(crlf)); - BUFFER_GET(2, it); - if (!it->pos) - return -ENOMEM; +#define H2_MSG_VERIFY(hid) \ +({ \ + bool ret = true; \ + TfwStr *tbl = msg->h_tbl->tbl; \ + if (unlikely(hid < TFW_HTTP_HDR_NONSINGULAR \ + && !TFW_STR_EMPTY(&tbl[hid]))) \ + { \ + ret = false; \ + } \ + /* \ + * Pseudo-headers must appear in the header block before \ + * regular headers; also, exactly one instance of ':method', \ + * ':scheme' and ':path' pseudo-headers must be contained in \ + * the request (see RFC 7540 section 8.1.2.1 and section \ + * 8.1.2.3 for details). \ + */ \ + if (test_bit(TFW_HTTP_B_H2_HDRS_FULL, req->flags) \ + && hid == TFW_HTTP_HDR_H2_AUTHORITY) \ + { \ + ret = false; \ + } \ + if (!test_bit(TFW_HTTP_B_H2_HDRS_FULL, req->flags) \ + && hid >= TFW_HTTP_HDR_REGULAR) \ + { \ + if (TFW_STR_EMPTY(&tbl[TFW_HTTP_HDR_H2_METHOD]) \ + || TFW_STR_EMPTY(&tbl[TFW_HTTP_HDR_H2_SCHEME]) \ + || TFW_STR_EMPTY(&tbl[TFW_HTTP_HDR_H2_PATH])) \ + { \ + ret = false; \ + } \ + else \ + { \ + __set_bit(TFW_HTTP_B_H2_HDRS_FULL, req->flags); \ + } \ + } \ + ret; \ +}) - crlf->data = it->pos; - crlf->len = 2; - crlf->flags |= TFW_STR_COMPLETE; +#define __FSM_H2_FIN(to, n, h_tag) \ +do { \ + p += n; \ + T_DBG3("%s: name fin, hid=%d, to=" #to "len=%lu, off=%lu\n", \ + __func__, hid, len, __data_off(p)); \ + if (unlikely(__data_off(p) < len)) \ + goto RGen_HdrOtherN; \ + __msg_hdr_chunk_fixup(data, len); \ + if (unlikely(!fin)) \ + __FSM_H2_POSTPONE(RGen_HdrOtherN); \ + it->tag = h_tag; \ + __FSM_H2_OK(to); \ +} while (0) - *(short *)it->pos = *(short *)"\r\n"; +#define __FSM_H2_NEXT_n(to, n) \ +do { \ + p += n; \ + T_DBG3("%s: name next, to=" #to "len=%lu, off=%lu\n", __func__, \ + len, __data_off(p)); \ + if (likely(__data_off(p) < len)) \ + goto to; \ + __msg_hdr_chunk_fixup(data, len); \ + if (unlikely(!fin)) \ + __FSM_H2_POSTPONE(to); \ + it->tag = TFW_TAG_HDR_RAW; \ + __FSM_H2_OK(RGen_HdrOtherV); \ +} while (0) - return T_OK; -} +#define __FSM_H2_NEXT(to) \ + __FSM_H2_NEXT_n(to, 1) -int -tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, - unsigned int *parsed) -{ - int r; - TfwHttpReq *req = (TfwHttpReq *)req_data; - TfwH2Ctx *ctx = tfw_h2_context(req->conn); - unsigned char type = ctx->hdr.type; +#define __FSM_H2_OTHER_n(n) \ + __FSM_H2_NEXT_n(RGen_HdrOtherN, n) - WARN_ON_ONCE(!len); - BUG_ON(type != HTTP2_HEADERS - && type != HTTP2_CONTINUATION - && type != HTTP2_DATA); +#define __FSM_H2_OTHER() \ + __FSM_H2_OTHER_n(1) - *parsed = 0; +#define __FSM_H2_HDR_COMPLETE(st_curr) \ +do { \ + T_DBG("%s: complete header, state=" #st_curr ", _hdr_tag=%u," \ + " c=%#x, p='%.*s', len=%lu, off=%lu\n", \ + __func__, parser->_hdr_tag, (char)c, \ + min(16U, (unsigned int)(data + len - p)), \ + p, len, p - data); \ + parser->state = NULL; \ + goto out; \ +} while (0) - if (type != HTTP2_DATA) - r = tfw_hpack_decode(&ctx->hpack, data, len, req, parsed); - else - r = tfw_h2_parse_body(data, len, req, parsed); +#define __FSM_H2_PSHDR_CHECK_lambda(pos, lambda) \ +do { \ + if (__data_off(pos) >= len) { \ + lambda; \ + } \ +} while (0) - if (r != T_OK && r != T_POSTPONE) - return r; +#define __FSM_H2_PSHDR_MOVE_lambda(n, lambda, st_next) \ +do { \ + p += n; \ + __FSM_H2_PSHDR_CHECK_lambda(p, lambda); \ + goto st_next; \ +} while (0) - /* - * Parsing of HTTP/2 frames' payload never gives T_OK result since - * request can be assembled from different number of frames; only - * stream's state can indicate the moment when request is completed. - * Note, due to persistent linkage Stream<->Request in HTTP/2 mode - * special flag is necessary for tracking the request completeness; if - * the request is not fully assembled and is not passed to forwarding - * procedures yet, it must be deleted during corresponding stream - * removal. - */ - if (tfw_h2_stream_req_complete(req->stream)) { - __set_bit(TFW_HTTP_B_FULLY_PARSED, req->flags); - return tfw_h2_parse_fin(req); - } +#define __FSM_H2_PSHDR_MOVE_FIN(st_curr, n, st_next) \ + __FSM_H2_PSHDR_MOVE_lambda(n, { \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + if (unlikely(!fin)) \ + __FSM_H2_POSTPONE(st_next); \ + __FSM_H2_HDR_COMPLETE(st_curr); \ + }, st_next) + +#define __FSM_H2_PSHDR_MOVE_FIN_fixup(st_curr, n, st_next) \ + __msg_hdr_chunk_fixup(p, n); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + __FSM_H2_PSHDR_MOVE_lambda(n, { \ + if (unlikely(!fin)) \ + __FSM_H2_POSTPONE(st_next); \ + __FSM_H2_HDR_COMPLETE(st_curr); \ + }, st_next) + +#define __FSM_H2_PSHDR_MOVE_DROP(st_curr, n, st_next) \ + __FSM_H2_PSHDR_MOVE_lambda(n, { \ + if (unlikely(!fin)) { \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + __FSM_H2_POSTPONE(st_next); \ + } \ + __FSM_H2_DROP(st_curr); \ + }, st_next) - return T_POSTPONE; -} +#define __FSM_H2_PSHDR_MOVE_DROP_nofixup(st_curr, n, st_next) \ + __FSM_H2_PSHDR_MOVE_lambda(n, { \ + if (unlikely(!fin)) \ + __FSM_H2_POSTPONE(st_next); \ + __FSM_H2_DROP(st_curr); \ + }, st_next) + +#define __FSM_H2_PSHDR_COMPLETE(st, n) \ +do { \ + p += n; \ + __FSM_H2_PSHDR_CHECK_lambda(p, { \ + if (unlikely(!fin)) \ + __FSM_H2_DROP(st); \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + __FSM_H2_HDR_COMPLETE(st); \ + }); \ + __FSM_H2_DROP(st); \ +} while (0) + +#define __FSM_H2_METHOD_MOVE(st_curr, n, st_next) \ + __FSM_H2_PSHDR_MOVE_lambda(n, { \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + if (unlikely(!fin)) \ + __FSM_H2_POSTPONE(st_next); \ + req->method = _TFW_HTTP_METH_UNKNOWN; \ + __FSM_H2_HDR_COMPLETE(st_curr); \ + }, st_next) + +#define __FSM_H2_METHOD_COMPLETE(st_curr, n, mid) \ + __FSM_H2_PSHDR_MOVE_lambda(n, { \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + if (unlikely(!fin)) \ + __FSM_H2_POSTPONE(Req_MethodUnknown); \ + req->method = mid; \ + __FSM_H2_HDR_COMPLETE(st_curr); \ + }, Req_MethodUnknown) /* - * ------------------------------------------------------------------------ - * HTTP response parsing - * ------------------------------------------------------------------------ + * Special set of macros for slow-path parsing of pseudo-headers value + * (char-by-char). */ -static int -__resp_parse_age(TfwHttpResp *resp, unsigned char *data, size_t len) -{ - int r = CSTR_NEQ; - __FSM_DECLARE_VARS(resp); +#define __FSM_H2_SCHEME_STATE_MOVE(st, ch, st_next) \ + __FSM_STATE(st, cold) { \ + if (likely(TFW_LC(c) == (ch))) \ + __FSM_H2_PSHDR_MOVE_DROP(st, 1, st_next); \ + __FSM_H2_DROP(st); \ + } - __FSM_START(parser->_i_st); +#define __FSM_H2_SCHEME_STATE_COMPLETE(st, ch) \ + __FSM_STATE(st, cold) { \ + if (likely(TFW_LC(c) == (ch))) \ + __FSM_H2_PSHDR_COMPLETE(st, 1); \ + __FSM_H2_DROP(st); \ + } - __FSM_STATE(Resp_I_Age) { - __fsm_sz = __data_remain(p); - __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); - if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); - if (__fsm_n < 0) - return __fsm_n; - resp->cache_ctl.age = parser->_acc; - parser->_acc = 0; - __FSM_I_MOVE_n(Resp_I_EoL, __fsm_n); +#define __FSM_H2_METH_STATE_MOVE(st, ch, st_next) \ + __FSM_STATE(st, cold) { \ + if (likely(c == (ch))) \ + __FSM_H2_METHOD_MOVE(st, 1, st_next); \ + __FSM_JMP(Req_MethodUnknown); \ } - __FSM_STATE(Resp_I_EoL) { - if (IS_WS(c)) - __FSM_I_MOVE(Resp_I_EoL); - if (IS_CRLF(c)) { - resp->cache_ctl.flags |= TFW_HTTP_CC_HDR_AGE; - return __data_off(p); - } - return CSTR_NEQ; +#define __FSM_H2_METH_STATE_COMPLETE(st, ch, mid) \ + __FSM_STATE(st, cold) { \ + if (likely(c == (ch))) \ + __FSM_H2_METHOD_COMPLETE(st, 1, mid); \ + __FSM_JMP(Req_MethodUnknown); \ } -done: - return r; +#define TFW_H2_PARSE_HDR_VAL(st_curr, hm, func, hid, saveval) \ +__FSM_STATE(st_curr) { \ + BUG_ON(p != data); \ + if (!H2_MSG_VERIFY(hid)) \ + __FSM_H2_DROP(st_curr); \ + if (!parser->_i_st) \ + TRY_STR_INIT(); \ + __fsm_n = func(hm, p, len, fin); \ + T_DBG3("%s: parse header value, " #func ": ret=%d data_len=%lu" \ + " hid=%u\n", __fsm_n, len, parser->_hdr_tag); \ + switch (__fsm_n) { \ + case CSTR_EQ: \ + if (saveval) { \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + } \ + parser->_i_st = NULL; \ + parser->_hdr_tag = hid; \ + __FSM_H2_HDR_COMPLETE(st_curr); \ + case CSTR_POSTPONE: \ + __FSM_H2_POSTPONE(st_curr); \ + case CSTR_BADLEN: \ + case CSTR_NEQ: \ + __FSM_H2_DROP(st_curr); \ + default: \ + /* Unexpected values. */ \ + WARN_ON_ONCE(1); \ + __FSM_H2_DROP(st_curr); \ + } \ } -STACK_FRAME_NON_STANDARD(__resp_parse_age); -/** - * Parse response Cache-Control, RFC 2616 14.9 +/* + * HTTP/2 automaton transition with alphabet checking for headers' name. + * Improbable states only, so cold label. */ -static int -__resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) -{ - int r = CSTR_NEQ; - __FSM_DECLARE_VARS(resp); +#define __FSM_H2_TX_AF(st, ch, st_next) \ +__FSM_STATE(st, cold) { \ + if (likely(c == ch)) \ + __FSM_H2_NEXT(st_next); \ + __FSM_JMP(RGen_HdrOtherN); \ +} - __FSM_START(parser->_i_st); +#define __FSM_H2_TX_AF_FIN(st, ch, st_next, tag) \ +__FSM_STATE(st, cold) { \ + if (likely(c == ch)) \ + __FSM_H2_FIN(st_next, 1, tag); \ + __FSM_JMP(RGen_HdrOtherN); \ +} - __FSM_STATE(Resp_I_CC) { - WARN_ON_ONCE(parser->_acc); - switch (TFW_LC(c)) { - case 'm': - __FSM_I_JMP(Resp_I_CC_m); - case 'n': - __FSM_I_JMP(Resp_I_CC_n); - case 'p': - __FSM_I_JMP(Resp_I_CC_p); - case 's': - __FSM_I_JMP(Resp_I_CC_s); - } - __FSM_I_JMP(Resp_I_Ext); - } +#define __FSM_H2_TX_AF_DROP(st, ch) \ +__FSM_STATE(st, cold) { \ + if (likely(c == ch)) \ + __FSM_H2_DROP(st); \ + __FSM_JMP(RGen_HdrOtherN); \ +} - __FSM_STATE(Resp_I_CC_m) { - TRY_STR("max-age=", Resp_I_CC_m, Resp_I_CC_MaxAgeV); - TRY_STR_LAMBDA("must-revalidate", { - parser->_acc = TFW_HTTP_CC_MUST_REVAL; - }, Resp_I_CC_m, Resp_I_Flag); - TRY_STR_INIT(); - __FSM_I_JMP(Resp_I_Ext); - } +/* + * As above, but drops message if expected character is not matched; + * applicable for HTTP/2 pseudo-header names, since only a limited + * number of strictly defined pseudo-headers are allowed (see RFC 7540 + * section 8.1.2.3 and section 8.1.2.4 for details). + */ +#define __FSM_H2_TXD_AF(st, ch, st_next) \ +__FSM_STATE(st, cold) { \ + if (likely(c == ch)) \ + __FSM_H2_NEXT(st_next); \ + __FSM_H2_DROP(st); \ +} - __FSM_STATE(Resp_I_CC_n) { - TRY_STR_LAMBDA("no-cache", { - parser->_acc = TFW_HTTP_CC_NO_CACHE; - }, Resp_I_CC_n, Resp_I_Flag); - TRY_STR_LAMBDA("no-store", { - parser->_acc = TFW_HTTP_CC_NO_STORE; - }, Resp_I_CC_n, Resp_I_Flag); - TRY_STR_LAMBDA("no-transform", { - parser->_acc = TFW_HTTP_CC_NO_TRANSFORM; - }, Resp_I_CC_n, Resp_I_Flag); - TRY_STR_INIT(); - __FSM_I_JMP(Resp_I_Ext); - } +#define __FSM_H2_TXD_AF_FIN(st, ch, st_next, tag) \ +__FSM_STATE(st, cold) { \ + if (likely(c == ch)) \ + __FSM_H2_FIN(st_next, 1, tag); \ + __FSM_H2_DROP(st); \ +} - __FSM_STATE(Resp_I_CC_p) { - TRY_STR_LAMBDA("public", { - parser->_acc = TFW_HTTP_CC_PUBLIC; - }, Resp_I_CC_p, Resp_I_Flag); - TRY_STR_LAMBDA("private", { - parser->_acc = TFW_HTTP_CC_PRIVATE; - }, Resp_I_CC_p, Resp_I_Flag); - TRY_STR_LAMBDA("proxy-revalidate", { - parser->_acc = TFW_HTTP_CC_PROXY_REVAL; - }, Resp_I_CC_p, Resp_I_Flag); - TRY_STR_INIT(); - __FSM_I_JMP(Resp_I_Ext); +#define __FSM_H2_REQ_NEXT_STATE(v_stage) \ + if (v_stage) \ + { \ + switch (it->tag) { \ + case TFW_TAG_HDR_H2_METHOD: \ + goto Req_HdrPsMethodV; \ + case TFW_TAG_HDR_H2_SCHEME: \ + goto Req_HdrPsSchemeV; \ + case TFW_TAG_HDR_H2_AUTHORITY: \ + goto Req_HdrPsAuthorityV; \ + case TFW_TAG_HDR_H2_PATH: \ + goto Req_HdrPsPathV; \ + case TFW_TAG_HDR_ACCEPT: \ + goto Req_HdrAcceptV; \ + case TFW_TAG_HDR_AUTHORIZATION: \ + goto Req_HdrAuthorizationV; \ + case TFW_TAG_HDR_CACHE_CONTROL: \ + goto Req_HdrCache_ControlV; \ + case TFW_TAG_HDR_CONTENT_LENGTH: \ + goto Req_HdrContent_LengthV; \ + case TFW_TAG_HDR_CONTENT_TYPE: \ + goto Req_HdrContent_TypeV; \ + case TFW_TAG_HDR_COOKIE: \ + goto Req_HdrCookieV; \ + case TFW_TAG_HDR_HOST: \ + goto Req_HdrHostV; \ + case TFW_TAG_HDR_IF_MODIFIED_SINCE: \ + goto Req_HdrIf_Modified_SinceV; \ + case TFW_TAG_HDR_IF_NONE_MATCH: \ + goto Req_HdrIf_None_MatchV; \ + case TFW_TAG_HDR_PRAGMA: \ + goto Req_HdrPragmaV; \ + case TFW_TAG_HDR_REFERER: \ + goto Req_HdrRefererV; \ + case TFW_TAG_HDR_X_FORWARDED_FOR: \ + goto Req_HdrX_Forwarded_ForV; \ + case TFW_TAG_HDR_USER_AGENT: \ + goto Req_HdrUser_AgentV; \ + case TFW_TAG_HDR_RAW: \ + goto RGen_HdrOtherV; \ + default: \ + __FSM_H2_DROP(Req_HdrForbidden); \ + } \ } - __FSM_STATE(Resp_I_Flag) { - WARN_ON_ONCE(!parser->_acc); - if (IS_WS(c) || c == ',' || IS_CRLF(c)) { - resp->cache_ctl.flags |= parser->_acc; - parser->_acc = 0; - __FSM_I_JMP(Resp_I_EoT); - } - __FSM_I_JMP(Resp_I_Ext); - } +/* + * Auxiliary macros for parsing message header values (as @__FSM_I_* + * macros, but intended for HTTP/2 messages parsing). + */ +#define __FSM_H2_I_MOVE_LAMBDA_n(to, n, lambda) \ +do { \ + p += n; \ + if (__data_off(p) < len) \ + goto to; \ + if (likely(fin)) { \ + lambda; \ + __FSM_EXIT(CSTR_EQ); \ + } \ + parser->_i_st = &&to; \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + __FSM_EXIT(CSTR_POSTPONE); \ +} while (0) + +#define __FSM_H2_I_MOVE_NEQ_LAMBDA_n(to, n, lambda) \ +do { \ + p += n; \ + if (likely(__data_off(p) < len)) \ + goto to; \ + if (likely(fin)) { \ + lambda; \ + __FSM_EXIT(CSTR_NEQ); \ + } \ + parser->_i_st = &&to; \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + __FSM_EXIT(CSTR_POSTPONE); \ +} while (0) + +#define __FSM_H2_I_MOVE_n(to, n) \ + __FSM_H2_I_MOVE_LAMBDA_n(to, n, {}) + +#define __FSM_H2_I_MOVE(to) __FSM_H2_I_MOVE_n(to, 1) + +#define __FSM_H2_I_MOVE_NEQ(to, n) \ + __FSM_H2_I_MOVE_NEQ_LAMBDA_n(to, n, {}) + +#define __FSM_H2_I_MATCH(alphabet) \ +do { \ + __fsm_n = __data_remain(p); \ + __fsm_sz = tfw_match_##alphabet(p, __fsm_n); \ +} while (0) + +#define __FSM_H2_I_MATCH_MOVE_LAMBDA(alphabet, to, lambda) \ +do { \ + __FSM_H2_I_MATCH(alphabet); \ + if (__fsm_sz == __fsm_n) { \ + if (likely(fin)) { \ + lambda; \ + __FSM_EXIT(CSTR_EQ); \ + } \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + parser->_i_st = &&to; \ + __FSM_EXIT(CSTR_POSTPONE); \ + } \ +} while (0) + +#define __FSM_H2_I_MATCH_MOVE(alphabet, to) \ + __FSM_H2_I_MATCH_MOVE_LAMBDA(alphabet, to, {}) + +/* + * The macros below control chunks within a string for HTTP/2 parsing (see + * description for @__FSM_I_MOVE_LAMBDA_fixup_f() and others for details). + */ +#define __FSM_H2_I_MOVE_LAMBDA_fixup_f(to, n, field, lambda, flag) \ +do { \ + BUG_ON(!(field)->data); \ + __msg_field_fixup_pos(field, p, n); \ + __FSM_I_field_chunk_flags(field, TFW_STR_HDR_VALUE | flag); \ + if (__data_off(p + n) < len) { \ + p += n; \ + goto to; \ + } \ + if (likely(fin)) \ + lambda; \ + parser->_i_st = &&to; \ + __FSM_EXIT(CSTR_POSTPONE); \ +} while (0) + +#define __FSM_H2_I_MOVE_LAMBDA_fixup(to, n, lambda, flag) \ + __FSM_H2_I_MOVE_LAMBDA_fixup_f(to, n, &parser->hdr, lambda, flag) + +#define __FSM_H2_I_MOVE_fixup(to, n, flag) \ + __FSM_H2_I_MOVE_LAMBDA_fixup(to, n, { \ + __FSM_EXIT(CSTR_EQ); \ + }, flag) + +#define __FSM_H2_I_MOVE_NEQ_fixup_f(to, n, field, flag) \ +do { \ + BUG_ON(!(field)->data); \ + if (likely(__data_off(p + n) < len)) { \ + __msg_field_fixup_pos(field, p, n); \ + __FSM_I_field_chunk_flags(field, TFW_STR_HDR_VALUE | flag); \ + p += n; \ + goto to; \ + } \ + if (likely(fin)) \ + __FSM_EXIT(CSTR_NEQ); \ + __msg_field_fixup_pos(field, p, n); \ + __FSM_I_field_chunk_flags(field, TFW_STR_HDR_VALUE | flag); \ + parser->_i_st = &&to; \ + __FSM_EXIT(CSTR_POSTPONE); \ +} while (0) + +#define __FSM_H2_I_MOVE_NEQ_fixup(to, n, flag) \ + __FSM_H2_I_MOVE_NEQ_fixup_f(to, n, &parser->hdr, flag) + +#define __FSM_H2_I_MATCH_MOVE_LAMBDA_fixup(alphabet, to, lambda, flag) \ +do { \ + __FSM_H2_I_MATCH(alphabet); \ + if (likely(__fsm_sz == __fsm_n)) { \ + __msg_hdr_chunk_fixup(p, __fsm_sz); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE | flag); \ + if (likely(fin)) \ + lambda; \ + parser->_i_st = &&to; \ + __FSM_EXIT(CSTR_POSTPONE); \ + } \ +} while (0) + +#define __FSM_H2_I_MATCH_MOVE_fixup(alphabet, to, flag) \ + __FSM_H2_I_MATCH_MOVE_LAMBDA_fixup(alphabet, to, { \ + __FSM_EXIT(CSTR_EQ); \ + }, flag) + +#define __FSM_H2_I_MATCH_MOVE_NEQ_fixup(alphabet, to, flag) \ +do { \ + __FSM_H2_I_MATCH(alphabet); \ + if (unlikely(__fsm_sz == __fsm_n)) { \ + if (likely(fin)) \ + __FSM_EXIT(CSTR_NEQ); \ + __msg_hdr_chunk_fixup(p, __fsm_sz); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE | flag); \ + parser->_i_st = &&to; \ + __FSM_EXIT(CSTR_POSTPONE); \ + } \ +} while (0) + +/** + * Parsing helpers for HTTP/2 messages (same as @TRY_STR_* macros but + * optimized for HTTP/2 parsing). + */ +#define H2_TRY_STR_2LAMBDA(str, lambda1, lambda2, curr_st, next_st) \ + if (!chunk->data) \ + chunk->data = p; \ + __fsm_n = __try_str(&parser->hdr, chunk, p, __data_remain(p), \ + str, sizeof(str) - 1); \ + if (__fsm_n > 0) { \ + if (likely(chunk->len == sizeof(str) - 1)) { \ + lambda1; \ + TRY_STR_INIT(); \ + p += __fsm_n; \ + if (__data_off(p) < len) \ + goto next_st; \ + if (likely(fin)) \ + lambda2; \ + parser->_i_st = &&next_st; \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + __FSM_EXIT(CSTR_POSTPONE); \ + } \ + if (likely(fin)) \ + return CSTR_NEQ; \ + parser->_i_st = &&curr_st; \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + __FSM_EXIT(CSTR_POSTPONE); \ + } + +#define H2_TRY_STR_LAMBDA(str, lambda, curr_st, next_st) \ + H2_TRY_STR_2LAMBDA(str, {}, lambda, curr_st, next_st) + +#define H2_TRY_STR(str, curr_st, next_st) \ + H2_TRY_STR_LAMBDA(str, { \ + __FSM_EXIT(CSTR_EQ); \ + }, curr_st, next_st) + +/** + * The same as @H2_TRY_STR_2LAMBDA(), but with explicit chunks control; + * besides, @str must be of plain @TfwStr{} type and variable @fld is + * used (instead of hard coded header field). + */ +#define H2_TRY_STR_2LAMBDA_fixup(str, fld, lambda1, lambda2, curr_st, next_st) \ + BUG_ON(!TFW_STR_PLAIN(str)); \ + if (!chunk->data) \ + chunk->data = p; \ + __fsm_n = __try_str(fld, chunk, p, __data_remain(p), \ + (str)->data, (str)->len); \ + if (__fsm_n > 0) { \ + if (likely(chunk->len == (str)->len)) { \ + lambda1; \ + TRY_STR_INIT(); \ + __msg_field_fixup_pos(fld, p, __fsm_n); \ + __FSM_I_field_chunk_flags(fld, TFW_STR_HDR_VALUE); \ + if (__data_off(p + __fsm_n) < len) { \ + p += __fsm_n; \ + goto next_st; \ + } \ + if (likely(fin)) \ + lambda2; \ + parser->_i_st = &&next_st; \ + __FSM_EXIT(CSTR_POSTPONE); \ + } \ + if (likely(fin)) \ + __FSM_EXIT(CSTR_NEQ); \ + parser->_i_st = &&curr_st; \ + __msg_field_fixup_pos(fld, p, __fsm_n); \ + __FSM_I_field_chunk_flags(fld, TFW_STR_HDR_VALUE); \ + __FSM_EXIT(CSTR_POSTPONE); \ + } + +#define H2_TRY_STR_LAMBDA_fixup(str, fld, lambda, curr_st, next_st) \ + H2_TRY_STR_2LAMBDA_fixup(str, fld, {}, lambda, curr_st, next_st) + +/* + * ------------------------------------------------------------------------ + * HTTP/2 request parsing + * ------------------------------------------------------------------------ + */ +static int +__h2_req_parse_accept(TfwHttpReq *req, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(req); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_Accept) { + H2_TRY_STR_LAMBDA("text/html", { + __set_bit(TFW_HTTP_B_ACCEPT_HTML, req->flags); + __FSM_EXIT(CSTR_EQ); + }, Req_I_Accept, Req_I_AcceptHtml); + H2_TRY_STR_LAMBDA("*/*", { + __set_bit(TFW_HTTP_B_ACCEPT_HTML, req->flags); + __FSM_EXIT(CSTR_EQ); + }, Req_I_Accept, Req_I_AcceptHtml); + TRY_STR_INIT(); + __FSM_I_JMP(Req_I_AcceptOther); + } + + __FSM_STATE(Req_I_AcceptHtml) { + if (IS_WS(c) || c == ',' || c == ';') { + __set_bit(TFW_HTTP_B_ACCEPT_HTML, req->flags); + __FSM_I_JMP(I_EoT); + } + /* Fall through. */ + } + + __FSM_STATE(Req_I_AcceptOther) { + __FSM_H2_I_MATCH_MOVE(uri, Req_I_AcceptOther); + c = *(p + __fsm_sz); + if (IS_WS(c) || c == ',') + __FSM_H2_I_MOVE_n(I_EoT, __fsm_sz + 1); + return CSTR_NEQ; + } + + /* End of term. */ + __FSM_STATE(I_EoT) { + if (IS_WS(c) || c == ',') + __FSM_H2_I_MOVE(I_EoT); + if (c == ';') + /* Skip weight parameter. */ + __FSM_H2_I_MOVE(Req_I_AcceptOther); + if (IS_TOKEN(c)) + __FSM_I_JMP(Req_I_Accept); + return CSTR_NEQ; + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_accept); + +static int +__h2_req_parse_authorization(TfwHttpReq *req, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(req); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_Auth) { + /* + * RFC 7235 requires handling quoted-string in auth-param, + * so almost any character can appear in the field. + */ + __FSM_H2_I_MATCH_MOVE_LAMBDA(ctext_vchar, Req_I_Auth, { + req->cache_ctl.flags |= TFW_HTTP_CC_HDR_AUTHORIZATION; + }); + return CSTR_NEQ; + } + +done: + return r; +} + +static int +__h2_req_parse_cache_control(TfwHttpReq *req, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(req); + +#define __FSM_H2_I_MOVE_RESET_ACC(to, n) \ + __FSM_H2_I_MOVE_LAMBDA_n(to, n, { \ + parser->_acc = 0; \ + }) + + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_CC) { + WARN_ON_ONCE(parser->_acc); + switch (TFW_LC(c)) { + case 'm': + __FSM_I_JMP(Req_I_CC_m); + case 'n': + __FSM_I_JMP(Req_I_CC_n); + case 'o': + __FSM_I_JMP(Req_I_CC_o); + } + __FSM_I_JMP(Req_I_CC_Ext); + } + + __FSM_STATE(Req_I_CC_m) { + H2_TRY_STR_LAMBDA("max-age=", { + req->cache_ctl.max_age = parser->_acc; + req->cache_ctl.flags |= TFW_HTTP_CC_MAX_AGE; + __FSM_EXIT(CSTR_EQ); + }, Req_I_CC_m, Req_I_CC_MaxAgeV); + H2_TRY_STR_LAMBDA("min-fresh=", { + req->cache_ctl.min_fresh = parser->_acc; + req->cache_ctl.flags |= TFW_HTTP_CC_MIN_FRESH; + __FSM_EXIT(CSTR_EQ); + }, Req_I_CC_m, Req_I_CC_MinFreshV); + H2_TRY_STR_LAMBDA("max-stale", { + req->cache_ctl.max_stale = UINT_MAX; + req->cache_ctl.flags |= TFW_HTTP_CC_MAX_STALE; + __FSM_EXIT(CSTR_EQ); + }, Req_I_CC_m, Req_I_CC_MaxStale); + TRY_STR_INIT(); + __FSM_I_JMP(Req_I_CC_Ext); + } + + __FSM_STATE(Req_I_CC_n) { + H2_TRY_STR_2LAMBDA("no-cache", { + parser->_acc = TFW_HTTP_CC_NO_CACHE; + }, { + req->cache_ctl.flags |= TFW_HTTP_CC_NO_CACHE; + __FSM_EXIT(CSTR_EQ); + }, Req_I_CC_n, Req_I_CC_Flag); + H2_TRY_STR_2LAMBDA("no-store", { + parser->_acc = TFW_HTTP_CC_NO_STORE; + }, { + req->cache_ctl.flags |= TFW_HTTP_CC_NO_STORE; + __FSM_EXIT(CSTR_EQ); + }, Req_I_CC_n, Req_I_CC_Flag); + H2_TRY_STR_2LAMBDA("no-transform", { + parser->_acc = TFW_HTTP_CC_NO_TRANSFORM; + }, { + req->cache_ctl.flags |= TFW_HTTP_CC_NO_TRANSFORM; + __FSM_EXIT(CSTR_EQ); + }, Req_I_CC_n, Req_I_CC_Flag); + TRY_STR_INIT(); + __FSM_I_JMP(Req_I_CC_Ext); + } + + __FSM_STATE(Req_I_CC_o) { + H2_TRY_STR_2LAMBDA("only-if-cached", { + parser->_acc = TFW_HTTP_CC_OIFCACHED; + }, { + req->cache_ctl.flags |= TFW_HTTP_CC_OIFCACHED; + __FSM_EXIT(CSTR_EQ); + }, Req_I_CC_o, Req_I_CC_Flag); + TRY_STR_INIT(); + __FSM_I_JMP(Req_I_CC_Ext); + } + + __FSM_STATE(Req_I_CC_Flag) { + WARN_ON_ONCE(!parser->_acc); + if (IS_WS(c) || c == ',') { + req->cache_ctl.flags |= parser->_acc; + __FSM_I_JMP(Req_I_EoT); + } + __FSM_I_JMP(Req_I_CC_Ext); + } + + __FSM_STATE(Req_I_CC_MaxAgeV) { + __fsm_sz = __data_remain(p); + __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); + if (__fsm_n == CSTR_POSTPONE) { + if (likely(fin)) { + req->cache_ctl.max_age = parser->_acc; + req->cache_ctl.flags |= TFW_HTTP_CC_MAX_AGE; + parser->_acc = 0; + return CSTR_EQ; + } + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + } + if (__fsm_n < 0) + return __fsm_n; + req->cache_ctl.max_age = parser->_acc; + req->cache_ctl.flags |= TFW_HTTP_CC_MAX_AGE; + __FSM_H2_I_MOVE_RESET_ACC(Req_I_EoT, __fsm_n); + } + + __FSM_STATE(Req_I_CC_MinFreshV) { + __fsm_sz = __data_remain(p); + __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); + if (__fsm_n == CSTR_POSTPONE) { + if (likely(fin)) { + req->cache_ctl.min_fresh = parser->_acc; + req->cache_ctl.flags |= TFW_HTTP_CC_MIN_FRESH; + parser->_acc = 0; + return CSTR_EQ; + } + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + } + if (__fsm_n < 0) + return __fsm_n; + req->cache_ctl.min_fresh = parser->_acc; + req->cache_ctl.flags |= TFW_HTTP_CC_MIN_FRESH; + __FSM_H2_I_MOVE_RESET_ACC(Req_I_EoT, __fsm_n); + } + + __FSM_STATE(Req_I_CC_MaxStale) { + if (c == '=') + __FSM_H2_I_MOVE_RESET_ACC(Req_I_CC_MaxStaleV, 1); + if (IS_WS(c) || c == ',') { + req->cache_ctl.max_stale = UINT_MAX; + req->cache_ctl.flags |= TFW_HTTP_CC_MAX_STALE; + __FSM_I_JMP(Req_I_EoT); + } + __FSM_I_JMP(Req_I_CC_Ext); + } + + __FSM_STATE(Req_I_CC_MaxStaleV) { + __fsm_sz = __data_remain(p); + __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); + if (__fsm_n == CSTR_POSTPONE) { + if (likely(fin)) { + req->cache_ctl.max_stale = parser->_acc; + req->cache_ctl.flags |= TFW_HTTP_CC_MAX_STALE; + parser->_acc = 0; + return CSTR_EQ; + } + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + } + if (__fsm_n < 0) + return __fsm_n; + req->cache_ctl.max_stale = parser->_acc; + req->cache_ctl.flags |= TFW_HTTP_CC_MAX_STALE; + __FSM_H2_I_MOVE_RESET_ACC(Req_I_EoT, __fsm_n); + } + + __FSM_STATE(Req_I_CC_Ext) { + /* TODO: process cache extensions. */ + __FSM_H2_I_MATCH_MOVE_LAMBDA(qetoken, Req_I_CC_Ext, { + parser->_acc = 0; + }); + c = *(p + __fsm_sz); + if (IS_WS(c) || c == ',') + __FSM_H2_I_MOVE_RESET_ACC(Req_I_EoT, __fsm_sz + 1); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_EoT) { + if (IS_WS(c) || c == ',') + __FSM_H2_I_MOVE_RESET_ACC(Req_I_EoT, 1); + parser->_acc = 0; + if (IS_TOKEN(c)) + __FSM_I_JMP(Req_I_CC); + return CSTR_NEQ; + } + +done: + return r; + +#undef __FSM_H2_I_MOVE_RESET_ACC +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_cache_control); + +static int +__h2_req_parse_content_length(TfwHttpMsg *msg, unsigned char *data, size_t len, + bool fin) +{ + int ret; + + ret = parse_ulong_ws(data, len, &msg->content_length); + + T_DBG3("%s: content_length=%lu, ret=%d\n", __func__, + msg->content_length, ret); + + if (ret == CSTR_POSTPONE) { + if (fin) + return CSTR_EQ; + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + return CSTR_POSTPONE; + } + + return ret >= 0 ? CSTR_NEQ : ret; +} + +static int +__h2_req_parse_content_type(TfwHttpMsg *hm, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + TfwHttpReq *req = (TfwHttpReq *)hm; + __FSM_DECLARE_VARS(hm); + +#define __FSM_H2_I_MOVE_FIN_fixup(to, n, flag) \ + __FSM_H2_I_MOVE_LAMBDA_fixup(to, n, { \ + goto finalize; \ + }, flag) + +#define __FSM_H2_I_MATCH_MOVE_FIN_fixup(alphabet, to, flag) \ + __FSM_H2_I_MATCH_MOVE_LAMBDA_fixup(alphabet, to, { \ + p += __fsm_sz; \ + goto finalize; \ + }, flag) + + __FSM_START(parser->_i_st); + + __FSM_STATE(I_ContType) { + if (req->method != TFW_HTTP_METH_POST) + __FSM_I_JMP(I_EoL); + /* Fall through. */ + } + + __FSM_STATE(I_ContTypeMediaType) { + static const TfwStr s_multipart_form_data = + TFW_STR_STRING("multipart/form-data"); + H2_TRY_STR_LAMBDA_fixup(&s_multipart_form_data, &parser->hdr, { + __set_bit(TFW_HTTP_B_CT_MULTIPART, req->flags); + __FSM_EXIT(CSTR_EQ); + }, I_ContTypeMediaType, I_ContTypeMaybeMultipart); + if (chunk->len >= sizeof("multipart/") - 1) { + TRY_STR_INIT(); + __FSM_I_JMP(I_ContTypeOtherSubtype); + } else { + TRY_STR_INIT(); + __FSM_I_JMP(I_ContTypeOtherType); + } + } + + __FSM_STATE(I_ContTypeMaybeMultipart) { + if (c == ';') { + __set_bit(TFW_HTTP_B_CT_MULTIPART, req->flags); + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamOWS, 1, 0); + } + if (IS_WS(c)) + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeMultipartOWS, 1, 0); + __FSM_I_JMP(I_ContTypeOtherSubtype); + } + + __FSM_STATE(I_ContTypeMultipartOWS) { + if (IS_WS(c)) + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeMultipartOWS, 1, 0); + if (c == ';') { + __set_bit(TFW_HTTP_B_CT_MULTIPART, req->flags); + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamOWS, 1, 0); + } + return CSTR_NEQ; + } + + __FSM_STATE(I_ContTypeParamOWS) { + if (IS_WS(c)) + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamOWS, 1, 0); + /* Fall through. */ + } + + __FSM_STATE(I_ContTypeParam) { + static const TfwStr s_boundary = TFW_STR_STRING("boundary="); + if (!test_bit(TFW_HTTP_B_CT_MULTIPART, req->flags)) + __FSM_I_JMP(I_ContTypeParamOther); + + H2_TRY_STR_2LAMBDA_fixup(&s_boundary, &parser->hdr, { + /* + * Requests with multipart/form-data payload should have + * only one boundary parameter. + */ + if (__test_and_set_bit( + TFW_HTTP_B_CT_MULTIPART_HAS_BOUNDARY, + req->flags)) + { + __FSM_EXIT(CSTR_NEQ); + } + }, { + __FSM_EXIT(CSTR_EQ); + }, I_ContTypeParam, I_ContTypeBoundaryValue); + TRY_STR_INIT(); + /* Fall through. */ + } + + __FSM_STATE(I_ContTypeParamOther) { + /* + * If the header value is finished here, that means that value + * is ended just after parameter name; thus, parameter value is + * missing, and the header is invalid. + */ + __FSM_H2_I_MATCH_MOVE_NEQ_fixup(token, I_ContTypeParamOther, 0); + if (*(p + __fsm_sz) != '=') + return CSTR_NEQ; + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamValue, __fsm_sz + 1, 0); + } + + __FSM_STATE(I_ContTypeBoundaryValue) { + req->multipart_boundary_raw.len = 0; + req->multipart_boundary.len = 0; + /* + * msg->parser.hdr.data can't be used as a base here, since its + * value can change due to reallocation during msg->parser.hdr + * growth. Let's store chunk number instead for now. + */ + req->multipart_boundary_raw.data = + (char *)(size_t)parser->hdr.nchunks; + if (*p == '"') { + req->multipart_boundary_raw.len += 1; + __FSM_H2_I_MOVE_NEQ_fixup(I_ContTypeBoundaryValueQuoted, + 1, 0); + } + /* Fall through. */ + } + + __FSM_STATE(I_ContTypeBoundaryValueUnquoted) { + __fsm_n = __data_remain(p); + __fsm_sz = tfw_match_token(p, __fsm_n); + if (__fsm_sz > 0) { + __msg_hdr_chunk_fixup(p, __fsm_sz); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE | TFW_STR_VALUE); + req->multipart_boundary_raw.len += __fsm_sz; + req->multipart_boundary.len += __fsm_sz; + } + if (__fsm_sz == __fsm_n) { + if (likely(fin)) { + req->multipart_boundary_raw.nchunks = + parser->hdr.nchunks - + (size_t)req->multipart_boundary_raw.data; + p += __fsm_sz; + goto finalize; + } + parser->_i_st = &&I_ContTypeBoundaryValueUnquoted; + return CSTR_POSTPONE; + } + + p += __fsm_sz; + req->multipart_boundary_raw.nchunks = parser->hdr.nchunks - + (size_t)req->multipart_boundary_raw.data; + /* __fsm_sz != __fsm_n, therefore __data_remain(p) > 0 */ + __FSM_I_JMP(I_ContTypeParamValueOWS); + } + + __FSM_STATE(I_ContTypeBoundaryValueQuoted) { + __fsm_n = __data_remain(p); + __fsm_sz = tfw_match_token(p, __fsm_n); + if (__fsm_sz > 0) { + __msg_hdr_chunk_fixup(p, __fsm_sz); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE | TFW_STR_VALUE); + req->multipart_boundary_raw.len += __fsm_sz; + req->multipart_boundary.len += __fsm_sz; + } + if (unlikely(__fsm_sz == __fsm_n)) { + if (likely(fin)) { + /* Missing closing '"'. */ + return CSTR_NEQ; + } + parser->_i_st = &&I_ContTypeBoundaryValueQuoted; + return CSTR_POSTPONE; + } + p += __fsm_sz; + if (*p == '\\') { + req->multipart_boundary_raw.len += 1; + __FSM_H2_I_MOVE_NEQ_fixup( + I_ContTypeBoundaryValueEscapedChar, + 1, 0); + } + if (IS_CRLF(*p)) { + /* Missing closing '"'. */ + return CSTR_NEQ; + } + if (*p != '"') { + /* TODO: faster qdtext/quoted-pair matcher. */ + req->multipart_boundary_raw.len += 1; + req->multipart_boundary.len += 1; + __FSM_H2_I_MOVE_NEQ_fixup(I_ContTypeBoundaryValueQuoted, + 1, TFW_STR_VALUE); + } + + /* *p == '"' */ + __msg_hdr_chunk_fixup(p, 1); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + p += 1; + req->multipart_boundary_raw.len += 1; + req->multipart_boundary_raw.nchunks = parser->hdr.nchunks - + (size_t)req->multipart_boundary_raw.data; + + if (unlikely(__data_remain(p) == 0)) { + if (fin) + goto finalize; + parser->_i_st = &&I_ContTypeParamValueOWS; + return CSTR_POSTPONE; + } + __FSM_I_JMP(I_ContTypeParamValueOWS); + } + + __FSM_STATE(I_ContTypeBoundaryValueEscapedChar) { + if (IS_CRLF(*p)) + return CSTR_NEQ; + req->multipart_boundary_raw.len += 1; + req->multipart_boundary.len += 1; + __FSM_H2_I_MOVE_NEQ_fixup(I_ContTypeBoundaryValueQuoted, 1, + TFW_STR_VALUE); + } + + __FSM_STATE(I_ContTypeParamValue) { + if (*p == '"') + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamValueQuoted, + 1, 0); + /* Fall through. */ + } + + __FSM_STATE(I_ContTypeParamValueUnquoted) { + __FSM_H2_I_MATCH_MOVE_FIN_fixup(token, + I_ContTypeParamValueUnquoted, + TFW_STR_VALUE); + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamValueOWS, __fsm_sz, 0); + } + + __FSM_STATE(I_ContTypeParamValueOWS) { + if (IS_WS(c)) + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamValueOWS, 1, 0); + if (c == ';') + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamOWS, 1, 0); + return CSTR_NEQ; + } + + __FSM_STATE(I_ContTypeParamValueQuoted) { + __FSM_H2_I_MATCH_MOVE_NEQ_fixup(token, + I_ContTypeParamValueQuoted, + TFW_STR_VALUE); + if (__fsm_sz > 0) { + __msg_hdr_chunk_fixup(p, __fsm_sz); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE | TFW_STR_VALUE); + } + p += __fsm_sz; + if (*p == '\\') + __FSM_H2_I_MOVE_NEQ_fixup( + I_ContTypeParamValueEscapedChar, + 1, 0); + if (*p == '"') + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamValueOWS, + 1, 0); + if (IS_CRLF(*p)) { + /* Missing closing '"'. */ + return CSTR_NEQ; + } + /* TODO: faster qdtext/quoted-pair matcher. */ + __FSM_H2_I_MOVE_NEQ_fixup(I_ContTypeParamValueQuoted, 1, 0); + } + + __FSM_STATE(I_ContTypeParamValueEscapedChar) { + if (IS_CRLF(*p)) + return CSTR_NEQ; + __FSM_H2_I_MOVE_NEQ_fixup(I_ContTypeParamValueQuoted, 1, + TFW_STR_VALUE); + } + + __FSM_STATE(I_ContTypeOtherType) { + __FSM_H2_I_MATCH_MOVE_FIN_fixup(token, I_ContTypeOtherType, 0); + if (c != '/') + return CSTR_NEQ; + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeOtherSubtype, + __fsm_sz + 1, 0); + } + + __FSM_STATE(I_ContTypeOtherSubtype) { + __FSM_H2_I_MATCH_MOVE_FIN_fixup(token, I_ContTypeOtherSubtype, 0); + __msg_hdr_chunk_fixup(p, __fsm_sz); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + p += __fsm_sz; + /* Fall through. */ + } + + __FSM_STATE(I_ContTypeOtherTypeOWS) { + if (IS_WS(c)) + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeOtherTypeOWS, 1, 0); + if (c == ';') + __FSM_H2_I_MOVE_FIN_fixup(I_ContTypeParamOWS, 1, 0); + return CSTR_NEQ; + } + + __FSM_STATE(I_EoL) { + __FSM_H2_I_MATCH_MOVE_FIN_fixup(ctext_vchar, I_EoL, 0); + return CSTR_NEQ; + } + +done: + return r; + +finalize: + if (req->multipart_boundary_raw.len > 0) { + req->multipart_boundary_raw.chunks = parser->hdr.chunks + + (size_t)req->multipart_boundary_raw.data; + + /* + * Raw value of multipart boundary is going to be used during + * Content-Type field composing. So to prevent memcpy'ing + * intersecting buffers, we have to make a separate copy. + */ + if (__strdup_multipart_boundaries(req)) + return CSTR_NEQ; + } + + return CSTR_EQ; + +#undef __FSM_H2_I_MOVE_FIN_fixup +#undef __FSM_H2_I_MATCH_MOVE_FIN_fixup +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_content_type); + +static int +__h2_req_parse_cookie(TfwHttpMsg *hm, unsigned char *data, size_t len, bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(hm); + + /* + * Cookie header is parsed according to RFC 6265 4.2.1. + * + * Here we build header value string manually to split it in chunks: + * chunk bounds are at least at name start, value start and value end. + * This simplifies cookie search, http_sticky uses it. + */ + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_CookieStart) { + __FSM_H2_I_MATCH_MOVE_NEQ_fixup(token, Req_I_CookieName, + TFW_STR_NAME); + /* + * Name should contain at least 1 character. + * Store "=" with cookie parameter name. + */ + if (likely(__fsm_sz && *(p + __fsm_sz) == '=')) + __FSM_H2_I_MOVE_fixup(Req_I_CookieVal, __fsm_sz + 1, + TFW_STR_NAME); + return CSTR_NEQ; + } + + /* + * At this state we know that we saw at least one character as + * cookie-name and now we can pass zero length token. Cookie-value + * can have zero length. + */ + __FSM_STATE(Req_I_CookieName) { + __FSM_H2_I_MATCH_MOVE_NEQ_fixup(token, Req_I_CookieName, + TFW_STR_NAME); + if (*(p + __fsm_sz) != '=') + return CSTR_NEQ; + /* Store "=" with cookie parameter name. */ + __FSM_H2_I_MOVE_fixup(Req_I_CookieVal, __fsm_sz + 1, + TFW_STR_NAME); + } + + __FSM_STATE(Req_I_CookieVal) { + __FSM_H2_I_MATCH_MOVE_fixup(cookie, Req_I_CookieVal, + TFW_STR_VALUE); + c = *(p + __fsm_sz); + if (c == ';') { + if (likely(__fsm_sz)) { + /* Save cookie-value w/o ';'. */ + __msg_hdr_chunk_fixup(p, __fsm_sz); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE + | TFW_STR_VALUE); + } + p += __fsm_sz; + __FSM_I_JMP(Req_I_CookieSemicolon); + } + return CSTR_NEQ; + } + + /* ';' was already matched. */ + __FSM_STATE(Req_I_CookieSemicolon) { + /* + * Fixup current delimiters chunk and move to next parameter + * if we can eat ';' and SP at once. + */ + if (likely(__data_available(p, 2))) { + if (likely(*(p + 1) == ' ')) + __FSM_H2_I_MOVE_NEQ_fixup(Req_I_CookieStart, + 2, 0); + return CSTR_NEQ; + } + /* + * After ';' must be SP and another cookie-pair. Thus, if this + * is the last parsed part of the header value, the header is + * invalid. + */ + if (likely(fin)) + __FSM_EXIT(CSTR_NEQ); + /* + * Only ';' is available now: fixup ';' as independent chunk, + * SP will be fixed up at next enter to the FSM. + */ + __msg_hdr_chunk_fixup(p, 1); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + parser->_i_st = &&Req_I_CookieSP; + __FSM_EXIT(CSTR_POSTPONE); + } + + __FSM_STATE(Req_I_CookieSP) { + if (unlikely(c != ' ')) + return CSTR_NEQ; + /* Fixup delimiters chunk and move to the next parameter. */ + __FSM_H2_I_MOVE_NEQ_fixup(Req_I_CookieStart, 1, 0); + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_cookie); + +#define __FSM_H2_TX_ETAG(st, ch, st_next) \ +__FSM_STATE(st) { \ + if (likely(c == (ch))) \ + __FSM_H2_I_MOVE_NEQ_fixup(st_next, 1, 0); \ + return CSTR_NEQ; \ +} + +static int +__h2_req_parse_if_nmatch(TfwHttpMsg *hm, unsigned char *data, size_t len, + bool fin) +{ + int weak, r = CSTR_NEQ; + __FSM_DECLARE_VARS(hm); + + /* + * ETag value and closing DQUOTE are placed into separate chunks (see + * comments in @__parse_etag() for details). + */ + __FSM_START(parser->_i_st); + + __FSM_STATE(I_Etag) { + TfwHttpReq *req = (TfwHttpReq *)hm; + + if (likely(c == '"')) { + req->cond.flags |= TFW_HTTP_COND_ETAG_LIST; + __FSM_H2_I_MOVE_NEQ_fixup(I_Etag_Val, 1, 0); + } + + if (likely(__data_available(p, 3)) + && (*p == 'W') && (*(p + 1) == '/') && (*(p + 2) == '"')) + { + __FSM_H2_I_MOVE_NEQ_fixup(I_Etag_Weak, 3, 0); + } + if (c == 'W') + __FSM_H2_I_MOVE_NEQ_fixup(I_Etag_W, 1, 0); + + if (c == '*') { + if (req->cond.flags & TFW_HTTP_COND_ETAG_LIST) + return CSTR_NEQ; + + req->cond.flags |= TFW_HTTP_COND_ETAG_ANY; + __FSM_H2_I_MOVE_fixup(I_EoL, 1, 0); + } + + if (IS_WS(c)) + __FSM_H2_I_MOVE_NEQ_fixup(I_Etag, 1, 0); + return CSTR_NEQ; + } + + __FSM_H2_TX_ETAG(I_Etag_W, '/', I_Etag_We); + __FSM_H2_TX_ETAG(I_Etag_We, '"', I_Etag_Weak); + + __FSM_STATE(I_Etag_Weak) { + parser->hdr.flags |= TFW_STR_ETAG_WEAK; + __FSM_JMP(I_Etag_Val); + } + + __FSM_STATE(I_Etag_Val) { + weak = parser->hdr.flags & TFW_STR_ETAG_WEAK; + __FSM_H2_I_MATCH_MOVE_NEQ_fixup(token, I_Etag_Val, + (TFW_STR_VALUE | weak)); + c = *(p + __fsm_sz); + if (likely(c == '"')) { + parser->hdr.flags &= ~TFW_STR_ETAG_WEAK; + __FSM_H2_I_MOVE_fixup(I_EoT, __fsm_sz + 1, + TFW_STR_VALUE | weak); + } + return CSTR_NEQ; + } + + __FSM_STATE(I_EoT) { + if (IS_WS(c)) + __FSM_H2_I_MOVE_fixup(I_EoT, 1, 0); + if (c == ',') + __FSM_H2_I_MOVE_NEQ_fixup(I_Etag, 1, 0); + return CSTR_NEQ; + } + + __FSM_STATE(I_EoL) { + if (IS_WS(c)) + __FSM_H2_I_MOVE_fixup(I_EoL, 1, 0); + return CSTR_NEQ; + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_if_none_match); + +static int +__h2_req_parse_host(TfwHttpReq *req, unsigned char *data, size_t len, bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(req); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_H_Start) { + if (likely(isalnum(c) || c == '.' || c == '-')) + __FSM_H2_I_MOVE(Req_I_H); + if (likely(c == '[')) + __FSM_H2_I_MOVE_NEQ(Req_I_H_v6, 1); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_H) { + /* See Req_AuthorityGen processing. */ + if (likely(isalnum(c) || c == '.' || c == '-')) + __FSM_H2_I_MOVE(Req_I_H); + if (c == ':') + __FSM_H2_I_MOVE(Req_I_H_Port); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_H_v6) { + /* See Req_AuthorityIPv6 processing. */ + if (likely(isxdigit(c) || c == ':')) + __FSM_H2_I_MOVE_NEQ(Req_I_H_v6, 1); + if (likely(c == ']')) + __FSM_H2_I_MOVE(Req_I_H_v6_End); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_H_v6_End) { + if (likely(c == ':')) + __FSM_H2_I_MOVE(Req_I_H_Port); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_H_Port) { + /* See Req_Port processing. */ + if (likely(isdigit(c))) + __FSM_H2_I_MOVE(Req_I_H_Port); + return CSTR_NEQ; + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_host); + +static int +__h2_req_parse_referer(TfwHttpMsg *hm, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(hm); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_Referer) { + __FSM_H2_I_MATCH_MOVE(uri, Req_I_Referer); + if (IS_WS(*(p + __fsm_sz))) + __FSM_H2_I_MOVE_n(Req_I_EoT, __fsm_sz + 1); + return CSTR_NEQ; + } + __FSM_STATE(Req_I_EoT) { + if (IS_WS(c)) + __FSM_H2_I_MOVE(Req_I_EoT); + return CSTR_NEQ; + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_referer); + +static int +__h2_parse_http_date(TfwHttpMsg *hm, unsigned char *data, size_t len, bool fin) +{ + static const unsigned long colon_a[] ____cacheline_aligned = { + /* ':' (0x3a)(58) Colon */ + 0x0400000000000000UL, 0, 0, 0 + }; + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(hm); + +#define H2_TRY_STR_NEQ_LAMBDA(str, lambda, curr_st, next_st) \ + H2_TRY_STR_2LAMBDA(str, lambda, { \ + __FSM_EXIT(CSTR_NEQ); \ + }, curr_st, next_st) + + __FSM_START_ALT(parser->_i_st); + + __FSM_STATE(I_Date) { + /* Skip redundant weekday with comma (e.g. "Sun,"). */ + __fsm_sz = __data_remain(p); + __fsm_n = __skip_weekday(p, __fsm_sz); + if (unlikely(__fsm_sz == __fsm_n)) { + if (likely(fin)) + __FSM_EXIT(CSTR_NEQ); + parser->_i_st = &&I_Date; + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + __FSM_EXIT(CSTR_POSTPONE); + } + if (unlikely(__fsm_n == CSTR_NEQ)) + return CSTR_NEQ; + p += __fsm_n + 1; + __FSM_I_JMP(I_DateDay); + } + + __FSM_STATE(I_DateDay) { + __fsm_sz = __data_remain(p); + /* Parse a 2-digit day. */ + __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); + if (unlikely(__fsm_n == CSTR_POSTPONE)) { + if (likely(fin)) + return CSTR_NEQ; + parser->_i_st = &&I_DateDay; + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + } + if (__fsm_n < 0) + return __fsm_n; + if (parser->_acc < 1 || parser->_acc > 31) + return CSTR_BADLEN; + /* Add seconds in full passed days. */ + parser->_date = (parser->_acc - 1) * SEC24H; + parser->_acc = 0; + p += __fsm_n; + __FSM_I_JMP(I_DateMonthSP); + } + + __FSM_STATE(I_DateMonthSP) { + if (likely(c == ' ')) + __FSM_H2_I_MOVE_NEQ(I_DateMonth, 1); + return CSTR_NEQ; + } + + __FSM_STATE(I_DateMonth) { + switch (TFW_LC(c)) { + case 'a': + __FSM_I_JMP(I_DateMonth_A); + case 'j': + __FSM_I_JMP(I_DateMonth_J); + case 'm': + __FSM_I_JMP(I_DateMonth_M); + } + __FSM_I_JMP(I_DateMonth_Other); + } + + __FSM_STATE(I_DateMonth_A) { + H2_TRY_STR_NEQ_LAMBDA("apr ", { + parser->_date += SB_APR; + }, I_DateMonth_A, I_DateYear); + H2_TRY_STR_NEQ_LAMBDA("aug ", { + parser->_date += SB_AUG; + }, I_DateMonth_A, I_DateYear); + TRY_STR_INIT(); + return CSTR_NEQ; + } + + __FSM_STATE(I_DateMonth_J) { + H2_TRY_STR_NEQ_LAMBDA("jan ", { }, I_DateMonth_J, I_DateYear); + H2_TRY_STR_NEQ_LAMBDA("jun ", { + parser->_date += SB_JUN; + }, I_DateMonth_J, I_DateYear); + H2_TRY_STR_NEQ_LAMBDA("jul ", { + parser->_date += SB_JUL; + }, I_DateMonth_J, I_DateYear); + TRY_STR_INIT(); + return CSTR_NEQ; + } + + __FSM_STATE(I_DateMonth_M) { + H2_TRY_STR_NEQ_LAMBDA("mar ", { + /* Add SEC24H for leap year on year parsing. */ + parser->_date += SB_MAR; + }, I_DateMonth_M, I_DateYear); + H2_TRY_STR_NEQ_LAMBDA("may ", { + parser->_date += SB_MAY; + }, I_DateMonth_M, I_DateYear); + TRY_STR_INIT(); + return CSTR_NEQ; + } + + __FSM_STATE(I_DateMonth_Other) { + H2_TRY_STR_NEQ_LAMBDA("feb ", { + parser->_date += SB_FEB; + }, I_DateMonth_Other, I_DateYear); + H2_TRY_STR_NEQ_LAMBDA("sep ", { + parser->_date += SB_SEP; + }, I_DateMonth_Other, I_DateYear); + H2_TRY_STR_NEQ_LAMBDA("oct ", { + parser->_date += SB_OCT; + }, I_DateMonth_Other, I_DateYear); + H2_TRY_STR_NEQ_LAMBDA("nov ", { + parser->_date += SB_NOV; + }, I_DateMonth_Other, I_DateYear); + H2_TRY_STR_NEQ_LAMBDA("dec ", { + parser->_date += SB_DEC; + }, I_DateMonth_Other, I_DateYear); + TRY_STR_INIT(); + return CSTR_NEQ; + } + + /* 4-digit year. */ + __FSM_STATE(I_DateYear) { + __fsm_sz = __data_remain(p); + __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); + if (unlikely(__fsm_n == CSTR_POSTPONE)) { + if (likely(fin)) + return CSTR_NEQ; + parser->_i_st = &&I_DateYear; + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + } + if (__fsm_n < 0) + return __fsm_n; + parser->_date = __year_day_secs(parser->_acc, parser->_date); + if (parser->_date < 0) + return CSTR_NEQ; + parser->_acc = 0; + __FSM_H2_I_MOVE_NEQ(I_DateHourSP, __fsm_n); + } + + __FSM_STATE(I_DateHourSP) { + if (likely(c == ' ')) + __FSM_H2_I_MOVE_NEQ(I_DateHour, 1); + return CSTR_NEQ; + } + + __FSM_STATE(I_DateHour) { + __fsm_sz = __data_remain(p); + __fsm_n = parse_int_a(p, __fsm_sz, colon_a, &parser->_acc); + if (unlikely(__fsm_n == CSTR_POSTPONE)) { + if (likely(fin)) + return CSTR_NEQ; + parser->_i_st = &&I_DateHour; + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + } + if (__fsm_n < 0) + return __fsm_n; + parser->_date += parser->_acc * 3600; + parser->_acc = 0; + __FSM_H2_I_MOVE_NEQ(I_DateMinCln, __fsm_n); + } + + __FSM_STATE(I_DateMinCln) { + if (likely(c == ':')) + __FSM_H2_I_MOVE_NEQ(I_DateMin, 1); + return CSTR_NEQ; + } + + __FSM_STATE(I_DateMin) { + __fsm_sz = __data_remain(p); + __fsm_n = parse_int_a(p, __fsm_sz, colon_a, &parser->_acc); + if (unlikely(__fsm_n == CSTR_POSTPONE)) { + if (likely(fin)) + return CSTR_NEQ; + parser->_i_st = &&I_DateMin; + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + } + if (__fsm_n < 0) + return __fsm_n; + parser->_date += parser->_acc * 60; + parser->_acc = 0; + __FSM_H2_I_MOVE_NEQ(I_DateSecCln, __fsm_n); + } + + __FSM_STATE(I_DateSecCln) { + if (likely(c == ':')) + __FSM_H2_I_MOVE_NEQ(I_DateSec, 1); + return CSTR_NEQ; + } + + __FSM_STATE(I_DateSec) { + __fsm_sz = __data_remain(p); + __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); + if (unlikely(__fsm_n == CSTR_POSTPONE)) { + if (likely(fin)) + return CSTR_NEQ; + parser->_i_st = &&I_DateSec; + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + } + if (__fsm_n < 0) + return __fsm_n; + parser->_date += parser->_acc; + parser->_acc = 0; + __FSM_H2_I_MOVE_NEQ(I_DateSecSP, __fsm_n); + } + + __FSM_STATE(I_DateSecSP) { + if (likely(c == ' ')) + __FSM_H2_I_MOVE_NEQ(I_DateZone, 1); + return CSTR_NEQ; + } + + __FSM_STATE(I_DateZone) { + H2_TRY_STR("gmt", I_DateZone, I_EoL); + TRY_STR_INIT(); + return CSTR_NEQ; + } + + __FSM_STATE(I_EoL) { + /* Skip the rest of the line. */ + __FSM_H2_I_MATCH_MOVE_LAMBDA(nctl, I_EoL, { + T_DBG3("%s: parsed date %lu", __func__, parser->_date); + }); + return CSTR_NEQ; + } + +done: + return r; + +#undef H2_TRY_STR_NEQ_LAMBDA +} +STACK_FRAME_NON_STANDARD(__h2_parse_http_date); + +static int +__h2_req_parse_if_msince(TfwHttpMsg *msg, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + TfwHttpReq *req = (TfwHttpReq *)msg; + TfwHttpParser *parser = &msg->stream->parser; + + if (!(req->cond.flags & TFW_HTTP_COND_IF_MSINCE)) + r = __h2_parse_http_date(msg, data, len, fin); + + if (r < 0 && r != CSTR_POSTPONE) { + /* On error just swallow the rest of the line. */ + parser->_date = 0; + parser->_acc = 0; + parser->_i_st = __I_EoL; + r = __h2_parse_http_date(msg, data, len, fin); + } + + if (r >= 0) { + req->cond.m_date = parser->_date; + req->cond.flags |= TFW_HTTP_COND_IF_MSINCE; + + return CSTR_EQ; + } + + return r; +} + +static int +__h2_req_parse_pragma(TfwHttpMsg *hm, unsigned char *data, size_t len, bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(hm); + + __FSM_START(parser->_i_st); + + __FSM_STATE(I_Pragma) { + H2_TRY_STR_LAMBDA("no-cache", { + msg->cache_ctl.flags |= TFW_HTTP_CC_PRAGMA_NO_CACHE; + __FSM_EXIT(CSTR_EQ); + }, I_Pragma, I_Pragma_NoCache); + TRY_STR_INIT(); + __FSM_I_JMP(I_Pragma_Ext); + } + + __FSM_STATE(I_Pragma_NoCache) { + if (IS_WS(c) || c == ',') + msg->cache_ctl.flags |= TFW_HTTP_CC_PRAGMA_NO_CACHE; + __FSM_I_JMP(I_Pragma_Ext); + } + + __FSM_STATE(I_Pragma_Ext) { + __FSM_H2_I_MATCH_MOVE(qetoken, I_Pragma_Ext); + c = *(p + __fsm_sz); + if (IS_WS(c) || c == ',') + __FSM_H2_I_MOVE_n(I_EoT, __fsm_sz + 1); + return CSTR_NEQ; + } + + __FSM_STATE(I_EoT) { + if (IS_WS(c) || c == ',') + __FSM_H2_I_MOVE(I_EoT); + __FSM_I_JMP(I_Pragma_Ext); + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_pragma); + +static int +__h2_req_parse_user_agent(TfwHttpMsg *hm, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(hm); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_UserAgent) { + __FSM_H2_I_MATCH_MOVE(ctext_vchar, Req_I_UserAgent); + return CSTR_NEQ; + } +done: + return r; +} + +static int +__h2_req_parse_x_forwarded_for(TfwHttpMsg *hm, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(hm); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_XFF) { + /* Eat OWS before the node ID. */ + if (unlikely(IS_WS(c))) + __FSM_H2_I_MOVE_NEQ_fixup(Req_I_XFF, 1, 0); + /* + * Eat IP address or host name. + * + * TODO: parse/validate IP addresses and textual IDs. + * Currently we just validate separate characters, but the + * whole value may be invalid (e.g. "---[_..[["). + */ + __FSM_H2_I_MATCH_MOVE_fixup(xff, Req_I_XFF_Node_Id, TFW_STR_VALUE); + if (unlikely(!__fsm_sz)) + return CSTR_NEQ; + __FSM_H2_I_MOVE_fixup(Req_I_XFF_Sep, __fsm_sz, TFW_STR_VALUE); + } + + /* + * At this state we know that we saw at least one character as + * a host address and now we can pass zero length token. + */ + __FSM_STATE(Req_I_XFF_Node_Id) { + __FSM_H2_I_MATCH_MOVE_fixup(xff, Req_I_XFF_Node_Id, TFW_STR_VALUE); + __msg_hdr_chunk_fixup(p, __fsm_sz); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE | TFW_STR_VALUE); + p += __fsm_sz; + __FSM_I_JMP(Req_I_XFF_Sep); + } + + __FSM_STATE(Req_I_XFF_Sep) { + /* OWS before comma is unusual. */ + if (unlikely(IS_WS(c))) + __FSM_H2_I_MOVE_fixup(Req_I_XFF_Sep, 1, 0); + /* + * Multiple subsequent commas look suspicious, so we don't + * stay in this state after the first comma is met. + */ + if (likely(c == ',')) + __FSM_H2_I_MOVE_NEQ_fixup(Req_I_XFF, 1, 0); + + return CSTR_NEQ; + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_x_forwarded_for); + +static int +__h2_req_parse_mark(TfwHttpReq *req, unsigned char *data, size_t len, bool fin) +{ + TfwStr *str; + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(req); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Req_I_UriMarkStart) { + if (WARN_ON_ONCE(c != '/')) + __FSM_EXIT(CSTR_NEQ); + + __msg_field_open(&req->mark, p); + /* Place initial slash into separate chunk. */ + __FSM_H2_I_MOVE_LAMBDA_fixup_f(Req_I_UriMarkName, 1, &req->mark, + { + /* + * The end of ':path' header has been met; thus, we can + * just go out, and the parsed '/' will be fixed up in + * the outside state after returning. + */ + TFW_STR_INIT(&req->mark); + return __data_off(p + 1); + }, 0); + } + + __FSM_STATE(Req_I_UriMarkName) { + str = tfw_http_sess_mark_name(); + H2_TRY_STR_2LAMBDA_fixup(str, &req->mark, { + parser->to_read = tfw_http_sess_mark_size(); + }, { + __FSM_EXIT(CSTR_NEQ); + }, Req_I_UriMarkName, Req_I_UriMarkValue); + /* + * In case of HTTP/2 processing we need not set @req->uri_path + * here; instead, the value of ':path' pseudo-header in + * @req->h_tbl (currently @parser->hdr) is used. If mark isn't + * matched here, concatenate descriptors accumulated in + * @req->mark with the descriptor of ':path' pseudo-header + * (that is @parser->hdr) - the latter will be finished in the + * 'Req_Path' state. Note, that if we are here, we must not be + * postponed in the outside state after returning. + */ + if (tfw_strcat(req->pool, &parser->hdr, &req->mark)) + __FSM_EXIT(CSTR_NEQ); + + TFW_STR_INIT(&req->mark); + return __data_off(p); + } + + __FSM_STATE(Req_I_UriMarkValue) { + __fsm_n = min_t(long, parser->to_read, __data_remain(p)); + parser->to_read -= __fsm_n; + if (parser->to_read) { + if (fin) + __FSM_EXIT(CSTR_NEQ); + __msg_field_fixup_pos(&req->mark, p, __fsm_n); + __FSM_I_field_chunk_flags(&req->mark, TFW_STR_VALUE); + parser->_i_st = &&Req_I_UriMarkValue; + __FSM_EXIT(CSTR_POSTPONE); + } + parser->to_read = -1; + __msg_field_finish_pos(&req->mark, p, __fsm_n); + __FSM_I_field_chunk_flags(&req->mark, TFW_STR_VALUE); + return __data_off(p + __fsm_n); + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_mark); + +int +tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, + bool fin, bool value_stage) +{ + int ret = T_OK; + TfwMsgParseIter *it = &req->pit; + __FSM_DECLARE_VARS(req); + + T_DBG("%s: len=%lu, data=%.*s%s, req=[%p]\n", len, min(500, (int)len), + data, len > 500 ? "..." : "", req); + + __FSM_START(parser->state); + + /* + * If next state is not defined here, that means the name has not been + * parsed and is statically indexed, thus we should determine the state + * from the header tag. + */ + __FSM_H2_REQ_NEXT_STATE(value_stage); + + /* ---------------- (Pseudo-)headers name ---------------- */ + + __FSM_STATE(RGen_Hdr, hot) { + + tfw_http_msg_hdr_open(msg, p); + + switch (c) { + case ':': + if (likely(__data_available(p, 7) + && C4_INT(p + 1, 'm', 'e', 't', 'h') + && *(p + 5) == 'o' + && *(p + 6) == 'd')) + { + __FSM_H2_FIN(Req_HdrPsMethodV, 7, + TFW_TAG_HDR_H2_METHOD); + } + if (likely(__data_available(p, 7) + && C4_INT(p + 1, 's', 'c', 'h', 'e') + && *(p + 5) == 'm' + && *(p + 6) == 'e')) + { + __FSM_H2_FIN(Req_HdrPsSchemeV, 7, + TFW_TAG_HDR_H2_SCHEME); + } + if (likely(__data_available(p, 10) + && C8_INT(p + 1, 'a', 'u', 't', 'h', + 'o', 'r', 'i', 't') + && *(p + 9) == 'y')) + { + __FSM_H2_FIN(Req_HdrPsAuthorityV, 10, + TFW_TAG_HDR_H2_AUTHORITY); + } + if (likely(__data_available(p, 5) + && C4_INT(p + 1, 'p', 'a', 't', 'h'))) + { + __FSM_H2_FIN(Req_HdrPsPathV, 5, + TFW_TAG_HDR_H2_PATH); + } + __FSM_H2_NEXT(Req_HdrPseudo); + case 'a': + if (likely(__data_available(p, 6) + && C4_INT(p + 1, 'c', 'c', 'e', 'p') + && *(p + 5) == 't')) + { + __FSM_H2_FIN(Req_HdrAcceptV, 6, + TFW_TAG_HDR_ACCEPT); + } + if (likely(__data_available(p, 13) + && C8_INT(p + 1, 'u', 't', 'h', 'o', + 'r', 'i', 'z', 'a') + && C4_INT(p + 9, 't', 'i', 'o', 'n'))) + { + __FSM_H2_FIN(Req_HdrAuthorizationV, 13, + TFW_TAG_HDR_AUTHORIZATION); + } + __FSM_H2_NEXT(Req_HdrA); + case 'c': + if (unlikely(!__data_available(p, 13))) + __FSM_H2_NEXT(Req_HdrC); + switch (PI(p + 1)) { + case TFW_CHAR4_INT('a', 'c', 'h', 'e'): + if (likely(C8_INT(p + 5, '-', 'c', 'o', 'n', + 't', 'r', 'o', 'l'))) + { + __FSM_H2_FIN(Req_HdrCache_ControlV, 13, + TFW_TAG_HDR_CACHE_CONTROL); + } + __FSM_H2_OTHER_n(5); + case TFW_CHAR4_INT('o', 'n', 'n', 'e'): + if (likely(C4_INT(p + 5, 'c', 't', 'i', 'o') + && *(p + 9) == 'n')) + { + /* + * Messages with connection-specific + * headers must be treated as malformed + * in HTTP/2 (see RFC 7540 section + * 8.1.2.2 for details). + */ + __FSM_H2_DROP(Req_HdrConnection); + } + __FSM_H2_OTHER_n(5); + case TFW_CHAR4_INT('o', 'n', 't', 'e'): + if (likely(*(p + 5) == 'n' + && *(p + 6) == 't' + && *(p + 7) == '-')) + { + __FSM_H2_NEXT_n(Req_HdrContent_, 8); + } + __FSM_H2_OTHER_n(5); + case TFW_CHAR4_INT('o', 'o', 'k', 'i'): + if (likely(*(p + 5) == 'e')) + { + __FSM_H2_FIN(Req_HdrCookieV, 6, + TFW_TAG_HDR_COOKIE); + } + __FSM_H2_OTHER_n(5); + default: + __FSM_H2_OTHER(); + } + case 'h': + if (likely(__data_available(p, 4) + && *(p + 1) == 'o' + && *(p + 2) == 's' + && *(p + 3) == 't')) + { + __FSM_H2_FIN(Req_HdrHostV, 4, + TFW_TAG_HDR_HOST); + } + __FSM_H2_NEXT(Req_HdrH); + case 'i': + if (likely(__data_available(p, 17) + && *(p + 1) == 'f' + && *(p + 2) == '-' + && C8_INT(p + 3, 'm', 'o', 'd', 'i', + 'f', 'i', 'e', 'd') + && *(p + 11) == '-' + && C4_INT(p + 12, 's', 'i', 'n', 'c') + && *(p + 16) == 'e')) + { + __FSM_H2_FIN(Req_HdrIf_Modified_SinceV, 17, + TFW_TAG_HDR_IF_MODIFIED_SINCE); + } + if (likely(__data_available(p, 13) + && *(p + 1) == 'f' + && *(p + 2) == '-' + && C4_INT(p + 3, 'n', 'o', 'n', 'e') + && *(p + 7) == '-' + && C4_INT(p + 8, 'm', 'a', 't', 'c') + && *(p + 12) == 'h')) + { + __FSM_H2_FIN(Req_HdrIf_None_MatchV, 13, + TFW_TAG_HDR_IF_NONE_MATCH); + } + __FSM_H2_NEXT(Req_HdrI); + case 'k': + if (likely(__data_available(p, 10) + && C4_INT(p, 'k', 'e', 'e', 'p') + && *(p + 4) == '-' + && C4_INT(p + 5, 'a', 'l', 'i', 'v') + && *(p + 9) == 'e')) + { + __FSM_H2_DROP(Req_HdrKeep_Alive); + } + __FSM_H2_NEXT(Req_HdrK); + case 'p': + if (likely(__data_available(p, 6) + && C4_INT(p + 1, 'r', 'a', 'g', 'm') + && *(p + 5) == 'a')) + { + __FSM_H2_FIN(Req_HdrPragmaV, 6, + TFW_TAG_HDR_PRAGMA); + } + __FSM_H2_NEXT(Req_HdrP); + case 'r': + if (likely(__data_available(p, 7) + && C4_INT(p + 1, 'e', 'f', 'e', 'r') + && *(p + 5) == 'e' + && *(p + 6) == 'r')) + { + __FSM_H2_FIN(Req_HdrRefererV, 7, + TFW_TAG_HDR_REFERER); + } + __FSM_H2_NEXT(Req_HdrR); + case 't': + if (likely(__data_available(p, 17) + && C8_INT(p, 't', 'r', 'a', 'n', + 's', 'f', 'e', 'r') + && *(p + 8) == '-' + && C8_INT(p + 9, 'e', 'n', 'c', 'o', + 'd', 'i', 'n', 'g'))) + { + __FSM_H2_DROP(Req_HdrTransfer_Encoding); + } + __FSM_H2_NEXT(Req_HdrT); + case 'x': + if (likely(__data_available(p, 15) + && C8_INT(p + 1, '-', 'f', 'o', 'r', 'w', + 'a', 'r', 'd') + && C4_INT(p + 9, 'e', 'd', '-', 'f') + && *(p + 13) == 'o' + && *(p + 15) == 'r')) + { + __FSM_H2_FIN(Req_HdrX_Forwarded_ForV, 15, + TFW_TAG_HDR_X_FORWARDED_FOR); + } + __FSM_H2_NEXT(Req_HdrX); + case 'u': + if (likely(__data_available(p, 10) + && C4_INT(p, 'u', 's', 'e', 'r') + && *(p + 4) == '-' + && C4_INT(p + 5, 'a', 'g', 'e', 'n') + && *(p + 9) == 't')) + { + __FSM_H2_FIN(Req_HdrUser_AgentV, 10, + TFW_TAG_HDR_USER_AGENT); + } + __FSM_H2_NEXT(Req_HdrU); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + + __FSM_STATE(Req_HdrContent_) { + switch (c) { + case 'l': + if (likely(__data_available(p, 6) + && C4_INT(p + 1, 'e', 'n', 'g', 't') + && *(p + 5) == 'h')) + { + __FSM_H2_FIN(Req_HdrContent_LengthV, 6, + TFW_TAG_HDR_CONTENT_LENGTH); + } + __FSM_H2_NEXT(Req_HdrContent_L); + case 't': + if (likely(__data_available(p, 4) + && *(p + 1) == 'y' + && *(p + 2) == 'p' + && *(p + 3) == 'e')) + { + __FSM_H2_FIN(Req_HdrContent_TypeV, 4, + TFW_TAG_HDR_CONTENT_TYPE); + } + __FSM_H2_NEXT(Req_HdrContent_T); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + + __FSM_STATE(RGen_HdrOtherN) { + __fsm_n = __data_remain(p); + __fsm_sz = tfw_match_token(p, __fsm_n); + if (unlikely(__fsm_sz != __fsm_n)) + __FSM_H2_DROP(RGen_HdrOtherN); + __msg_hdr_chunk_fixup(data, len); + if (unlikely(!fin)) + __FSM_H2_POSTPONE(RGen_HdrOtherN); + it->tag = TFW_TAG_HDR_RAW; + __FSM_H2_OK(RGen_HdrOtherV); + } + + /* ---------------- Pseudo-header values ---------------- */ + + __FSM_STATE(Req_HdrPsMethodV, hot) { + if (!H2_MSG_VERIFY(TFW_HTTP_HDR_H2_METHOD)) + __FSM_H2_DROP(Req_HdrPsMethodV); + + parser->_hdr_tag = TFW_HTTP_HDR_H2_METHOD; + if (likely(__data_available(p, 3) + && *p == 'G' + && *(p + 1) == 'E' + && *(p + 2) == 'T')) + { + __FSM_H2_METHOD_COMPLETE(Req_HdrPsMethodV, 3, + TFW_HTTP_METH_GET); + } + if (likely(__data_available(p, 4) + && PI(p) == TFW_CHAR4_INT('P', 'O', 'S', 'T'))) + { + __FSM_H2_METHOD_COMPLETE(Req_HdrPsMethodV, 4, + TFW_HTTP_METH_POST); + } + __FSM_JMP(Req_RareMethods_3); + } + + __FSM_STATE(Req_HdrPsSchemeV, hot) { + if (!H2_MSG_VERIFY(TFW_HTTP_HDR_H2_SCHEME)) + __FSM_H2_DROP(Req_HdrPsSchemeV); + + parser->_hdr_tag = TFW_HTTP_HDR_H2_SCHEME; + if (likely(__data_available(p, 5) + && C4_INT_LCM(p, 'h', 't', 't', 'p') + && TFW_LC(*(p + 4)) == 's')) + { + __FSM_H2_PSHDR_COMPLETE(Req_HdrPsSchemeV, 5); + } + __FSM_JMP(Req_Scheme_1CharStep); + } + + __FSM_STATE(Req_HdrPsAuthorityV) { + if (!H2_MSG_VERIFY(TFW_HTTP_HDR_H2_AUTHORITY)) + __FSM_H2_DROP(Req_HdrPsAuthorityV); + + parser->_hdr_tag = TFW_HTTP_HDR_H2_AUTHORITY; + __set_bit(TFW_HTTP_B_URI_FULL, req->flags); + if (likely(isalnum(c) || c == '.' || c == '-')) + __FSM_H2_PSHDR_MOVE_FIN(Req_HdrPsAuthorityV, 1, + Req_AuthorityGen); + else if (c == '[') + __FSM_H2_PSHDR_MOVE_DROP(Req_HdrPsAuthorityV, + 1, Req_AuthorityIPv6); + __FSM_H2_DROP(Req_HdrPsAuthorityV); + } + + __FSM_STATE(Req_AuthorityGen) { + if (likely(isalnum(c) || c == '.' || c == '-')) + __FSM_H2_PSHDR_MOVE_FIN(Req_AuthorityGen, 1, + Req_AuthorityGen); + /* + * The value of HTTP/2 authority pseudo-header must not + * include 'userinfo' component (see RFC 7540 section + * 8.1.2.3 for details). + */ + if (unlikely(c == '@')) + __FSM_H2_DROP(Req_AuthorityGen); + __FSM_JMP(Req_AuthorityEnd); + } + + __FSM_STATE(Req_AuthorityIPv6) { + if (likely(isxdigit(c) || c == ':')) { + __FSM_H2_PSHDR_MOVE_DROP(Req_AuthorityIPv6, + 1, Req_AuthorityIPv6); + } else if(c == ']') { + __FSM_H2_PSHDR_MOVE_FIN(Req_AuthorityIPv6, 1, + Req_AuthorityEnd); + } + __FSM_H2_DROP(Req_AuthorityIPv6); + } + + __FSM_STATE(Req_AuthorityEnd) { + if (c == ':') + __FSM_H2_PSHDR_MOVE_FIN(Req_AuthorityEnd, 1, Req_Port); + __FSM_H2_DROP(Req_AuthorityEnd); + } + + __FSM_STATE(Req_Port) { + if (likely(isdigit(c))) + __FSM_H2_PSHDR_MOVE_FIN(Req_Port, 1, Req_Port); + __FSM_H2_DROP(Req_Port); + } + + __FSM_STATE(Req_HdrPsPathV, hot) { + if (!H2_MSG_VERIFY(TFW_HTTP_HDR_H2_PATH)) + __FSM_H2_DROP(Req_HdrPsPathV); + + parser->_hdr_tag = TFW_HTTP_HDR_H2_PATH; + if (likely(c == '/')) + __FSM_JMP(Req_Mark); + + if (unlikely(c == '*')) { + __FSM_H2_PSHDR_CHECK_lambda(p + 1, { + if ((req->method + && req->method != TFW_HTTP_METH_OPTIONS) + || !fin) + { + __FSM_H2_DROP(Req_HdrPsPathV); + } + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + __FSM_H2_HDR_COMPLETE(Req_HdrPsPathV); + }); + } + __FSM_H2_DROP(Req_HdrPsPathV); + } + + __FSM_STATE(Req_Mark, hot) { + if (!tfw_http_sess_max_misses()) + __FSM_H2_PSHDR_MOVE_FIN_fixup(Req_Mark, 1, Req_Path); + + TRY_STR_INIT(); + WARN_ON_ONCE(parser->_i_st); + + __fsm_n = __h2_req_parse_mark(req, p, __data_remain(p), fin); + if (__fsm_n == CSTR_POSTPONE) + __FSM_H2_POSTPONE(Req_Mark); + if (__fsm_n < 0) { + __FSM_H2_DROP(Req_Mark); + } + WARN_ON_ONCE(!__fsm_n); + parser->_i_st = NULL; + + if (TFW_STR_EMPTY(&req->mark)) { + /* + * All @__fsm_n data is already fixed up in + * @__h2_req_parse_mark() (into @parser->hdr), except + * the case of final chunk of the ':path' header, but + * in this case the reaming data will be fixed up below, + * in @__FSM_H2_PSHDR_MOVE_FIN(), just before the exit. + */ + WARN_ON_ONCE(__fsm_n > 1); + __FSM_H2_PSHDR_MOVE_FIN(Req_Mark, __fsm_n, Req_Path); + } + __FSM_H2_PSHDR_MOVE_DROP_nofixup(Req_Mark, __fsm_n, Req_MarkEnd); + } + + __FSM_STATE(Req_MarkEnd, hot) { + if (likely(c == '/')) + __FSM_H2_PSHDR_MOVE_FIN_fixup(Req_MarkEnd, 1, Req_Path); + __FSM_H2_DROP(Req_MarkEnd); + } + + __FSM_STATE(Req_Path) { + __fsm_n = __data_remain(p); + __fsm_sz = tfw_match_uri(p, __fsm_n); + if (likely(__fsm_sz != __fsm_n)) + __FSM_H2_DROP(Req_Path); + __msg_hdr_chunk_fixup(p, __fsm_sz); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + if (unlikely(!fin)) + __FSM_H2_POSTPONE(Req_Path); + __FSM_H2_HDR_COMPLETE(Req_Path); + } + + /* ---------------- Header values ---------------- */ + + /* 'accept' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrAcceptV, req, __h2_req_parse_accept, + TFW_HTTP_HDR_RAW, 1); + + /* 'authorization' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrAuthorizationV, req, + __h2_req_parse_authorization, TFW_HTTP_HDR_RAW, 1); + + /* 'cache-control' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrCache_ControlV, req, + __h2_req_parse_cache_control, TFW_HTTP_HDR_RAW, 1); + + /* 'content-length' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrContent_LengthV, msg, + __h2_req_parse_content_length, + TFW_HTTP_HDR_CONTENT_LENGTH, 1); + + /* 'content-type' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrContent_TypeV, msg, + __h2_req_parse_content_type, + TFW_HTTP_HDR_CONTENT_TYPE, 0); + + /* 'host' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrHostV, req, __h2_req_parse_host, + TFW_HTTP_HDR_HOST, 1); + + /* 'if-none-match' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrIf_None_MatchV, msg, + __h2_req_parse_if_nmatch, + TFW_HTTP_HDR_IF_NONE_MATCH, 0); + + /* 'if-modified-since' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrIf_Modified_SinceV, msg, + __h2_req_parse_if_msince, + TFW_HTTP_HDR_RAW, 1); + + /* 'pragma' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrPragmaV, msg, __h2_req_parse_pragma, + TFW_HTTP_HDR_RAW, 1); + + /* 'referer' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrRefererV, msg, __h2_req_parse_referer, + TFW_HTTP_HDR_REFERER, 1); + + /* 'x-forwarded-for' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrX_Forwarded_ForV, msg, + __h2_req_parse_x_forwarded_for, + TFW_HTTP_HDR_X_FORWARDED_FOR, 0); + + /* 'user-agent' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrUser_AgentV, msg, __h2_req_parse_user_agent, + TFW_HTTP_HDR_USER_AGENT, 1); + + /* 'cookie' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrCookieV, msg, __h2_req_parse_cookie, + TFW_HTTP_HDR_COOKIE, 0); + + __FSM_STATE(RGen_HdrOtherV) { + if (!H2_MSG_VERIFY(TFW_HTTP_HDR_RAW)) + __FSM_H2_DROP(RGen_HdrOtherV); + + __fsm_n = __data_remain(p); + __fsm_sz = tfw_match_ctext_vchar(p, __fsm_n); + if (unlikely(__fsm_sz != __fsm_n)) + __FSM_H2_DROP(RGen_HdrOtherV); + + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + + if (unlikely(!fin)) + __FSM_H2_POSTPONE(RGen_HdrOtherV); + + parser->_hdr_tag = TFW_HTTP_HDR_RAW; + __FSM_H2_HDR_COMPLETE(RGen_HdrOtherV); + } + + /* ---------------- Slow path ---------------- */ + + barrier(); + + /* Improbable states of (pseudo-)header names processing. */ + + __FSM_STATE(Req_HdrPseudo, cold) { + switch (c) { + case 'a': + __FSM_H2_NEXT(Req_HdrPsA); + case 'm': + __FSM_H2_NEXT(Req_HdrPsM); + case 'p': + __FSM_H2_NEXT(Req_HdrPsP); + case 's': + __FSM_H2_NEXT(Req_HdrPsS); + default: + __FSM_H2_DROP(Req_HdrPseudo); + } + } + + __FSM_H2_TXD_AF(Req_HdrPsA, 'u', Req_HdrPsAu); + __FSM_H2_TXD_AF(Req_HdrPsAu, 't', Req_HdrPsAut); + __FSM_H2_TXD_AF(Req_HdrPsAut, 'h', Req_HdrPsAuth); + __FSM_H2_TXD_AF(Req_HdrPsAuth, 'o', Req_HdrPsAutho); + __FSM_H2_TXD_AF(Req_HdrPsAutho, 'r', Req_HdrPsAuthor); + __FSM_H2_TXD_AF(Req_HdrPsAuthor, 'i', Req_HdrPsAuthori); + __FSM_H2_TXD_AF(Req_HdrPsAuthori, 't', Req_HdrPsAuthorit); + __FSM_H2_TXD_AF_FIN(Req_HdrPsAuthorit, 'y', Req_HdrPsAuthorityV, + TFW_TAG_HDR_H2_AUTHORITY); + + __FSM_H2_TXD_AF(Req_HdrPsM, 'e', Req_HdrPsMe); + __FSM_H2_TXD_AF(Req_HdrPsMe, 't', Req_HdrPsMet); + __FSM_H2_TXD_AF(Req_HdrPsMet, 'h', Req_HdrPsMeth); + __FSM_H2_TXD_AF(Req_HdrPsMeth, 'o', Req_HdrPsMetho); + __FSM_H2_TXD_AF_FIN(Req_HdrPsMetho, 'd', Req_HdrPsMethodV, + TFW_TAG_HDR_H2_METHOD); + + __FSM_H2_TXD_AF(Req_HdrPsP, 'a', Req_HdrPsPa); + __FSM_H2_TXD_AF(Req_HdrPsPa, 't', Req_HdrPsPat); + __FSM_H2_TXD_AF_FIN(Req_HdrPsPat, 'h', Req_HdrPsPathV, + TFW_TAG_HDR_H2_PATH); + + __FSM_H2_TXD_AF(Req_HdrPsS, 'c', Req_HdrPsSc); + __FSM_H2_TXD_AF(Req_HdrPsSc, 'h', Req_HdrPsSch); + __FSM_H2_TXD_AF(Req_HdrPsSch, 'e', Req_HdrPsSche); + __FSM_H2_TXD_AF(Req_HdrPsSche, 'm', Req_HdrPsSchem); + __FSM_H2_TXD_AF_FIN(Req_HdrPsSchem, 'e', Req_HdrPsSchemeV, + TFW_TAG_HDR_H2_SCHEME); + + __FSM_STATE(Req_HdrA, cold) { + switch (c) { + case 'c': + __FSM_H2_NEXT(Req_HdrAc); + case 'u': + __FSM_H2_NEXT(Req_HdrAu); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + + __FSM_H2_TX_AF(Req_HdrAc, 'c', Req_HdrAcc); + __FSM_H2_TX_AF(Req_HdrAcc, 'e', Req_HdrAcce); + __FSM_H2_TX_AF(Req_HdrAcce, 'p', Req_HdrAccep); + __FSM_H2_TX_AF_FIN(Req_HdrAccep, 't', Req_HdrAcceptV, + TFW_TAG_HDR_ACCEPT); + + __FSM_H2_TX_AF(Req_HdrAu, 't', Req_HdrAut); + __FSM_H2_TX_AF(Req_HdrAut, 'h', Req_HdrAuth); + __FSM_H2_TX_AF(Req_HdrAuth, 'o', Req_HdrAutho); + __FSM_H2_TX_AF(Req_HdrAutho, 'r', Req_HdrAuthor); + __FSM_H2_TX_AF(Req_HdrAuthor, 'i', Req_HdrAuthori); + __FSM_H2_TX_AF(Req_HdrAuthori, 'z', Req_HdrAuthoriz); + __FSM_H2_TX_AF(Req_HdrAuthoriz, 'a', Req_HdrAuthoriza); + __FSM_H2_TX_AF(Req_HdrAuthoriza, 't', Req_HdrAuthorizat); + __FSM_H2_TX_AF(Req_HdrAuthorizat, 'i', Req_HdrAuthorizati); + __FSM_H2_TX_AF(Req_HdrAuthorizati, 'o', Req_HdrAuthorizatio); + __FSM_H2_TX_AF_FIN(Req_HdrAuthorizatio, 'n', Req_HdrAuthorizationV, + TFW_TAG_HDR_AUTHORIZATION); + + __FSM_STATE(Req_HdrC, cold) { + switch (c) { + case 'a': + __FSM_H2_NEXT(Req_HdrCa); + case 'o': + __FSM_H2_NEXT(Req_HdrCo); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + + __FSM_H2_TX_AF(Req_HdrCa, 'c', Req_HdrCac); + __FSM_H2_TX_AF(Req_HdrCac, 'h', Req_HdrCach); + __FSM_H2_TX_AF(Req_HdrCach, 'e', Req_HdrCache); + __FSM_H2_TX_AF(Req_HdrCache, '-', Req_HdrCache_); + __FSM_H2_TX_AF(Req_HdrCache_, 'c', Req_HdrCache_C); + __FSM_H2_TX_AF(Req_HdrCache_C, 'o', Req_HdrCache_Co); + __FSM_H2_TX_AF(Req_HdrCache_Co, 'n', Req_HdrCache_Con); + __FSM_H2_TX_AF(Req_HdrCache_Con, 't', Req_HdrCache_Cont); + __FSM_H2_TX_AF(Req_HdrCache_Cont, 'r', Req_HdrCache_Contr); + __FSM_H2_TX_AF(Req_HdrCache_Contr, 'o', Req_HdrCache_Contro); + __FSM_H2_TX_AF_FIN(Req_HdrCache_Contro, 'l', Req_HdrCache_ControlV, + TFW_TAG_HDR_CACHE_CONTROL); + + __FSM_STATE(Req_HdrCo, cold) { + switch (c) { + case 'n': + __FSM_H2_NEXT(Req_HdrCon); + case 'o': + __FSM_H2_NEXT(Req_HdrCoo); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + + __FSM_STATE(Req_HdrCon, cold) { + switch (c) { + case 'n': + __FSM_H2_NEXT(Req_HdrConn); + case 't': + __FSM_H2_NEXT(Req_HdrCont); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + __FSM_H2_TX_AF(Req_HdrConn, 'e', Req_HdrConne); + __FSM_H2_TX_AF(Req_HdrConne, 'c', Req_HdrConnec); + __FSM_H2_TX_AF(Req_HdrConnec, 't', Req_HdrConnect); + __FSM_H2_TX_AF(Req_HdrConnect, 'i', Req_HdrConnecti); + __FSM_H2_TX_AF(Req_HdrConnecti, 'o', Req_HdrConnectio); + __FSM_H2_TX_AF_DROP(Req_HdrConnectio, 'n'); + + __FSM_H2_TX_AF(Req_HdrCont, 'e', Req_HdrConte); + __FSM_H2_TX_AF(Req_HdrConte, 'n', Req_HdrConten); + __FSM_H2_TX_AF(Req_HdrConten, 't', Req_HdrContent); + __FSM_H2_TX_AF(Req_HdrContent, '-', Req_HdrContent_); + + __FSM_H2_TX_AF(Req_HdrContent_L, 'e', Req_HdrContent_Le); + __FSM_H2_TX_AF(Req_HdrContent_Le, 'n', Req_HdrContent_Len); + __FSM_H2_TX_AF(Req_HdrContent_Len, 'g', Req_HdrContent_Leng); + __FSM_H2_TX_AF(Req_HdrContent_Leng, 't', Req_HdrContent_Lengt); + __FSM_H2_TX_AF_FIN(Req_HdrContent_Lengt, 'h', Req_HdrContent_LengthV, + TFW_TAG_HDR_CONTENT_LENGTH); + + __FSM_H2_TX_AF(Req_HdrContent_T, 'y', Req_HdrContent_Ty); + __FSM_H2_TX_AF(Req_HdrContent_Ty, 'p', Req_HdrContent_Typ); + __FSM_H2_TX_AF_FIN(Req_HdrContent_Typ, 'e', Req_HdrContent_TypeV, + TFW_TAG_HDR_CONTENT_TYPE); + + __FSM_H2_TX_AF(Req_HdrH, 'o', Req_HdrHo); + __FSM_H2_TX_AF(Req_HdrHo, 's', Req_HdrHos); + __FSM_H2_TX_AF_FIN(Req_HdrHos, 't', Req_HdrHostV, TFW_TAG_HDR_HOST); + + __FSM_H2_TX_AF(Req_HdrI, 'f', Req_HdrIf); + __FSM_H2_TX_AF(Req_HdrIf, '-', Req_HdrIf_); + __FSM_STATE(Req_HdrIf_, cold) { + switch (c) { + case 'm': + __FSM_H2_NEXT(Req_HdrIf_M); + case 'n': + __FSM_H2_NEXT(Req_HdrIf_N); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + + __FSM_H2_TX_AF(Req_HdrIf_M, 'o', Req_HdrIf_Mo); + __FSM_H2_TX_AF(Req_HdrIf_Mo, 'd', Req_HdrIf_Mod); + __FSM_H2_TX_AF(Req_HdrIf_Mod, 'i', Req_HdrIf_Modi); + __FSM_H2_TX_AF(Req_HdrIf_Modi, 'f', Req_HdrIf_Modif); + __FSM_H2_TX_AF(Req_HdrIf_Modif, 'i', Req_HdrIf_Modifi); + __FSM_H2_TX_AF(Req_HdrIf_Modifi, 'e', Req_HdrIf_Modifie); + __FSM_H2_TX_AF(Req_HdrIf_Modifie, 'd', Req_HdrIf_Modified); + __FSM_H2_TX_AF(Req_HdrIf_Modified, '-', Req_HdrIf_Modified_); + __FSM_H2_TX_AF(Req_HdrIf_Modified_, 's', Req_HdrIf_Modified_S); + __FSM_H2_TX_AF(Req_HdrIf_Modified_S, 'i', Req_HdrIf_Modified_Si); + __FSM_H2_TX_AF(Req_HdrIf_Modified_Si, 'n', Req_HdrIf_Modified_Sin); + __FSM_H2_TX_AF(Req_HdrIf_Modified_Sin, 'c', Req_HdrIf_Modified_Sinc); + __FSM_H2_TX_AF_FIN(Req_HdrIf_Modified_Sinc, 'e', + Req_HdrIf_Modified_SinceV, + TFW_TAG_HDR_IF_MODIFIED_SINCE); + + __FSM_H2_TX_AF(Req_HdrIf_N, 'o', Req_HdrIf_No); + __FSM_H2_TX_AF(Req_HdrIf_No, 'n', Req_HdrIf_Non); + __FSM_H2_TX_AF(Req_HdrIf_Non, 'e', Req_HdrIf_None); + __FSM_H2_TX_AF(Req_HdrIf_None, '-', Req_HdrIf_None_); + __FSM_H2_TX_AF(Req_HdrIf_None_, 'm', Req_HdrIf_None_M); + __FSM_H2_TX_AF(Req_HdrIf_None_M, 'a', Req_HdrIf_None_Ma); + __FSM_H2_TX_AF(Req_HdrIf_None_Ma, 't', Req_HdrIf_None_Mat); + __FSM_H2_TX_AF(Req_HdrIf_None_Mat, 'c', Req_HdrIf_None_Matc); + __FSM_H2_TX_AF_FIN(Req_HdrIf_None_Matc, 'h', Req_HdrIf_None_MatchV, + TFW_TAG_HDR_IF_NONE_MATCH); + + __FSM_H2_TX_AF(Req_HdrK, 'e', Req_HdrKe); + __FSM_H2_TX_AF(Req_HdrKe, 'e', Req_HdrKee); + __FSM_H2_TX_AF(Req_HdrKee, 'p', Req_HdrKeep); + __FSM_H2_TX_AF(Req_HdrKeep, '-', Req_HdrKeep_); + __FSM_H2_TX_AF(Req_HdrKeep_, 'a', Req_HdrKeep_A); + __FSM_H2_TX_AF(Req_HdrKeep_A, 'l', Req_HdrKeep_Al); + __FSM_H2_TX_AF(Req_HdrKeep_Al, 'i', Req_HdrKeep_Ali); + __FSM_H2_TX_AF(Req_HdrKeep_Ali, 'v', Req_HdrKeep_Aliv); + __FSM_H2_TX_AF_DROP(Req_HdrKeep_Aliv, 'e'); + + __FSM_H2_TX_AF(Req_HdrP, 'r', Req_HdrPr); + __FSM_H2_TX_AF(Req_HdrPr, 'a', Req_HdrPra); + __FSM_H2_TX_AF(Req_HdrPra, 'g', Req_HdrPrag); + __FSM_H2_TX_AF(Req_HdrPrag, 'm', Req_HdrPragm); + __FSM_H2_TX_AF_FIN(Req_HdrPragm, 'a', Req_HdrPragmaV, + TFW_TAG_HDR_PRAGMA); + + __FSM_H2_TX_AF(Req_HdrR, 'e', Req_HdrRe); + __FSM_H2_TX_AF(Req_HdrRe, 'f', Req_HdrRef); + __FSM_H2_TX_AF(Req_HdrRef, 'e', Req_HdrRefe); + __FSM_H2_TX_AF(Req_HdrRefe, 'r', Req_HdrRefer); + __FSM_H2_TX_AF(Req_HdrRefer, 'e', Req_HdrRefere); + __FSM_H2_TX_AF_FIN(Req_HdrRefere, 'r', Req_HdrRefererV, + TFW_TAG_HDR_REFERER); + + __FSM_H2_TX_AF(Req_HdrT, 'r', Req_HdrTr); + __FSM_H2_TX_AF(Req_HdrTr, 'a', Req_HdrTra); + __FSM_H2_TX_AF(Req_HdrTra, 'n', Req_HdrTran); + __FSM_H2_TX_AF(Req_HdrTran, 's', Req_HdrTrans); + __FSM_H2_TX_AF(Req_HdrTrans, 'f', Req_HdrTransf); + __FSM_H2_TX_AF(Req_HdrTransf, 'e', Req_HdrTransfe); + __FSM_H2_TX_AF(Req_HdrTransfe, 'r', Req_HdrTransfer); + __FSM_H2_TX_AF(Req_HdrTransfer, '-', Req_HdrTransfer_); + __FSM_H2_TX_AF(Req_HdrTransfer_, 'e', Req_HdrTransfer_E); + __FSM_H2_TX_AF(Req_HdrTransfer_E, 'n', Req_HdrTransfer_En); + __FSM_H2_TX_AF(Req_HdrTransfer_En, 'c', Req_HdrTransfer_Enc); + __FSM_H2_TX_AF(Req_HdrTransfer_Enc, 'o', Req_HdrTransfer_Enco); + __FSM_H2_TX_AF(Req_HdrTransfer_Enco, 'd', Req_HdrTransfer_Encod); + __FSM_H2_TX_AF(Req_HdrTransfer_Encod, 'i', Req_HdrTransfer_Encodi); + __FSM_H2_TX_AF(Req_HdrTransfer_Encodi, 'n', Req_HdrTransfer_Encodin); + __FSM_H2_TX_AF_DROP(Req_HdrTransfer_Encodin, 'g'); + + __FSM_H2_TX_AF(Req_HdrX, '-', Req_HdrX_); + __FSM_H2_TX_AF(Req_HdrX_, 'f', Req_HdrX_F); + __FSM_H2_TX_AF(Req_HdrX_F, 'o', Req_HdrX_Fo); + __FSM_H2_TX_AF(Req_HdrX_Fo, 'r', Req_HdrX_For); + __FSM_H2_TX_AF(Req_HdrX_For, 'w', Req_HdrX_Forw); + __FSM_H2_TX_AF(Req_HdrX_Forw, 'a', Req_HdrX_Forwa); + __FSM_H2_TX_AF(Req_HdrX_Forwa, 'r', Req_HdrX_Forwar); + __FSM_H2_TX_AF(Req_HdrX_Forwar, 'd', Req_HdrX_Forward); + __FSM_H2_TX_AF(Req_HdrX_Forward, 'e', Req_HdrX_Forwarde); + __FSM_H2_TX_AF(Req_HdrX_Forwarde, 'd', Req_HdrX_Forwarded); + __FSM_H2_TX_AF(Req_HdrX_Forwarded, '-', Req_HdrX_Forwarded_); + __FSM_H2_TX_AF(Req_HdrX_Forwarded_, 'f', Req_HdrX_Forwarded_F); + __FSM_H2_TX_AF(Req_HdrX_Forwarded_F, 'o', Req_HdrX_Forwarded_Fo); + __FSM_H2_TX_AF_FIN(Req_HdrX_Forwarded_Fo, 'r', Req_HdrX_Forwarded_ForV, + TFW_TAG_HDR_X_FORWARDED_FOR); + + __FSM_H2_TX_AF(Req_HdrU, 's', Req_HdrUs); + __FSM_H2_TX_AF(Req_HdrUs, 'e', Req_HdrUse); + __FSM_H2_TX_AF(Req_HdrUse, 'r', Req_HdrUser); + __FSM_H2_TX_AF(Req_HdrUser, '-', Req_HdrUser_); + __FSM_H2_TX_AF(Req_HdrUser_, 'a', Req_HdrUser_A); + __FSM_H2_TX_AF(Req_HdrUser_A, 'g', Req_HdrUser_Ag); + __FSM_H2_TX_AF(Req_HdrUser_Ag, 'e', Req_HdrUser_Age); + __FSM_H2_TX_AF(Req_HdrUser_Age, 'n', Req_HdrUser_Agen); + __FSM_H2_TX_AF_FIN(Req_HdrUser_Agen, 't', Req_HdrUser_AgentV, + TFW_TAG_HDR_USER_AGENT); + + __FSM_H2_TX_AF(Req_HdrCoo, 'k', Req_HdrCook); + __FSM_H2_TX_AF(Req_HdrCook, 'i', Req_HdrCooki); + __FSM_H2_TX_AF_FIN(Req_HdrCooki, 'e', Req_HdrCookieV, + TFW_TAG_HDR_COOKIE); + + /* Improbable states of method values processing. */ + + __FSM_STATE(Req_RareMethods_3, cold) { + if (__data_available(p, 3)) { + if (*p == 'P' && *(p + 1) == 'U' && *(p + 2) == 'T') { + __FSM_H2_METHOD_COMPLETE(Req_HdrPsMethodV, 3, + TFW_HTTP_METH_PUT); + } + __FSM_JMP(Req_RareMethods_4); + } + __FSM_JMP(Req_Method_1CharStep); + } + + __FSM_STATE(Req_RareMethods_4, cold) { + if (__data_available(p, 4)) { + if (C4_INT(p, 'H', 'E', 'A', 'D')) + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_4, 4, + TFW_HTTP_METH_HEAD); + if (C4_INT(p, 'C', 'O', 'P', 'Y')) + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_4, 4, + TFW_HTTP_METH_COPY); + if (C4_INT(p, 'L', 'O', 'C', 'K')) + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_4, 4, + TFW_HTTP_METH_LOCK); + if (C4_INT(p, 'M', 'O', 'V', 'E')) + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_4, 4, + TFW_HTTP_METH_MOVE); + __FSM_JMP(Req_RareMethods_5); + } + __FSM_JMP(Req_Method_1CharStep); + } + + __FSM_STATE(Req_RareMethods_5, cold) { + if (__data_available(p, 5)) { + if (C4_INT(p, 'P', 'U', 'R', 'G') && *(p + 4) == 'E') + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_5, 5, + TFW_HTTP_METH_PURGE); + if (C4_INT(p, 'M', 'K', 'C', 'O') && *(p + 4) == 'L') + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_5, 5, + TFW_HTTP_METH_MKCOL); + if (C4_INT(p, 'P', 'A', 'T', 'C') && *(p + 4) == 'H') + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_5, 5, + TFW_HTTP_METH_PATCH); + if (C4_INT(p, 'T', 'R', 'A', 'C') && *(p + 4) == 'E') + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_5, 5, + TFW_HTTP_METH_TRACE); + __FSM_JMP(Req_RareMethods_6); + } + __FSM_JMP(Req_Method_1CharStep); + } + + __FSM_STATE(Req_RareMethods_6, cold) { + if (__data_available(p, 6)) { + if (C4_INT(p, 'D', 'E', 'L', 'E') + && *(p + 4) == 'T' + && *(p + 5) == 'E') + { + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_6, 6, + TFW_HTTP_METH_DELETE); + } + if (C4_INT(p, 'U', 'N', 'L', 'O') + && *(p + 4) == 'C' + && *(p + 5) == 'K') + { + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_6, 6, + TFW_HTTP_METH_UNLOCK); + } + __FSM_JMP(Req_RareMethods_7); + } + __FSM_JMP(Req_Method_1CharStep); + } + + __FSM_STATE(Req_RareMethods_7, cold) { + if (__data_available(p, 7)) { + if (C4_INT(p, 'O', 'P', 'T', 'I') + && *(p + 4) == 'O' + && *(p + 5) == 'N' + && *(p + 6) == 'S') + { + __FSM_H2_METHOD_COMPLETE(Req_RareMethods_7, 7, + TFW_HTTP_METH_OPTIONS); + } + __FSM_JMP(Req_RareMethods); + } + __FSM_JMP(Req_Method_1CharStep); + } + + __FSM_STATE(Req_RareMethods, cold) { + if (!__data_available(p, 8)) + __FSM_JMP(Req_Method_1CharStep); + if (!C4_INT(p, 'P', 'R', 'O', 'P')) + __FSM_JMP(Req_MethodUnknown); + p += 4; + if(C4_INT(p, 'F', 'I', 'N', 'D')) + __FSM_H2_METHOD_COMPLETE(Req_RareMethods, 4, + TFW_HTTP_METH_PROPFIND); + if (!__data_available(p, 5)) + __FSM_JMP(Req_MethodUnknown); + if (C4_INT(p, 'P', 'A', 'T', 'C') && *(p + 4) == 'H') + __FSM_H2_METHOD_COMPLETE(Req_RareMethods, 5, + TFW_HTTP_METH_PROPPATCH); + __FSM_JMP(Req_MethodUnknown); + } + + __FSM_STATE(Req_Method_1CharStep, cold) { + switch (c) { + case 'G': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethG); + case 'H': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethH); + case 'P': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethP); + case 'C': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethC); + case 'D': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethD); + case 'L': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethL); + case 'M': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethM); + case 'O': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethO); + case 'T': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethT); + case 'U': + __FSM_H2_METHOD_MOVE(Req_Method_1CharStep, 1, + Req_MethU); + } + __FSM_JMP(Req_MethodUnknown); + } + /* GET */ + __FSM_H2_METH_STATE_MOVE(Req_MethG, 'E', Req_MethGe); + __FSM_H2_METH_STATE_COMPLETE(Req_MethGe, 'T', TFW_HTTP_METH_GET); + /* P* */ + __FSM_STATE(Req_MethP, cold) { + switch (c) { + case 'O': + __FSM_H2_METHOD_MOVE(Req_MethP, 1, Req_MethPo); + case 'A': + __FSM_H2_METHOD_MOVE(Req_MethP, 1, Req_MethPa); + case 'R': + __FSM_H2_METHOD_MOVE(Req_MethP, 1, Req_MethPr); + case 'U': + __FSM_H2_METHOD_MOVE(Req_MethP, 1, Req_MethPu); + } + __FSM_JMP(Req_MethodUnknown); + } + /* POST */ + __FSM_H2_METH_STATE_MOVE(Req_MethPo, 'S', Req_MethPos); + __FSM_H2_METH_STATE_COMPLETE(Req_MethPos, 'T', TFW_HTTP_METH_POST); + /* PATCH */ + __FSM_H2_METH_STATE_MOVE(Req_MethPa, 'T', Req_MethPat); + __FSM_H2_METH_STATE_MOVE(Req_MethPat, 'C', Req_MethPatc); + __FSM_H2_METH_STATE_COMPLETE(Req_MethPatc, 'H', TFW_HTTP_METH_PATCH); + /* PROP* */ + __FSM_H2_METH_STATE_MOVE(Req_MethPr, 'O', Req_MethPro); + __FSM_H2_METH_STATE_MOVE(Req_MethPro, 'P', Req_MethProp); + __FSM_STATE(Req_MethProp, cold) { + switch (c) { + case 'F': + __FSM_H2_METHOD_MOVE(Req_MethProp, 1, Req_MethPropf); + case 'P': + __FSM_H2_METHOD_MOVE(Req_MethProp, 1, Req_MethPropp); + } + __FSM_JMP(Req_MethodUnknown); + } + /* PROPFIND */ + __FSM_H2_METH_STATE_MOVE(Req_MethPropf, 'I', Req_MethPropfi); + __FSM_H2_METH_STATE_MOVE(Req_MethPropfi, 'N', Req_MethPropfin); + __FSM_H2_METH_STATE_COMPLETE(Req_MethPropfin, 'D', + TFW_HTTP_METH_PROPFIND); + /* PROPPATCH */ + __FSM_H2_METH_STATE_MOVE(Req_MethPropp, 'A', Req_MethProppa); + __FSM_H2_METH_STATE_MOVE(Req_MethProppa, 'T', Req_MethProppat); + __FSM_H2_METH_STATE_MOVE(Req_MethProppat, 'C', Req_MethProppatc); + __FSM_H2_METH_STATE_COMPLETE(Req_MethProppatc, 'H', + TFW_HTTP_METH_PROPPATCH); + /* PU* */ + __FSM_STATE(Req_MethPu, cold) { + switch (c) { + case 'R': + __FSM_H2_METHOD_MOVE(Req_MethPu, 1, Req_MethPur); + case 'T': + /* PUT */ + __FSM_H2_METHOD_COMPLETE(Req_MethPu, 1, + TFW_HTTP_METH_PUT); + } + __FSM_JMP(Req_MethodUnknown); + } + /* PURGE */ + __FSM_H2_METH_STATE_MOVE(Req_MethPur, 'G', Req_MethPurg); + __FSM_H2_METH_STATE_COMPLETE(Req_MethPurg, 'E', TFW_HTTP_METH_PURGE); + /* HEAD */ + __FSM_H2_METH_STATE_MOVE(Req_MethH, 'E', Req_MethHe); + __FSM_H2_METH_STATE_MOVE(Req_MethHe, 'A', Req_MethHea); + __FSM_H2_METH_STATE_COMPLETE(Req_MethHea, 'D', TFW_HTTP_METH_HEAD); + /* COPY */ + __FSM_H2_METH_STATE_MOVE(Req_MethC, 'O', Req_MethCo); + __FSM_H2_METH_STATE_MOVE(Req_MethCo, 'P', Req_MethCop); + __FSM_H2_METH_STATE_COMPLETE(Req_MethCop, 'Y', TFW_HTTP_METH_COPY); + /* DELETE */ + __FSM_H2_METH_STATE_MOVE(Req_MethD, 'E', Req_MethDe); + __FSM_H2_METH_STATE_MOVE(Req_MethDe, 'L', Req_MethDel); + __FSM_H2_METH_STATE_MOVE(Req_MethDel, 'E', Req_MethDele); + __FSM_H2_METH_STATE_MOVE(Req_MethDele, 'T', Req_MethDelet); + __FSM_H2_METH_STATE_COMPLETE(Req_MethDelet, 'E', TFW_HTTP_METH_DELETE); + /* LOCK */ + __FSM_H2_METH_STATE_MOVE(Req_MethL, 'O', Req_MethLo); + __FSM_H2_METH_STATE_MOVE(Req_MethLo, 'C', Req_MethLoc); + __FSM_H2_METH_STATE_COMPLETE(Req_MethLoc, 'K', TFW_HTTP_METH_LOCK); + /* M* */ + __FSM_STATE(Req_MethM, cold) { + switch (c) { + case 'K': + __FSM_H2_METHOD_MOVE(Req_MethM, 1, Req_MethMk); + case 'O': + __FSM_H2_METHOD_MOVE(Req_MethM, 1, Req_MethMo); + } + __FSM_JMP(Req_MethodUnknown); + } + /* MKCOL */ + __FSM_H2_METH_STATE_MOVE(Req_MethMk, 'C', Req_MethMkc); + __FSM_H2_METH_STATE_MOVE(Req_MethMkc, 'O', Req_MethMkco); + __FSM_H2_METH_STATE_COMPLETE(Req_MethMkco, 'L', TFW_HTTP_METH_MKCOL); + /* MOVE */ + __FSM_H2_METH_STATE_MOVE(Req_MethMo, 'V', Req_MethMov); + __FSM_H2_METH_STATE_COMPLETE(Req_MethMov, 'E', TFW_HTTP_METH_MOVE); + /* OPTIONS */ + __FSM_H2_METH_STATE_MOVE(Req_MethO, 'P', Req_MethOp); + __FSM_H2_METH_STATE_MOVE(Req_MethOp, 'T', Req_MethOpt); + __FSM_H2_METH_STATE_MOVE(Req_MethOpt, 'I', Req_MethOpti); + __FSM_H2_METH_STATE_MOVE(Req_MethOpti, 'O', Req_MethOptio); + __FSM_H2_METH_STATE_MOVE(Req_MethOptio, 'N', Req_MethOption); + __FSM_H2_METH_STATE_COMPLETE(Req_MethOption, 'S', TFW_HTTP_METH_OPTIONS); + /* TRACE */ + __FSM_H2_METH_STATE_MOVE(Req_MethT, 'R', Req_MethTr); + __FSM_H2_METH_STATE_MOVE(Req_MethTr, 'A', Req_MethTra); + __FSM_H2_METH_STATE_MOVE(Req_MethTra, 'C', Req_MethTrac); + __FSM_H2_METH_STATE_COMPLETE(Req_MethTrac, 'E', TFW_HTTP_METH_TRACE); + /* UNLOCK */ + __FSM_H2_METH_STATE_MOVE(Req_MethU, 'N', Req_MethUn); + __FSM_H2_METH_STATE_MOVE(Req_MethUn, 'L', Req_MethUnl); + __FSM_H2_METH_STATE_MOVE(Req_MethUnl, 'O', Req_MethUnlo); + __FSM_H2_METH_STATE_MOVE(Req_MethUnlo, 'C', Req_MethUnloc); + __FSM_H2_METH_STATE_COMPLETE(Req_MethUnloc, 'K', TFW_HTTP_METH_UNLOCK); + + __FSM_STATE(Req_MethodUnknown, cold) { + __fsm_n = __data_remain(p); + __fsm_sz = tfw_match_token(p, __fsm_n); + if (likely(__fsm_sz == __fsm_n)) { + __msg_hdr_chunk_fixup(data, len); + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + if (unlikely(!fin)) + __FSM_H2_POSTPONE(Req_MethodUnknown); + req->method = _TFW_HTTP_METH_UNKNOWN; + __FSM_H2_HDR_COMPLETE(Req_MethodUnknown); + } + __FSM_H2_DROP(Req_MethodUnknown); + } + + /* Improbable states of shceme value processing. */ + + __FSM_H2_SCHEME_STATE_MOVE(Req_Scheme_1CharStep, 'h', Req_SchemeH); + __FSM_H2_SCHEME_STATE_MOVE(Req_SchemeH, 't', Req_SchemeHt); + __FSM_H2_SCHEME_STATE_MOVE(Req_SchemeHt, 't', Req_SchemeHtt); + __FSM_H2_SCHEME_STATE_MOVE(Req_SchemeHtt, 'p', Req_SchemeHttp); + __FSM_H2_SCHEME_STATE_COMPLETE(Req_SchemeHttp, 's'); + +out: + return ret; +} + +static int +tfw_h2_parse_body(char *data, unsigned long len, TfwHttpReq *req, + unsigned int *parsed) +{ + unsigned long m_len; + TfwH2Ctx *ctx = tfw_h2_context(req->conn); + TfwHttpMsg *msg = (TfwHttpMsg *)req; + TfwHttpParser *parser = &msg->stream->parser; + int ret = T_POSTPONE; + + if (parser->to_read == -1) { + if (WARN_ON_ONCE(!ctx->plen)) + return T_DROP; + + parser->to_read = ctx->plen; + + T_DBG3("%s: init, content_length=%lu, to_read=%ld\n", + req->content_length, parser->to_read); + + if (!req->body.data) + tfw_http_msg_set_str_data(msg, &req->body, data); + } + + BUG_ON(parser->to_read < 0); + m_len = min_t(long, parser->to_read, len); + parser->to_read -= m_len; + + if (parser->to_read) { + T_DBG3("%s: postpone, to_read=%ld, m_len=%lu, len=%lu\n", + parser->to_read, m_len, len); + __msg_field_fixup(&req->body, data + len); + goto out; + } + + WARN_ON_ONCE(m_len != len); + T_DBG3("%s: to_read=%ld, m_len=%lu, len=%lu\n", parser->to_read, + m_len, len); + + if (tfw_http_msg_add_str_data(msg, &req->body, data, m_len)) + return T_DROP; + + parser->to_read = -1; + req->body.flags |= TFW_STR_COMPLETE; + ret = T_OK; + +out: + *parsed += m_len; + return ret; +} + +int +tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, + unsigned int *parsed) +{ + int r; + TfwHttpReq *req = (TfwHttpReq *)req_data; + TfwH2Ctx *ctx = tfw_h2_context(req->conn); + unsigned char type = ctx->hdr.type; + + WARN_ON_ONCE(!len); + BUG_ON(type != HTTP2_HEADERS + && type != HTTP2_CONTINUATION + && type != HTTP2_DATA); + + *parsed = 0; + + if (likely(type != HTTP2_DATA)) { + r = tfw_hpack_decode(&ctx->hpack, data, len, req, parsed); + req->pit.hb_len += *parsed; + } else { + r = tfw_h2_parse_body(data, len, req, parsed); + } + + if (r != T_OK && r != T_POSTPONE) + return r; + /* + * Parsing of HTTP/2 frames' payload never gives T_OK result since + * request can be assembled from different number of frames; only + * stream's state can indicate the moment when request is completed. + * Note, due to persistent linkage Stream<->Request in HTTP/2 mode + * special flag is necessary for tracking the request completeness; if + * the request is not fully assembled and is not passed to forwarding + * procedures yet, it must be deleted during corresponding stream + * removal. + */ + if (tfw_h2_stream_req_complete(req->stream)) { + TfwHttpHdrTbl *ht = req->h_tbl; + + if (req->content_length + && req->content_length != req->body.len) + return T_DROP; + + __set_bit(TFW_HTTP_B_FULLY_PARSED, req->flags); + + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY], + &req->host); + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_PATH], + &req->uri_path); + return T_OK; + } + + return T_POSTPONE; +} + +/* + * ------------------------------------------------------------------------ + * HTTP response parsing + * ------------------------------------------------------------------------ + */ +static int +__resp_parse_age(TfwHttpResp *resp, unsigned char *data, size_t len) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(resp); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Resp_I_Age) { + __fsm_sz = __data_remain(p); + __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); + if (__fsm_n == CSTR_POSTPONE) + __msg_hdr_chunk_fixup(data, len); + if (__fsm_n < 0) + return __fsm_n; + resp->cache_ctl.age = parser->_acc; + parser->_acc = 0; + __FSM_I_MOVE_n(Resp_I_EoL, __fsm_n); + } + + __FSM_STATE(Resp_I_EoL) { + if (IS_WS(c)) + __FSM_I_MOVE(Resp_I_EoL); + if (IS_CRLF(c)) { + resp->cache_ctl.flags |= TFW_HTTP_CC_HDR_AGE; + return __data_off(p); + } + return CSTR_NEQ; + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__resp_parse_age); + +/** + * Parse response Cache-Control, RFC 2616 14.9 + */ +static int +__resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(resp); + + __FSM_START(parser->_i_st); + + __FSM_STATE(Resp_I_CC) { + WARN_ON_ONCE(parser->_acc); + switch (TFW_LC(c)) { + case 'm': + __FSM_I_JMP(Resp_I_CC_m); + case 'n': + __FSM_I_JMP(Resp_I_CC_n); + case 'p': + __FSM_I_JMP(Resp_I_CC_p); + case 's': + __FSM_I_JMP(Resp_I_CC_s); + } + __FSM_I_JMP(Resp_I_Ext); + } + + __FSM_STATE(Resp_I_CC_m) { + TRY_STR("max-age=", Resp_I_CC_m, Resp_I_CC_MaxAgeV); + TRY_STR_LAMBDA("must-revalidate", { + parser->_acc = TFW_HTTP_CC_MUST_REVAL; + }, Resp_I_CC_m, Resp_I_Flag); + TRY_STR_INIT(); + __FSM_I_JMP(Resp_I_Ext); + } + + __FSM_STATE(Resp_I_CC_n) { + TRY_STR_LAMBDA("no-cache", { + parser->_acc = TFW_HTTP_CC_NO_CACHE; + }, Resp_I_CC_n, Resp_I_Flag); + TRY_STR_LAMBDA("no-store", { + parser->_acc = TFW_HTTP_CC_NO_STORE; + }, Resp_I_CC_n, Resp_I_Flag); + TRY_STR_LAMBDA("no-transform", { + parser->_acc = TFW_HTTP_CC_NO_TRANSFORM; + }, Resp_I_CC_n, Resp_I_Flag); + TRY_STR_INIT(); + __FSM_I_JMP(Resp_I_Ext); + } + + __FSM_STATE(Resp_I_CC_p) { + TRY_STR_LAMBDA("public", { + parser->_acc = TFW_HTTP_CC_PUBLIC; + }, Resp_I_CC_p, Resp_I_Flag); + TRY_STR_LAMBDA("private", { + parser->_acc = TFW_HTTP_CC_PRIVATE; + }, Resp_I_CC_p, Resp_I_Flag); + TRY_STR_LAMBDA("proxy-revalidate", { + parser->_acc = TFW_HTTP_CC_PROXY_REVAL; + }, Resp_I_CC_p, Resp_I_Flag); + TRY_STR_INIT(); + __FSM_I_JMP(Resp_I_Ext); + } + + __FSM_STATE(Resp_I_Flag) { + WARN_ON_ONCE(!parser->_acc); + if (IS_WS(c) || c == ',' || IS_CRLF(c)) { + resp->cache_ctl.flags |= parser->_acc; + parser->_acc = 0; + __FSM_I_JMP(Resp_I_EoT); + } + __FSM_I_JMP(Resp_I_Ext); + } __FSM_STATE(Resp_I_CC_s) { TRY_STR("s-maxage=", Resp_I_CC_s, Resp_I_CC_SMaxAgeV); @@ -4606,8 +7652,8 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, if (unlikely(!__data_available(p, 9))) { /* Slow path. */ if (c == 'H') { - __msg_field_open(&resp->s_line, p); - __FSM_MOVE_f(Resp_HttpVerT1, &resp->s_line); + tfw_http_msg_hdr_open(msg, p); + __FSM_MOVE(Resp_HttpVerT1); } TFW_PARSER_BLOCK(Resp_HttpVer); } @@ -4616,17 +7662,15 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case TFW_CHAR8_INT('H', 'T', 'T', 'P', '/', '1', '.', '1'): resp->version = TFW_HTTP_VER_11; if (*(p + 8) == ' ') { - __msg_field_open(&resp->s_line, p); - __FSM_MOVE_nf(Resp_StatusCode, 9, - &resp->s_line); + tfw_http_msg_hdr_open(msg, p); + __FSM_MOVE_n(Resp_StatusCode, 9); } TFW_PARSER_BLOCK(Resp_HttpVer); case TFW_CHAR8_INT('H', 'T', 'T', 'P', '/', '1', '.', '0'): resp->version = TFW_HTTP_VER_10; if (*(p + 8) == ' ') { - __msg_field_open(&resp->s_line, p); - __FSM_MOVE_nf(Resp_StatusCode, 9, - &resp->s_line); + tfw_http_msg_hdr_open(msg, p); + __FSM_MOVE_n(Resp_StatusCode, 9); } /* fall through */ default: @@ -4641,8 +7685,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, switch (__fsm_n) { case CSTR_POSTPONE: /* Not all the header data is parsed. */ - __FSM_MOVE_nf(Resp_StatusCode, __fsm_sz, - &resp->s_line); + __FSM_MOVE_n(Resp_StatusCode, __fsm_sz); case CSTR_BADLEN: case CSTR_NEQ: /* bad status value */ @@ -4658,8 +7701,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, { __set_bit(TFW_HTTP_B_VOID_BODY, resp->flags); } - __FSM_MOVE_nf(Resp_ReasonPhrase, __fsm_n, - &resp->s_line); + __FSM_MOVE_n(Resp_ReasonPhrase, __fsm_n); } } @@ -4669,11 +7711,12 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, * reason-phrase = *( HTAB / SP / VCHAR / obs-text ) */ __FSM_STATE(Resp_ReasonPhrase) { - __FSM_MATCH_MOVE_f(ctext_vchar, Resp_ReasonPhrase, - &resp->s_line); - if (IS_CRLF(*(p + __fsm_sz))) { - __msg_field_finish(&resp->s_line, p + __fsm_sz); - __FSM_MOVE_nofixup_n(RGen_EoL, __fsm_sz); + __FSM_MATCH_MOVE(ctext_vchar, Resp_ReasonPhrase); + p += __fsm_sz; + if (IS_CRLF(*p)) { + parser->_hdr_tag = TFW_HTTP_STATUS_LINE; + __msg_hdr_chunk_fixup(data, __data_off(p)); + __FSM_JMP(RGen_EoL); } TFW_PARSER_BLOCK(Resp_ReasonPhrase); } @@ -4692,11 +7735,43 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, switch (TFW_LC(c)) { case 'a': + if (likely(__data_available(p, 28) + && C8_INT_LCM(p, 'a', 'c', 'c', 'e', + 's', 's', '-', 'c') + && C8_INT_LCM(p + 8, 'o', 'n', 't', 'r', + 'o', 'l', '-', 'a') + && C8_INT_LCM(p + 16, 'l', 'l', 'o', 'w', + '-', 'o', 'r', 'i') + && C4_INT3_LCM(p + 24, 'g', 'i', 'n', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(20); + __FSM_MOVE_n(RGen_LWS, 28); + } + if (likely(__data_available(p, 14) + && C8_INT_LCM(p + 1, 'c', 'c', 'e', 'p', + 't', '-', 'r', 'a') + && C4_INT_LCM(p + 9, 'n', 'g', 'e', 's') + && *(p + 13) == ':')) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(18); + __FSM_MOVE_n(RGen_LWS, 14); + } + if (likely(__data_available(p, 6) + && C4_INT_LCM(p + 1, 'l', 'l', 'o', 'w') + && *(p + 5) == ':')) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(22); + __FSM_MOVE_n(RGen_LWS, 6); + } if (likely(__data_available(p, 4) && C4_INT3_LCM(p, 'a', 'g', 'e', ':'))) { parser->_i_st = &&Resp_HdrAgeV; - __FSM_MOVE_n(RGen_OWS, 4); + __msg_hdr_set_hpack_index(21); + __FSM_MOVE_n(RGen_LWS, 4); } __FSM_MOVE(Resp_HdrA); case 'c': @@ -4712,7 +7787,8 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, ':'))) { parser->_i_st = &&Resp_HdrCache_CtrlV; - __FSM_MOVE_n(RGen_OWS, 14); + __msg_hdr_set_hpack_index(24); + __FSM_MOVE_n(RGen_LWS, 14); } __FSM_MOVE_n(RGen_HdrOther, 5); case TFW_CHAR4_INT('o', 'n', 'n', 'e'): @@ -4721,7 +7797,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && *(p + 10) == ':')) { parser->_i_st = &&Resp_HdrConnectionV; - __FSM_MOVE_n(RGen_OWS, 11); + __FSM_MOVE_n(RGen_LWS, 11); } __FSM_MOVE_n(RGen_HdrOther, 5); case TFW_CHAR4_INT('o', 'n', 't', 'e'): @@ -4740,7 +7816,8 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C4_INT3_LCM(p + 1, 'a', 't', 'e', ':'))) { parser->_i_st = &&Resp_HdrDateV; - __FSM_MOVE_n(RGen_OWS, 5); + __msg_hdr_set_hpack_index(33); + __FSM_MOVE_n(RGen_LWS, 5); } __FSM_MOVE(Resp_HdrD); case 'e': @@ -4748,14 +7825,16 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C4_INT3_LCM(p + 1, 't', 'a', 'g', ':'))) { parser->_i_st = &&Resp_HdrEtagV; - __FSM_MOVE_n(RGen_OWS, 5); + __msg_hdr_set_hpack_index(34); + __FSM_MOVE_n(RGen_LWS, 5); } if (likely(__data_available(p, 8) && C8_INT7_LCM(p, 'e', 'x', 'p', 'i', 'r', 'e', 's', ':'))) { parser->_i_st = &&Resp_HdrExpiresV; - __FSM_MOVE_n(RGen_OWS, 8); + __msg_hdr_set_hpack_index(36); + __FSM_MOVE_n(RGen_LWS, 8); } __FSM_MOVE(Resp_HdrE); case 'k': @@ -4767,7 +7846,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && *(p + 10) == ':')) { parser->_i_st = &&Resp_HdrKeep_AliveV; - __FSM_MOVE_n(RGen_OWS, 11); + __FSM_MOVE_n(RGen_LWS, 11); } __FSM_MOVE(Resp_HdrK); case 'l': @@ -4779,29 +7858,90 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && *(p + 13) == ':')) { parser->_i_st = &&Resp_HdrLast_ModifiedV; - __FSM_MOVE_n(RGen_OWS, 14); + __msg_hdr_set_hpack_index(44); + __FSM_MOVE_n(RGen_LWS, 14); + } + if (likely(__data_available(p, 9) + && C8_INT7_LCM(p + 1, 'o', 'c', 'a', 't', + 'i', 'o', 'n', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(46); + __FSM_MOVE_n(RGen_LWS, 9); + } + if (likely(__data_available(p, 5) + && C4_INT3_LCM(p + 1, 'i', 'n', 'k', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(45); + __FSM_MOVE_n(RGen_LWS, 5); } __FSM_MOVE(Resp_HdrL); - case 'p': + if (likely(__data_available(p, 19) + && C8_INT_LCM(p + 1, 'r', 'o', 'x', 'y', + '-', 'a', 'u', 't') + && C8_INT_LCM(p + 9, 'h', 'e', 'n', 't', + 'i', 'c', 'a', 't') + && TFW_LC(*(p + 17)) == 'e' + && *(p + 18) == ':')) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(48); + __FSM_MOVE_n(RGen_LWS, 19); + } if (likely(__data_available(p, 7)) && C4_INT_LCM(p + 1, 'r', 'a', 'g', 'm') && TFW_LC(*(p + 5)) == 'a' && *(p + 6) == ':') { parser->_i_st = &&Resp_HdrPragmaV; - __FSM_MOVE_n(RGen_OWS, 7); + __FSM_MOVE_n(RGen_LWS, 7); } __FSM_MOVE(Resp_HdrP); - + case 'r': + if (likely(__data_available(p, 12) + && C8_INT_LCM(p, 'r', 'e', 't', 'r', + 'y', '-', 'a', 'f') + && C4_INT3_LCM(p + 8, 't', 'e', 'r', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(53); + __FSM_MOVE_n(RGen_LWS, 12); + } + __FSM_MOVE(Resp_HdrR); case 's': + if (likely(__data_available(p, 26) + && C8_INT_LCM(p + 1, 't', 'r', 'i', 'c', + 't', '-', 't', 'r') + && C8_INT_LCM(p + 9, 'a', 'n', 's', 'p', + 'o', 'r', 't', '-') + && C8_INT_LCM(p + 17, 's', 'e', 'c', 'u', + 'r', 'i', 't', 'y') + && *(p + 25) == ':')) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(56); + __FSM_MOVE_n(RGen_LWS, 26); + } + if (likely(__data_available(p, 11) + && C8_INT_LCM(p + 1, 'e', 't', '-', 'c', + 'o', 'o', 'k', 'i') + && TFW_LC(*(p + 9)) == 'e' + && *(p + 10) == ':')) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(55); + __FSM_MOVE_n(RGen_LWS, 11); + } if (likely(__data_available(p, 7) && C4_INT_LCM(p + 1, 'e', 'r', 'v', 'e') && TFW_LC(*(p + 5)) == 'r' && *(p + 6) == ':')) { parser->_i_st = &&Resp_HdrServerV; - __FSM_MOVE_n(RGen_OWS, 7); + __msg_hdr_set_hpack_index(54); + __FSM_MOVE_n(RGen_LWS, 7); } __FSM_MOVE(Resp_HdrS); case 't': @@ -4814,9 +7954,37 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && *(p + 17) == ':')) { parser->_i_st = &&Resp_HdrTransfer_EncodingV; - __FSM_MOVE_n(RGen_OWS, 18); + __FSM_MOVE_n(RGen_LWS, 18); } __FSM_MOVE(Resp_HdrT); + case 'v': + if (likely(__data_available(p, 5) + && C4_INT3_LCM(p + 1, 'a', 'r', 'y', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(59); + __FSM_MOVE_n(RGen_LWS, 5); + } + if (likely(__data_available(p, 4) + && C4_INT3_LCM(p, 'v', 'i', 'a', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(60); + __FSM_MOVE_n(RGen_LWS, 4); + } + __FSM_MOVE(Resp_HdrV); + case 'w': + if (likely(__data_available(p, 17) + && C8_INT_LCM(p + 1, 'w', 'w', '-', 'a', + 'u', 't', 'h', 'e') + && C8_INT7_LCM(p + 9, 'n', 't', 'i', 'c', + 'a', 't', 'e', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(61); + __FSM_MOVE_n(RGen_LWS, 17); + } + __FSM_MOVE(Resp_HdrW); default: __FSM_JMP(RGen_HdrOther); } @@ -4825,22 +7993,71 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, /* Content-* headers. */ __FSM_STATE(Resp_HdrContent_) { switch (TFW_LC(c)) { + case 'd': + if (likely(__data_available(p, 12) + && C8_INT_LCM(p, 'd', 'i', 's', 'p', + 'o', 's', 'i', 't') + && C4_INT3_LCM(p + 8, 'i', 'o', 'n', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(25); + __FSM_MOVE_n(RGen_LWS, 12); + } + __FSM_MOVE(Resp_HdrContent_D); + case 'e': + if (likely(__data_available(p, 9) + && C8_INT7_LCM(p + 1, 'n', 'c', 'o', 'd', + 'i', 'n', 'g', ':'))) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(26); + __FSM_MOVE_n(RGen_LWS, 9); + } + __FSM_MOVE(Resp_HdrContent_E); case 'l': + if (likely(__data_available(p, 9))) { + if (C8_INT7_LCM(p + 1, 'a', 'n', 'g', 'u', + 'a', 'g', 'e', ':')) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(27); + __FSM_MOVE_n(RGen_LWS, 9); + } + if (C8_INT7_LCM(p + 1, 'o', 'c', 'a', 't', + 'i', 'o', 'n', ':')) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(29); + __FSM_MOVE_n(RGen_LWS, 9); + } + } if (likely(__data_available(p, 7) && C4_INT_LCM(p + 1, 'e', 'n', 'g', 't') && TFW_LC(*(p + 5)) == 'h' && *(p + 6) == ':')) { parser->_i_st = &&Resp_HdrContent_LengthV; - __FSM_MOVE_n(RGen_OWS, 7); + __msg_hdr_set_hpack_index(28); + __FSM_MOVE_n(RGen_LWS, 7); } __FSM_MOVE(Resp_HdrContent_L); + case 'r': + if (likely(__data_available(p, 6) + && C4_INT_LCM(p + 1, 'a', 'n', 'g', 'e') + && *(p + 5) == ':')) + { + parser->_i_st = &&RGen_HdrOtherV; + __msg_hdr_set_hpack_index(30); + __FSM_MOVE_n(RGen_LWS, 6); + } + __FSM_MOVE(Resp_HdrContent_R); case 't': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'y', 'p', 'e', ':'))) { parser->_i_st = &&Resp_HdrContent_TypeV; - __FSM_MOVE_n(RGen_OWS, 5); + __msg_hdr_set_hpack_index(31); + __FSM_MOVE_n(RGen_LWS, 5); } __FSM_MOVE(Resp_HdrContent_T); default: @@ -4913,30 +8130,107 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, /* ---------------- Improbable states ---------------- */ /* Parse HTTP version and SP (1.1 and 1.0 are supported). */ - __FSM_TX_f(Resp_HttpVerT1, 'T', Resp_HttpVerT2, &resp->s_line); - __FSM_TX_f(Resp_HttpVerT2, 'T', Resp_HttpVerP, &resp->s_line); - __FSM_TX_f(Resp_HttpVerP, 'P', Resp_HttpVerSlash, &resp->s_line); - __FSM_TX_f(Resp_HttpVerSlash, '/', Resp_HttpVer11, &resp->s_line); - __FSM_TX_f(Resp_HttpVer11, '1', Resp_HttpVerDot, &resp->s_line); - __FSM_TX_f(Resp_HttpVerDot, '.', Resp_HttpVer12, &resp->s_line); + __FSM_TX(Resp_HttpVerT1, 'T', Resp_HttpVerT2); + __FSM_TX(Resp_HttpVerT2, 'T', Resp_HttpVerP); + __FSM_TX(Resp_HttpVerP, 'P', Resp_HttpVerSlash); + __FSM_TX(Resp_HttpVerSlash, '/', Resp_HttpVer11); + __FSM_TX(Resp_HttpVer11, '1', Resp_HttpVerDot); + __FSM_TX(Resp_HttpVerDot, '.', Resp_HttpVer12); __FSM_STATE(Resp_HttpVer12) { switch (c) { case '1': resp->version = TFW_HTTP_VER_11; - __FSM_MOVE_f(Resp_SSpace, &resp->s_line); + __FSM_MOVE(Resp_SSpace); case '0': resp->version = TFW_HTTP_VER_10; - __FSM_MOVE_f(Resp_SSpace, &resp->s_line); + __FSM_MOVE(Resp_SSpace); default: TFW_PARSER_BLOCK(Resp_HttpVer12); } } - __FSM_TX_f(Resp_SSpace, ' ', Resp_StatusCode, &resp->s_line); + __FSM_TX(Resp_SSpace, ' ', Resp_StatusCode); + + __FSM_STATE(Resp_HdrA) { + switch (TFW_LC(c)) { + case 'c': + __FSM_MOVE(Resp_HdrAc); + case 'l': + __FSM_MOVE(Resp_HdrAl); + case 'g': + __FSM_MOVE(Resp_HdrAg); + default: + __FSM_JMP(RGen_HdrOther); + } + } + __FSM_TX_AF(Resp_HdrAc, 'c', Resp_HdrAcc); + __FSM_TX_AF(Resp_HdrAcc, 'e', Resp_HdrAcce); + + __FSM_STATE(Resp_HdrAcce) { + switch (TFW_LC(c)) { + case 'p': + __FSM_MOVE(Resp_HdrAccep); + case 's': + __FSM_MOVE(Resp_HdrAcces); + default: + __FSM_JMP(RGen_HdrOther); + } + } + + /* Access-Control-Allow-Origin header processing. */ + __FSM_TX_AF(Resp_HdrAcces, 's', Resp_HdrAccess); + __FSM_TX_AF(Resp_HdrAccess, '-', Resp_HdrAccess_); + __FSM_TX_AF(Resp_HdrAccess_, 'c', Resp_HdrAccess_C); + __FSM_TX_AF(Resp_HdrAccess_C, 'o', Resp_HdrAccess_Co); + __FSM_TX_AF(Resp_HdrAccess_Co, 'n', Resp_HdrAccess_Con); + __FSM_TX_AF(Resp_HdrAccess_Con, 't', Resp_HdrAccess_Cont); + __FSM_TX_AF(Resp_HdrAccess_Cont, 'r', Resp_HdrAccess_Contr); + __FSM_TX_AF(Resp_HdrAccess_Contr, 'o', Resp_HdrAccess_Contro); + __FSM_TX_AF(Resp_HdrAccess_Contro, 'l', Resp_HdrAccess_Control); + __FSM_TX_AF(Resp_HdrAccess_Control, '-', Resp_HdrAccess_Control_); + __FSM_TX_AF(Resp_HdrAccess_Control_, 'a', Resp_HdrAccess_Control_A); + __FSM_TX_AF(Resp_HdrAccess_Control_A, 'l', Resp_HdrAccess_Control_Al); + __FSM_TX_AF(Resp_HdrAccess_Control_Al, 'l', Resp_HdrAccess_Control_All); + __FSM_TX_AF(Resp_HdrAccess_Control_All, 'o', + Resp_HdrAccess_Control_Allo); + __FSM_TX_AF(Resp_HdrAccess_Control_Allo, 'w', + Resp_HdrAccess_Control_Allow); + __FSM_TX_AF(Resp_HdrAccess_Control_Allow, '-', + Resp_HdrAccess_Control_Allow_); + __FSM_TX_AF(Resp_HdrAccess_Control_Allow_, 'o', + Resp_HdrAccess_Control_Allow_O); + __FSM_TX_AF(Resp_HdrAccess_Control_Allow_O, 'r', + Resp_HdrAccess_Control_Allow_Or); + __FSM_TX_AF(Resp_HdrAccess_Control_Allow_Or, 'i', + Resp_HdrAccess_Control_Allow_Ori); + __FSM_TX_AF(Resp_HdrAccess_Control_Allow_Ori, 'g', + Resp_HdrAccess_Control_Allow_Orig); + __FSM_TX_AF(Resp_HdrAccess_Control_Allow_Orig, 'i', + Resp_HdrAccess_Control_Allow_Origi); + __FSM_TX_AF(Resp_HdrAccess_Control_Allow_Origi, 'n', + Resp_HdrAccess_Control_Allow_Origin); + __FSM_TX_AF_OWS_HP(Resp_HdrAccess_Control_Allow_Origin, + RGen_HdrOtherV, 20); + + /* Accept-Ranges header processing. */ + __FSM_TX_AF(Resp_HdrAccep, 't', Resp_HdrAccept); + __FSM_TX_AF(Resp_HdrAccept, '-', Resp_HdrAccept_); + __FSM_TX_AF(Resp_HdrAccept_, 'r', Resp_HdrAccept_R); + __FSM_TX_AF(Resp_HdrAccept_R, 'a', Resp_HdrAccept_Ra); + __FSM_TX_AF(Resp_HdrAccept_Ra, 'n', Resp_HdrAccept_Ran); + __FSM_TX_AF(Resp_HdrAccept_Ran, 'g', Resp_HdrAccept_Rang); + __FSM_TX_AF(Resp_HdrAccept_Rang, 'e', Resp_HdrAccept_Range); + __FSM_TX_AF(Resp_HdrAccept_Range, 's', Resp_HdrAccept_Ranges); + __FSM_TX_AF_OWS_HP(Resp_HdrAccept_Ranges, RGen_HdrOtherV, 18); + + /* Allow header processing. */ + __FSM_TX_AF(Resp_HdrAl, 'l', Resp_HdrAll); + __FSM_TX_AF(Resp_HdrAll, 'o', Resp_HdrAllo); + __FSM_TX_AF(Resp_HdrAllo, 'w', Resp_HdrAllow); + __FSM_TX_AF_OWS_HP(Resp_HdrAllow, RGen_HdrOtherV, 22); /* Age header processing. */ - __FSM_TX_AF(Resp_HdrA, 'g', Resp_HdrAg); __FSM_TX_AF(Resp_HdrAg, 'e', Resp_HdrAge); - __FSM_TX_AF_OWS(Resp_HdrAge, Resp_HdrAgeV); + __FSM_TX_AF_OWS_HP(Resp_HdrAge, Resp_HdrAgeV, 21); __FSM_STATE(Resp_HdrC) { switch (TFW_LC(c)) { @@ -4961,7 +8255,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __FSM_TX_AF(Resp_HdrCache_Cont, 'r', Resp_HdrCache_Contr); __FSM_TX_AF(Resp_HdrCache_Contr, 'o', Resp_HdrCache_Contro); __FSM_TX_AF(Resp_HdrCache_Contro, 'l', Resp_HdrCache_Control); - __FSM_TX_AF_OWS(Resp_HdrCache_Control, Resp_HdrCache_CtrlV); + __FSM_TX_AF_OWS_HP(Resp_HdrCache_Control, Resp_HdrCache_CtrlV, 24); /* Connection header processing. */ __FSM_TX_AF(Resp_HdrCo, 'n', Resp_HdrCon); @@ -4989,25 +8283,85 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __FSM_TX_AF(Resp_HdrConten, 't', Resp_HdrContent); __FSM_TX_AF(Resp_HdrContent, '-', Resp_HdrContent_); + /* Content-Disposition header processing. */ + __FSM_TX_AF(Resp_HdrContent_D, 'i', Resp_HdrContent_Di); + __FSM_TX_AF(Resp_HdrContent_Di, 's', Resp_HdrContent_Dis); + __FSM_TX_AF(Resp_HdrContent_Dis, 'p', Resp_HdrContent_Disp); + __FSM_TX_AF(Resp_HdrContent_Disp, 'o', Resp_HdrContent_Dispo); + __FSM_TX_AF(Resp_HdrContent_Dispo, 's', Resp_HdrContent_Dispos); + __FSM_TX_AF(Resp_HdrContent_Dispos, 'i', Resp_HdrContent_Disposi); + __FSM_TX_AF(Resp_HdrContent_Disposi, 't', Resp_HdrContent_Disposit); + __FSM_TX_AF(Resp_HdrContent_Disposit, 'i', Resp_HdrContent_Dispositi); + __FSM_TX_AF(Resp_HdrContent_Dispositi, 'o', Resp_HdrContent_Dispositio); + __FSM_TX_AF(Resp_HdrContent_Dispositio, 'n', Resp_HdrContent_Disposition); + __FSM_TX_AF_OWS_HP(Resp_HdrContent_Disposition, RGen_HdrOtherV, 25); + + /* Content-Encoding header processing. */ + __FSM_TX_AF(Resp_HdrContent_E, 'n', Resp_HdrContent_En); + __FSM_TX_AF(Resp_HdrContent_En, 'c', Resp_HdrContent_Enc); + __FSM_TX_AF(Resp_HdrContent_Enc, 'o', Resp_HdrContent_Enco); + __FSM_TX_AF(Resp_HdrContent_Enco, 'd', Resp_HdrContent_Encod); + __FSM_TX_AF(Resp_HdrContent_Encod, 'i', Resp_HdrContent_Encodi); + __FSM_TX_AF(Resp_HdrContent_Encodi, 'n', Resp_HdrContent_Encodin); + __FSM_TX_AF(Resp_HdrContent_Encodin, 'g', Resp_HdrContent_Encoding); + __FSM_TX_AF_OWS_HP(Resp_HdrContent_Encoding, RGen_HdrOtherV, 26); + + __FSM_STATE(Resp_HdrContent_L) { + switch (TFW_LC(c)) { + case 'a': + __FSM_MOVE(Resp_HdrContent_La); + case 'e': + __FSM_MOVE(Resp_HdrContent_Le); + case 'o': + __FSM_MOVE(Resp_HdrContent_Lo); + default: + __FSM_JMP(RGen_HdrOther); + } + } + + /* Content-Language header processing. */ + __FSM_TX_AF(Resp_HdrContent_La, 'n', Resp_HdrContent_Lan); + __FSM_TX_AF(Resp_HdrContent_Lan, 'g', Resp_HdrContent_Lang); + __FSM_TX_AF(Resp_HdrContent_Lang, 'u', Resp_HdrContent_Langu); + __FSM_TX_AF(Resp_HdrContent_Langu, 'a', Resp_HdrContent_Langua); + __FSM_TX_AF(Resp_HdrContent_Langua, 'g', Resp_HdrContent_Languag); + __FSM_TX_AF(Resp_HdrContent_Languag, 'e', Resp_HdrContent_Language); + __FSM_TX_AF_OWS_HP(Resp_HdrContent_Language, RGen_HdrOtherV, 27); + /* Content-Length header processing. */ - __FSM_TX_AF(Resp_HdrContent_L, 'e', Resp_HdrContent_Le); __FSM_TX_AF(Resp_HdrContent_Le, 'n', Resp_HdrContent_Len); __FSM_TX_AF(Resp_HdrContent_Len, 'g', Resp_HdrContent_Leng); __FSM_TX_AF(Resp_HdrContent_Leng, 't', Resp_HdrContent_Lengt); __FSM_TX_AF(Resp_HdrContent_Lengt, 'h', Resp_HdrContent_Length); - __FSM_TX_AF_OWS(Resp_HdrContent_Length, Resp_HdrContent_LengthV); + __FSM_TX_AF_OWS_HP(Resp_HdrContent_Length, Resp_HdrContent_LengthV, 28); + + /* Content-Location header processing. */ + __FSM_TX_AF(Resp_HdrContent_Lo, 'c', Resp_HdrContent_Loc); + __FSM_TX_AF(Resp_HdrContent_Loc, 'a', Resp_HdrContent_Loca); + __FSM_TX_AF(Resp_HdrContent_Loca, 't', Resp_HdrContent_Locat); + __FSM_TX_AF(Resp_HdrContent_Locat, 'i', Resp_HdrContent_Locati); + __FSM_TX_AF(Resp_HdrContent_Locati, 'o', Resp_HdrContent_Locatio); + __FSM_TX_AF(Resp_HdrContent_Locatio, 'n', Resp_HdrContent_Location); + __FSM_TX_AF_OWS_HP(Resp_HdrContent_Location, RGen_HdrOtherV, 29); + + /* Content-Range header processing. */ + __FSM_TX_AF(Resp_HdrContent_R, 'a', Resp_HdrContent_Ra); + __FSM_TX_AF(Resp_HdrContent_Ra, 'n', Resp_HdrContent_Ran); + __FSM_TX_AF(Resp_HdrContent_Ran, 'g', Resp_HdrContent_Rang); + __FSM_TX_AF(Resp_HdrContent_Rang, 'e', Resp_HdrContent_Range); + __FSM_TX_AF_OWS_HP(Resp_HdrContent_Range, RGen_HdrOtherV, 30); /* Content-Type header processing. */ __FSM_TX_AF(Resp_HdrContent_T, 'y', Resp_HdrContent_Ty); __FSM_TX_AF(Resp_HdrContent_Ty, 'p', Resp_HdrContent_Typ); __FSM_TX_AF(Resp_HdrContent_Typ, 'e', Resp_HdrContent_Type); - __FSM_TX_AF_OWS(Resp_HdrContent_Type, Resp_HdrContent_TypeV); + __FSM_TX_AF_OWS_HP(Resp_HdrContent_Type, Resp_HdrContent_TypeV, 31); /* Date header processing. */ __FSM_TX_AF(Resp_HdrD, 'a', Resp_HdrDa); __FSM_TX_AF(Resp_HdrDa, 't', Resp_HdrDat); __FSM_TX_AF(Resp_HdrDat, 'e', Resp_HdrDate); - __FSM_TX_AF_OWS(Resp_HdrDate, Resp_HdrDateV); + __FSM_TX_AF_OWS_HP(Resp_HdrDate, Resp_HdrDateV, 33); __FSM_STATE(Resp_HdrE) { switch (TFW_LC(c)) { @@ -5022,7 +8376,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, /* ETag header processing. */ __FSM_TX_AF(Resp_HdrEt, 'a', Resp_HdrEta); __FSM_TX_AF(Resp_HdrEta, 'g', Resp_HdrEtag); - __FSM_TX_AF_OWS(Resp_HdrEtag, Resp_HdrEtagV); + __FSM_TX_AF_OWS_HP(Resp_HdrEtag, Resp_HdrEtagV, 34); /* Expires header processing. */ __FSM_TX_AF(Resp_HdrEx, 'p', Resp_HdrExp); @@ -5030,7 +8384,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __FSM_TX_AF(Resp_HdrExpi, 'r', Resp_HdrExpir); __FSM_TX_AF(Resp_HdrExpir, 'e', Resp_HdrExpire); __FSM_TX_AF(Resp_HdrExpire, 's', Resp_HdrExpires); - __FSM_TX_AF_OWS(Resp_HdrExpires, Resp_HdrExpiresV); + __FSM_TX_AF_OWS_HP(Resp_HdrExpires, Resp_HdrExpiresV, 36); /* Keep-Alive header processing. */ __FSM_TX_AF(Resp_HdrK, 'e', Resp_HdrKe); @@ -5044,8 +8398,20 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __FSM_TX_AF(Resp_HdrKeep_Aliv, 'e', Resp_HdrKeep_Alive); __FSM_TX_AF_OWS(Resp_HdrKeep_Alive, Resp_HdrKeep_AliveV); + __FSM_STATE(Resp_HdrL) { + switch (TFW_LC(c)) { + case 'a': + __FSM_MOVE(Resp_HdrLa); + case 'i': + __FSM_MOVE(Resp_HdrLi); + case 'o': + __FSM_MOVE(Resp_HdrLo); + default: + __FSM_JMP(RGen_HdrOther); + } + } + /* Last-Modified header processing. */ - __FSM_TX_AF(Resp_HdrL, 'a', Resp_HdrLa); __FSM_TX_AF(Resp_HdrLa, 's', Resp_HdrLas); __FSM_TX_AF(Resp_HdrLas, 't', Resp_HdrLast); __FSM_TX_AF(Resp_HdrLast, '-', Resp_HdrLast_); @@ -5057,23 +8423,142 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __FSM_TX_AF(Resp_HdrLast_Modif, 'i', Resp_HdrLast_Modifi); __FSM_TX_AF(Resp_HdrLast_Modifi, 'e', Resp_HdrLast_Modifie); __FSM_TX_AF(Resp_HdrLast_Modifie, 'd', Resp_HdrLast_Modified); - __FSM_TX_AF_OWS(Resp_HdrLast_Modified, Resp_HdrLast_ModifiedV); + __FSM_TX_AF_OWS_HP(Resp_HdrLast_Modified, Resp_HdrLast_ModifiedV, 44); + + /* Link header processing. */ + __FSM_TX_AF(Resp_HdrLi, 'n', Resp_HdrLin); + __FSM_TX_AF(Resp_HdrLin, 'k', Resp_HdrLink); + __FSM_TX_AF_OWS_HP(Resp_HdrLink, RGen_HdrOtherV, 45); + + /* Location header processing. */ + __FSM_TX_AF(Resp_HdrLo, 'c', Resp_HdrLoc); + __FSM_TX_AF(Resp_HdrLoc, 'a', Resp_HdrLoca); + __FSM_TX_AF(Resp_HdrLoca, 't', Resp_HdrLocat); + __FSM_TX_AF(Resp_HdrLocat, 'i', Resp_HdrLocati); + __FSM_TX_AF(Resp_HdrLocati, 'o', Resp_HdrLocatio); + __FSM_TX_AF(Resp_HdrLocatio, 'n', Resp_HdrLocation); + __FSM_TX_AF_OWS_HP(Resp_HdrLocation, RGen_HdrOtherV, 46); - /* Pragma header processing. */ __FSM_TX_AF(Resp_HdrP, 'r', Resp_HdrPr); - __FSM_TX_AF(Resp_HdrPr, 'a', Resp_HdrPra); + __FSM_STATE(Resp_HdrPr) { + switch (TFW_LC(c)) { + case 'a': + __FSM_MOVE(Resp_HdrPra); + case 'o': + __FSM_MOVE(Resp_HdrPro); + default: + __FSM_JMP(RGen_HdrOther); + } + } + + /* Proxy-Authenticate header processing. */ + __FSM_TX_AF(Resp_HdrPro, 'x', Resp_HdrProx); + __FSM_TX_AF(Resp_HdrProx, 'y', Resp_HdrProxy); + __FSM_TX_AF(Resp_HdrProxy, '-', Resp_HdrProxy_); + __FSM_TX_AF(Resp_HdrProxy_, 'a', Resp_HdrProxy_A); + __FSM_TX_AF(Resp_HdrProxy_A, 'u', Resp_HdrProxy_Au); + __FSM_TX_AF(Resp_HdrProxy_Au, 't', Resp_HdrProxy_Aut); + __FSM_TX_AF(Resp_HdrProxy_Aut, 'h', Resp_HdrProxy_Auth); + __FSM_TX_AF(Resp_HdrProxy_Auth, 'e', Resp_HdrProxy_Authe); + __FSM_TX_AF(Resp_HdrProxy_Authe, 'n', Resp_HdrProxy_Authen); + __FSM_TX_AF(Resp_HdrProxy_Authen, 't', Resp_HdrProxy_Authent); + __FSM_TX_AF(Resp_HdrProxy_Authent, 'i', Resp_HdrProxy_Authenti); + __FSM_TX_AF(Resp_HdrProxy_Authenti, 'c', Resp_HdrProxy_Authentic); + __FSM_TX_AF(Resp_HdrProxy_Authentic, 'a', Resp_HdrProxy_Authentica); + __FSM_TX_AF(Resp_HdrProxy_Authentica, 't', Resp_HdrProxy_Authenticat); + __FSM_TX_AF(Resp_HdrProxy_Authenticat, 'e', Resp_HdrProxy_Authenticate); + __FSM_TX_AF_OWS_HP(Resp_HdrProxy_Authenticate, RGen_HdrOtherV, 48); + + /* Pragma header processing. */ __FSM_TX_AF(Resp_HdrPra, 'g', Resp_HdrPrag); __FSM_TX_AF(Resp_HdrPrag, 'm', Resp_HdrPragm); __FSM_TX_AF(Resp_HdrPragm, 'a', Resp_HdrPragma); __FSM_TX_AF_OWS(Resp_HdrPragma, Resp_HdrPragmaV); + /* Retry-After header processing. */ + __FSM_TX_AF(Resp_HdrR, 'e', Resp_HdrRe); + __FSM_TX_AF(Resp_HdrRe, 't', Resp_HdrRet); + __FSM_TX_AF(Resp_HdrRet, 'r', Resp_HdrRetr); + __FSM_TX_AF(Resp_HdrRetr, 'y', Resp_HdrRetry); + __FSM_TX_AF(Resp_HdrRetry, '-', Resp_HdrRetry_); + __FSM_TX_AF(Resp_HdrRetry_, 'a', Resp_HdrRetry_A); + __FSM_TX_AF(Resp_HdrRetry_A, 'f', Resp_HdrRetry_Af); + __FSM_TX_AF(Resp_HdrRetry_Af, 't', Resp_HdrRetry_Aft); + __FSM_TX_AF(Resp_HdrRetry_Aft, 'e', Resp_HdrRetry_Afte); + __FSM_TX_AF(Resp_HdrRetry_Afte, 'r', Resp_HdrRetry_After); + __FSM_TX_AF_OWS_HP(Resp_HdrRetry_After, RGen_HdrOtherV, 53); + + __FSM_STATE(Resp_HdrS) { + switch (TFW_LC(c)) { + case 'e': + __FSM_MOVE(Resp_HdrSe); + case 't': + __FSM_MOVE(Resp_HdrSt); + default: + __FSM_JMP(RGen_HdrOther); + } + } + + /* Strict-Transport-Security header processing. */ + __FSM_TX_AF(Resp_HdrSt, 'r', Resp_HdrStr); + __FSM_TX_AF(Resp_HdrStr, 'i', Resp_HdrStri); + __FSM_TX_AF(Resp_HdrStri, 'c', Resp_HdrStric); + __FSM_TX_AF(Resp_HdrStric, 't', Resp_HdrStrict); + __FSM_TX_AF(Resp_HdrStrict, '-', Resp_HdrStrict_); + __FSM_TX_AF(Resp_HdrStrict_, 't', Resp_HdrStrict_T); + __FSM_TX_AF(Resp_HdrStrict_T, 'r', Resp_HdrStrict_Tr); + __FSM_TX_AF(Resp_HdrStrict_Tr, 'a', Resp_HdrStrict_Tra); + __FSM_TX_AF(Resp_HdrStrict_Tra, 'n', Resp_HdrStrict_Tran); + __FSM_TX_AF(Resp_HdrStrict_Tran, 's', Resp_HdrStrict_Trans); + __FSM_TX_AF(Resp_HdrStrict_Trans, 'p', Resp_HdrStrict_Transp); + __FSM_TX_AF(Resp_HdrStrict_Transp, 'o', Resp_HdrStrict_Transpo); + __FSM_TX_AF(Resp_HdrStrict_Transpo, 'r', Resp_HdrStrict_Transpor); + __FSM_TX_AF(Resp_HdrStrict_Transpor, 't', Resp_HdrStrict_Transport); + __FSM_TX_AF(Resp_HdrStrict_Transport, '-', Resp_HdrStrict_Transport_); + __FSM_TX_AF(Resp_HdrStrict_Transport_, 's', Resp_HdrStrict_Transport_S); + __FSM_TX_AF(Resp_HdrStrict_Transport_S, 'e', + Resp_HdrStrict_Transport_Se); + __FSM_TX_AF(Resp_HdrStrict_Transport_Se, 'c', + Resp_HdrStrict_Transport_Sec); + __FSM_TX_AF(Resp_HdrStrict_Transport_Sec, 'u', + Resp_HdrStrict_Transport_Secu); + __FSM_TX_AF(Resp_HdrStrict_Transport_Secu, 'r', + Resp_HdrStrict_Transport_Secur); + __FSM_TX_AF(Resp_HdrStrict_Transport_Secur, 'i', + Resp_HdrStrict_Transport_Securi); + __FSM_TX_AF(Resp_HdrStrict_Transport_Securi, 't', + Resp_HdrStrict_Transport_Securit); + __FSM_TX_AF(Resp_HdrStrict_Transport_Securit, 'y', + Resp_HdrStrict_Transport_Security); + __FSM_TX_AF_OWS_HP(Resp_HdrStrict_Transport_Security, + RGen_HdrOtherV, 56); + + __FSM_STATE(Resp_HdrSe) { + switch (TFW_LC(c)) { + case 'r': + __FSM_MOVE(Resp_HdrSer); + case 't': + __FSM_MOVE(Resp_HdrSet); + default: + __FSM_JMP(RGen_HdrOther); + } + } + /* Server header processing. */ - __FSM_TX_AF(Resp_HdrS, 'e', Resp_HdrSe); - __FSM_TX_AF(Resp_HdrSe, 'r', Resp_HdrSer); __FSM_TX_AF(Resp_HdrSer, 'v', Resp_HdrServ); __FSM_TX_AF(Resp_HdrServ, 'e', Resp_HdrServe); __FSM_TX_AF(Resp_HdrServe, 'r', Resp_HdrServer); - __FSM_TX_AF_OWS(Resp_HdrServer, Resp_HdrServerV); + __FSM_TX_AF_OWS_HP(Resp_HdrServer, Resp_HdrServerV, 54); + + /* Set-Cookie header processing. */ + __FSM_TX_AF(Resp_HdrSet, '-', Resp_HdrSet_); + __FSM_TX_AF(Resp_HdrSet_, 'c', Resp_HdrSet_C); + __FSM_TX_AF(Resp_HdrSet_C, 'o', Resp_HdrSet_Co); + __FSM_TX_AF(Resp_HdrSet_Co, 'o', Resp_HdrSet_Coo); + __FSM_TX_AF(Resp_HdrSet_Coo, 'k', Resp_HdrSet_Cook); + __FSM_TX_AF(Resp_HdrSet_Cook, 'i', Resp_HdrSet_Cooki); + __FSM_TX_AF(Resp_HdrSet_Cooki, 'e', Resp_HdrSet_Cookie); + __FSM_TX_AF_OWS_HP(Resp_HdrSet_Cookie, RGen_HdrOtherV, 55); /* Transfer-Encoding header processing. */ __FSM_TX_AF(Resp_HdrT, 'r', Resp_HdrTr); @@ -5094,6 +8579,44 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __FSM_TX_AF(Resp_HdrTransfer_Encodin, 'g', Resp_HdrTransfer_Encoding); __FSM_TX_AF_OWS(Resp_HdrTransfer_Encoding, Resp_HdrTransfer_EncodingV); + __FSM_STATE(Resp_HdrV) { + switch (TFW_LC(c)) { + case 'a': + __FSM_MOVE(Resp_HdrVa); + case 'i': + __FSM_MOVE(Resp_HdrVi); + default: + __FSM_JMP(RGen_HdrOther); + } + } + + /* Vary header processing. */ + __FSM_TX_AF(Resp_HdrVa, 'r', Resp_HdrVar); + __FSM_TX_AF(Resp_HdrVar, 'y', Resp_HdrVary); + __FSM_TX_AF_OWS_HP(Resp_HdrVary, RGen_HdrOtherV, 59); + + /* Via header processing. */ + __FSM_TX_AF(Resp_HdrVi, 'a', Resp_HdrVia); + __FSM_TX_AF_OWS_HP(Resp_HdrVia, RGen_HdrOtherV, 60); + + /* WWW-Authenticate header processing. */ + __FSM_TX_AF(Resp_HdrW, 'w', Resp_HdrWW); + __FSM_TX_AF(Resp_HdrWW, 'w', Resp_HdrWWW); + __FSM_TX_AF(Resp_HdrWWW, '-', Resp_HdrWWW_); + __FSM_TX_AF(Resp_HdrWWW_, 'a', Resp_HdrWWW_A); + __FSM_TX_AF(Resp_HdrWWW_A, 'u', Resp_HdrWWW_Au); + __FSM_TX_AF(Resp_HdrWWW_Au, 't', Resp_HdrWWW_Aut); + __FSM_TX_AF(Resp_HdrWWW_Aut, 'h', Resp_HdrWWW_Auth); + __FSM_TX_AF(Resp_HdrWWW_Auth, 'e', Resp_HdrWWW_Authe); + __FSM_TX_AF(Resp_HdrWWW_Authe, 'n', Resp_HdrWWW_Authen); + __FSM_TX_AF(Resp_HdrWWW_Authen, 't', Resp_HdrWWW_Authent); + __FSM_TX_AF(Resp_HdrWWW_Authent, 'i', Resp_HdrWWW_Authenti); + __FSM_TX_AF(Resp_HdrWWW_Authenti, 'c', Resp_HdrWWW_Authentic); + __FSM_TX_AF(Resp_HdrWWW_Authentic, 'a', Resp_HdrWWW_Authentica); + __FSM_TX_AF(Resp_HdrWWW_Authentica, 't', Resp_HdrWWW_Authenticat); + __FSM_TX_AF(Resp_HdrWWW_Authenticat, 'e', Resp_HdrWWW_Authenticate); + __FSM_TX_AF_OWS_HP(Resp_HdrWWW_Authenticate, RGen_HdrOtherV, 61); + __FSM_FINISH(resp); return r; diff --git a/tempesta_fw/http_parser.h b/tempesta_fw/http_parser.h index c0d401e159..e789852f68 100644 --- a/tempesta_fw/http_parser.h +++ b/tempesta_fw/http_parser.h @@ -105,7 +105,9 @@ void tfw_http_init_parser_resp(TfwHttpResp *resp); int tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, unsigned int *parsed); -int tfw_h2_parse_hdr(const char *data, unsigned long len, TfwHttpReq *req); + +int tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, + bool fin, bool value_stage); int tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, unsigned int *parsed); int tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, diff --git a/tempesta_fw/http_sess.c b/tempesta_fw/http_sess.c index 323c8e1f5b..10256d18f9 100644 --- a/tempesta_fw/http_sess.c +++ b/tempesta_fw/http_sess.c @@ -344,7 +344,7 @@ tfw_http_sticky_get(TfwHttpReq *req, TfwStr *cookie_val) hdr = &req->h_tbl->tbl[TFW_HTTP_HDR_COOKIE]; if (TFW_STR_EMPTY(hdr)) return 0; - tfw_http_msg_clnthdr_val(hdr, TFW_HTTP_HDR_COOKIE, &value); + tfw_http_msg_clnthdr_val(req, hdr, TFW_HTTP_HDR_COOKIE, &value); return search_cookie(req->pool, &value, cookie_val); } @@ -386,7 +386,7 @@ __sticky_calc(TfwHttpReq *req, StickyVal *sv) /* User-Agent header field is not mandatory and may be missing. */ hdr = &req->h_tbl->tbl[TFW_HTTP_HDR_USER_AGENT]; if (!TFW_STR_EMPTY(hdr)) - tfw_http_msg_clnthdr_val(hdr, TFW_HTTP_HDR_USER_AGENT, + tfw_http_msg_clnthdr_val(req, hdr, TFW_HTTP_HDR_USER_AGENT, &ua_value); shash_desc->tfm = tfw_sticky_shash; @@ -429,19 +429,22 @@ tfw_http_sticky_calc(TfwHttpReq *req, StickyVal *sv) static int tfw_http_sticky_add(TfwHttpResp *resp) { - static const unsigned int len = sizeof(StickyVal) * 2; int r; + static const unsigned int len = sizeof(StickyVal) * 2; + bool to_h2 = TFW_MSG_H2(resp->req); + char *name = to_h2 ? S_SET_COOKIE : S_F_SET_COOKIE; + unsigned int nm_len = to_h2 ? SLEN(S_SET_COOKIE) : SLEN(S_F_SET_COOKIE); TfwHttpSess *sess = resp->req->sess; unsigned long ts_be64 = cpu_to_be64(sess->ts); char buf[len]; TfwStr set_cookie = { .chunks = (TfwStr []) { - { .data = S_F_SET_COOKIE, .len = SLEN(S_F_SET_COOKIE) }, + { .data = name, .len = nm_len }, { .data = tfw_cfg_sticky.name_eq.data, .len = tfw_cfg_sticky.name_eq.len }, { .data = buf, .len = len }, }, - .len = SLEN(S_F_SET_COOKIE) + tfw_cfg_sticky.name_eq.len + len, + .len = nm_len + tfw_cfg_sticky.name_eq.len + len, .eolen = 2, .nchunks = 3 }; @@ -450,12 +453,19 @@ tfw_http_sticky_add(TfwHttpResp *resp) bin2hex(buf, &ts_be64, sizeof(ts_be64)); bin2hex(&buf[sizeof(ts_be64) * 2], sess->hmac, sizeof(sess->hmac)); - T_DBG("%s: \"" S_F_SET_COOKIE "%.*s=%.*s\"\n", __func__, + T_DBG("%s: name=%s, val='%.*s=%.*s'\n", __func__, name, PR_TFW_STR(&tfw_cfg_sticky.name), len, buf); - r = tfw_http_msg_hdr_add((TfwHttpMsg *)resp, &set_cookie); - if (r) - T_WARN("Cannot add \"" S_F_SET_COOKIE "%.*s=%.*s\"\n", + if (to_h2) { + TFW_STR_INDEX_SET(&set_cookie, 55); + r = __hdr_h2_add(resp, &set_cookie); + } + else { + r = tfw_http_msg_hdr_add((TfwHttpMsg *)resp, &set_cookie); + } + + if (unlikely(r)) + T_WARN("Cannot add '%s' header: val='%.*s=%.*s'\n", name, PR_TFW_STR(&tfw_cfg_sticky.name), len, buf); return r; } diff --git a/tempesta_fw/msg.h b/tempesta_fw/msg.h index 5dd5f67707..9dc20ba3c4 100644 --- a/tempesta_fw/msg.h +++ b/tempesta_fw/msg.h @@ -65,32 +65,53 @@ typedef struct { * Iterator for HTTP/2 message processing. * * @pool - allocation pool for target buffer of decoded headers; - * @hdr - pointer to message header which is currently processed; + * @parsed_hdr - pointer to the message header which is currently processed; + * @hb_len - length of the message HTTP/2 headers block; + * @hdrs_len - accumulated length of message's decoded and parsed headers; + * @hdrs_cnt - count of all headers from message headers block; * @start_pos - pointer to the beginning of decoded headers' buffer; used * as start point during exporting buffer pages to message's * target skb; * @__off - offset for iterator reinitializing before next processing * stage; + * @hdr - descriptor of currently decoded header in target buffer; * @pos - pointer to the currently allocated chunk of decoded headers' * buffer; * @rspace - space remained in the allocated chunk; * @next - pointer to the decoded header part (name/value) to be * - parsed next; * @nm_len - length of the decoded header's name; - * @hdr_tag - ID of currently processed decoded header. + * @nm_num - chunks number of the decoded header's name; + * @hdr_tag - tag of currently processed decoded header. */ typedef struct { TfwPool *pool; - TfwStr *hdr; + TfwStr *parsed_hdr; + unsigned long hb_len; + unsigned long hdrs_len; + unsigned int hdrs_cnt; char *start_pos; char __off[0]; + TfwStr hdr; char *pos; unsigned long rspace; TfwStr *next; unsigned long nm_len; - unsigned int hdr_tag; + unsigned int nm_num; + unsigned int tag; } TfwMsgParseIter; +/** + * Iterator for message HTTP/2 transformation process. + * + * @it - skb creation/writing iterator; + * @acc_len - accumulated length of transformed message; + */ +typedef struct { + TfwMsgIter iter; + unsigned long acc_len; +} TfwMsgTransIter; + int tfw_msg_write(TfwMsgIter *it, const TfwStr *data); int tfw_msg_iter_setup(TfwMsgIter *it, struct sk_buff **skb_head, size_t data_len, unsigned int tx_flags); diff --git a/tempesta_fw/ss_skb.c b/tempesta_fw/ss_skb.c index efb57a061e..851b7640d7 100644 --- a/tempesta_fw/ss_skb.c +++ b/tempesta_fw/ss_skb.c @@ -832,7 +832,7 @@ __skb_fragment(struct sk_buff *skb_head, struct sk_buff *skb, char *pspt, return abs(len); } -static inline int +int skb_fragment(struct sk_buff *skb_head, struct sk_buff *skb, char *pspt, int len, TfwStr *it) { @@ -1006,6 +1006,65 @@ ss_skb_cutoff_data(struct sk_buff *skb_head, const TfwStr *str, int skip, return 0; } +int +skb_next_data(struct sk_buff *skb, char *last_ptr, TfwStr *it) +{ + int i; + long off; + unsigned int f_size; + struct skb_shared_info *si = skb_shinfo(skb); + + if (unlikely(skb_has_frag_list(skb))) { + WARN_ON(skb_has_frag_list(skb)); + return -EINVAL; + } + + f_size = skb_headlen(skb); + off = last_ptr - (char *)skb->data; + + T_DBG("%s: last_ptr=[%p], skb->data=[%p], si->nr_frags=%u, f_size=%u," + " off=ld\n", __func__, last_ptr, skb->data, si->nr_frags, f_size, + off); + + if (off >= 0 && off < f_size) { + if (f_size - off > 1) { + it->data = last_ptr + 1; + it->skb = skb; + return 0; + } + + __it_next_data(skb, 0, it); + + return 0; + } + + for (i = 0; i < si->nr_frags; ++i) { + const skb_frag_t *frag = &si->frags[i]; + + f_size = skb_frag_size(frag); + off = last_ptr - (char *)skb_frag_address(frag); + + T_DBG3("%s: frags search, skb_frag_address(frag)=[%p]," + " f_size=%u, off=ld\n", __func__, skb_frag_address(frag), + f_size, off); + + if (off < 0 || off >= f_size) + continue; + + if (f_size - off > 1) { + it->data = last_ptr + 1; + it->skb = skb; + return 0; + } + + __it_next_data(skb, i + 1, it); + + return 0; + } + + return -ENOENT; +} + /** * Process a socket buffer like standard skb_seq_read(), but return when the * @actor finishes processing, so a caller gets control w/o looping when an @@ -1443,3 +1502,128 @@ ss_skb_to_sgvec_with_new_pages(struct sk_buff *skb, struct scatterlist *sgl, return out_frags; } + +/* + * Remove @offset amount of data from head side of the skb list, and insert + * new page @pg instead. + */ +int +ss_skb_replace_page(struct sk_buff **skb_head, struct page *pg, + unsigned long pg_len, unsigned long offset) +{ + struct sk_buff *skb; + unsigned int head_len; + struct skb_shared_info *si; + int m_len, i = 0; + + /* Remove skbs which are fully included in the @offset quantity. */ + while ((*skb_head)->len < offset) { + skb = ss_skb_dequeue(skb_head); + offset -= skb->len; + __kfree_skb(skb); + if (WARN_ON_ONCE(!*skb_head)) + return -EIO; + } + skb = *skb_head; + T_DBG3("%s: offset=%#lx, skb=[%pK], (head=[%pK], data=[%pK], tail=[%pK]" + " end=[%pK], len=%u, data_len=%u, nr_frags=%u)\n", __func__, + offset, skb, skb->head, skb->data, skb_tail_pointer(skb), + skb_end_pointer(skb), skb->len, skb->data_len, + skb_shinfo(skb)->nr_frags); + + /* + * Erase @offset amount of data from linear part and fragments of + * the first remaining skb. + */ + si = skb_shinfo(skb); + m_len = min_t(int, skb_headlen(skb), offset); + if (m_len) { + __skb_pull(skb, m_len); + offset -= m_len; + } + while (offset) { + skb_frag_t *frag; + + if (WARN_ON_ONCE(i >= si->nr_frags)) + return -EIO; + + frag = &si->frags[i]; + m_len = min_t(int, skb_frag_size(frag), offset); + + T_DBG3("%s: skb=[%p], m_len=%d, fragsize=%d\n", __func__, skb, + len, skb_frag_size(frag)); + + if (unlikely(m_len == skb_frag_size(frag))) { + ss_skb_adjust_data_len(skb, m_len); + __skb_frag_unref(frag); + ++i; + } + else { + frag->page_offset += m_len; + skb_frag_size_sub(frag, m_len); + skb->len -= m_len; + skb->data_len -= m_len; + } + offset -= m_len; + } + + /* + * If one or more fragments have been fully removed, we can just + * insert fragment with new data into the freed slot, without + * extending fragments/skbs. + */ + if (i > 0) { + si->nr_frags -= i; + + /* + * If all data have been removed from the skb or only first + * fragment (from zero slot) have been removed, we don't + * need to shift anything. + */ + if (skb->len && i > 1) { + WARN_ON_ONCE(!si->nr_frags); + memmove(&si->frags[1], &si->frags[i], + si->nr_frags * sizeof(skb_frag_t)); + } + ++si->nr_frags; + + goto done; + } + + head_len = skb_headlen(skb); + + /* + * Extend fragments to make room for additional fragments (maximum 2) + * to hold new data page and skb linear part (if it still exists after + * @offset removal). + */ + if (__extend_pgfrags(*skb_head, skb, 0, 1 + !!head_len)) + return -ENOMEM; + + if (head_len) { + struct page *head_pg = virt_to_head_page(skb->head); + char *head_ptr = (char *)page_address(head_pg); + unsigned int head_off = (char *)skb->data - head_ptr; + + __skb_fill_page_desc(skb, 1, head_pg, head_off, head_len); + get_page(head_pg); + + skb->tail -= head_len; + skb->data_len += head_len; + + /* + * Prevent @skb->tail from moving forward, since we have linked + * all the data from the linear part (which is after @skb->tail + * now) with the second fragment. + */ + skb->tail_lock = 1; + } + +done: + /* Set up new fragment with data page into the zero slot. */ + __skb_fill_page_desc(skb, 0, pg, 0, pg_len); + ss_skb_adjust_data_len(skb, pg_len); + + return 0; +} + diff --git a/tempesta_fw/ss_skb.h b/tempesta_fw/ss_skb.h index cf911aeaf6..e68f572602 100644 --- a/tempesta_fw/ss_skb.h +++ b/tempesta_fw/ss_skb.h @@ -172,6 +172,8 @@ char *ss_skb_fmt_src_addr(const struct sk_buff *skb, char *out_buf); int ss_skb_alloc_data(struct sk_buff **skb_head, size_t len, unsigned int tx_flags); struct sk_buff *ss_skb_split(struct sk_buff *skb, int len); +int skb_fragment(struct sk_buff *skb_head, struct sk_buff *skb, char *pspt, + int len, TfwStr *it); int ss_skb_get_room(struct sk_buff *skb_head, struct sk_buff *skb, char *pspt, unsigned int len, TfwStr *it); int ss_skb_expand_head_tail(struct sk_buff *skb_head, struct sk_buff *skb, @@ -180,6 +182,7 @@ int ss_skb_chop_head_tail(struct sk_buff *skb_head, struct sk_buff *skb, size_t head, size_t tail); int ss_skb_cutoff_data(struct sk_buff *skb_head, const TfwStr *hdr, int skip, int tail); +int skb_next_data(struct sk_buff *skb, char *last_ptr, TfwStr *it); int ss_skb_process(struct sk_buff *skb, ss_skb_actor_t actor, void *objdata, unsigned int *chunks, unsigned int *processed); @@ -189,5 +192,7 @@ void ss_skb_init_for_xmit(struct sk_buff *skb); void ss_skb_dump(struct sk_buff *skb); int ss_skb_to_sgvec_with_new_pages(struct sk_buff *skb, struct scatterlist *sgl, struct page ***old_pages); +int ss_skb_replace_page(struct sk_buff **skb_head, struct page *pg, + unsigned long pg_len, unsigned long offset); #endif /* __TFW_SS_SKB_H__ */ diff --git a/tempesta_fw/str.h b/tempesta_fw/str.h index 89682989ec..a0aac746f7 100644 --- a/tempesta_fw/str.h +++ b/tempesta_fw/str.h @@ -186,6 +186,8 @@ size_t tfw_ultohex(unsigned long ai, char *buf, unsigned int len); * casually from sleepable context, e.g. on configuration phase. * ------------------------------------------------------------------------ */ +#define TFW_STR_FBITS 10 +#define TFW_STR_FMASK ((1U << TFW_STR_FBITS) - 1) #define __TFW_STR_CN_MAX UINT_MAX /* * Str consists from compound or plain strings. @@ -204,6 +206,16 @@ size_t tfw_ultohex(unsigned long ai, char *buf, unsigned int len); #define TFW_STR_ETAG_WEAK 0x20 /* Trailer header. */ #define TFW_STR_TRAILER 0x40 +/* + * The string/chunk is a header fully indexed in HPACK static + * table (used only for HTTP/1.1=>HTTP/2 message transformation). + */ +#define TFW_STR_FULL_INDEX 0x80 +/* + * The string/chunk is a part of header value (used only for + * HTTP/2=>HTTP/1.1 and HTTP/2=>HTTP/2 message transformations). + */ +#define TFW_STR_HDR_VALUE 0x80 /* * @ptr - pointer to string data or array of nested strings; @@ -214,7 +226,11 @@ size_t tfw_ultohex(unsigned long ai, char *buf, unsigned int len); * it should be 0 if the string has no EOL at all, 1 for LF and * 2 for CRLF); * @nchunks - number of chunks of compound string; - * @flags - flags; + * @flags - double-byte field for flags; the least significant 10 bits are + * used for common flags' bits, and the most significant 6 bits + * are intended for HPACK static index (in cases when the HTTP + * header represented in @TfwStr is found in corresponding HPACK + * static table). */ typedef struct tfwstr_t { union { @@ -230,6 +246,8 @@ typedef struct tfwstr_t { #define TFW_STR_STRING(val) ((TfwStr){.data = (val), NULL, \ sizeof(val) - 1, 0, 0, 0}) +#define TFW_STR_F_STRING(val, flags) ((TfwStr){.data = (val), NULL, \ + sizeof(val) - 1, 0, flags, 0}) #define DEFINE_TFW_STR(name, val) TfwStr name = TFW_STR_STRING(val) #define TFW_STR_FROM_CSTR(s) ((TfwStr){.data = (char*)(s), \ NULL, strlen(s), 0, 0, 0}) @@ -237,6 +255,11 @@ typedef struct tfwstr_t { /* Use this with "%.*s" in printing calls. */ #define PR_TFW_STR(s) (int)min(20UL, (s)->len), (s)->data +/* Get/set HPACK static index from/into @s. */ +#define TFW_STR_INDEX(s) ((s)->flags >> TFW_STR_FBITS) +#define TFW_STR_INDEX_SET(s, i) ((s)->flags = ((s)->flags & TFW_STR_FMASK) \ + | ((i) << TFW_STR_FBITS)) + #define TFW_STR_INIT(s) memset((s), 0, sizeof(TfwStr)) #define TFW_STR_EMPTY(s) (!((s)->nchunks | (s)->len)) diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index 684a6e403f..899425cca9 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -61,7 +61,7 @@ test_hpack_req_alloc(void) BUG_ON(!req); req->pit.pool = __tfw_pool_new(0); BUG_ON(!req->pit.pool); - req->pit.hdr = &req->stream->parser.hdr; + req->pit.parsed_hdr = &req->stream->parser.hdr; __set_bit(TFW_HTTP_B_H2, req->flags); return req; @@ -170,15 +170,15 @@ TEST(hpack, dec_table_dynamic) hp = &ctx.hpack; - it.hdr = s1; + it.hdr = *s1; it.nm_len = 10; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); - it.hdr = s2; + it.hdr = *s2; it.nm_len = 15; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); - it.hdr = s3; + it.hdr = *s3; it.nm_len = 12; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); @@ -244,11 +244,11 @@ TEST(hpack, dec_table_mixed) hp = &ctx.hpack; - it.hdr = s1; + it.hdr = *s1; it.nm_len = s1_name->len; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); - it.hdr = s2; + it.hdr = *s2; it.nm_len = s2_name->len; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); @@ -257,7 +257,7 @@ TEST(hpack, dec_table_mixed) if (entry_1) { name = entry_1->name; value = entry_1->value; - it.hdr = s4; + it.hdr = *s4; it.nm_len = s1_name->len; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, entry_1, &it)); } @@ -267,7 +267,7 @@ TEST(hpack, dec_table_mixed) if (entry_2) { name = entry_2->name; value = entry_2->value; - it.hdr = s5; + it.hdr = *s5; it.nm_len = s2_name->len; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, entry_2, &it)); } @@ -275,7 +275,7 @@ TEST(hpack, dec_table_mixed) entry_3 = tfw_hpack_find_index(&hp->dec_tbl, 24); EXPECT_NOT_NULL(entry_3); if (entry_3) { - it.hdr = s3; + it.hdr = *s3; it.nm_len = s3_name->len; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, entry_3, &it)); } @@ -359,7 +359,7 @@ TEST(hpack, dec_table_wrap) s_name.data = name->ptr; HDR_COMPOUND_STR(s, &s_name, s_value); - it.hdr = s; + it.hdr = *s; it.nm_len = s_name.len; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); @@ -799,7 +799,7 @@ TEST(hpack, dec_huffman) TEST(hpack, enc_table_hdr_write) { char *buf; - unsigned long hdr_len, n_len = 0, v_off = 0, v_len = 0; + unsigned long hdr_len, n_len, v_off, v_len; #define HDR_NAME_1 "x-forwarded-for" #define HDR_VALUE_1 "test.com, foo.com, example.com" @@ -848,62 +848,54 @@ TEST(hpack, enc_table_hdr_write) collect_compound_str(s4, s4_value); collect_compound_str(s5, s5_value); - hdr_len = tfw_http_msg_hdr_length(s1, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s1, &n_len, &v_off, &v_len); EXPECT_EQ(n_len, strlen(HDR_NAME_1)); EXPECT_EQ(v_len, strlen(HDR_VALUE_1)); EXPECT_EQ(v_off, off1); EXPECT_EQ(hdr_len, t_s1_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_http_msg_hdr_write(s1, n_len, v_off, v_len, buf); + tfw_h2_msg_hdr_write(s1, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s1, buf, hdr_len)); - n_len = v_off = v_len = 0; - - hdr_len = tfw_http_msg_hdr_length(s2, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s2, &n_len, &v_off, &v_len); EXPECT_EQ(n_len, strlen(HDR_NAME_2)); EXPECT_EQ(v_len, strlen(HDR_VALUE_2)); EXPECT_EQ(v_off, off2); EXPECT_EQ(hdr_len, t_s2_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_http_msg_hdr_write(s2, n_len, v_off, v_len, buf); + tfw_h2_msg_hdr_write(s2, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s2, buf, hdr_len)); - n_len = v_off = v_len = 0; - - hdr_len = tfw_http_msg_hdr_length(s3, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s3, &n_len, &v_off, &v_len); EXPECT_EQ(n_len, strlen(HDR_NAME_3)); EXPECT_EQ(v_len, strlen(HDR_VALUE_3)); EXPECT_EQ(v_off, off3); EXPECT_EQ(hdr_len, t_s3_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_http_msg_hdr_write(s3, n_len, v_off, v_len, buf); + tfw_h2_msg_hdr_write(s3, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s3, buf, hdr_len)); - n_len = v_off = v_len = 0; - - hdr_len = tfw_http_msg_hdr_length(s4, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s4, &n_len, &v_off, &v_len); EXPECT_EQ(n_len, strlen(HDR_NAME_4)); EXPECT_EQ(v_len, strlen(HDR_VALUE_4)); EXPECT_EQ(v_off, off4); EXPECT_EQ(hdr_len, t_s4_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_http_msg_hdr_write(s4, n_len, v_off, v_len, buf); + tfw_h2_msg_hdr_write(s4, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s4, buf, hdr_len)); - n_len = v_off = v_len = 0; - - hdr_len = tfw_http_msg_hdr_length(s5, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s5, &n_len, &v_off, &v_len); EXPECT_EQ(n_len, strlen(HDR_NAME_5)); EXPECT_EQ(v_len, strlen(HDR_VALUE_5)); EXPECT_EQ(v_off, off5); EXPECT_EQ(hdr_len, t_s5_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_http_msg_hdr_write(s5, n_len, v_off, v_len, buf); + tfw_h2_msg_hdr_write(s5, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s5, buf, hdr_len)); #undef HDR_NAME_1 diff --git a/tempesta_fw/t/unit/test_http_parser.c b/tempesta_fw/t/unit/test_http_parser.c index ca0bab4458..2c71ccdbb8 100644 --- a/tempesta_fw/t/unit/test_http_parser.c +++ b/tempesta_fw/t/unit/test_http_parser.c @@ -948,22 +948,29 @@ TEST(http_parser, fills_hdr_tbl_for_req) ht = req->h_tbl; /* Special headers: */ - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_HOST], + tfw_http_msg_clnthdr_val(req, + &ht->tbl[TFW_HTTP_HDR_HOST], TFW_HTTP_HDR_HOST, &h_host); - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_CONNECTION], + tfw_http_msg_clnthdr_val(req, + &ht->tbl[TFW_HTTP_HDR_CONNECTION], TFW_HTTP_HDR_CONNECTION, &h_connection); - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_CONTENT_TYPE], + tfw_http_msg_clnthdr_val(req, + &ht->tbl[TFW_HTTP_HDR_CONTENT_TYPE], TFW_HTTP_HDR_CONTENT_TYPE, &h_conttype); - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], + tfw_http_msg_clnthdr_val(req, + &ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], TFW_HTTP_HDR_X_FORWARDED_FOR, &h_xff); - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_USER_AGENT], + tfw_http_msg_clnthdr_val(req, + &ht->tbl[TFW_HTTP_HDR_USER_AGENT], TFW_HTTP_HDR_USER_AGENT, &h_user_agent); - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_TRANSFER_ENCODING], + tfw_http_msg_clnthdr_val(req, + &ht->tbl[TFW_HTTP_HDR_TRANSFER_ENCODING], TFW_HTTP_HDR_TRANSFER_ENCODING, &h_te); - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_COOKIE], + tfw_http_msg_clnthdr_val(req, + &ht->tbl[TFW_HTTP_HDR_COOKIE], TFW_HTTP_HDR_COOKIE, &h_cookie); /* Common (raw) headers: 16 total with 10 dummies. */ @@ -1070,8 +1077,8 @@ TEST(http_parser, fills_hdr_tbl_for_resp) { ht = resp->h_tbl; - EXPECT_TRUE(tfw_str_eq_cstr(&resp->s_line, "HTTP/1.1 200 OK", - strlen("HTTP/1.1 200 OK"), 0)); + EXPECT_TRUE(tfw_str_eq_cstr(&ht->tbl[TFW_HTTP_STATUS_LINE], + "HTTP/1.1 200 OK", strlen("HTTP/1.1 200 OK"), 0)); /* Special headers: */ tfw_http_msg_srvhdr_val(&ht->tbl[TFW_HTTP_HDR_CONNECTION], @@ -2099,7 +2106,7 @@ TEST(http_parser, referer) "\r\n") { ht = req->h_tbl; - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_REFERER], + tfw_http_msg_clnthdr_val(req, &ht->tbl[TFW_HTTP_HDR_REFERER], TFW_HTTP_HDR_REFERER, &h_referer); EXPECT_TRUE(tfw_str_eq_cstr(&h_referer, s_referer1, @@ -2111,7 +2118,7 @@ TEST(http_parser, referer) "\r\n") { ht = req->h_tbl; - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_REFERER], + tfw_http_msg_clnthdr_val(req, &ht->tbl[TFW_HTTP_HDR_REFERER], TFW_HTTP_HDR_REFERER, &h_referer); EXPECT_TRUE(tfw_str_eq_cstr(&h_referer, s_referer2, @@ -2124,7 +2131,7 @@ TEST(http_parser, referer) "\r\n") { ht = req->h_tbl; - tfw_http_msg_clnthdr_val(&ht->tbl[TFW_HTTP_HDR_REFERER], + tfw_http_msg_clnthdr_val(req, &ht->tbl[TFW_HTTP_HDR_REFERER], TFW_HTTP_HDR_REFERER, &h_referer); EXPECT_TRUE(tfw_str_eq_cstr(&h_referer, s_referer3, diff --git a/tempesta_fw/t/unit/test_http_sticky.c b/tempesta_fw/t/unit/test_http_sticky.c index 6a041edc4b..90ef336f60 100644 --- a/tempesta_fw/t/unit/test_http_sticky.c +++ b/tempesta_fw/t/unit/test_http_sticky.c @@ -614,7 +614,6 @@ http_parse_resp_helper(void) memset(mock.resp->h_tbl->tbl, 0, __HHTBL_SZ(1) * sizeof(TfwStr)); TFW_STR_INIT(&mock.resp->crlf); TFW_STR_INIT(&mock.resp->body); - TFW_STR_INIT(&mock.resp->s_line); return http_parse_helper((TfwHttpMsg *)mock.resp, tfw_http_parse_resp); } diff --git a/tempesta_fw/t/unit/test_tfw_str.c b/tempesta_fw/t/unit/test_tfw_str.c index 035fd1d9ce..02b2ea4f0f 100644 --- a/tempesta_fw/t/unit/test_tfw_str.c +++ b/tempesta_fw/t/unit/test_tfw_str.c @@ -1263,7 +1263,7 @@ TEST(tfw_str_collect_cmp, collect_chunks) * other fields of the output TfwStr. */ EXPECT_EQ(out.eolen, 0); - EXPECT_EQ(out.flags, 0); + EXPECT_EQ(out.flags & TFW_STR_FMASK, 0); /* * Try to start at other chunks too. From dc7170215a9d8351c6b4f981a4bb46bf815a081a Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Wed, 30 Oct 2019 17:42:57 +0300 Subject: [PATCH 02/64] HTTP/2 Parser implementation (#309): 1. Changes in HPACK decoder to copy only Huffman-decoded and dynamically indexed headers; 2. Appropriate changes in HPACK-decoder/parser unit-tests. --- tempesta_fw/hpack.c | 394 +++++++------- tempesta_fw/hpack.h | 27 +- tempesta_fw/http_frame.c | 3 - tempesta_fw/http_msg.c | 40 +- tempesta_fw/http_msg.h | 1 + tempesta_fw/msg.h | 4 - tempesta_fw/pool.c | 23 +- tempesta_fw/t/unit/test_hpack.c | 773 ++++++++++++++++++---------- tempesta_fw/t/unit/tfw_str_helper.c | 8 +- tempesta_fw/t/unit/tfw_str_helper.h | 6 +- 10 files changed, 771 insertions(+), 508 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index b7aff15695..0f025c9b9d 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -942,75 +942,105 @@ static const HTState ht_decode[] ____cacheline_aligned = { {6, 0}, /* 2: EOS */ }; -#define HP_STR(str) \ - (&(TfwHPackStr){ \ - .ptr = str, \ - .len = sizeof(str) - 1, \ - .count = -1 \ - }) +#define HP_HDR_NAME(name, h_tag) \ + ((TfwHPackEntry){ \ + .hdr = &(TfwStr){ \ + .chunks = &(TfwStr){ \ + .data = name, \ + .len = SLEN(name), \ + }, \ + .len = SLEN(name), \ + .nchunks = 1 \ + }, \ + .name_len = SLEN(name), \ + .name_num = 1, \ + .tag = h_tag \ +}) + +#define HP_HDR_FULL(name, value, h_tag) \ + ((TfwHPackEntry){ \ + .hdr = &(TfwStr){ \ + .chunks = (TfwStr []){ \ + { \ + .data = name, \ + .len = SLEN(name) \ + }, \ + { \ + .data = value, \ + .len = SLEN(value), \ + .flags = TFW_STR_HDR_VALUE \ + } \ + }, \ + .len = SLEN(name) + SLEN(value), \ + .nchunks = 2 \ + }, \ + .name_len = SLEN(name), \ + .name_num = 1, \ + .tag = h_tag \ +}) static const TfwHPackEntry static_table[] ____cacheline_aligned = { - { HP_STR(":authority"), NULL, TFW_TAG_HDR_H2_AUTHORITY }, - { HP_STR(":method"), HP_STR("GET"), TFW_TAG_HDR_H2_METHOD }, - { HP_STR(":method"), HP_STR("POST"), TFW_TAG_HDR_H2_METHOD }, - { HP_STR(":path"), HP_STR("/"), TFW_TAG_HDR_H2_PATH }, - { HP_STR(":path"), HP_STR("/index.html"), TFW_TAG_HDR_H2_PATH }, - { HP_STR(":scheme"), HP_STR("http"), TFW_TAG_HDR_H2_SCHEME }, - { HP_STR(":scheme"), HP_STR("https"),TFW_TAG_HDR_H2_SCHEME }, - { HP_STR(":status"), HP_STR("200"), TFW_TAG_HDR_H2_STATUS }, - { HP_STR(":status"), HP_STR("204"), TFW_TAG_HDR_H2_STATUS }, - { HP_STR(":status"), HP_STR("206"), TFW_TAG_HDR_H2_STATUS }, - { HP_STR(":status"), HP_STR("304"), TFW_TAG_HDR_H2_STATUS }, - { HP_STR(":status"), HP_STR("400"), TFW_TAG_HDR_H2_STATUS }, - { HP_STR(":status"), HP_STR("404"), TFW_TAG_HDR_H2_STATUS }, - { HP_STR(":status"), HP_STR("500"), TFW_TAG_HDR_H2_STATUS }, - { HP_STR("accept-charset"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("accept-encoding"), HP_STR("gzip, deflate"), TFW_TAG_HDR_RAW }, - { HP_STR("accept-language"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("accept-ranges"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("accept"), NULL, TFW_TAG_HDR_ACCEPT }, - { HP_STR("access-control-allow-origin"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("age"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("allow"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("authorization"), NULL, TFW_TAG_HDR_AUTHORIZATION }, - { HP_STR("cache-control"), NULL, TFW_TAG_HDR_CACHE_CONTROL }, - { HP_STR("content-disposition"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("content-encoding"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("content-language"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("content-length"), NULL, TFW_TAG_HDR_CONTENT_LENGTH }, - { HP_STR("content-location"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("content-range"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("content-type"), NULL, TFW_TAG_HDR_CONTENT_TYPE }, - { HP_STR("cookie"), NULL, TFW_TAG_HDR_COOKIE }, - { HP_STR("date"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("etag"), NULL, TFW_TAG_HDR_ETAG }, - { HP_STR("expect"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("expires"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("from"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("host"), NULL, TFW_TAG_HDR_HOST }, - { HP_STR("if-match"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("if-modified-since"), NULL, TFW_TAG_HDR_IF_MODIFIED_SINCE }, - { HP_STR("if-none-match"), NULL, TFW_TAG_HDR_IF_NONE_MATCH }, - { HP_STR("if-range"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("if-unmodified-since"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("last-modified"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("link"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("location"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("max-forwards"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("proxy-authenticate"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("proxy-authorization"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("range"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("referer"), NULL, TFW_TAG_HDR_REFERER }, - { HP_STR("refresh"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("retry-after"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("server"), NULL, TFW_TAG_HDR_SERVER }, - { HP_STR("set-cookie"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("strict-transport-security"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("transfer-encoding"), NULL, TFW_TAG_HDR_TRANSFER_ENCODING }, - { HP_STR("user-agent"), NULL, TFW_TAG_HDR_USER_AGENT }, - { HP_STR("vary"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("via"), NULL, TFW_TAG_HDR_RAW }, - { HP_STR("www-authenticate"), NULL, TFW_TAG_HDR_RAW } + HP_HDR_NAME(":authority", TFW_TAG_HDR_H2_AUTHORITY), + HP_HDR_FULL(":method", "GET", TFW_TAG_HDR_H2_METHOD), + HP_HDR_FULL(":method", "POST", TFW_TAG_HDR_H2_METHOD), + HP_HDR_FULL(":path", "/", TFW_TAG_HDR_H2_PATH), + HP_HDR_FULL(":path", "/index.html", TFW_TAG_HDR_H2_PATH), + HP_HDR_FULL(":scheme", "http", TFW_TAG_HDR_H2_SCHEME), + HP_HDR_FULL(":scheme", "https", TFW_TAG_HDR_H2_SCHEME), + HP_HDR_FULL(":status", "200", TFW_TAG_HDR_H2_STATUS), + HP_HDR_FULL(":status", "204", TFW_TAG_HDR_H2_STATUS), + HP_HDR_FULL(":status", "206", TFW_TAG_HDR_H2_STATUS), + HP_HDR_FULL(":status", "304", TFW_TAG_HDR_H2_STATUS), + HP_HDR_FULL(":status", "400", TFW_TAG_HDR_H2_STATUS), + HP_HDR_FULL(":status", "404", TFW_TAG_HDR_H2_STATUS), + HP_HDR_FULL(":status", "500", TFW_TAG_HDR_H2_STATUS), + HP_HDR_NAME("accept-charset", TFW_TAG_HDR_RAW), + HP_HDR_FULL("accept-encoding", "gzip, deflate", TFW_TAG_HDR_RAW), + HP_HDR_NAME("accept-language", TFW_TAG_HDR_RAW), + HP_HDR_NAME("accept-ranges", TFW_TAG_HDR_RAW), + HP_HDR_NAME("accept", TFW_TAG_HDR_ACCEPT), + HP_HDR_NAME("access-control-allow-origin", TFW_TAG_HDR_RAW), + HP_HDR_NAME("age", TFW_TAG_HDR_RAW), + HP_HDR_NAME("allow", TFW_TAG_HDR_RAW), + HP_HDR_NAME("authorization", TFW_TAG_HDR_AUTHORIZATION), + HP_HDR_NAME("cache-control", TFW_TAG_HDR_CACHE_CONTROL), + HP_HDR_NAME("content-disposition", TFW_TAG_HDR_RAW), + HP_HDR_NAME("content-encoding", TFW_TAG_HDR_RAW), + HP_HDR_NAME("content-language", TFW_TAG_HDR_RAW), + HP_HDR_NAME("content-length", TFW_TAG_HDR_CONTENT_LENGTH), + HP_HDR_NAME("content-location", TFW_TAG_HDR_RAW), + HP_HDR_NAME("content-range", TFW_TAG_HDR_RAW), + HP_HDR_NAME("content-type", TFW_TAG_HDR_CONTENT_TYPE), + HP_HDR_NAME("cookie", TFW_TAG_HDR_COOKIE), + HP_HDR_NAME("date", TFW_TAG_HDR_RAW), + HP_HDR_NAME("etag", TFW_TAG_HDR_ETAG), + HP_HDR_NAME("expect", TFW_TAG_HDR_RAW), + HP_HDR_NAME("expires", TFW_TAG_HDR_RAW), + HP_HDR_NAME("from", TFW_TAG_HDR_RAW), + HP_HDR_NAME("host", TFW_TAG_HDR_HOST), + HP_HDR_NAME("if-match", TFW_TAG_HDR_RAW), + HP_HDR_NAME("if-modified-since", TFW_TAG_HDR_IF_MODIFIED_SINCE), + HP_HDR_NAME("if-none-match", TFW_TAG_HDR_IF_NONE_MATCH), + HP_HDR_NAME("if-range", TFW_TAG_HDR_RAW), + HP_HDR_NAME("if-unmodified-since", TFW_TAG_HDR_RAW), + HP_HDR_NAME("last-modified", TFW_TAG_HDR_RAW), + HP_HDR_NAME("link", TFW_TAG_HDR_RAW), + HP_HDR_NAME("location", TFW_TAG_HDR_RAW), + HP_HDR_NAME("max-forwards", TFW_TAG_HDR_RAW), + HP_HDR_NAME("proxy-authenticate", TFW_TAG_HDR_RAW), + HP_HDR_NAME("proxy-authorization", TFW_TAG_HDR_RAW), + HP_HDR_NAME("range", TFW_TAG_HDR_RAW), + HP_HDR_NAME("referer", TFW_TAG_HDR_REFERER), + HP_HDR_NAME("refresh", TFW_TAG_HDR_RAW), + HP_HDR_NAME("retry-after", TFW_TAG_HDR_RAW), + HP_HDR_NAME("server", TFW_TAG_HDR_SERVER), + HP_HDR_NAME("set-cookie", TFW_TAG_HDR_RAW), + HP_HDR_NAME("strict-transport-security", TFW_TAG_HDR_RAW), + HP_HDR_NAME("transfer-encoding", TFW_TAG_HDR_TRANSFER_ENCODING), + HP_HDR_NAME("user-agent", TFW_TAG_HDR_USER_AGENT), + HP_HDR_NAME("vary", TFW_TAG_HDR_RAW), + HP_HDR_NAME("via", TFW_TAG_HDR_RAW), + HP_HDR_NAME("www-authenticate", TFW_TAG_HDR_RAW) }; #define HPACK_STATIC_ENTRIES (sizeof(static_table) / sizeof(TfwHPackEntry)) @@ -1133,8 +1163,6 @@ do { \ #define BUFFER_HDR_INIT(length, it) \ do { \ - if (!(it)->start_pos) \ - (it)->start_pos = (it)->pos; \ (it)->hdr.data = (it)->pos; \ (it)->hdr.len = length; \ (it)->next = &(it)->hdr; \ @@ -1477,68 +1505,30 @@ tfw_huffman_decode(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, return T_DROP; } -static inline TfwHPackStr * -tfw_hpack_create_str(unsigned long len) -{ - TfwHPackStr *str; - - if (!(str = kmalloc(sizeof(TfwHPackStr) + len, GFP_ATOMIC))) - return NULL; - str->ptr = (char *)(str + 1); - str->len = len; - str->count = 1; - - ++act_hp_str_n; - - return str; -} - -static inline void -tfw_hpack_str_get(TfwHPackStr *hp_str) -{ - if (hp_str->count == -1) - return; - - ++hp_str->count; -} - -static inline void -tfw_hpack_str_put(TfwHPackStr *hp_str) -{ - if (hp_str->count == -1) - return; - - WARN_ON_ONCE(hp_str->count <= 0); - if (--hp_str->count == 0) { - --act_hp_str_n; - kfree(hp_str); - } -} - -static TfwHPackEntry * -tfw_hpack_create_entry(TfwPool *__restrict pool, TfwMsgParseIter *__restrict it) +static int +tfw_hpack_set_entry(TfwPool *__restrict h_pool, TfwMsgParseIter *__restrict it, + TfwHPackEntry *__restrict entry, bool *__restrict np) { char *data; TfwHPackEntry *entry; const TfwStr *d, *s, *end, *d_hdr, *s_hdr = it->parsed_hdr; unsigned long size = sizeof(TfwHPackEntry); - if (WARN_ON_ONCE(TFW_STR_EMPTY(s_hdr))) - return NULL; + if (WARN_ON_ONCE(TFW_STR_PLAIN(s_hdr) || TFW_STR_DUP(s_hdr))) + return -EINVAL; size += (s_hdr->nchunks + 1) * sizeof(TfwStr) + s_hdr->len; T_DBG3("%s: size=%lu, s_hdr->nchunks=%u, s_hdr->len=%lu\n", __func__, size, s_hdr->nchunks, s_hdr->len); - if (!(entry = tfw_pool_alloc(pool, size))) - return NULL; + if (!(d_hdr = __tfw_pool_alloc(h_pool, size, true, np))) + return -ENOMEM; - d_hdr = entry->hdr; *d_hdr = *s_hdr; d_hdr->chunks = d_hdr + 1; data = (char *)(TFW_STR_LAST(d_hdr) + 1); - d = TFW_STR_CHUNK(d_hdr, 0); + d = d_hdr->chunks; TFW_STR_FOR_EACH_CHUNK(s, s_hdr, end) { *d = *s; d->data = data; @@ -1555,52 +1545,34 @@ tfw_hpack_create_entry(TfwPool *__restrict pool, TfwMsgParseIter *__restrict it) __func__, d_hdr->nchunks, d_hdr->len, d_hdr->flags, it->nm_len, it->nm_num, it->tag); + entry->hdr = d_hdr; entry->name_len = it->nm_len; entry->name_num = it->nm_num; entry->tag = it->tag; + entry->last = false; return entry; } -static inline void -tfw_hpack_free_entry(TfwHPackEntry *entry) -{ - tfw_hpack_str_put(entry->name); - tfw_hpack_str_put(entry->value); -} - static int tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, TfwMsgParseIter *__restrict it) { + int r; + bool new_page; unsigned int delta; - unsigned long name_len; - TfwHPackEntry *entry; - const TfwHPackStr *name, *value; unsigned int window, size, new_size; + unsigned long hdr_len = it->parsed_hdr->len; unsigned int count = tbl->n; unsigned int curr = tbl->curr; unsigned int length = tbl->length; - TfwHPackEntry *entries = tbl->entries; - int r = 0; - - if (!(entry = tfw_hpack_create_entry(tbl->pool, it))) - return -ENOMEM; - - name = entry.name; - value = entry.value; - name_len = name->len; + TfwHPackEntry *entry, *prev_entry, *entries = tbl->entries; /* Check for integer overflow occurred during @delta calculation. */ - if ((delta = HPACK_ENTRY_OVERHEAD + name_len) < name_len - || (delta += value->len) < value->len) - { - T_WARN("HPACK decoder: very big header (name->len = %lu," - " valueb->len = %lu)." - " The entry cannot be added into dynamic table\n", - name_len, value->len); - r = -EINVAL; - goto out; + if ((delta = HPACK_ENTRY_OVERHEAD + hdr_len) < hdr_len) { + T_WARN("HPACK decoder: very big header (hdr_len = %lu). The" + " entry cannot be added into dynamic table\n", hdr_len); + return -EINVAL; } size = tbl->size; @@ -1632,13 +1604,11 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, cp = entries + early; do { - TfwHPackStr *np = cp->name; - TfwHPackStr *vp = cp->value; - - size -= HPACK_ENTRY_OVERHEAD + np->len + vp->len; + size -= HPACK_ENTRY_OVERHEAD + cp->hdr->len; T_DBG3("%s: dropped index: %u\n", __func__, early); - tfw_hpack_free_entry(cp); + if (cp->last) + tfw_pool_clean(tbl->h_pool, cp->hdr); early++; cp++; count--; @@ -1669,7 +1639,9 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, do { T_DBG3("%s: drop index: %u\n", __func__, curr); - tfw_hpack_free_entry(cp); + if (cp->last) + tfw_pool_clean(tbl->h_pool, + cp->hdr); curr++; cp++; if (unlikely(curr == length)) { @@ -1681,52 +1653,49 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, tbl->curr = 0; tbl->size = 0; } - goto out; + return 0; } } else if (unlikely(count == length)) { TfwHPackEntry *previous = entries; TfwPool *pool = tbl->pool; - unsigned long block, wrap; + unsigned long block, wrap, tail; T_DBG3("%s: reallocation index structures...", __func__); if (length) { block = length * sizeof(TfwHPackEntry); + new_block = block << 1; entries = tfw_pool_realloc_no_copy(pool, entries, - block, block << 1); - if (unlikely(!entries)) { - r = -ENOMEM; - goto out; - } + block, new_block); + if (unlikely(!entries)) + return -ENOMEM; length <<= 1; wrap = curr * sizeof(TfwHPackEntry); - block -= wrap; + tail = block - wrap; if (!curr && entries == previous) { curr = count; } else if (entries == previous) { - memcpy_fast(entries + wrap + block, - previous + wrap, block); + memcpy_fast(entries + new_block - tail, + entries + wrap, tail); } else { - if (block) + if (tail) memcpy_fast(entries, previous + wrap, - block); + tail); if (wrap) - memcpy_fast(entries + block, previous, + memcpy_fast(entries + tail, previous, wrap); - tfw_pool_clean(pool); + tfw_pool_clean(pool, NULL); curr = count; } } else { length = 32; block = length * sizeof(TfwHPackEntry); entries = tfw_pool_alloc(pool, block); - if (unlikely(!entries)) { - r = -ENOMEM; - goto out; - } + if (unlikely(!entries)) + return -ENOMEM; } T_DBG3("%s: table extended, length=%u, curr=%u\n", __func__, length, curr); @@ -1735,8 +1704,18 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, tbl->entries = entries; } - entries += curr; - *entries = entry; + entry = entries + curr; + if ((r = tfw_hpack_set_entry(tbl->h_pool, it, entry, &new_page))) + return r; + /* + * If the new entry is placed into the new page, and previous entry + * exists, then mark it as last entry in previous page (in order to + * free unused pages during entries eviction stage). + */ + if (count && new_page) { + prev_entry = curr ? (entry - 1) : (entries + length - 1); + prev_entry->last = true; + } curr++; if (unlikely(curr == length)) @@ -1749,11 +1728,6 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, __func__, tbl->curr, tbl->n, tbl->length); return 0; -out: - T_DBG3("%s: item not added (r=%d)\n", __func__, r); - tfw_hpack_free_entry(&entry); - - return r; } static TfwHPackEntry * @@ -1779,8 +1753,10 @@ tfw_hpack_find_index(TfwHPackDTbl *__restrict tbl, unsigned long index) } T_DBG3("%s: tbl->length=%u, tbl->curr=%u, curr=%u, index=%lu\n", __func__, tbl->length, tbl->curr, curr, index); + entry = tbl->entries + curr; - BUG_ON(!entry->name || !entry->value); + WARN_ON_ONCE(!entry->hdr->nchunks || entry->name_num != 1); + return entry; } @@ -1813,16 +1789,15 @@ tfw_hpack_set_length(TfwHPack *__restrict hp, unsigned long new_size) length, new_size); cp = entries + early; do { - TfwHPackStr *np = cp->name; - TfwHPackStr *vp = cp->value; + unsigned long hdr_len = cp->hdr->len; - WARN_ON_ONCE(!np->len); - WARN_ON_ONCE(!vp->len); - size -= HPACK_ENTRY_OVERHEAD + np->len + vp->len; + WARN_ON_ONCE(!hdr_len); + size -= HPACK_ENTRY_OVERHEAD + hdr_len; T_DBG3("%s: drop index, early=%u, count=%u," " length=%u\n", __func__, early, count, length); - tfw_hpack_free_entry(cp); + if (cp->last) + tfw_pool_clean(tbl->h_pool, cp->hdr); early++; cp++; count--; @@ -1862,26 +1837,33 @@ tfw_hpack_init(TfwHPack *__restrict hp, unsigned int htbl_sz) dt->window = hp->max_window = htbl_sz; if (!(dt->pool = __tfw_pool_new(0))) return -ENOMEM; + if (!(dt->h_pool = __tfw_pool_new(0))) + goto err_dt; et->window = htbl_sz; spin_lock_init(&et->lock); et->rb_size = HPACK_ENC_TABLE_MAX_SIZE; - if (!(et->pool = __tfw_pool_new(HPACK_ENC_TABLE_MAX_SIZE))) { - tfw_pool_destroy(dt->pool); - return -ENOMEM; - } + if (!(et->pool = __tfw_pool_new(HPACK_ENC_TABLE_MAX_SIZE))) + goto err_et; et->rbuf = __tfw_pool_alloc(et->pool, HPACK_ENC_TABLE_MAX_SIZE, true, &np); BUG_ON(np || !et->rbuf); return 0; + +err_et: + tfw_pool_destroy(dt->h_pool); +err_dt: + tfw_pool_destroy(dt->pool); + + return -ENOMEM; } void tfw_hpack_clean(TfwHPack *__restrict hp) { tfw_pool_destroy(hp->enc_tbl.pool); - tfw_hpack_set_length(hp, 0); + tfw_pool_destroy(hp->dec_tbl.h_pool); tfw_pool_destroy(hp->dec_tbl.pool); WARN_ON_ONCE(act_hp_str_n); } @@ -1952,37 +1934,33 @@ tfw_hpack_hdr_name_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, char *data; TfwStr *d; const TfwStr *s, *end; - TfwHttpParser *parser = &req->stream->parser; - TfwStr *d_hdr = &parser->hdr; - TfwStr *s_hdr = entry->hdr; TfwMsgParseIter *it = &req->pit; + TfwStr *d_hdr = it->parsed_hdr; + TfwStr *s_hdr = entry->hdr; + unsigned int num = entry->name_num; unsigned long sz = entry->name_len; WARN_ON_ONCE(!TFW_STR_EMPTY(d_hdr)); + if (WARN_ON_ONCE(!num || num > s_hdr->nchunks)) + return -EINVAL; if (!(data = tfw_pool_alloc_not_align(it->pool, sz))) return T_BAD; d_hdr->len = sz; - d_hdr->flags = s_hdr->flags; - - if (TFW_STR_PLAIN(s_hdr)) { - d_hdr->data = data; - goto done; - } - d_hdr->nchunks = num; + d_hdr->flags = s_hdr->flags; if (!(d_hdr->chunks = tfw_pool_alloc(req->pool, num * sizeof(TfwStr)))) return T_BAD; /* * Since headers in static table cannot be changed, we need to copy only - * descriptors (because they will grow during further processing). + * descriptors (i.e. only high-level and the name descriptors), because + * they will grow during further processing. */ - d = __TFW_STR_CH(d_hdr, 0); + d = d_hdr->chunks; if (hp->index <= HPACK_STATIC_ENTRIES) { - WARN_ON_ONCE(!s_hdr->nchunks); *d = *s_hdr->chunks; goto done; } @@ -2042,10 +2020,10 @@ tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, * (without full copying) only references for statically indexed * headers (also, see comment above). */ - d_size = s_hdr->nchunks * sizeof(TfwStr); if (!(data = tfw_pool_alloc_not_align(it->pool, s_hdr->len))) return T_BAD; + d_size = s_hdr->nchunks * sizeof(TfwStr); if (!(d_hdr->chunks = tfw_pool_alloc(req->pool, d_size))) return T_BAD; @@ -2053,7 +2031,7 @@ tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, d_hdr->flags = s_hdr->flags; d_hdr->nchunks = s_hdr->nchunks; - d = TFW_STR_CHUNK(d_hdr, 0); + d = d_hdr->chunks; TFW_STR_FOR_EACH_CHUNK(s, s_hdr, end) { *d = *s; d->data = data; @@ -2604,10 +2582,10 @@ do { \ #define HPACK_NODE_COND_OFF(tbl, node) \ ((node) ? HPACK_NODE_OFF(tbl, node) : -1) -#define HPACK_NODE_ALIGN(sz) (((sz) + 7) & ~7UL) +#define HPACK_ALIGN(sz) (((sz) + 7) & ~7UL) #define HPACK_NODE_SIZE(node) \ - HPACK_NODE_ALIGN(sizeof(TfwHPackNode) + ((TfwHPackNode *)node)->hdr_len) + HPACK_ALIGN(sizeof(TfwHPackNode) + ((TfwHPackNode *)node)->hdr_len) #define HPACK_NODE_NEXT(node) \ ((TfwHPackNode *)((char *)(node) + HPACK_NODE_SIZE(node))) @@ -3442,7 +3420,7 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, && tfw_hpack_rbuf_calc(tbl, window - node_size, del_list, &it)) return -E2BIG; - node_len = HPACK_NODE_ALIGN(sizeof(TfwHPackNode) + hdr_len); + node_len = HPACK_ALIGN(sizeof(TfwHPackNode) + hdr_len); if (it.rb_size < it.rb_len + node_len) { WARN_ON_ONCE(it.rb_size == HPACK_ENC_TABLE_MAX_SIZE); @@ -3735,7 +3713,7 @@ tfw_hpack_idx_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict str, return 0; } -//comment is needed +//!!! comment is needed static int tfw_hpack_lit_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, @@ -3808,7 +3786,7 @@ tfw_hpack_lit_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, return 0; } -//comment is needed +//!!! comment is needed static int tfw_hpack_lit_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, bool name_indexed) @@ -3892,7 +3870,7 @@ tfw_hpack_lit_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return 0; } -//comment is needed (about @fc too) +//!!! comment is needed (about @fc too) static inline int tfw_hpack_lit_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, @@ -4040,7 +4018,7 @@ tfw_hpack_lit_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return 0; } -//comment is needed (including the @tgt description) +//!!! comment is needed (including the @tgt description) int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, TfwStr *__restrict hdr, TfwH2TransOp op) diff --git a/tempesta_fw/hpack.h b/tempesta_fw/hpack.h index 879860ffe6..edb2822242 100644 --- a/tempesta_fw/hpack.h +++ b/tempesta_fw/hpack.h @@ -105,19 +105,6 @@ typedef struct { TFW_HPACK_ETBL_COMMON; } TfwHPackETblIter; -/** - * HPack strings representation. - * - * @ptr - pointer to the actual string data; - * @len - length of the string; - * @count - number of users of the string instance. - */ -typedef struct { - char *ptr; - unsigned long len; - int count; -} TfwHPackStr; - typedef enum { TFW_H2_TRANS_ADD = 0, TFW_H2_TRANS_EXPAND, @@ -153,23 +140,28 @@ typedef enum { /** * Representation of the entry in HPack decoder index. * + * @hdr - pointer to the header data descriptor; * @name_len - length of the header's name part; * @name_num - chunks count of the header's name part; * @tag - tag of the indexed header; - * @hdr - descriptor of the header data. + * @last - flag bit indicating that corresponding header is the last on the page. */ typedef struct { + TfwStr *hdr; unsigned long name_len; - unsigned int name_num; + unsigned long name_num; unsigned int tag; - TfwStr hdr[0]; + unsigned char last : 1; } TfwHPackEntry; /** * HPack decoder dynamic index table. * * @entries - dynamic table of entries; - * @pool - memory pool for dynamic table; + * @pool - memory pool for constantly sized entries (i.e. the entry + * descriptors); + * @h_pool - memory pool for entries of variable size (headers themselves + * - and @TfwStr descriptors for them); * @n - actual number of entries in the table; * @curr - circular buffer index of recent entry; * @length - current length of the dynamic table (in entries); @@ -180,6 +172,7 @@ typedef struct { typedef struct { TfwHPackEntry *entries; TfwPool *pool; + TfwPool *h_pool; unsigned int n; unsigned int curr; unsigned int length; diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index c787ec678e..824153d2ab 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -922,9 +922,6 @@ tfw_h2_apply_settings_entry(TfwH2Ctx *ctx, unsigned short id, return T_OK; } - /* - * TODO #309: apply settings entry. - */ return T_OK; } diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 8c670207ad..b2ba3dac61 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -251,6 +251,35 @@ __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client) } EXPORT_SYMBOL(__http_msg_hdr_val); +void +__h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name) +{ + TfwStr *c, *end; + + if (unlikely(TFW_STR_EMPTY(hdr))) { + TFW_STR_INIT(out_val); + return; + } + + BUG_ON(TFW_STR_DUP(hdr)); + BUG_ON(TFW_STR_EMPTY(hdr)); + + *out_name = *hdr; + + if (unlikely(TFW_STR_PLAIN(hdr))) { + WARN_ON_ONCE(hdr->flags & TFW_STR_HDR_VALUE) + return; + } + + TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { + if (c->flags & TFW_STR_HDR_VALUE) { + out_name->len -= c->len; + out_name->nchunks--; + } + } +} +EXPORT_SYMBOL(__h2_msg_hdr_name); + void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val) { @@ -277,6 +306,7 @@ __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val) /* Empty header value part. */ TFW_STR_INIT(out_val); } +EXPORT_SYMBOL(__h2_msg_hdr_val); /** * Slow check of generic (raw) header for singularity. @@ -504,7 +534,15 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) * Both the headers, the new one and existing one, can already be * compound. */ - id = __hdr_lookup(hm, &parser->hdr); + if (TFW_MSG_H2(hm)) { + TfwStr h_name; + + __h2_msg_hdr_name(&parser->hdr, &h_name); + id = __h2_hdr_lookup(hm, &h_name); + } + else { + id = __hdr_lookup(hm, &parser->hdr); + } /* Allocate some more room if not enough to store the header. */ if (unlikely(id == ht->size)) { diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 09ba9a0fb0..a839e0f258 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -50,6 +50,7 @@ __tfw_http_msg_set_str_data(TfwStr *str, void *data, struct sk_buff *skb) __tfw_http_msg_set_str_data(str, data, \ ss_skb_peek_tail(&hm->msg.skb_head)) +void __h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name); void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val); void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client); diff --git a/tempesta_fw/msg.h b/tempesta_fw/msg.h index 9dc20ba3c4..7e9c8ce560 100644 --- a/tempesta_fw/msg.h +++ b/tempesta_fw/msg.h @@ -69,9 +69,6 @@ typedef struct { * @hb_len - length of the message HTTP/2 headers block; * @hdrs_len - accumulated length of message's decoded and parsed headers; * @hdrs_cnt - count of all headers from message headers block; - * @start_pos - pointer to the beginning of decoded headers' buffer; used - * as start point during exporting buffer pages to message's - * target skb; * @__off - offset for iterator reinitializing before next processing * stage; * @hdr - descriptor of currently decoded header in target buffer; @@ -90,7 +87,6 @@ typedef struct { unsigned long hb_len; unsigned long hdrs_len; unsigned int hdrs_cnt; - char *start_pos; char __off[0]; TfwStr hdr; char *pos; diff --git a/tempesta_fw/pool.c b/tempesta_fw/pool.c index 14124b5a5d..fac99f6427 100644 --- a/tempesta_fw/pool.c +++ b/tempesta_fw/pool.c @@ -212,25 +212,36 @@ tfw_pool_free(TfwPool *p, void *ptr, size_t n) EXPORT_SYMBOL(tfw_pool_free); /** - * Delete all chunks between the last (i.e. current in use) and the first one + * Delete all chunks between the first (i.e. current in use) and the last one * (which is the holder of @TfwPool itself). This is a garbage collection * procedure, which is applicable only for cases when pool is used for one - * dynamically resizable (via @__tfw_pool_realloc()) instance. + * dynamically resizable (via @__tfw_pool_realloc()) instance. Also, this + * function can be used when caller definitely know, that all data behind the + * @ptr can be evicted (including the page which contains the @ptr). */ void -tfw_pool_clean(TfwPool *p) +tfw_pool_clean(TfwPool *pool, void *ptr) { TfwPoolChunk *c, *next; - if (!p) + if (!pool) return; - for (c = p->curr->next; c; c = next) { + for (c = pool->curr->next; c; c = next) { if (!(next = c->next)) break; + if (ptr) { + if ((char *)ptr < (char *)TFW_POOL_CHUNK_BASE(c) + || (char *)ptr >= (char *)TFW_POOL_CHUNK_BASE(c) + + c->off) + { + continue; + } + ptr = NULL; + } tfw_pool_free_pages(TFW_POOL_CHUNK_BASE(c), c->order); } - p->curr->next = c; + pool->curr->next = c; } EXPORT_SYMBOL(tfw_pool_clean); diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index 899425cca9..eb0da68bf7 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -36,10 +36,9 @@ .len = (nm)->len - (hdr_res)->len, \ .nchunks = (nm)->nchunks - 1 \ }; \ - collect_compound_str(hdr_res, &nm_fin); \ + collect_compound_str(hdr_res, &nm_fin, 0); \ } \ - collect_compound_str2(hdr_res, ":", 1); \ - collect_compound_str(hdr_res, val); \ + collect_compound_str(hdr_res, val, TFW_STR_HDR_VALUE); \ }) #define HDR_COMPOUND_STR_LIT(hdr_res, nm_lit, val_lit) \ @@ -47,7 +46,7 @@ do { \ TFW_STR(name, nm_lit); \ TFW_STR(value, val_lit); \ BUG_ON(!name || !value); \ - HP_HDR_COMPOUND_STR(hdr_res, name, value); \ + HDR_COMPOUND_STR(hdr_res, name, value); \ } while (0) static TfwH2Ctx ctx; @@ -89,62 +88,73 @@ test_h2_teardown(void) TEST(hpack, dec_table_static) { TfwHPack *hp; + TfwStr *hdr, h_val; const TfwHPackEntry *entry; - TfwHPackStr *name, *value; + const char *s_ius = "if-unmodified-since"; + const int ius_len = strlen(s_ius); + const char *s_wa = "www-authenticate"; + const int wa_len = strlen(s_wa); + const char *s_auth = ":authority"; + const int auth_len = strlen(s_auth); + const char *s_aenc = "accept-encoding"; + const int aenc_len = strlen(s_aenc); + const char *s_aenc_v = "gzip, deflate"; + const int aenc_v_len = strlen(s_aenc_v); + const char *s_tenc = "transfer-encoding"; + const int tenc_len = strlen(s_tenc); hp = &ctx.hpack; entry = tfw_hpack_find_index(&hp->dec_tbl, 43); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - EXPECT_EQ(strlen("if-unmodified-since"), name->len); - EXPECT_OK(memcmp_fast("if-unmodified-since", name->ptr, - name->len)); - EXPECT_NULL(entry->value); + hdr = entry->hdr; + EXPECT_EQ(hdr->nchunks, 1); + EXPECT_EQ(ius_len, hdr->len); + EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_ius, ius_len, 0)); EXPECT_EQ(entry->tag, TFW_HTTP_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 61); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - EXPECT_EQ(strlen("www-authenticate"), name->len); - EXPECT_OK(memcmp_fast("www-authenticate", name->ptr, - name->len)); - EXPECT_NULL(entry->value); + hdr = entry->hdr; + EXPECT_EQ(hdr->nchunks, 1); + EXPECT_EQ(wa_len, hdr->len); + EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_wa, wa_len, 0)); EXPECT_EQ(entry->tag, TFW_HTTP_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 1); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - EXPECT_EQ(strlen(":authority"), name->len); - EXPECT_OK(memcmp_fast(":authority", name->ptr, name->len)); - EXPECT_NULL(entry->value); + hdr = entry->hdr; + EXPECT_EQ(hdr->nchunks, 1); + EXPECT_EQ(auth_len, hdr->len); + EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_auth, auth_len, 0)); EXPECT_EQ(entry->tag, TFW_HTTP_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 16); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - value = entry->value; - EXPECT_EQ(strlen("accept-encoding"), name->len); - EXPECT_OK(memcmp_fast("accept-encoding", name->ptr, name->len)); - EXPECT_OK(memcmp_fast("gzip, deflate", value->ptr, value->len)); + hdr = entry->hdr; + EXPECT_EQ(hdr->nchunks, 2); + EXPECT_EQ(aenc_len, entry->nm_len); + EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_aenc, aenc_len, + TFW_STR_EQ_PREFIX)); + __h2_msg_hdr_val(hdr, &h_val); + EXPECT_TRUE(tfw_str_eq_cstr(&h_val, s_aenc_v, aenc_v_len, 0)); EXPECT_EQ(entry->tag, TFW_HTTP_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 57); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - EXPECT_EQ(strlen("transfer-encoding"), name->len); - EXPECT_OK(memcmp_fast("transfer-encoding", name->ptr, - name->len)); - EXPECT_NULL(entry->value); + hdr = entry->hdr; + EXPECT_EQ(hdr->nchunks, 1); + EXPECT_EQ(tenc_len, hdr->len); + EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_tenc, tenc_len, 0)); EXPECT_EQ(entry->tag, TFW_HTTP_HDR_TRANSFER_ENCODING); } } @@ -152,16 +162,15 @@ TEST(hpack, dec_table_static) TEST(hpack, dec_table_dynamic) { TfwHPack *hp; - TfwMsgParseIter it; const TfwHPackEntry *entry; - TfwStr *s1, *s2, *s3; - TfwHPackStr *name, *value; + TfwStr h_val, *hdr, *s1, *s2, *s3; + TfwMsgParseIter *it = &test_req->pit; unsigned int new_len = 0; TFW_STR(s1_name, "custom-key"); TFW_STR(s1_value, "custom-value"); - TFW_STR(s2_name, "X-Forwarded-For"); + TFW_STR(s2_name, "x-forwarded-for"); TFW_STR(s2_value, "example.com"); - TFW_STR(s3_name, "X-Custom-Hdr"); + TFW_STR(s3_name, "x-custom-hdr"); TFW_STR(s3_value, "custom header values"); HDR_COMPOUND_STR(s1, s1_name, s1_value); @@ -170,47 +179,30 @@ TEST(hpack, dec_table_dynamic) hp = &ctx.hpack; - it.hdr = *s1; - it.nm_len = 10; - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); + *it->parsed_hdr = *s1; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - it.hdr = *s2; - it.nm_len = 15; - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); + *it->parsed_hdr = *s2; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - it.hdr = *s3; - it.nm_len = 12; - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); + *it->parsed_hdr = *s3; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); entry = tfw_hpack_find_index(&hp->dec_tbl, 64); EXPECT_NOT_NULL(entry); - if (entry) { - name = entry->name; - value = entry->value; - EXPECT_TRUE(tfw_str_eq_cstr(s1_name, name->ptr, name->len, 0)); - EXPECT_TRUE(tfw_str_eq_cstr(s1_value, value->ptr, - value->len, 0)); - } + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s1) == 0); entry = tfw_hpack_find_index(&hp->dec_tbl, 63); EXPECT_NOT_NULL(entry); - if (entry) { - name = entry->name; - value = entry->value; - EXPECT_TRUE(tfw_str_eq_cstr(s2_name, name->ptr, name->len, 0)); - EXPECT_TRUE(tfw_str_eq_cstr(s2_value, value->ptr, - value->len, 0)); - } + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s2) == 0); entry = tfw_hpack_find_index(&hp->dec_tbl, 62); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - value = entry->value; - new_len += name->len + value->len + 32; - EXPECT_TRUE(tfw_str_eq_cstr(s3_name, name->ptr, name->len, 0)); - EXPECT_TRUE(tfw_str_eq_cstr(s3_value, value->ptr, - value->len, 0)); + new_len += entry->hdr->len + 32; + EXPECT_TRUE(tfw_strcmp(entry->hdr, s3) == 0); } EXPECT_OK(tfw_hpack_set_length(hp, new_len)); @@ -220,13 +212,12 @@ TEST(hpack, dec_table_dynamic) EXPECT_NOT_NULL(tfw_hpack_find_index(&hp->dec_tbl, 62)); } -TEST(hpack, dec_table_mixed) +TEST(hpack, dec_table_dynamic_inc) { TfwHPack *hp; - TfwMsgParseIter it; - TfwHPackStr *name, *value; TfwStr *s1, *s2, *s3, *s4, *s5; - const TfwHPackEntry *entry, *entry_1, *entry_2, *entry_3; + const TfwHPackEntry *entry; + TfwMsgParseIter *it = &test_req->pit; TFW_STR(s1_name, "custom-header-1"); TFW_STR(s1_value, "custom value 1"); TFW_STR(s2_name, "custom-header-2"); @@ -244,87 +235,76 @@ TEST(hpack, dec_table_mixed) hp = &ctx.hpack; - it.hdr = *s1; - it.nm_len = s1_name->len; - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); - - it.hdr = *s2; - it.nm_len = s2_name->len; - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); - - entry_1 = tfw_hpack_find_index(&hp->dec_tbl, 63); - EXPECT_NOT_NULL(entry_1); - if (entry_1) { - name = entry_1->name; - value = entry_1->value; - it.hdr = *s4; - it.nm_len = s1_name->len; - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, entry_1, &it)); - } + *it->parsed_hdr = *s1; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - entry_2 = tfw_hpack_find_index(&hp->dec_tbl, 63); - EXPECT_NOT_NULL(entry_2); - if (entry_2) { - name = entry_2->name; - value = entry_2->value; - it.hdr = *s5; - it.nm_len = s2_name->len; - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, entry_2, &it)); - } + *it->parsed_hdr = *s2; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - entry_3 = tfw_hpack_find_index(&hp->dec_tbl, 24); - EXPECT_NOT_NULL(entry_3); - if (entry_3) { - it.hdr = *s3; - it.nm_len = s3_name->len; - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, entry_3, &it)); - } + entry = tfw_hpack_find_index(&hp->dec_tbl, 62); + EXPECT_NOT_NULL(entry); + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s2) == 0); + + entry = tfw_hpack_find_index(&hp->dec_tbl, 63); + EXPECT_NOT_NULL(entry); + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s1) == 0); + + *it->parsed_hdr = *s3; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); + + *it->parsed_hdr = *s4; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); + + entry = tfw_hpack_find_index(&hp->dec_tbl, 62); + EXPECT_NOT_NULL(entry); + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s4) == 0); + + entry = tfw_hpack_find_index(&hp->dec_tbl, 63); + EXPECT_NOT_NULL(entry); + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s3) == 0); + + *it->parsed_hdr = *s5; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); + + /* Verify the correctness of the indexes order. */ + entry = tfw_hpack_find_index(&hp->dec_tbl, 66); + EXPECT_NOT_NULL(entry); + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s1) == 0); + + entry = tfw_hpack_find_index(&hp->dec_tbl, 65); + EXPECT_NOT_NULL(entry); + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s2) == 0); entry = tfw_hpack_find_index(&hp->dec_tbl, 64); EXPECT_NOT_NULL(entry); - if (entry) { - /* - * Check that entries with 66 and 64 indexes use the same - * dynamic @name instance. - */ - EXPECT_EQ(entry_1->name, entry->name); - EXPECT_EQ(entry->name->count, 2); - EXPECT_EQ(entry->value->count, 1); - } + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s3) == 0); entry = tfw_hpack_find_index(&hp->dec_tbl, 63); EXPECT_NOT_NULL(entry); - if (entry) { - /* - * Check that entries with 65 and 63 indexes use the same - * dynamic @name instance. - */ - EXPECT_EQ(entry_2->name, entry->name); - EXPECT_EQ(entry->name->count, 2); - EXPECT_EQ(entry->value->count, 1); - } + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s4) == 0); entry = tfw_hpack_find_index(&hp->dec_tbl, 62); EXPECT_NOT_NULL(entry); - if (entry) { - /* - * Check that entries with 62 and 24(static) indexes use the - * same static @name instance. - */ - EXPECT_EQ(entry_3->name, entry->name); - EXPECT_EQ(entry->name->count, -1); - EXPECT_EQ(entry->value->count, 1); - } + if (entry) + EXPECT_TRUE(tfw_strcmp(entry->hdr, s5) == 0); } TEST(hpack, dec_table_wrap) { int shift; TfwHPack *hp = &ctx.hpack; + TfwMsgParseIter *it = &test_req->pit; TFW_STR(s_value, "custom value"); for (shift = 0; shift < 14; ++shift) { - TfwMsgParseIter it; TfwHPackEntry *last_entries; const TfwHPackEntry *entries, *entry; int i, start_idx = 17, stop_idx = start_idx + shift + 1; @@ -344,8 +324,7 @@ TEST(hpack, dec_table_wrap) * table. */ for (i = start_idx; i < stop_idx; ++i) { - TfwStr s_name = {}; - TfwHPackStr *name; + TfwStr *hdr; entry = tfw_hpack_find_index(&hp->dec_tbl, i); EXPECT_NOT_NULL(entry); @@ -353,16 +332,12 @@ TEST(hpack, dec_table_wrap) if (i >= end_idx - shift) last_entries[i - (end_idx - shift)] = *entry; - EXPECT_NULL(entry->value); - name = entry->name; - s_name.len = name->len; - s_name.data = name->ptr; - HDR_COMPOUND_STR(s, &s_name, s_value); + hdr = entry->hdr; + EXPECT_EQ(hdr->nchunks, 1); + HDR_COMPOUND_STR(s, hdr, s_value); - it.hdr = *s; - it.nm_len = s_name.len; - - EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, NULL, &it)); + *it->parsed_hdr = *s; + EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); } @@ -371,7 +346,7 @@ TEST(hpack, dec_table_wrap) * Evict first @shift entries, i.e shrink table to only * one existing entry. */ - EXPECT_OK(tfw_hpack_set_length(hp, s->len - 1 + 32)); + EXPECT_OK(tfw_hpack_set_length(hp, s->len + 32)); EXPECT_OK(tfw_hpack_set_length(hp, HPACK_TABLE_DEF_SIZE)); @@ -393,13 +368,9 @@ TEST(hpack, dec_table_wrap) const TfwHPackEntry *l_entry = &last_entries[i]; const TfwHPackEntry *t_entry = &entries[i]; - EXPECT_NOT_NULL(l_entry->name); - if (l_entry->name) { - EXPECT_EQ(l_entry->name->len, - t_entry->name->len); - EXPECT_OK(memcmp_fast(l_entry->name->ptr, - t_entry->name->ptr, - t_entry->name->len)); + EXPECT_NOT_NULL(l_entry->hdr); + if (l_entry->hdr) { + EXPECT_EQ(l_entry->hdr, t_entry-hdr); } } @@ -412,12 +383,20 @@ TEST(hpack, dec_raw) { int r; TfwHPack *hp; - const char *pos; - TfwMsgParseIter *it; + TfwHttpHdrTbl *ht; + TfwStr h_name, h_value; unsigned int parsed; unsigned long test_len1, test_len2, test_len3; - const char *test_data1 = "custom-key:custom-value\r\n"; +#define HDR_NAME_1 "custom-key" +#define HDR_VALUE_1 "custom-value" +#define HDR_NAME_2 "x-custom-hdr" +#define HDR_VALUE_2 "test foo example value" +#define HDR_NAME_3 "x-forwarded-for" +#define HDR_VALUE_3 " 127.0.0.1, example.com" + + const char *test_name1 = HDR_NAME_1; + const char *test_value1 = HDR_VALUE_1; unsigned long hdr_len1 = 25; const char *hdr_data1 = "\x40" /* == With indexing == */ @@ -429,7 +408,8 @@ TEST(hpack, dec_raw) "\x6D\x2D\x76\x61\x6C" /* */ "\x75\x65"; /* */ - const char *test_data2 = "x-custom-hdr:test foo example value\r\n"; + const char *test_name2 = HDR_NAME_2; + const char *test_value2 = HDR_VALUE_2; unsigned long hdr_len2 = 37; const char *hdr_data2 = "\x00" /* == Without indexing == */ @@ -444,7 +424,8 @@ TEST(hpack, dec_raw) "\x65\x20\x76\x61\x6C" /* */ "\x75\x65"; /* */ - const char *test_data3 = "x-forwarded-for: 127.0.0.1, example.com\r\n"; + const char *test_name3 = HDR_NAME_3; + const char *test_value3 = HDR_VALUE_3; unsigned long hdr_len3 = 41; const char *hdr_data3 = "\x10" /* == Never indexing == */ @@ -460,11 +441,6 @@ TEST(hpack, dec_raw) "\x63\x6F\x6D"; /* */ hp = &ctx.hpack; - it = &test_req->pit; - - test_len1 = strlen(test_data1); - test_len2 = strlen(test_data2); - test_len3 = strlen(test_data3); r = tfw_hpack_decode(hp, hdr_data1, hdr_len1, test_req, &parsed); EXPECT_EQ(r, T_OK); @@ -478,40 +454,98 @@ TEST(hpack, dec_raw) EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len3); - pos = it->start_pos; - EXPECT_OK(memcmp_fast(pos, test_data1, test_len1)); + ht = test_req->h_tbl + + __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW], &h_name); + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW], &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(strlen(test_name1), h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, + strlen(test_name1), 0)); + EXPECT_EQ(strlen(test_value1), h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value1, + strlen(test_value1), 0)); + } + + __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_name); + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(strlen(test_name2), h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, + strlen(test_name2), 0)); + EXPECT_EQ(strlen(test_value2), h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value2, + strlen(test_value2), 0)); + } - pos += test_len1; - EXPECT_OK(memcmp_fast(pos, test_data2, test_len2)); + __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_name); + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(strlen(test_name3), h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name3, + strlen(test_name3), 0)); + EXPECT_EQ(strlen(test_value3), h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, + strlen(test_value3), 0)); + } - pos += test_len2; - EXPECT_OK(memcmp_fast(pos, test_data3, test_len3)); +#undef HDR_NAME_1 +#undef HDR_VALUE_1 +#undef HDR_NAME_2 +#undef HDR_VALUE_2 +#undef HDR_NAME_3 +#undef HDR_VALUE_3 } TEST(hpack, dec_indexed) { int r; TfwHPack *hp; - const char *pos; - TfwMsgParseIter *it; + TfwHttpHdrTbl *ht; unsigned int parsed; const TfwHPackEntry *entry; - const TfwHPackStr *name, *value; - - const char *test_data1 = "x-forwarded-for: test.com, foo.com," - " example.com\r\n"; - const char *test_data2 = "accept-encoding:gzip, deflate\r\n"; - const char *test_data3 = "accept-encoding:deflate, gzip;q=1.0," - " *;q=0.5\r\n"; - const char *test_data4 = "x-forwarded-for:127.0.0.1\r\n"; - const char *test_data5 = "host:localhost\r\n"; - const char *test_data6 = "transfer-encoding:chunked\r\n"; - unsigned long test_len1 = strlen(test_data1); - unsigned long test_len2 = strlen(test_data2); - unsigned long test_len3 = strlen(test_data3); - unsigned long test_len4 = strlen(test_data4); - unsigned long test_len5 = strlen(test_data5); - unsigned long test_len6 = strlen(test_data6); + const TfwStr h_name, h_value, *hdr, *dup; + +#define HDR_NAME_1 "x-forwarded-for" +#define HDR_VALUE_1 " test.com, foo.com, example.com" +#define HDR_NAME_2 "accept-encoding" +#define HDR_VALUE_2 "gzip, deflate" +#define HDR_NAME_3 "accept-encoding" +#define HDR_VALUE_3 "deflate, gzip;q=1.0, *;q=0.5" +#define HDR_NAME_4 "x-forwarded-for" +#define HDR_VALUE_4 "127.0.0.1" +#define HDR_NAME_5 "host" +#define HDR_VALUE_5 "localhost" +#define HDR_NAME_6 "transfer-encoding" +#define HDR_VALUE_6 "chunked" + + const char *test_name1 = HDR_NAME_1; + const char *test_value1 = HDR_VALUE_1; + const char *test_name2 = HDR_NAME_2; + const char *test_value2 = HDR_VALUE_2; + const char *test_value3 = HDR_VALUE_3; + const char *test_value4 = HDR_VALUE_4; + const char *test_name5 = HDR_NAME_5; + const char *test_value5 = HDR_VALUE_5; + const char *test_name6 = HDR_NAME_6; + const char *test_value6 = HDR_VALUE_6; + + unsigned long test_len_nm1 = strlen(test_name1); + unsigned long test_len_val1 = strlen(test_value1); + unsigned long test_len_nm2 = strlen(test_name2); + unsigned long test_len_val2 = strlen(test_value2); + unsigned long test_len_val3 = strlen(test_value3); + unsigned long test_len_val4 = strlen(test_value4); + unsigned long test_len_nm5 = strlen(test_name5); + unsigned long test_len_val5 = strlen(test_value5); + unsigned long test_len_nm6 = strlen(test_name6); + unsigned long test_len_val6 = strlen(test_value6); unsigned long hdr_len1 = 49; const char *hdr_data1 = @@ -578,7 +612,6 @@ TEST(hpack, dec_indexed) "\x65\x64"; /* */ hp = &ctx.hpack; - it = &test_req->pit; /* * Processing prepared HTTP/2 headers in HPACK decoding @@ -612,30 +645,116 @@ TEST(hpack, dec_indexed) EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len7); + ht = test_req->h_tbl; + /* * Verify that decoded headers had been correctly written into - * the special target buffer. + * the headers table. */ - pos = it->start_pos; - EXPECT_OK(memcmp_fast(pos, test_data1, test_len1)); - - pos += test_len1; - EXPECT_OK(memcmp_fast(pos, test_data1, test_len1)); - - pos += test_len1; - EXPECT_OK(memcmp_fast(pos, test_data2, test_len2)); - - pos += test_len2; - EXPECT_OK(memcmp_fast(pos, test_data3, test_len3)); + hdr = &ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR]; + EXPECT_TRUE(TFW_STR_DUP(hdr)); + EXPECT_EQ(hdr->nchunks, 3); + if (hdr->nchunks == 3) { + dup = hdr->chunks; + __h2_msg_hdr_name(dup, &h_name); + __h2_msg_hdr_val(dup, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm1, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, + test_len_nm1, 0)); + EXPECT_EQ(test_len_val1, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value1, + test_len_val1, 0)); + } + dup = hdr->chunks + 1; + __h2_msg_hdr_name(dup, &h_name); + __h2_msg_hdr_val(dup, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm1, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, + test_len_nm1, 0)); + EXPECT_EQ(test_len_val1, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value1, + test_len_val1, 0)); + } + dup = hdr->chunks + 2; + __h2_msg_hdr_name(dup, &h_name); + __h2_msg_hdr_val(dup, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm1, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, + test_len_nm1, 0)); + EXPECT_EQ(test_len_val4, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value4, + test_len_val4, 0)); + } + } - pos += test_len3; - EXPECT_OK(memcmp_fast(pos, test_data4, test_len4)); + hdr = &ht->tbl[TFW_HTTP_HDR_RAW]; + EXPECT_TRUE(TFW_STR_DUP(hdr)); + EXPECT_EQ(hdr->nchunks, 2); + if (hdr->nchunks == 2) { + dup = hdr->chunks; + __h2_msg_hdr_name(dup, &h_name); + __h2_msg_hdr_val(dup, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm2, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, + test_len_nm2, 0)); + EXPECT_EQ(test_len_val2, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value2, + test_len_val2, 0)); + } + dup = hdr->chunks + 1; + __h2_msg_hdr_name(dup, &h_name); + __h2_msg_hdr_val(dup, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm2, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, + test_len_nm2, 0)); + EXPECT_EQ(test_len_val3, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, + test_len_val3, 0)); + } + } - pos += test_len4; - EXPECT_OK(memcmp_fast(pos, test_data5, test_len5)); + hdr = &ht->tbl[TFW_HTTP_HDR_HOST]; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm5, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name5, + test_len_nm5, 0)); + EXPECT_EQ(test_len_val5, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value5, + test_len_val5, 0)); + } - pos += test_len5; - EXPECT_OK(memcmp_fast(pos, test_data6, test_len6)); + hdr = &ht->tbl[TFW_HTTP_HDR_TRANSFER_ENCODING]; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm6, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name6, + test_len_nm6, 0)); + EXPECT_EQ(test_len_val6, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value6, + test_len_val6, 0)); + } /* * Verify that decoded headers had been placed into decoder index @@ -646,60 +765,115 @@ TEST(hpack, dec_indexed) * have 'without indexing' code in the head part, which means they * hadn't been indexed too. */ + entry = tfw_hpack_find_index(&hp->dec_tbl, 65); + EXPECT_NULL(entry); + entry = tfw_hpack_find_index(&hp->dec_tbl, 64); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - value = entry->value; - EXPECT_EQ(strlen("x-forwarded-for"), name->len); - EXPECT_OK(memcmp_fast(test_data1, name->ptr, name->len)); - EXPECT_EQ(strlen(" test.com, foo.com, example.com"), - value->len); - EXPECT_OK(memcmp_fast(test_data1 + 15 + 1, value->ptr, - value->len)); + hdr = entry->hdr; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm1, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, + test_len_nm1, 0)); + EXPECT_EQ(test_len_val1, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value1, + test_len_val1, 0)); + } + EXPECT_EQ(entry->nm_len, test_len_nm1); + EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_X_FORWARDED_FOR); } entry = tfw_hpack_find_index(&hp->dec_tbl, 63); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - value = entry->value; - EXPECT_EQ(strlen("accept-encoding"), name->len); - EXPECT_OK(memcmp_fast(test_data3, name->ptr, name->len)); - EXPECT_EQ(strlen("deflate, gzip;q=1.0, *;q=0.5"), value->len); - EXPECT_OK(memcmp_fast(test_data3 + 15 + 1, value->ptr, - value->len)); + hdr = entry->hdr; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm2, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, + test_len_nm2, 0)); + EXPECT_EQ(test_len_val3, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, + test_len_val3, 0)); + } + EXPECT_EQ(entry->nm_len, test_len_nm2); + EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 62); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - value = entry->value; - EXPECT_EQ(strlen("x-forwarded-for"), name->len); - EXPECT_OK(memcmp_fast(test_data4, name->ptr, name->len)); - EXPECT_EQ(strlen("127.0.0.1"), value->len); - EXPECT_OK(memcmp_fast(test_data4 + 15 + 1, value->ptr, - value->len)); + hdr = entry->hdr; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm1, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, + test_len_nm1, 0)); + EXPECT_EQ(test_len_val4, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value4, + test_len_val4, 0)); + } + EXPECT_EQ(entry->nm_len, test_len_nm1); + EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } + +#undef HDR_NAME_1 +#undef HDR_VALUE_1 +#undef HDR_NAME_2 +#undef HDR_VALUE_2 +#undef HDR_NAME_3 +#undef HDR_VALUE_3 +#undef HDR_NAME_4 +#undef HDR_VALUE_4 +#undef HDR_NAME_5 +#undef HDR_VALUE_5 +#undef HDR_NAME_6 +#undef HDR_VALUE_6 } TEST(hpack, dec_huffman) { int r; TfwHPack *hp; - const char *pos; - TfwMsgParseIter *it; + TfwHttpHdrTbl *ht; unsigned int parsed; const TfwHPackEntry *entry; - const TfwHPackStr *name, *value; - - const char *test_data1 = "custom-key:custom-value\r\n"; - unsigned long test_len1 = strlen(test_data1); - const char *test_data2 = "cache-control:no-cache\r\n"; - unsigned long test_len2 = strlen(test_data2); - const char *test_data3 = ":authority:www.example.com\r\n"; - unsigned long test_len3 = strlen(test_data3); + const TfwStr h_name, h_value, *hdr; + +#define HDR_NAME_1 "custom-key" +#define HDR_VALUE_1 "custom-value" +#define HDR_NAME_2 "cache-control" +#define HDR_VALUE_2 "no-cache" +#define HDR_NAME_3 ":authority" +#define HDR_VALUE_3 "www.example.com" + + const char *test_name1 = HDR_NAME_1; + const char *test_value1 = HDR_VALUE_1; + const char *test_name2 = HDR_NAME_2; + const char *test_value2 = HDR_VALUE_2; + const char *test_name3 = HDR_NAME_3; + const char *test_value3 = HDR_VALUE_3; + + unsigned long test_len_nm1 = strlen(test_name1); + unsigned long test_len_val1 = strlen(test_value1); + unsigned long test_len_nm2 = strlen(test_name2); + unsigned long test_len_val2 = strlen(test_value2); + unsigned long test_len_nm3 = strlen(test_name3); + unsigned long test_len_val3 = strlen(test_value3); unsigned long hdr_len1 = 20; const char *hdr_data1 = @@ -736,8 +910,11 @@ TEST(hpack, dec_huffman) "\xF4\xFF"; /* */ hp = &ctx.hpack; - it = &test_req->pit; + /* + * Processing prepared Huffman-encoded HTTP/2 headers in HPACK + * decoding procedure. + */ r = tfw_hpack_decode(hp, hdr_data1, hdr_len1, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len1); @@ -750,49 +927,119 @@ TEST(hpack, dec_huffman) EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len3); - pos = it->start_pos; - EXPECT_OK(memcmp_fast(pos, test_data1, test_len1)); + ht = test_req->h_tbl; - pos += test_len1; - EXPECT_OK(memcmp_fast(pos, test_data2, test_len2)); + /* + * Verify that Huffman-decoded headers had been correctly written + * into the headers table. + */ + hdr = &ht->tbl[TFW_HTTP_HDR_RAW]; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm1, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, + test_len_nm1, 0)); + EXPECT_EQ(test_len_val1, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value1, + test_len_val1, 0)); + } - pos += test_len2; - EXPECT_OK(memcmp_fast(pos, test_data3, test_len3)); + hdr = &ht->tbl[TFW_HTTP_HDR_RAW + 1]; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm2, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, + test_len_nm2, 0)); + EXPECT_EQ(test_len_val2, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value2, + test_len_val2, 0)); + } + + hdr = &ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm3, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name3, + test_len_nm3, 0)); + EXPECT_EQ(test_len_val3, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, + test_len_val3, 0)); + } + /* + * Verify that Huffman-decoded headers had been correctly placed into + * decoder index table with appropriate indexes. + */ entry = tfw_hpack_find_index(&hp->dec_tbl, 64); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - value = entry->value; - EXPECT_EQ(strlen("custom-key"), name->len); - EXPECT_OK(memcmp_fast(test_data1, name->ptr, name->len)); - EXPECT_EQ(strlen("custom-value"), value->len); - EXPECT_OK(memcmp_fast(test_data1 + 10 + 1, value->ptr, - value->len)); + hdr = entry->hdr; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm1, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, + test_len_nm1, 0)); + EXPECT_EQ(test_len_val1, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value1, + test_len_val1, 0)); + } + EXPECT_EQ(entry->nm_len, test_len_nm1); + EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 63); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - value = entry->value; - EXPECT_EQ(strlen("cache-control"), name->len); - EXPECT_OK(memcmp_fast(test_data2, name->ptr, name->len)); - EXPECT_EQ(strlen("no-cache"), value->len); - EXPECT_OK(memcmp_fast(test_data2 + 13 + 1, value->ptr, - value->len)); + hdr = entry->hdr; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm2, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, + test_len_nm2, 0)); + EXPECT_EQ(test_len_val2, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value2, + test_len_val2, 0)); + } + EXPECT_EQ(entry->nm_len, test_len_nm2); + EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_CACHE_CONTROL); } entry = tfw_hpack_find_index(&hp->dec_tbl, 62); EXPECT_NOT_NULL(entry); if (entry) { - name = entry->name; - value = entry->value; - EXPECT_EQ(strlen(":authority"), name->len); - EXPECT_OK(memcmp_fast(test_data3, name->ptr, name->len)); - EXPECT_EQ(strlen("www.example.com"), value->len); - EXPECT_OK(memcmp_fast(test_data3 + 10 + 1, value->ptr, - value->len)); + hdr = entry->hdr; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm3, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name3, + test_len_nm3, 0)); + EXPECT_EQ(test_len_val3, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, + test_len_val3, 0)); + } + EXPECT_EQ(entry->nm_len, test_len_nm3); + EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_H2_AUTHORITY); } } @@ -1022,10 +1269,6 @@ TEST(hpack, enc_table_index) #undef HDR_VALUE_2 #undef HDR_NAME_3 #undef HDR_VALUE_3 -#undef HDR_NAME_4 -#undef HDR_VALUE_4 -#undef HDR_NAME_5 -#undef HDR_VALUE_5 } TEST(hpack, enc_table_rbtree) @@ -1418,7 +1661,7 @@ TEST_SUITE(hpack) TEST_RUN(hpack, dec_table_static); TEST_RUN(hpack, dec_table_dynamic); - TEST_RUN(hpack, dec_table_mixed); + TEST_RUN(hpack, dec_table_dynamic_inc); TEST_RUN(hpack, dec_table_wrap); TEST_RUN(hpack, dec_raw); TEST_RUN(hpack, dec_indexed); diff --git a/tempesta_fw/t/unit/tfw_str_helper.c b/tempesta_fw/t/unit/tfw_str_helper.c index 82e5ad7577..05e720bd19 100644 --- a/tempesta_fw/t/unit/tfw_str_helper.c +++ b/tempesta_fw/t/unit/tfw_str_helper.c @@ -109,7 +109,8 @@ make_compound_str2(const char *data1, const char *data2) } TfwStr * -collect_compound_str(TfwStr *res_str, const TfwStr *in_str) +collect_compound_str(TfwStr *res_str, const TfwStr *in_str, + unsigned short flags) { const TfwStr *c, *end; TfwStr *c_new, *c_start = NULL; @@ -119,6 +120,7 @@ collect_compound_str(TfwStr *res_str, const TfwStr *in_str) BUG_ON(!c_new); *c_new = *c; + c_new->flags |= flags; res_str->len += c_new->len; if (!c_start) @@ -128,13 +130,15 @@ collect_compound_str(TfwStr *res_str, const TfwStr *in_str) } TfwStr * -collect_compound_str2(TfwStr *res_str, char *str, unsigned long len) +collect_compound_str2(TfwStr *res_str, char *str, unsigned long len, + unsigned short flags) { TfwStr *c_new = tfw_str_add_compound(str_pool, res_str); BUG_ON(!c_new); c_new->len = len; c_new->data = str; + c_new->flags = flags; res_str->len += len; return c_new; diff --git a/tempesta_fw/t/unit/tfw_str_helper.h b/tempesta_fw/t/unit/tfw_str_helper.h index 06a760ef33..bd955c4c6c 100644 --- a/tempesta_fw/t/unit/tfw_str_helper.h +++ b/tempesta_fw/t/unit/tfw_str_helper.h @@ -27,8 +27,10 @@ void free_all_str(void); TfwStr *make_plain_str(const char *data); TfwStr *make_compound_str(const char *data); TfwStr *make_compound_str2(const char *data1, const char *data2); -TfwStr *collect_compound_str(TfwStr *res_str, const TfwStr *in_str); -TfwStr *collect_compound_str2(TfwStr *res_str, char *str, unsigned long len); +TfwStr *collect_compound_str(TfwStr *res_str, const TfwStr *in_str, + unsigned short flags); +TfwStr *collect_compound_str2(TfwStr *res_str, char *str, unsigned long len, + unsigned short flags); #define TFW_STR(name, literal) TfwStr *name = make_compound_str(literal) #define TFW_STR2(name, literal1, literal2) TfwStr *name = make_compound_str2(literal1, literal2) From b5658bec4e6e6f9f9b0a92fafc052674b47b796b Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Thu, 31 Oct 2019 01:09:24 +0300 Subject: [PATCH 03/64] HTTP/2 Parser implementation: some comments added for HPACK functionality (#309). --- tempesta_fw/hpack.c | 76 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index 0f025c9b9d..aed5781ae5 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -1554,6 +1554,25 @@ tfw_hpack_set_entry(TfwPool *__restrict h_pool, TfwMsgParseIter *__restrict it, return entry; } +/* + * The procedure for adding new header into the HPACK decoder table. + * Note, that our decoder dynamic table must satisfy several main requirements: + * 1. Provide fast direct access to entries by index; + * 2. Be able to increase, since the real size of our table is always + * greater than it's standardized pseudo-size (RFC 7541 section 4.1); + * 3. Store the records with variable size (headers strings and their + * @TfwStr descriptors). + * To meet this specification, the current decoder dynamic table is using two + * pools (see @pool and @h_pool members in @TfwHPackDTbl structure description): + * the first one is intended for storage of constant length entries (because we + * need a quick access by index for the entries of decoder table) and it is + * always a single resizable chunk in the memory, relocatable between different + * pages in the pool (in case of storage growth); the purpose of the second pool + * is to store records with variable size (the headers strings and their @TfwStr + * descriptors) - this storage area cannot be relocated during growth due to + * internal pointers of @TfwStr, but can be shared between different pages of + * the pool; in general scheme the first module refers to the second one. + */ static int tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, TfwMsgParseIter *__restrict it) @@ -3713,9 +3732,12 @@ tfw_hpack_idx_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict str, return 0; } -//!!! comment is needed +/* + * Add header @hdr in HTTP/2 HPACK format into the response @resp, before + * the @first data. + */ static int -tfw_hpack_lit_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, +tfw_hpack_hdr_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, bool name_indexed) { @@ -3786,9 +3808,13 @@ tfw_hpack_lit_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, return 0; } -//!!! comment is needed + +/* + * Expand the response @resp with the new @hdr in HTTP/2 HPACK format, via + * extending of skb/frags chain. + */ static int -tfw_hpack_lit_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, +tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, bool name_indexed) { int ret; @@ -3870,9 +3896,12 @@ tfw_hpack_lit_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return 0; } -//!!! comment is needed (about @fc too) +/* + * Substitute the header @tgt with the new header @hdr in HTTP/2 HPACK format + * in the response @resp. + */ static inline int -tfw_hpack_lit_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, +tfw_hpack_hdr_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, bool name_indexed) { @@ -3885,7 +3914,7 @@ tfw_hpack_lit_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, fc = TFW_STR_CHUNK(tgt, 0); - if ((ret = tfw_hpack_lit_add(resp, fc, hdr, idx, name_indexed))) + if ((ret = tfw_hpack_hdr_add(resp, fc, hdr, idx, name_indexed))) return ret; if ((ret = ss_skb_cutoff_data(skb_head, tgt, 0, tfw_str_eolen(tgt)))) @@ -3894,8 +3923,12 @@ tfw_hpack_lit_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, return 0; } +/* + * Transform the HTTP/1.1 header @hdr in-place into HTTP/2 HPACK format in the + * response @resp. + */ static int -tfw_hpack_lit_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, +tfw_hpack_hdr_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, bool name_indexed) { int r; @@ -4018,7 +4051,16 @@ tfw_hpack_lit_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return 0; } -//!!! comment is needed (including the @tgt description) +/* + * Perform encoding of the header @hdr (or @tgt) into the HTTP/2 HPACK format. + * The four operation types can be executed here: addition, expansion, + * substitution and in-place transformation. In addition or expansion cases + * @tgt must be NULL, since the new header is inserted into the message and + * there is no target to replace. In case of substitution the header @tgt + * must be replaced by new header @hdr, so the both must be present. In-place + * transformation changes the header @tgt directly in the message, thus the + * @hdr is not needed and must be NULL. + */ int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, TfwStr *__restrict hdr, TfwH2TransOp op) @@ -4083,13 +4125,13 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, switch (op) { case TFW_H2_TRANS_ADD: - return tfw_hpack_lit_add(resp, fc, hdr, &idx, true); + return tfw_hpack_hdr_add(resp, fc, hdr, &idx, true); case TFW_H2_TRANS_EXPAND: - return tfw_hpack_lit_expand(resp, hdr, &idx, true); + return tfw_hpack_hdr_expand(resp, hdr, &idx, true); case TFW_H2_TRANS_SUB: - return tfw_hpack_lit_sub(resp, tgt, hdr, &idx, true); + return tfw_hpack_hdr_sub(resp, tgt, hdr, &idx, true); case TFW_H2_TRANS_INPLACE: - return tfw_hpack_lit_inplace(resp, tgt, &idx, true); + return tfw_hpack_hdr_inplace(resp, tgt, &idx, true); default: BUG(); } @@ -4102,13 +4144,13 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, switch (op) { case TFW_H2_TRANS_ADD: - return tfw_hpack_lit_add(resp, fc, hdr, &idx, false); + return tfw_hpack_hdr_add(resp, fc, hdr, &idx, false); case TFW_H2_TRANS_EXPAND: - return tfw_hpack_lit_expand(resp, hdr, &idx, false); + return tfw_hpack_hdr_expand(resp, hdr, &idx, false); case TFW_H2_TRANS_SUB: - return tfw_hpack_lit_sub(resp, tgt, hdr, &idx, false); + return tfw_hpack_hdr_sub(resp, tgt, hdr, &idx, false); case TFW_H2_TRANS_INPLACE: - return tfw_hpack_lit_inplace(resp, tgt, &idx, false); + return tfw_hpack_hdr_inplace(resp, tgt, &idx, false); default: BUG(); } From 9a57b1f5a3348e3b05ea6b9142f6cd64a5cd8520 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Thu, 31 Oct 2019 15:45:26 +0300 Subject: [PATCH 04/64] HTTP/2 Parser implementation: numerous corrections and cleanups (#309). --- tempesta_fw/hpack.c | 222 +++++++++++++------------- tempesta_fw/hpack.h | 2 +- tempesta_fw/http.c | 269 ++++++++++++++++---------------- tempesta_fw/http_msg.c | 6 +- tempesta_fw/pool.h | 2 +- tempesta_fw/t/unit/test_hpack.c | 96 ++++++------ 6 files changed, 301 insertions(+), 296 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index aed5781ae5..40cdac3a0f 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -942,105 +942,103 @@ static const HTState ht_decode[] ____cacheline_aligned = { {6, 0}, /* 2: EOS */ }; -#define HP_HDR_NAME(name, h_tag) \ - ((TfwHPackEntry){ \ - .hdr = &(TfwStr){ \ - .chunks = &(TfwStr){ \ - .data = name, \ - .len = SLEN(name), \ - }, \ +#define HP_HDR_NAME(name) \ + (&(TfwStr){ \ + .chunks = &(TfwStr){ \ + .data = name, \ .len = SLEN(name), \ - .nchunks = 1 \ }, \ - .name_len = SLEN(name), \ - .name_num = 1, \ - .tag = h_tag \ -}) + .len = SLEN(name), \ + .nchunks = 1 \ + }) + +#define HP_HDR_FULL(name, value) \ + (&(TfwStr){ \ + .chunks = (TfwStr []){ \ + { .data = name, .len = SLEN(name) }, \ + { .data = value, .len = SLEN(value), \ + .flags = TFW_STR_HDR_VALUE } \ + }, \ + .len = SLEN(name) + SLEN(value), \ + .nchunks = 2 \ + }) -#define HP_HDR_FULL(name, value, h_tag) \ +#define HP_ENTRY(name, h_tag, hdr_expr) \ ((TfwHPackEntry){ \ - .hdr = &(TfwStr){ \ - .chunks = (TfwStr []){ \ - { \ - .data = name, \ - .len = SLEN(name) \ - }, \ - { \ - .data = value, \ - .len = SLEN(value), \ - .flags = TFW_STR_HDR_VALUE \ - } \ - }, \ - .len = SLEN(name) + SLEN(value), \ - .nchunks = 2 \ - }, \ + .hdr = hdr_expr, \ .name_len = SLEN(name), \ .name_num = 1, \ .tag = h_tag \ }) +#define HP_ENTRY_NAME(name, h_tag) \ + HP_ENTRY(name, h_tag, HP_HDR_NAME(name)) + +#define HP_ENTRY_FULL(name, value, h_tag) \ + HP_ENTRY(name, h_tag, HP_HDR_FULL(name, value)) + static const TfwHPackEntry static_table[] ____cacheline_aligned = { - HP_HDR_NAME(":authority", TFW_TAG_HDR_H2_AUTHORITY), - HP_HDR_FULL(":method", "GET", TFW_TAG_HDR_H2_METHOD), - HP_HDR_FULL(":method", "POST", TFW_TAG_HDR_H2_METHOD), - HP_HDR_FULL(":path", "/", TFW_TAG_HDR_H2_PATH), - HP_HDR_FULL(":path", "/index.html", TFW_TAG_HDR_H2_PATH), - HP_HDR_FULL(":scheme", "http", TFW_TAG_HDR_H2_SCHEME), - HP_HDR_FULL(":scheme", "https", TFW_TAG_HDR_H2_SCHEME), - HP_HDR_FULL(":status", "200", TFW_TAG_HDR_H2_STATUS), - HP_HDR_FULL(":status", "204", TFW_TAG_HDR_H2_STATUS), - HP_HDR_FULL(":status", "206", TFW_TAG_HDR_H2_STATUS), - HP_HDR_FULL(":status", "304", TFW_TAG_HDR_H2_STATUS), - HP_HDR_FULL(":status", "400", TFW_TAG_HDR_H2_STATUS), - HP_HDR_FULL(":status", "404", TFW_TAG_HDR_H2_STATUS), - HP_HDR_FULL(":status", "500", TFW_TAG_HDR_H2_STATUS), - HP_HDR_NAME("accept-charset", TFW_TAG_HDR_RAW), - HP_HDR_FULL("accept-encoding", "gzip, deflate", TFW_TAG_HDR_RAW), - HP_HDR_NAME("accept-language", TFW_TAG_HDR_RAW), - HP_HDR_NAME("accept-ranges", TFW_TAG_HDR_RAW), - HP_HDR_NAME("accept", TFW_TAG_HDR_ACCEPT), - HP_HDR_NAME("access-control-allow-origin", TFW_TAG_HDR_RAW), - HP_HDR_NAME("age", TFW_TAG_HDR_RAW), - HP_HDR_NAME("allow", TFW_TAG_HDR_RAW), - HP_HDR_NAME("authorization", TFW_TAG_HDR_AUTHORIZATION), - HP_HDR_NAME("cache-control", TFW_TAG_HDR_CACHE_CONTROL), - HP_HDR_NAME("content-disposition", TFW_TAG_HDR_RAW), - HP_HDR_NAME("content-encoding", TFW_TAG_HDR_RAW), - HP_HDR_NAME("content-language", TFW_TAG_HDR_RAW), - HP_HDR_NAME("content-length", TFW_TAG_HDR_CONTENT_LENGTH), - HP_HDR_NAME("content-location", TFW_TAG_HDR_RAW), - HP_HDR_NAME("content-range", TFW_TAG_HDR_RAW), - HP_HDR_NAME("content-type", TFW_TAG_HDR_CONTENT_TYPE), - HP_HDR_NAME("cookie", TFW_TAG_HDR_COOKIE), - HP_HDR_NAME("date", TFW_TAG_HDR_RAW), - HP_HDR_NAME("etag", TFW_TAG_HDR_ETAG), - HP_HDR_NAME("expect", TFW_TAG_HDR_RAW), - HP_HDR_NAME("expires", TFW_TAG_HDR_RAW), - HP_HDR_NAME("from", TFW_TAG_HDR_RAW), - HP_HDR_NAME("host", TFW_TAG_HDR_HOST), - HP_HDR_NAME("if-match", TFW_TAG_HDR_RAW), - HP_HDR_NAME("if-modified-since", TFW_TAG_HDR_IF_MODIFIED_SINCE), - HP_HDR_NAME("if-none-match", TFW_TAG_HDR_IF_NONE_MATCH), - HP_HDR_NAME("if-range", TFW_TAG_HDR_RAW), - HP_HDR_NAME("if-unmodified-since", TFW_TAG_HDR_RAW), - HP_HDR_NAME("last-modified", TFW_TAG_HDR_RAW), - HP_HDR_NAME("link", TFW_TAG_HDR_RAW), - HP_HDR_NAME("location", TFW_TAG_HDR_RAW), - HP_HDR_NAME("max-forwards", TFW_TAG_HDR_RAW), - HP_HDR_NAME("proxy-authenticate", TFW_TAG_HDR_RAW), - HP_HDR_NAME("proxy-authorization", TFW_TAG_HDR_RAW), - HP_HDR_NAME("range", TFW_TAG_HDR_RAW), - HP_HDR_NAME("referer", TFW_TAG_HDR_REFERER), - HP_HDR_NAME("refresh", TFW_TAG_HDR_RAW), - HP_HDR_NAME("retry-after", TFW_TAG_HDR_RAW), - HP_HDR_NAME("server", TFW_TAG_HDR_SERVER), - HP_HDR_NAME("set-cookie", TFW_TAG_HDR_RAW), - HP_HDR_NAME("strict-transport-security", TFW_TAG_HDR_RAW), - HP_HDR_NAME("transfer-encoding", TFW_TAG_HDR_TRANSFER_ENCODING), - HP_HDR_NAME("user-agent", TFW_TAG_HDR_USER_AGENT), - HP_HDR_NAME("vary", TFW_TAG_HDR_RAW), - HP_HDR_NAME("via", TFW_TAG_HDR_RAW), - HP_HDR_NAME("www-authenticate", TFW_TAG_HDR_RAW) + HP_ENTRY_NAME(":authority", TFW_TAG_HDR_H2_AUTHORITY), + HP_ENTRY_FULL(":method", "GET", TFW_TAG_HDR_H2_METHOD), + HP_ENTRY_FULL(":method", "POST", TFW_TAG_HDR_H2_METHOD), + HP_ENTRY_FULL(":path", "/", TFW_TAG_HDR_H2_PATH), + HP_ENTRY_FULL(":path", "/index.html", TFW_TAG_HDR_H2_PATH), + HP_ENTRY_FULL(":scheme", "http", TFW_TAG_HDR_H2_SCHEME), + HP_ENTRY_FULL(":scheme", "https", TFW_TAG_HDR_H2_SCHEME), + HP_ENTRY_FULL(":status", "200", TFW_TAG_HDR_H2_STATUS), + HP_ENTRY_FULL(":status", "204", TFW_TAG_HDR_H2_STATUS), + HP_ENTRY_FULL(":status", "206", TFW_TAG_HDR_H2_STATUS), + HP_ENTRY_FULL(":status", "304", TFW_TAG_HDR_H2_STATUS), + HP_ENTRY_FULL(":status", "400", TFW_TAG_HDR_H2_STATUS), + HP_ENTRY_FULL(":status", "404", TFW_TAG_HDR_H2_STATUS), + HP_ENTRY_FULL(":status", "500", TFW_TAG_HDR_H2_STATUS), + HP_ENTRY_NAME("accept-charset", TFW_TAG_HDR_RAW), + HP_ENTRY_FULL("accept-encoding", "gzip, deflate", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("accept-language", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("accept-ranges", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("accept", TFW_TAG_HDR_ACCEPT), + HP_ENTRY_NAME("access-control-allow-origin", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("age", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("allow", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("authorization", TFW_TAG_HDR_AUTHORIZATION), + HP_ENTRY_NAME("cache-control", TFW_TAG_HDR_CACHE_CONTROL), + HP_ENTRY_NAME("content-disposition", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("content-encoding", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("content-language", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("content-length", TFW_TAG_HDR_CONTENT_LENGTH), + HP_ENTRY_NAME("content-location", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("content-range", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("content-type", TFW_TAG_HDR_CONTENT_TYPE), + HP_ENTRY_NAME("cookie", TFW_TAG_HDR_COOKIE), + HP_ENTRY_NAME("date", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("etag", TFW_TAG_HDR_ETAG), + HP_ENTRY_NAME("expect", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("expires", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("from", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("host", TFW_TAG_HDR_HOST), + HP_ENTRY_NAME("if-match", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("if-modified-since", TFW_TAG_HDR_IF_MODIFIED_SINCE), + HP_ENTRY_NAME("if-none-match", TFW_TAG_HDR_IF_NONE_MATCH), + HP_ENTRY_NAME("if-range", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("if-unmodified-since", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("last-modified", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("link", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("location", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("max-forwards", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("proxy-authenticate", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("proxy-authorization", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("range", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("referer", TFW_TAG_HDR_REFERER), + HP_ENTRY_NAME("refresh", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("retry-after", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("server", TFW_TAG_HDR_SERVER), + HP_ENTRY_NAME("set-cookie", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("strict-transport-security", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("transfer-encoding", TFW_TAG_HDR_TRANSFER_ENCODING), + HP_ENTRY_NAME("user-agent", TFW_TAG_HDR_USER_AGENT), + HP_ENTRY_NAME("vary", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("via", TFW_TAG_HDR_RAW), + HP_ENTRY_NAME("www-authenticate", TFW_TAG_HDR_RAW) }; #define HPACK_STATIC_ENTRIES (sizeof(static_table) / sizeof(TfwHPackEntry)) @@ -1511,8 +1509,8 @@ tfw_hpack_set_entry(TfwPool *__restrict h_pool, TfwMsgParseIter *__restrict it, TfwHPackEntry *__restrict entry, bool *__restrict np) { char *data; - TfwHPackEntry *entry; - const TfwStr *d, *s, *end, *d_hdr, *s_hdr = it->parsed_hdr; + TfwStr *d, *d_hdr; + const TfwStr *s, *end, *s_hdr = it->parsed_hdr; unsigned long size = sizeof(TfwHPackEntry); if (WARN_ON_ONCE(TFW_STR_PLAIN(s_hdr) || TFW_STR_DUP(s_hdr))) @@ -1551,7 +1549,7 @@ tfw_hpack_set_entry(TfwPool *__restrict h_pool, TfwMsgParseIter *__restrict it, entry->tag = it->tag; entry->last = false; - return entry; + return 0; } /* @@ -1677,7 +1675,7 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, } else if (unlikely(count == length)) { TfwHPackEntry *previous = entries; TfwPool *pool = tbl->pool; - unsigned long block, wrap, tail; + unsigned long block, new_block, wrap, tail; T_DBG3("%s: reallocation index structures...", __func__); if (length) { @@ -1711,8 +1709,8 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, } } else { length = 32; - block = length * sizeof(TfwHPackEntry); - entries = tfw_pool_alloc(pool, block); + new_block = length * sizeof(TfwHPackEntry); + entries = tfw_pool_alloc(pool, new_block); if (unlikely(!entries)) return -ENOMEM; } @@ -1749,14 +1747,14 @@ tfw_hpack_add_index(TfwHPackDTbl *__restrict tbl, return 0; } -static TfwHPackEntry * +static const TfwHPackEntry * tfw_hpack_find_index(TfwHPackDTbl *__restrict tbl, unsigned long index) { - TfwHPackEntry *entry = NULL; + const TfwHPackEntry *entry = NULL; if (index <= HPACK_STATIC_ENTRIES) { entry = static_table + index - 1; - BUG_ON(!entry->name); + BUG_ON(!entry->hdr); return entry; } @@ -1948,17 +1946,14 @@ tfw_hpack_process_hdr_value(TfwHttpReq *req) static int tfw_hpack_hdr_name_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, - TfwHPackEntry *__restrict entry) + const TfwHPackEntry *__restrict entry) { char *data; - TfwStr *d; - const TfwStr *s, *end; - TfwMsgParseIter *it = &req->pit; - TfwStr *d_hdr = it->parsed_hdr; - TfwStr *s_hdr = entry->hdr; - unsigned int num = entry->name_num; unsigned long sz = entry->name_len; + const TfwStr *s, *end, *s_hdr = entry->hdr; + TfwMsgParseIter *it = &req->pit; + TfwStr *d, *d_hdr = it->parsed_hdr; WARN_ON_ONCE(!TFW_STR_EMPTY(d_hdr)); if (WARN_ON_ONCE(!num || num > s_hdr->nchunks)) @@ -2000,16 +1995,14 @@ tfw_hpack_hdr_name_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, static int tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, - TfwHPackEntry *__restrict entry) + const TfwHPackEntry *__restrict entry) { char *data; - TfwStr *d; - const TfwStr *s, *end; unsigned long d_size; TfwMsgParseIter *it = &req->pit; + const TfwStr *s, *end, *s_hdr = entry->hdr; TfwHttpParser *parser = &req->stream->parser; - TfwStr *d_hdr = &parser->hdr; - TfwStr *s_hdr = entry->hdr; + TfwStr *d, *d_hdr = &parser->hdr; WARN_ON_ONCE(TFW_STR_PLAIN(s_hdr)); WARN_ON_ONCE(!TFW_STR_EMPTY(d_hdr)); @@ -2109,8 +2102,9 @@ tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, * HPACK decoder FSM for HTTP/2 message processing. */ int -tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *src, unsigned long n, - TfwHttpReq *__restrict req, unsigned int *__restrict parsed) +tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, + unsigned long n, TfwHttpReq *__restrict req, + unsigned int *__restrict parsed) { int r = T_POSTPONE; unsigned int state = hp->state; @@ -2270,7 +2264,7 @@ tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *src, unsigned long n, } case HPACK_STATE_INDEXED_NAME_TEXT: { - TfwHPackEntry *entry; + const TfwHPackEntry *entry; get_indexed_name: T_DBG3("%s: decode indexed (%lu) header name...\n", __func__, hp->index); @@ -2356,7 +2350,7 @@ tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *src, unsigned long n, } case HPACK_STATE_ALL_INDEXED: { - TfwHPackEntry *entry; + const TfwHPackEntry *entry; get_all_indexed: T_DBG3("%s: get entire header by index: %lu\n", __func__, hp->index); diff --git a/tempesta_fw/hpack.h b/tempesta_fw/hpack.h index edb2822242..436b103567 100644 --- a/tempesta_fw/hpack.h +++ b/tempesta_fw/hpack.h @@ -239,7 +239,7 @@ int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, TfwStr *__restrict hdr, TfwH2TransOp op); void tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl, unsigned short new_size); -int tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *src, +int tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, unsigned long n, TfwHttpReq *__restrict req, unsigned int *__restrict parsed); void tfw_hpack_enc_release(TfwHPack *__restrict hp, unsigned long *flags); diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index ede3538f55..88348d60e2 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -651,6 +651,17 @@ tfw_http_conn_req_clean(TfwHttpReq *req) tfw_http_conn_msg_free((TfwHttpMsg *)req); } + +/* + * Free the request and the paired response. + */ +static inline void +tfw_http_resp_pair_free(TfwHttpReq *req) +{ + tfw_http_conn_msg_free(req->pair); + tfw_http_conn_msg_free((TfwHttpMsg *)req); +} + /* * Close the client connection and free unpaired request. This function * is needed for cases when we cannot prepare response for this request. @@ -677,6 +688,89 @@ tfw_http_resp_build_error(TfwHttpReq *req) TFW_INC_STAT_BH(clnt.msgs_otherr); } +/* + * Static index determination for response ':status' pseudo-header (see RFC + * 7541 Appendix A for details). + */ +static inline unsigned short +tfw_h2_pseudo_index(unsigned short status) +{ + switch (status) { + case 200: + return 8; + case 204: + return 9; + case 206: + return 10; + case 304: + return 11; + case 400: + return 12; + case 404: + return 13; + case 500: + return 14; + default: + return 0; + } +} + +static int +tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) +{ + int ret; + unsigned short index = tfw_h2_pseudo_index(resp->status); + char buf[H2_STAT_VAL_LEN]; + TfwStr *s_line = NULL; + TfwStr s_hdr = { + .chunks = (TfwStr []){ + { .data = S_H2_STAT, .len = SLEN(S_H2_STAT) }, + { .data = buf, .len = H2_STAT_VAL_LEN } + }, + .len = SLEN(S_H2_STAT) + H2_STAT_VAL_LEN, + .nchunks = 2 + }; + + WARN_ON_ONCE(op != TFW_H2_TRANS_EXPAND && op != TFW_H2_TRANS_SUB); + + if (index) { + TFW_STR_INDEX_SET(&s_hdr, index); + s_hdr.flags |= TFW_STR_FULL_INDEX; + } + + if (op == TFW_H2_TRANS_SUB) + s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; + + if (!tfw_ultoa(resp->status, __TFW_STR_CH(&s_hdr, 1)->data, + H2_STAT_VAL_LEN)) + return -E2BIG; + + if ((ret = tfw_hpack_encode(resp, s_line, &s_hdr, op))) + return ret; + + return 0; +} + +static inline void +tfw_h2_resp_fwd(TfwHttpResp *resp) +{ + TfwHttpReq *req = resp->req; + TfwH2Ctx *ctx = tfw_h2_context(req->conn); + + if (tfw_cli_conn_send((TfwCliConn *)req->conn, (TfwMsg *)resp)) { + T_DBG("%s: cannot send data to client via HTTP/2\n", __func__); + TFW_INC_STAT_BH(serv.msgs_otherr); + tfw_connection_close(req->conn, true); + } + else { + TFW_INC_STAT_BH(serv.msgs_forwarded); + } + + tfw_hpack_enc_release(&ctx->hpack, resp->flags); + + tfw_http_resp_pair_free(req); +} + static void tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) { @@ -684,18 +778,17 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) struct sk_buff **skb_head; TfwFrameHdr frame_hdr; unsigned int stream_id; - TfwStr *start *date, *clen, *srv, *body; + TfwStr *start, *date, *clen, *srv, *body; char *pos_dt_nm, *pos_cl_nm, *pos_srv_nm; char *pos_dt_val, *pos_cl_val, *pos_srv_val; unsigned long len_dt_nm, len_cl_nm, len_srv_nm; unsigned long len_dt_val, len_cl_val, len_srv_val; unsigned long len = SLEN(S_H2_STAT) + H2_STAT_VAL_LEN; unsigned char buf[FRAME_HEADER_SIZE]; + TfwH2Ctx *ctx = tfw_h2_context(req->conn); TfwStr *msg = &http_predef_resps[code]; - TfwMsgIter it = {}; TfwStr hdr = { .chunks = (TfwStr []){ {}, {} }, - .len = 0, .nchunks = 2 }; TfwStr f_hdr = { @@ -745,8 +838,8 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) frame_hdr.type = HTTP2_HEADERS; frame_hdr.flags = HTTP2_F_END_HEADERS; if (!body->data) - frame_hd.flags |= HTTP2_F_END_STREAM; - tfw_h2_pack_frame_header(buf, frame_hdr); + frame_hdr.flags |= HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, &frame_hdr); /* Set frame header and payload for HEADERS. */ if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr)) @@ -761,7 +854,7 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) tfw_http_prep_date(pos_dt_val); __TFW_STR_CH(&hdr, 1)->data = pos_dt_val; __TFW_STR_CH(&hdr, 1)->len = len_dt_val; - hdr->len = len_dt_nm + len_dt_val; + hdr.len = len_dt_nm + len_dt_val; TFW_STR_INDEX_SET(&hdr, 33); if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; @@ -770,7 +863,7 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) __TFW_STR_CH(&hdr, 0)->len = len_cl_nm; __TFW_STR_CH(&hdr, 1)->data = pos_cl_val; __TFW_STR_CH(&hdr, 1)->len = len_cl_val; - hdr->len = len_cl_nm + len_cl_val; + hdr.len = len_cl_nm + len_cl_val; TFW_STR_INDEX_SET(&hdr, 28); if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; @@ -779,7 +872,7 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) __TFW_STR_CH(&hdr, 0)->len = len_srv_nm; __TFW_STR_CH(&hdr, 1)->data = pos_srv_val; __TFW_STR_CH(&hdr, 1)->len = len_srv_val; - hdr->len = len_srv_nm + len_srv_val; + hdr.len = len_srv_nm + len_srv_val; TFW_STR_INDEX_SET(&hdr, 54); if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; @@ -789,7 +882,7 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) frame_hdr.length = body->len; frame_hdr.type = HTTP2_DATA; frame_hdr.flags = HTTP2_F_END_STREAM; - tfw_h2_pack_frame_header(buf, frame_hdr); + tfw_h2_pack_frame_header(buf, &frame_hdr); if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr) || tfw_http_msg_expand_data(&resp->mit.iter, skb_head, body)) @@ -802,8 +895,8 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) return; err_setup: - TFW_DBG("%s: HTTP/2 response message transformation error:" - " conn=[%p]\n", __func__, req->conn); + T_DBG("%s: HTTP/2 response message transformation error: conn=[%p]\n", + __func__, req->conn); tfw_hpack_enc_release(&ctx->hpack, resp->flags); @@ -2245,16 +2338,6 @@ tfw_http_conn_release(TfwConn *conn) } } -/* - * Free the request and the paired response. - */ -static inline void -tfw_http_resp_pair_free(TfwHttpReq *req) -{ - tfw_http_conn_msg_free(req->pair); - tfw_http_conn_msg_free((TfwHttpMsg *)req); -} - /* * Drop client connection's resources. * @@ -2681,7 +2764,8 @@ __h2_set_req_loc_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) { TfwStr *orig_hdr; - TfwHttpHdrTbl *ht = req->h_tbl; + TfwHttpMsg *hm = (TfwHttpMsg *)req; + TfwHttpHdrTbl *ht = hm->h_tbl; TfwMsgParseIter *it = &req->pit; const TfwStr *s_val = TFW_STR_CHUNK(hdr, 2); @@ -2710,7 +2794,7 @@ __h2_set_req_loc_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, } } else { - hid = __h2_hdr_lookup(req, TFW_STR_CHUNK(hdr, 0)); + hid = __h2_hdr_lookup(hm, TFW_STR_CHUNK(hdr, 0)); if (hid == ht->off && !s_val) /* * The raw header not found, and there is nothing @@ -2718,7 +2802,7 @@ __h2_set_req_loc_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, */ return 0; if (unlikely(hid == ht->size)) - if (tfw_http_msg_grow_hdr_tbl(req)) + if (tfw_http_msg_grow_hdr_tbl(hm)) return -ENOMEM; if (hid == ht->off) { /* @@ -2761,7 +2845,7 @@ __h2_set_req_loc_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, if (TFW_STR_DUP(orig_hdr)) orig_hdr = __TFW_STR_CH(orig_hdr, 0); - it->hdrs_len += h_app->len; + it->hdrs_len += h_app.len; return tfw_strcat(req->pool, orig_hdr, &h_app); } /* @@ -2786,7 +2870,8 @@ tfw_h2_set_req_loc_hdrs(TfwHttpReq *req) for (i = 0; i < h_mods->sz; ++i) { TfwHdrModsDesc *d = &h_mods->hdrs[i]; - r = __h2_set_req_loc_hdrs(req, d->hdr, d->hid, d->append); + int r = __h2_set_req_loc_hdrs(req, d->hdr, d->hid, d->append); + if (r) { T_ERR("HTTP/2: can't update location-specific header in" " request [%p]\n", req); @@ -2807,10 +2892,11 @@ tfw_h2_adjust_req(TfwHttpReq *req) { int r; char *dst; - struct page *p; + struct page *pg; unsigned int hid; TfwStr method; - const TfwStr *fld, *fld_end, *hdr, *dup_end; + const TfwStr *fld, *fld_end, *hdr, *dup_end, *chunk, *end; + TfwHttpMsg *hm = (TfwHttpMsg *)req; TfwHttpHdrTbl *ht = req->h_tbl; TfwMsgParseIter *it = &req->pit; bool auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); @@ -2872,14 +2958,14 @@ do { \ #define WRITE_HDR(buf, hdr) \ ({ \ - TfwStr *c, *end; \ + const TfwStr *c, *end; \ bool val_found = false; \ TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { \ if (!val_found) { \ if (c->flags & TFW_STR_HDR_VALUE) { \ WRITE_LIT(buf, S_DLM); \ val_found = true; \ - } else if (c->data == ':') { \ + } else if (*c->data == ':') { \ /* For statically configured adjustments. */ \ WARN_ON_ONCE(c->len != SLEN(S_DLM)); \ WRITE_LIT(buf, S_DLM); \ @@ -2897,9 +2983,9 @@ do { \ it->hdrs_len, it->hdrs_cnt); /* Substitution/addition of 'x-forwarded-for' header. */ - hid = __h2_hdr_lookup(req, TFW_STR_CHUNK(&h_xff, 0)); + hid = __h2_hdr_lookup(hm, TFW_STR_CHUNK(&h_xff, 0)); if (unlikely(hid == ht->size)) - if (tfw_http_msg_grow_hdr_tbl(req)) + if (tfw_http_msg_grow_hdr_tbl(hm)) return -ENOMEM; if (hid < ht->off) __h2_hdrs_dup_decrease(req, &ht->tbl[hid]); @@ -3024,7 +3110,7 @@ do { \ T_DBG3("%s: req adjusted, it->hdrs_len=%lu, it->hdrs_cnt=%u, dst=[%p]," " p=[%p]\n", __func__, it->hdrs_len, it->hdrs_cnt, dst, (char *)page_address(p)); - WARN_ON_ONCE(dst - (char *)page_address(p) != it->hdrs_len); + WARN_ON_ONCE(dst - (char *)page_address(pg) != it->hdrs_len); r = ss_skb_replace_page(&req->msg.skb_head, pg, it->hdrs_len, it->hb_len); @@ -3110,7 +3196,7 @@ tfw_http_adjust_resp(TfwHttpResp *resp) } if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { - r = tfw_http_set_hdr_date(resp); + r = tfw_http_set_hdr_date(hm); if (r < 0) return r; } @@ -3144,25 +3230,6 @@ __tfw_http_resp_fwd(TfwCliConn *cli_conn, struct list_head *ret_queue) } } -static inline void -tfw_h2_resp_fwd(TfwHttpResp *resp) -{ - TfwHttpReq *req = resp->req; - - if (tfw_cli_conn_send(req->conn, (TfwMsg *)resp)) { - T_DBG("%s: cannot send data to client via HTTP/2\n", __func__); - TFW_INC_STAT_BH(serv.msgs_otherr); - tfw_connection_close(req->conn, true); - } - else { - TFW_INC_STAT_BH(serv.msgs_forwarded); - } - - tfw_hpack_enc_release(&ctx->hpack, resp->flags); - - tfw_http_resp_pair_free(req); -} - /* * Mark @resp as ready to transmit. Then, starting with the first request * in @seq_queue, pick consecutive requests that have response ready to @@ -3270,69 +3337,6 @@ tfw_http_resp_fwd(TfwHttpResp *resp) } } -/* - * Static index determination for response ':status' pseudo-header (see RFC - * 7541 Appendix A for details). - */ -static inline unsigned short -tfw_h2_pseudo_index(unsigned short status) -{ - switch (status) { - case 200: - return 8; - case 204: - return 9; - case 206: - return 10; - case 304: - return 11; - case 400: - return 12; - case 404: - return 13; - case 500: - return 14; - default: - return 0; - } -} - -static int -tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) -{ - int ret; - unsigned short index = tfw_h2_pseudo_index(resp->status); - char buf[H2_STAT_VAL_LEN]; - TfwStr *s_line = NULL; - TfwStr s_hdr = { - .chunks = (TfwStr []){ - { .data = S_H2_STAT, .len = SLEN(S_H2_STAT) }, - { .data = buf, .len = H2_STAT_VAL_LEN } - }, - .len = SLEN(S_H2_STAT) + H2_STAT_VAL_LEN, - .nchunks = 2 - }; - - WARN_ON_ONCE(op != TFW_H2_TRANS_EXPAND && op != TFW_H2_TRANS_SUB); - - if (index) { - TFW_STR_INDEX_SET(&s_hdr, index); - s_hdr.flags |= TFW_STR_FULL_INDEX; - } - - if (op == TFW_H2_TRANS_SUB) - s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; - - if (!tfw_ultoa(resp->status, __TFW_STR_CH(&s_hdr, 1)->data, - H2_STAT_VAL_LEN)) - return -E2BIG; - - if ((ret = tfw_hpack_encode(resp, s_line, &s_hdr, op))) - return ret; - - return 0; -} - static int tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, unsigned long h_len) @@ -3364,7 +3368,7 @@ tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, frame_hdr.length = b_len; frame_hdr.type = HTTP2_DATA; frame_hdr.flags = HTTP2_F_END_STREAM; - tfw_h2_pack_frame_header(buf, frame_hdr); + tfw_h2_pack_frame_header(buf, &frame_hdr); first = TFW_STR_CHUNK(&resp->crlf, 0); ret = ss_skb_get_room(resp->msg.skb_head, first->skb, @@ -3387,9 +3391,9 @@ tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, frame_hdr.type = HTTP2_HEADERS; frame_hdr.flags = HTTP2_F_END_HEADERS; if (!b_len) - frame_hd.flags |= HTTP2_F_END_STREAM; + frame_hdr.flags |= HTTP2_F_END_STREAM; - tfw_h2_pack_frame_header(buf, frame_hdr); + tfw_h2_pack_frame_header(buf, &frame_hdr); first = TFW_STR_CHUNK(s_line, 0); ret = ss_skb_get_room(resp->msg.skb_head, first->skb, first->data, s_hdr.len, &it); @@ -3430,9 +3434,9 @@ tfw_h2_add_hdr_via(TfwHttpResp *resp) TFW_STR_INDEX_SET(&via, 60); r = __hdr_h2_add(resp, &via); if (unlikely(r)) - TFW_ERR("HTTP/2: unable to add 'via' header (resp=[%p])\n", resp); + T_ERR("HTTP/2: unable to add 'via' header (resp=[%p])\n", resp); else - TFW_DBG3("%s: added 'via' header, resp=[%p]\n", resp); + T_DBG3("%s: added 'via' header, resp=[%p]\n", resp); return r; #undef NM_VIA #undef V_PROTO @@ -3449,13 +3453,13 @@ tfw_h2_set_hdr_date(TfwHttpResp *resp) char *s_date = *this_cpu_ptr(&g_buf); tfw_http_prep_date_from(s_date, resp->date); - r = tfw_h2_msg_hdr_sub(resp, "date", SLEN("date"), s_date, + r = tfw_h2_msg_hdr_sub((TfwHttpMsg *)resp, "date", SLEN("date"), s_date, SLEN(S_V_DATE), TFW_HTTP_HDR_RAW, 33); if (unlikely(r)) - TFW_ERR("HTTP/2: unable to add 'date' header to response" + T_ERR("HTTP/2: unable to add 'date' header to response" " [%p]\n", resp); else - TFW_DBG3("%s: added 'date' header, resp=[%p]\n", resp); + T_DBG3("%s: added 'date' header, resp=[%p]\n", resp); return r; } @@ -3488,7 +3492,8 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) { int r; unsigned int stream_id; - const TfwStr *field, *hdrs_end, *hdr, *dup_end; + TfwStr *field, *hdrs_end, *hdr, *dup_end; + TfwHttpMsg *hmresp = (TfwHttpMsg *)resp; TfwHttpReq *req = resp->req; TfwH2Ctx *ctx = tfw_h2_context(req->conn); TfwStr *s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; @@ -3504,19 +3509,19 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) goto out; /* Adjust HTTP/1.1 headers with transformation to HTTP/2 form. */ - r = tfw_http_sess_resp_process(resp) + r = tfw_http_sess_resp_process(resp); if (unlikely(r)) goto clean; - r = tfw_http_msg_del_hbh_hdrs(resp); + r = tfw_http_msg_del_hbh_hdrs(hmresp); if (unlikely(r)) goto clean; - r = TFW_H2_MSG_HDR_DEL(resp, "keep-alive", TFW_HTTP_HDR_KEEP_ALIVE); + r = TFW_H2_MSG_HDR_DEL(hmresp, "keep-alive", TFW_HTTP_HDR_KEEP_ALIVE); if (unlikely(r)) goto clean; - r = TFW_H2_MSG_HDR_DEL(resp, "connection", TFW_HTTP_HDR_CONNECTION); + r = TFW_H2_MSG_HDR_DEL(hmresp, "connection", TFW_HTTP_HDR_CONNECTION); if (unlikely(r)) goto clean; @@ -3534,7 +3539,7 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) goto clean; } - r = TFW_H2_MSG_HDR_SUB(resp, "server", TFW_NAME "/" TFW_VERSION, + r = TFW_H2_MSG_HDR_SUB(hmresp, "server", TFW_NAME "/" TFW_VERSION, TFW_HTTP_HDR_SERVER, 54); if (unlikely(r)) goto clean; @@ -3565,7 +3570,7 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) if (unlikely(r)) goto clean; - r = tfw_http_msg_del_str(resp, &resp->crlf); + r = tfw_http_msg_del_str(hmresp, &resp->crlf); if (unlikely(r)) goto clean; @@ -3576,7 +3581,7 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) return; clean: - tfw_http_conn_msg_free((TfwHttpMsg *)resp); + tfw_http_conn_msg_free(hmresp); tfw_http_send_resp(req, 500, "response dropped: processing error"); tfw_hpack_enc_release(&ctx->hpack, resp->flags); diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index b2ba3dac61..bb5971b042 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -254,10 +254,10 @@ EXPORT_SYMBOL(__http_msg_hdr_val); void __h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name) { - TfwStr *c, *end; + const TfwStr *c, *end; if (unlikely(TFW_STR_EMPTY(hdr))) { - TFW_STR_INIT(out_val); + TFW_STR_INIT(out_name); return; } @@ -267,7 +267,7 @@ __h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name) *out_name = *hdr; if (unlikely(TFW_STR_PLAIN(hdr))) { - WARN_ON_ONCE(hdr->flags & TFW_STR_HDR_VALUE) + WARN_ON_ONCE(hdr->flags & TFW_STR_HDR_VALUE); return; } diff --git a/tempesta_fw/pool.h b/tempesta_fw/pool.h index bbf1fdfe30..550551a0d6 100644 --- a/tempesta_fw/pool.h +++ b/tempesta_fw/pool.h @@ -72,7 +72,7 @@ typedef struct { TfwPool *__tfw_pool_new(size_t n); void *__tfw_pool_alloc(TfwPool *p, size_t n, bool align, bool *new_page); void tfw_pool_free(TfwPool *p, void *ptr, size_t n); -void tfw_pool_clean(TfwPool *p); +void tfw_pool_clean(TfwPool *p, void *ptr); void tfw_pool_destroy(TfwPool *p); void *__tfw_pool_realloc(TfwPool *p, void *ptr, size_t old_n, size_t new_n, bool copy); diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index eb0da68bf7..5650636c43 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -140,7 +140,7 @@ TEST(hpack, dec_table_static) if (entry) { hdr = entry->hdr; EXPECT_EQ(hdr->nchunks, 2); - EXPECT_EQ(aenc_len, entry->nm_len); + EXPECT_EQ(aenc_len, entry->name_len); EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_aenc, aenc_len, TFW_STR_EQ_PREFIX)); __h2_msg_hdr_val(hdr, &h_val); @@ -163,7 +163,7 @@ TEST(hpack, dec_table_dynamic) { TfwHPack *hp; const TfwHPackEntry *entry; - TfwStr h_val, *hdr, *s1, *s2, *s3; + TfwStr *s1, *s2, *s3; TfwMsgParseIter *it = &test_req->pit; unsigned int new_len = 0; TFW_STR(s1_name, "custom-key"); @@ -370,7 +370,7 @@ TEST(hpack, dec_table_wrap) EXPECT_NOT_NULL(l_entry->hdr); if (l_entry->hdr) { - EXPECT_EQ(l_entry->hdr, t_entry-hdr); + EXPECT_EQ(l_entry->hdr, t_entry->hdr); } } @@ -386,7 +386,6 @@ TEST(hpack, dec_raw) TfwHttpHdrTbl *ht; TfwStr h_name, h_value; unsigned int parsed; - unsigned long test_len1, test_len2, test_len3; #define HDR_NAME_1 "custom-key" #define HDR_VALUE_1 "custom-value" @@ -398,7 +397,7 @@ TEST(hpack, dec_raw) const char *test_name1 = HDR_NAME_1; const char *test_value1 = HDR_VALUE_1; unsigned long hdr_len1 = 25; - const char *hdr_data1 = + char *hdr_data1 = "\x40" /* == With indexing == */ "\x0A" /* Literal name (len = 10) */ "\x63\x75\x73\x74\x6F" /* custom-key */ @@ -411,7 +410,7 @@ TEST(hpack, dec_raw) const char *test_name2 = HDR_NAME_2; const char *test_value2 = HDR_VALUE_2; unsigned long hdr_len2 = 37; - const char *hdr_data2 = + char *hdr_data2 = "\x00" /* == Without indexing == */ "\x0C" /* Literal name (len = 12) */ "\x78\x2D\x63\x75\x73" /* x-custom-hdr */ @@ -427,7 +426,7 @@ TEST(hpack, dec_raw) const char *test_name3 = HDR_NAME_3; const char *test_value3 = HDR_VALUE_3; unsigned long hdr_len3 = 41; - const char *hdr_data3 = + char *hdr_data3 = "\x10" /* == Never indexing == */ "\x0F" /* Literal name (len = 15) */ "\x78\x2D\x66\x6F\x72" /* x-forwarded-for */ @@ -454,7 +453,7 @@ TEST(hpack, dec_raw) EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len3); - ht = test_req->h_tbl + ht = test_req->h_tbl; __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW], &h_name); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW], &h_value); @@ -510,7 +509,7 @@ TEST(hpack, dec_indexed) TfwHttpHdrTbl *ht; unsigned int parsed; const TfwHPackEntry *entry; - const TfwStr h_name, h_value, *hdr, *dup; + TfwStr h_name, h_value, *hdr, *dup; #define HDR_NAME_1 "x-forwarded-for" #define HDR_VALUE_1 " test.com, foo.com, example.com" @@ -548,7 +547,7 @@ TEST(hpack, dec_indexed) unsigned long test_len_val6 = strlen(test_value6); unsigned long hdr_len1 = 49; - const char *hdr_data1 = + char *hdr_data1 = "\x40" /* == With indexing == */ "\x0F" /* Literal name (len = 15) */ "\x78\x2D\x66\x6F\x72" /* x-forwarded-for */ @@ -564,13 +563,13 @@ TEST(hpack, dec_indexed) "\x6D"; /* */ unsigned long hdr_len2 = 1; - const char *hdr_data2 = "\xBE"; /* == Indexed (dynamic: 62) == */ + char *hdr_data2 = "\xBE"; /* == Indexed (dynamic: 62) == */ unsigned long hdr_len3 = 1; - const char *hdr_data3 = "\x90"; /* == Indexed (static: 16) == */ + char *hdr_data3 = "\x90"; /* == Indexed (static: 16) == */ unsigned long hdr_len4 = 30; - const char *hdr_data4 = + char *hdr_data4 = "\x50" /* == With indexing == */ /* (name indexed - static: 16) */ "\x1C" /* Literal value (len = 28) */ @@ -582,7 +581,7 @@ TEST(hpack, dec_indexed) "\x30\x2E\x35"; /* */ unsigned long hdr_len5 = 12; - const char *hdr_data5 = + char *hdr_data5 = "\x7F\x00" /* == With indexing == */ /* (name indexed - dynamic: 63) */ /* (multibyte integer encoding) */ @@ -592,7 +591,7 @@ TEST(hpack, dec_indexed) "\x2E\x30\x2E\x31"; /* */ unsigned long hdr_len6 = 12; - const char *hdr_data6 = + char *hdr_data6 = "\x0F\x17" /* == Without indexing == */ /* (name indexed - static: 38) */ /* (multibyte integer encoding) */ @@ -602,7 +601,7 @@ TEST(hpack, dec_indexed) "\x68\x6F\x73\x74"; /* */ unsigned long hdr_len7 = 10; - const char *hdr_data7 = + char *hdr_data7 = "\x0F\x2A" /* == Without indexing == */ /* (name indexed - static: 57) */ /* (multibyte integer encoding) */ @@ -784,8 +783,8 @@ TEST(hpack, dec_indexed) EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value1, test_len_val1, 0)); } - EXPECT_EQ(entry->nm_len, test_len_nm1); - EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->name_len, test_len_nm1); + EXPECT_EQ(entry->name_num, h_name.nchunks); EXPECT_EQ(entry->tag, TFW_TAG_HDR_X_FORWARDED_FOR); } @@ -805,8 +804,8 @@ TEST(hpack, dec_indexed) EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, test_len_val3, 0)); } - EXPECT_EQ(entry->nm_len, test_len_nm2); - EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->name_len, test_len_nm2); + EXPECT_EQ(entry->name_num, h_name.nchunks); EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } @@ -826,8 +825,8 @@ TEST(hpack, dec_indexed) EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value4, test_len_val4, 0)); } - EXPECT_EQ(entry->nm_len, test_len_nm1); - EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->name_len, test_len_nm1); + EXPECT_EQ(entry->name_num, h_name.nchunks); EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } @@ -852,7 +851,7 @@ TEST(hpack, dec_huffman) TfwHttpHdrTbl *ht; unsigned int parsed; const TfwHPackEntry *entry; - const TfwStr h_name, h_value, *hdr; + TfwStr h_name, h_value, *hdr; #define HDR_NAME_1 "custom-key" #define HDR_VALUE_1 "custom-value" @@ -876,7 +875,7 @@ TEST(hpack, dec_huffman) unsigned long test_len_val3 = strlen(test_value3); unsigned long hdr_len1 = 20; - const char *hdr_data1 = + char *hdr_data1 = "\x40" /* == With indexing == */ "\x88" /* Literal name (len = 8) */ /* (Huffman encoded) */ @@ -889,7 +888,7 @@ TEST(hpack, dec_huffman) "\xB8\xE8\xB4\xBF"; /* */ unsigned long hdr_len2 = 8; - const char *hdr_data2 = + char *hdr_data2 = "\x58" /* == With indexing == */ /* (name indexed - static: 24) */ /* */ @@ -899,7 +898,7 @@ TEST(hpack, dec_huffman) "\xBF"; /* */ unsigned long hdr_len3 = 14; - const char *hdr_data3 = + char *hdr_data3 = "\x41" /* == With indexing == */ /* (name indexed - static: 1) */ /* */ @@ -995,8 +994,8 @@ TEST(hpack, dec_huffman) EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value1, test_len_val1, 0)); } - EXPECT_EQ(entry->nm_len, test_len_nm1); - EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->name_len, test_len_nm1); + EXPECT_EQ(entry->name_num, h_name.nchunks); EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } @@ -1016,8 +1015,8 @@ TEST(hpack, dec_huffman) EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value2, test_len_val2, 0)); } - EXPECT_EQ(entry->nm_len, test_len_nm2); - EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->name_len, test_len_nm2); + EXPECT_EQ(entry->name_num, h_name.nchunks); EXPECT_EQ(entry->tag, TFW_TAG_HDR_CACHE_CONTROL); } @@ -1037,10 +1036,17 @@ TEST(hpack, dec_huffman) EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, test_len_val3, 0)); } - EXPECT_EQ(entry->nm_len, test_len_nm3); - EXPECT_EQ(entry->nm_num, h_name.nchunks); + EXPECT_EQ(entry->name_len, test_len_nm3); + EXPECT_EQ(entry->name_num, h_name.nchunks); EXPECT_EQ(entry->tag, TFW_TAG_HDR_H2_AUTHORITY); } + +#undef HDR_NAME_1 +#undef HDR_VALUE_1 +#undef HDR_NAME_2 +#undef HDR_VALUE_2 +#undef HDR_NAME_3 +#undef HDR_VALUE_3 } TEST(hpack, enc_table_hdr_write) @@ -1089,11 +1095,11 @@ TEST(hpack, enc_table_hdr_write) const char *t_s5 = HDR_NAME_5 HDR_VALUE_5; unsigned long t_s5_len = strlen(t_s5); - collect_compound_str(s1, s1_value); - collect_compound_str(s2, s2_value); - collect_compound_str(s3, s3_value); - collect_compound_str(s4, s4_value); - collect_compound_str(s5, s5_value); + collect_compound_str(s1, s1_value, 0); + collect_compound_str(s2, s2_value, 0); + collect_compound_str(s3, s3_value, 0); + collect_compound_str(s4, s4_value, 0); + collect_compound_str(s5, s5_value, 0); hdr_len = tfw_h2_msg_hdr_length(s1, &n_len, &v_off, &v_len); EXPECT_EQ(n_len, strlen(HDR_NAME_1)); @@ -1187,9 +1193,9 @@ TEST(hpack, enc_table_index) const char *t_s3 = HDR_NAME_3 HDR_VALUE_3; unsigned long t_s3_len = strlen(t_s3); - collect_compound_str(s1, s1_value); - collect_compound_str(s2, s2_value); - collect_compound_str(s3, s3_value); + collect_compound_str(s1, s1_value, 0); + collect_compound_str(s2, s2_value, 0); + collect_compound_str(s3, s3_value, 0); tbl = &ctx.hpack.enc_tbl; @@ -1302,11 +1308,11 @@ TEST(hpack, enc_table_rbtree) TFW_STR(s5, HDR_NAME_5 ":"); TFW_STR(s5_value, HDR_VALUE_5); - collect_compound_str(s1, s1_value); - collect_compound_str(s2, s2_value); - collect_compound_str(s3, s3_value); - collect_compound_str(s4, s4_value); - collect_compound_str(s5, s5_value); + collect_compound_str(s1, s1_value, 0); + collect_compound_str(s2, s2_value, 0); + collect_compound_str(s3, s3_value, 0); + collect_compound_str(s4, s4_value, 0); + collect_compound_str(s5, s5_value, 0); tbl = &ctx.hpack.enc_tbl; From 8f9a906391f5a970950f36ed6a1bc1e7c8e13170 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Thu, 31 Oct 2019 16:29:32 +0300 Subject: [PATCH 05/64] HTTP/2 Parser implementation: removal of outdated (already done) 'TODO' comments (#309). --- tempesta_fw/http.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 88348d60e2..ea7bf98d9d 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -13,24 +13,22 @@ * and allow quick processing with SIMD instructions. However, content of * HTTP/2-message (i.e. headers) could be encrypted with special HPACK * methods (static/dynamic indexing and Huffman coding), which doesn't - * allow to analyze the message content directly. o solve this issue for + * allow to analyze the message content directly. To solve this issue for * HTTP/2-requests processing, current implementation of HTTP-layer creates * HTTP/1.1-representation of HTTP/2 headers (during HPACK-decoding and * HTTP/2-parsing) and places that representation into the special pool * @TfwHttpReq.pit.pool. * - * TODO #309: during adjusting stage (before re-sending request to backend) - * in @tfw_h2_adjust_req() the obtained @TfwHttpReq.pit.pool with - * HTTP/1.1-representation must be used for request assembling in case of - * HTTP/2 => HTTP/1.1 transformation. - * - * TODO #309: actually, we don't need to create separate HTTP/1.1-representation + * Actually, we don't need to create separate HTTP/1.1-representation * for all HTTP/2 headers: some of them could be not encoded (not in Huffman * form - in ASCII instead) and some - could be just indexed (we can keep static * and dynamic indexes during internal request analysis and convert them into - * ASCII strings in-place - on demand), so, we can save memory/processor - * resources and create additional HTTP/1.1-representation only for Huffman - * encoded headers. + * ASCII strings in-place - on demand); thus, in current implementation we save + * memory/processor resources and create additional HTTP/1.1-representation only + * for Huffman encoded headers. On the final stage of request processing (in + * @tfw_h2_adjust_req() procedure, before re-sending request to backend) the + * obtained @TfwHttpReq.pit.pool with HTTP/1.1-representation is used for + * request assembling in case of HTTP/2 => HTTP/1.1 transformation. * * Described above approach was chosen instead of immediate HTTP/2 => HTTP/1.1 * transformation on HTTP/2-message decoding, because the latter one is not From fb7c80ffd28e0bbde40bfdac84f1eaa1285f5410 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Mon, 4 Nov 2019 15:13:21 +0300 Subject: [PATCH 06/64] HTTP/2 Parser implementation (#309): Corrections as a result of HPACK decoder/encoder/parser unit-tests debugging. --- tempesta_fw/hpack.c | 78 +++++--- tempesta_fw/http.c | 8 +- tempesta_fw/http_msg.c | 10 +- tempesta_fw/http_parser.c | 153 ++++++++------- tempesta_fw/ss_skb.c | 8 +- tempesta_fw/t/unit/test_hpack.c | 320 ++++++++++++++++++++++---------- 6 files changed, 375 insertions(+), 202 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index 40cdac3a0f..6225cf9230 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -1183,8 +1183,10 @@ do { \ do { \ WARN_ON_ONCE(TFW_STR_EMPTY(it->parsed_hdr)); \ it->nm_len = it->parsed_hdr->len; \ - it->nm_num = it->parsed_hdr->nchunks; \ - if (state & HPACK_FLAGS_HUFFMAN_NAME) { \ + it->nm_num = it->parsed_hdr->nchunks \ + ? it->parsed_hdr->nchunks \ + : 1; \ + if (state & HPACK_FLAGS_HUFFMAN_VALUE) { \ BUFFER_GET(length, it); \ if (!it->pos) { \ r = T_DROP; \ @@ -1752,15 +1754,13 @@ tfw_hpack_find_index(TfwHPackDTbl *__restrict tbl, unsigned long index) { const TfwHPackEntry *entry = NULL; + WARN_ON_ONCE(tbl->n > tbl->length); + if (index <= HPACK_STATIC_ENTRIES) { entry = static_table + index - 1; - BUG_ON(!entry->hdr); - return entry; + WARN_ON_ONCE(entry->name_num != 1); } - - WARN_ON_ONCE(tbl->n > tbl->length); - index -= HPACK_STATIC_ENTRIES; - if (index <= tbl->n) { + else if ((index -= HPACK_STATIC_ENTRIES) <= tbl->n) { unsigned int curr = tbl->curr; if (index <= curr) { @@ -1772,12 +1772,12 @@ tfw_hpack_find_index(TfwHPackDTbl *__restrict tbl, unsigned long index) __func__, tbl->length, tbl->curr, curr, index); entry = tbl->entries + curr; - WARN_ON_ONCE(!entry->hdr->nchunks || entry->name_num != 1); - - return entry; + WARN_ON_ONCE(!entry->name_num); } - return NULL; + WARN_ON_ONCE(entry && (!entry->hdr || !entry->hdr->nchunks)); + + return entry; } static int @@ -1905,11 +1905,12 @@ tfw_hpack_reinit(TfwHPack *__restrict hp, TfwMsgParseIter *__restrict it) static inline int tfw_hpack_process_hdr_name(TfwHttpReq *req) { - int ret = T_BAD; + const TfwStr *c, *end; TfwMsgParseIter *it = &req->pit; - const TfwStr *c, *end, *next = it->next; + const TfwStr *hdr = &it->hdr, *next = it->next; + int ret = T_BAD; - WARN_ON_ONCE(next != &it->hdr); + WARN_ON_ONCE(next != hdr); TFW_STR_FOR_EACH_CHUNK(c, next, end) { bool last = c + 1 == end; @@ -1924,13 +1925,28 @@ tfw_hpack_process_hdr_name(TfwHttpReq *req) static inline int tfw_hpack_process_hdr_value(TfwHttpReq *req) { - int ret = T_BAD; + const TfwStr *chunk, *end; TfwMsgParseIter *it = &req->pit; - const TfwStr *hdr = &it->hdr; - const TfwStr *chunk = it->next; - const TfwStr *end = hdr->chunks + hdr->nchunks; + const TfwStr *hdr = &it->hdr, *next = it->next; + int ret = T_BAD; + + BUG_ON(TFW_STR_DUP(hdr)); + if (TFW_STR_PLAIN(hdr)) { + WARN_ON_ONCE(hdr != next); + chunk = hdr; + end = hdr + 1; + } else { + /* + * In case of compound @hdr the @next can point either to the + * @hdr itself (if only header's value has been Huffman-decoded, + * i.e. in case of indexed or raw header's name), or to some + * chunk inside the @hdr (if both, the name and the value, has + * been Huffman-decoded). + */ + chunk = (hdr != next) ? next : next->chunks; + end = hdr->chunks + hdr->nchunks; + } - WARN_ON_ONCE(chunk == hdr); while (chunk < end) { bool last = chunk + 1 == end; @@ -2056,40 +2072,58 @@ tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, switch (entry->tag) { case TFW_TAG_HDR_H2_METHOD: parser->_hdr_tag = TFW_HTTP_HDR_H2_METHOD; + break; case TFW_TAG_HDR_H2_SCHEME: parser->_hdr_tag = TFW_HTTP_HDR_H2_SCHEME; + break; case TFW_TAG_HDR_H2_AUTHORITY: parser->_hdr_tag = TFW_HTTP_HDR_H2_AUTHORITY; + break; case TFW_TAG_HDR_H2_PATH: parser->_hdr_tag = TFW_HTTP_HDR_H2_PATH; + break; case TFW_TAG_HDR_ACCEPT: parser->_hdr_tag = TFW_HTTP_HDR_RAW; + break; case TFW_TAG_HDR_AUTHORIZATION: parser->_hdr_tag = TFW_HTTP_HDR_RAW; + break; case TFW_TAG_HDR_CACHE_CONTROL: parser->_hdr_tag = TFW_HTTP_HDR_RAW; + break; case TFW_TAG_HDR_CONTENT_LENGTH: parser->_hdr_tag = TFW_HTTP_HDR_CONTENT_LENGTH; + break; case TFW_TAG_HDR_CONTENT_TYPE: parser->_hdr_tag = TFW_HTTP_HDR_CONTENT_TYPE; + break; case TFW_TAG_HDR_COOKIE: parser->_hdr_tag = TFW_HTTP_HDR_COOKIE; + break; case TFW_TAG_HDR_HOST: parser->_hdr_tag = TFW_HTTP_HDR_HOST; + break; case TFW_TAG_HDR_IF_MODIFIED_SINCE: parser->_hdr_tag = TFW_HTTP_HDR_RAW; + break; case TFW_TAG_HDR_IF_NONE_MATCH: parser->_hdr_tag = TFW_HTTP_HDR_IF_NONE_MATCH; + break; case TFW_TAG_HDR_PRAGMA: parser->_hdr_tag = TFW_HTTP_HDR_RAW; + break; case TFW_TAG_HDR_REFERER: parser->_hdr_tag = TFW_HTTP_HDR_REFERER; + break; case TFW_TAG_HDR_X_FORWARDED_FOR: parser->_hdr_tag = TFW_HTTP_HDR_X_FORWARDED_FOR; + break; case TFW_TAG_HDR_USER_AGENT: parser->_hdr_tag = TFW_HTTP_HDR_USER_AGENT; + break; case TFW_TAG_HDR_RAW: parser->_hdr_tag = TFW_HTTP_HDR_RAW; + break; default: WARN_ON_ONCE(1); return T_DROP; @@ -3856,8 +3890,8 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, mit->acc_len += s_name->len; T_DBG3("%s: name str, acc_len=%lu, s_name.len=%lu, s_name.data" - "='%.*s'\n", __func__, mit->acc_len, s_name.len, - (int)s_name.len, s_name.data); + "='%.*s'\n", __func__, mit->acc_len, s_name->len, + (int)s_name->len, s_name->data); } /* * The @hdr must have the HTTP/2-format chunk structure (without the diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index ea7bf98d9d..4479a2fc61 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3106,8 +3106,8 @@ do { \ WRITE_LIT(dst, S_CRLF); T_DBG3("%s: req adjusted, it->hdrs_len=%lu, it->hdrs_cnt=%u, dst=[%p]," - " p=[%p]\n", __func__, it->hdrs_len, it->hdrs_cnt, dst, - (char *)page_address(p)); + " pg=[%p]\n", __func__, it->hdrs_len, it->hdrs_cnt, dst, + (char *)page_address(pg)); WARN_ON_ONCE(dst - (char *)page_address(pg) != it->hdrs_len); r = ss_skb_replace_page(&req->msg.skb_head, pg, it->hdrs_len, @@ -3434,7 +3434,7 @@ tfw_h2_add_hdr_via(TfwHttpResp *resp) if (unlikely(r)) T_ERR("HTTP/2: unable to add 'via' header (resp=[%p])\n", resp); else - T_DBG3("%s: added 'via' header, resp=[%p]\n", resp); + T_DBG3("%s: added 'via' header, resp=[%p]\n", __func__, resp); return r; #undef NM_VIA #undef V_PROTO @@ -3457,7 +3457,7 @@ tfw_h2_set_hdr_date(TfwHttpResp *resp) T_ERR("HTTP/2: unable to add 'date' header to response" " [%p]\n", resp); else - T_DBG3("%s: added 'date' header, resp=[%p]\n", resp); + T_DBG3("%s: added 'date' header, resp=[%p]\n", __func__, resp); return r; } diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index bb5971b042..ff314c6ba0 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -25,6 +25,7 @@ #if DBG_HTTP_PARSER == 0 #undef DEBUG #endif + #include "lib/str.h" #include "gfsm.h" #include "http_msg.h" @@ -104,8 +105,9 @@ static inline unsigned int __tfw_http_msg_spec_hid(const TfwStr *hdr, const TfwHdrDef array[]) { const TfwHdrDef *def; + size_t size = TFW_HTTP_HDR_RAW - TFW_HTTP_HDR_REGULAR; /* TODO: return error if @hdr can't be applied to response or client. */ - def = (TfwHdrDef *)__tfw_http_msg_find_hdr(hdr, array, TFW_HTTP_HDR_RAW, + def = (TfwHdrDef *)__tfw_http_msg_find_hdr(hdr, array, size, sizeof(TfwHdrDef)); return def ? def->id : TFW_HTTP_HDR_RAW; @@ -511,7 +513,6 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) if (TFW_STR_EMPTY(h)) /* Just store the special header in empty slot. */ goto done; - /* * Process duplicate header. * @@ -1484,8 +1485,9 @@ tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, ++it->frag; skb_fill_page_desc(it->skb, it->frag, page, 0, 0); - T_DBG3("message expanded by new frag %d," - " page=[%p], skb=[%p]\n", i, + T_DBG3("message expanded by new frag %u," + " page=[%p], skb=[%p]\n", + skb_shinfo(it->skb)->nr_frags, page_address(page), it->skb); } diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index cc0c07cbab..a79242c879 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -340,7 +340,7 @@ __FSM_STATE(st, cold) { \ if (likely(TFW_LC(c) == ch)) \ __FSM_MOVE(st_next); \ /* It should be checked in st_fallback if `c` is allowed */ \ - __FSM_JMP(RGen_HdrOther); \ + __FSM_JMP(RGen_HdrOtherN); \ } /* As above, but reads OWS through transitional state. */ @@ -351,7 +351,7 @@ __FSM_STATE(st, cold) { \ __FSM_MOVE(RGen_LWS); \ } \ /* It should be checked in st_fallback if `c` is allowed */ \ - __FSM_JMP(RGen_HdrOther); \ + __FSM_JMP(RGen_HdrOtherN); \ } /* As above, but with HPACK static index setting. */ @@ -363,7 +363,7 @@ __FSM_STATE(st, cold) { \ __FSM_MOVE(RGen_LWS); \ } \ /* It should be checked in st_fallback if `c` is allowed */ \ - __FSM_JMP(RGen_HdrOther); \ + __FSM_JMP(RGen_HdrOtherN); \ } /* Used for improbable states only, so use cold label. */ @@ -989,10 +989,6 @@ __FSM_STATE(st_curr) { \ * Note that some of these can be extremely large. */ #define RGEN_HDR_OTHER() \ -__FSM_STATE(RGen_HdrOther) { \ - parser->_hdr_tag = TFW_HTTP_HDR_RAW; \ - /* Fall through. */ \ -} \ __FSM_STATE(RGen_HdrOtherN) { \ __FSM_MATCH_MOVE(token, RGen_HdrOtherN); \ if (likely(*(p + __fsm_sz) == ':')) { \ @@ -1013,6 +1009,7 @@ __FSM_STATE(RGen_HdrOtherV) { \ __msg_hdr_chunk_fixup(p, __fsm_sz); \ mark_raw_hbh(msg, &parser->hdr); \ mark_trailer_hdr(msg, &parser->hdr); \ + parser->_hdr_tag = TFW_HTTP_HDR_RAW; \ __FSM_MOVE_nofixup_n(RGen_EoL, __fsm_sz); \ } @@ -1203,6 +1200,7 @@ __FSM_STATE(RGen_LWS, hot) { \ T_DBG3("parse LWS: __fsm_n=%d, __fsm_sz=%lu, len=%lu," \ " off=%lu\n", __fsm_n, __fsm_sz, len, __data_off(p)); \ if (__fsm_n == CSTR_POSTPONE) { \ + __msg_hdr_chunk_fixup(p, __fsm_sz); \ p += __fsm_sz; \ parser->state = &&RGen_LWS; \ __FSM_EXIT(TFW_POSTPONE); \ @@ -3399,7 +3397,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, parser->_i_st = &&Req_HdrCache_ControlV; __FSM_MOVE_n(RGen_LWS, 14); } - __FSM_MOVE_n(RGen_HdrOther, 5); + __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 'n', 'e'): if (likely(C4_INT_LCM(p + 5, 'c', 't', 'i', 'o') && TFW_LC(*(p + 9)) == 'n' @@ -3408,7 +3406,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, parser->_i_st = &&Req_HdrConnectionV; __FSM_MOVE_n(RGen_LWS, 11); } - __FSM_MOVE_n(RGen_HdrOther, 5); + __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 't', 'e'): if (likely(TFW_LC(*(p + 5)) == 'n' && TFW_LC(*(p + 6)) == 't' @@ -3416,7 +3414,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, { __FSM_MOVE_n(Req_HdrContent_, 8); } - __FSM_MOVE_n(RGen_HdrOther, 5); + __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'o', 'k', 'i'): if (likely(TFW_LC(*(p + 5)) == 'e' && *(p + 6) == ':')) @@ -3424,9 +3422,9 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, parser->_i_st = &&Req_HdrCookieV; __FSM_MOVE_n(RGen_LWS, 7); } - __FSM_MOVE_n(RGen_HdrOther, 5); + __FSM_MOVE_n(RGen_HdrOtherN, 5); default: - __FSM_MOVE(RGen_HdrOther); + __FSM_MOVE(RGen_HdrOtherN); } case 'h': if (likely(__data_available(p, 5) @@ -3537,7 +3535,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, } __FSM_MOVE(Req_HdrU); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -3563,7 +3561,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, } __FSM_MOVE(Req_HdrContent_T); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -3930,7 +3928,7 @@ Req_Method_1CharStep: __attribute__((cold)) case 'u': __FSM_MOVE(Req_HdrAu); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -3962,7 +3960,7 @@ Req_Method_1CharStep: __attribute__((cold)) case 'o': __FSM_MOVE(Req_HdrCo); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -3987,7 +3985,7 @@ Req_Method_1CharStep: __attribute__((cold)) case 'o': __FSM_MOVE(Req_HdrCoo); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -3999,7 +3997,7 @@ Req_Method_1CharStep: __attribute__((cold)) case 't': __FSM_MOVE(Req_HdrCont); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } __FSM_TX_AF(Req_HdrConn, 'e', Req_HdrConne); @@ -4040,7 +4038,7 @@ Req_Method_1CharStep: __attribute__((cold)) parser->_i_st = &&Req_HdrHostV; __FSM_MOVE(RGen_LWS); } - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } /* If-* header processing. */ @@ -4053,7 +4051,7 @@ Req_Method_1CharStep: __attribute__((cold)) case 'n': __FSM_MOVE(Req_HdrIf_N); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -4251,8 +4249,8 @@ do { \ #define __FSM_H2_FIN(to, n, h_tag) \ do { \ p += n; \ - T_DBG3("%s: name fin, hid=%d, to=" #to "len=%lu, off=%lu\n", \ - __func__, hid, len, __data_off(p)); \ + T_DBG3("%s: name fin, h_tag=%d, to=" #to " len=%lu, off=%lu\n", \ + __func__, h_tag, len, __data_off(p)); \ if (unlikely(__data_off(p) < len)) \ goto RGen_HdrOtherN; \ __msg_hdr_chunk_fixup(data, len); \ @@ -4265,7 +4263,7 @@ do { \ #define __FSM_H2_NEXT_n(to, n) \ do { \ p += n; \ - T_DBG3("%s: name next, to=" #to "len=%lu, off=%lu\n", __func__, \ + T_DBG3("%s: name next, to=" #to " len=%lu, off=%lu\n", __func__, \ len, __data_off(p)); \ if (likely(__data_off(p) < len)) \ goto to; \ @@ -4287,7 +4285,7 @@ do { \ #define __FSM_H2_HDR_COMPLETE(st_curr) \ do { \ - T_DBG("%s: complete header, state=" #st_curr ", _hdr_tag=%u," \ + T_DBG3("%s: complete header, state=" #st_curr ", _hdr_tag=%u," \ " c=%#x, p='%.*s', len=%lu, off=%lu\n", \ __func__, parser->_hdr_tag, (char)c, \ min(16U, (unsigned int)(data + len - p)), \ @@ -4303,47 +4301,56 @@ do { \ } \ } while (0) -#define __FSM_H2_PSHDR_MOVE_lambda(n, lambda, st_next) \ +#define __FSM_H2_PSHDR_MOVE_FIN(st_curr, n, st_next) \ do { \ p += n; \ - __FSM_H2_PSHDR_CHECK_lambda(p, lambda); \ - goto st_next; \ -} while (0) - -#define __FSM_H2_PSHDR_MOVE_FIN(st_curr, n, st_next) \ - __FSM_H2_PSHDR_MOVE_lambda(n, { \ + __FSM_H2_PSHDR_CHECK_lambda(p, { \ __msg_hdr_chunk_fixup(data, len); \ __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ if (unlikely(!fin)) \ __FSM_H2_POSTPONE(st_next); \ __FSM_H2_HDR_COMPLETE(st_curr); \ - }, st_next) + }); \ + goto st_next; \ +} while (0) #define __FSM_H2_PSHDR_MOVE_FIN_fixup(st_curr, n, st_next) \ +do { \ __msg_hdr_chunk_fixup(p, n); \ __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ - __FSM_H2_PSHDR_MOVE_lambda(n, { \ + p += n; \ + __FSM_H2_PSHDR_CHECK_lambda(p, { \ if (unlikely(!fin)) \ __FSM_H2_POSTPONE(st_next); \ __FSM_H2_HDR_COMPLETE(st_curr); \ - }, st_next) + }); \ + goto st_next; \ +} while (0) #define __FSM_H2_PSHDR_MOVE_DROP(st_curr, n, st_next) \ - __FSM_H2_PSHDR_MOVE_lambda(n, { \ +do { \ + p += n; \ + __FSM_H2_PSHDR_CHECK_lambda(p, { \ if (unlikely(!fin)) { \ __msg_hdr_chunk_fixup(data, len); \ __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ __FSM_H2_POSTPONE(st_next); \ } \ __FSM_H2_DROP(st_curr); \ - }, st_next) + }); \ + goto st_next; \ +} while (0) #define __FSM_H2_PSHDR_MOVE_DROP_nofixup(st_curr, n, st_next) \ - __FSM_H2_PSHDR_MOVE_lambda(n, { \ +do { \ + p += n; \ + __FSM_H2_PSHDR_CHECK_lambda(p, { \ if (unlikely(!fin)) \ __FSM_H2_POSTPONE(st_next); \ __FSM_H2_DROP(st_curr); \ - }, st_next) + }); \ + goto st_next; \ +} while (0) #define __FSM_H2_PSHDR_COMPLETE(st, n) \ do { \ @@ -4359,24 +4366,32 @@ do { \ } while (0) #define __FSM_H2_METHOD_MOVE(st_curr, n, st_next) \ - __FSM_H2_PSHDR_MOVE_lambda(n, { \ +do { \ + p += n; \ + __FSM_H2_PSHDR_CHECK_lambda(p, { \ __msg_hdr_chunk_fixup(data, len); \ __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ if (unlikely(!fin)) \ __FSM_H2_POSTPONE(st_next); \ req->method = _TFW_HTTP_METH_UNKNOWN; \ __FSM_H2_HDR_COMPLETE(st_curr); \ - }, st_next) + }); \ + goto st_next; \ +} while (0) #define __FSM_H2_METHOD_COMPLETE(st_curr, n, mid) \ - __FSM_H2_PSHDR_MOVE_lambda(n, { \ +do { \ + p += n; \ + __FSM_H2_PSHDR_CHECK_lambda(p, { \ __msg_hdr_chunk_fixup(data, len); \ __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ if (unlikely(!fin)) \ __FSM_H2_POSTPONE(Req_MethodUnknown); \ req->method = mid; \ __FSM_H2_HDR_COMPLETE(st_curr); \ - }, Req_MethodUnknown) + }); \ + goto Req_MethodUnknown; \ +} while (0) /* * Special set of macros for slow-path parsing of pseudo-headers value @@ -4419,7 +4434,7 @@ __FSM_STATE(st_curr) { \ TRY_STR_INIT(); \ __fsm_n = func(hm, p, len, fin); \ T_DBG3("%s: parse header value, " #func ": ret=%d data_len=%lu" \ - " hid=%u\n", __fsm_n, len, parser->_hdr_tag); \ + " hid=%d\n", __func__, __fsm_n, len, hid); \ switch (__fsm_n) { \ case CSTR_EQ: \ if (saveval) { \ @@ -6103,15 +6118,15 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, TfwMsgParseIter *it = &req->pit; __FSM_DECLARE_VARS(req); - T_DBG("%s: len=%lu, data=%.*s%s, req=[%p]\n", len, min(500, (int)len), - data, len > 500 ? "..." : "", req); + T_DBG("%s: len=%lu, data=%.*s%s, req=[%p]\n", __func__, len, + min(500, (int)len), data, len > 500 ? "..." : "", req); __FSM_START(parser->state); /* * If next state is not defined here, that means the name has not been - * parsed and is statically indexed, thus we should determine the state - * from the header tag. + * parsed and is indexed, thus we should determine the state from the + * header tag. */ __FSM_H2_REQ_NEXT_STATE(value_stage); @@ -6295,7 +6310,7 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, 'a', 'r', 'd') && C4_INT(p + 9, 'e', 'd', '-', 'f') && *(p + 13) == 'o' - && *(p + 15) == 'r')) + && *(p + 14) == 'r')) { __FSM_H2_FIN(Req_HdrX_Forwarded_ForV, 15, TFW_TAG_HDR_X_FORWARDED_FOR); @@ -7159,7 +7174,7 @@ tfw_h2_parse_body(char *data, unsigned long len, TfwHttpReq *req, parser->to_read = ctx->plen; - T_DBG3("%s: init, content_length=%lu, to_read=%ld\n", + T_DBG3("%s: init, content_length=%lu, to_read=%ld\n", __func__, req->content_length, parser->to_read); if (!req->body.data) @@ -7172,14 +7187,14 @@ tfw_h2_parse_body(char *data, unsigned long len, TfwHttpReq *req, if (parser->to_read) { T_DBG3("%s: postpone, to_read=%ld, m_len=%lu, len=%lu\n", - parser->to_read, m_len, len); + __func__, parser->to_read, m_len, len); __msg_field_fixup(&req->body, data + len); goto out; } WARN_ON_ONCE(m_len != len); - T_DBG3("%s: to_read=%ld, m_len=%lu, len=%lu\n", parser->to_read, - m_len, len); + T_DBG3("%s: to_read=%ld, m_len=%lu, len=%lu\n", __func__, + parser->to_read, m_len, len); if (tfw_http_msg_add_str_data(msg, &req->body, data, m_len)) return T_DROP; @@ -7790,7 +7805,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __msg_hdr_set_hpack_index(24); __FSM_MOVE_n(RGen_LWS, 14); } - __FSM_MOVE_n(RGen_HdrOther, 5); + __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 'n', 'e'): if (likely(C4_INT_LCM(p + 5, 'c', 't', 'i', 'o') && TFW_LC(*(p + 9)) == 'n' @@ -7799,7 +7814,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, parser->_i_st = &&Resp_HdrConnectionV; __FSM_MOVE_n(RGen_LWS, 11); } - __FSM_MOVE_n(RGen_HdrOther, 5); + __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 't', 'e'): if (likely(TFW_LC(*(p + 5)) == 'n' && TFW_LC(*(p + 6)) == 't' @@ -7807,9 +7822,9 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, { __FSM_MOVE_n(Resp_HdrContent_, 8); } - __FSM_MOVE_n(RGen_HdrOther, 5); + __FSM_MOVE_n(RGen_HdrOtherN, 5); default: - __FSM_MOVE(RGen_HdrOther); + __FSM_MOVE(RGen_HdrOtherN); } case 'd': if (likely(__data_available(p, 5) @@ -7986,7 +8001,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, } __FSM_MOVE(Resp_HdrW); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8061,7 +8076,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, } __FSM_MOVE(Resp_HdrContent_T); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8159,7 +8174,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 'g': __FSM_MOVE(Resp_HdrAg); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } __FSM_TX_AF(Resp_HdrAc, 'c', Resp_HdrAcc); @@ -8172,7 +8187,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 's': __FSM_MOVE(Resp_HdrAcces); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8239,7 +8254,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 'o': __FSM_MOVE(Resp_HdrCo); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8266,7 +8281,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 't': __FSM_MOVE(Resp_HdrCont); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } __FSM_TX_AF(Resp_HdrConn, 'e', Resp_HdrConne); @@ -8315,7 +8330,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 'o': __FSM_MOVE(Resp_HdrContent_Lo); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8370,7 +8385,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 'x': __FSM_MOVE(Resp_HdrEx); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } /* ETag header processing. */ @@ -8407,7 +8422,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 'o': __FSM_MOVE(Resp_HdrLo); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8447,7 +8462,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 'o': __FSM_MOVE(Resp_HdrPro); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8495,7 +8510,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 't': __FSM_MOVE(Resp_HdrSt); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8540,7 +8555,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 't': __FSM_MOVE(Resp_HdrSet); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -8586,7 +8601,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, case 'i': __FSM_MOVE(Resp_HdrVi); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } diff --git a/tempesta_fw/ss_skb.c b/tempesta_fw/ss_skb.c index 851b7640d7..a0637bf9c2 100644 --- a/tempesta_fw/ss_skb.c +++ b/tempesta_fw/ss_skb.c @@ -1023,7 +1023,7 @@ skb_next_data(struct sk_buff *skb, char *last_ptr, TfwStr *it) off = last_ptr - (char *)skb->data; T_DBG("%s: last_ptr=[%p], skb->data=[%p], si->nr_frags=%u, f_size=%u," - " off=ld\n", __func__, last_ptr, skb->data, si->nr_frags, f_size, + " off=%ld\n", __func__, last_ptr, skb->data, si->nr_frags, f_size, off); if (off >= 0 && off < f_size) { @@ -1045,8 +1045,8 @@ skb_next_data(struct sk_buff *skb, char *last_ptr, TfwStr *it) off = last_ptr - (char *)skb_frag_address(frag); T_DBG3("%s: frags search, skb_frag_address(frag)=[%p]," - " f_size=%u, off=ld\n", __func__, skb_frag_address(frag), - f_size, off); + " f_size=%u, off=%ld\n", __func__, + skb_frag_address(frag), f_size, off); if (off < 0 || off >= f_size) continue; @@ -1551,7 +1551,7 @@ ss_skb_replace_page(struct sk_buff **skb_head, struct page *pg, m_len = min_t(int, skb_frag_size(frag), offset); T_DBG3("%s: skb=[%p], m_len=%d, fragsize=%d\n", __func__, skb, - len, skb_frag_size(frag)); + m_len, skb_frag_size(frag)); if (unlikely(m_len == skb_frag_size(frag))) { ss_skb_adjust_data_len(skb, m_len); diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index 5650636c43..32873bc6f2 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -24,6 +24,13 @@ #include "helpers.h" #include "tfw_str_helper.h" +#define HDR_METHOD_NM ":method" +#define HDR_METHOD_VAL "GET" +#define HDR_SCHEME_NM ":scheme" +#define HDR_SCHEME_VAL "https" +#define HDR_PATH_NM ":path" +#define HDR_PATH_VAL "/" + #define HDR_COMPOUND_STR(hdr_res, nm, val) \ ({ \ TfwStr *c; \ @@ -112,7 +119,7 @@ TEST(hpack, dec_table_static) EXPECT_EQ(hdr->nchunks, 1); EXPECT_EQ(ius_len, hdr->len); EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_ius, ius_len, 0)); - EXPECT_EQ(entry->tag, TFW_HTTP_HDR_RAW); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 61); @@ -122,7 +129,7 @@ TEST(hpack, dec_table_static) EXPECT_EQ(hdr->nchunks, 1); EXPECT_EQ(wa_len, hdr->len); EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_wa, wa_len, 0)); - EXPECT_EQ(entry->tag, TFW_HTTP_HDR_RAW); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 1); @@ -132,7 +139,7 @@ TEST(hpack, dec_table_static) EXPECT_EQ(hdr->nchunks, 1); EXPECT_EQ(auth_len, hdr->len); EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_auth, auth_len, 0)); - EXPECT_EQ(entry->tag, TFW_HTTP_HDR_RAW); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_H2_AUTHORITY); } entry = tfw_hpack_find_index(&hp->dec_tbl, 16); @@ -145,7 +152,7 @@ TEST(hpack, dec_table_static) TFW_STR_EQ_PREFIX)); __h2_msg_hdr_val(hdr, &h_val); EXPECT_TRUE(tfw_str_eq_cstr(&h_val, s_aenc_v, aenc_v_len, 0)); - EXPECT_EQ(entry->tag, TFW_HTTP_HDR_RAW); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } entry = tfw_hpack_find_index(&hp->dec_tbl, 57); @@ -155,7 +162,7 @@ TEST(hpack, dec_table_static) EXPECT_EQ(hdr->nchunks, 1); EXPECT_EQ(tenc_len, hdr->len); EXPECT_TRUE(tfw_str_eq_cstr(hdr, s_tenc, tenc_len, 0)); - EXPECT_EQ(entry->tag, TFW_HTTP_HDR_TRANSFER_ENCODING); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_TRANSFER_ENCODING); } } @@ -163,7 +170,7 @@ TEST(hpack, dec_table_dynamic) { TfwHPack *hp; const TfwHPackEntry *entry; - TfwStr *s1, *s2, *s3; + TfwStr h_name, *s1, *s2, *s3; TfwMsgParseIter *it = &test_req->pit; unsigned int new_len = 0; TFW_STR(s1_name, "custom-key"); @@ -179,12 +186,24 @@ TEST(hpack, dec_table_dynamic) hp = &ctx.hpack; + __h2_msg_hdr_name(s1, &h_name); + it->nm_num = h_name.nchunks; + it->nm_len = h_name.len; + it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s1; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); + __h2_msg_hdr_name(s2, &h_name); + it->nm_num = h_name.nchunks; + it->nm_len = h_name.len; + it->tag = TFW_TAG_HDR_X_FORWARDED_FOR; *it->parsed_hdr = *s2; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); + __h2_msg_hdr_name(s3, &h_name); + it->nm_num = h_name.nchunks; + it->nm_len = h_name.len; + it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s3; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); @@ -200,10 +219,9 @@ TEST(hpack, dec_table_dynamic) entry = tfw_hpack_find_index(&hp->dec_tbl, 62); EXPECT_NOT_NULL(entry); - if (entry) { + if (entry) new_len += entry->hdr->len + 32; EXPECT_TRUE(tfw_strcmp(entry->hdr, s3) == 0); - } EXPECT_OK(tfw_hpack_set_length(hp, new_len)); @@ -215,7 +233,7 @@ TEST(hpack, dec_table_dynamic) TEST(hpack, dec_table_dynamic_inc) { TfwHPack *hp; - TfwStr *s1, *s2, *s3, *s4, *s5; + TfwStr h_name, *s1, *s2, *s3, *s4, *s5; const TfwHPackEntry *entry; TfwMsgParseIter *it = &test_req->pit; TFW_STR(s1_name, "custom-header-1"); @@ -235,9 +253,17 @@ TEST(hpack, dec_table_dynamic_inc) hp = &ctx.hpack; + __h2_msg_hdr_name(s1, &h_name); + it->nm_num = h_name.nchunks; + it->nm_len = h_name.len; + it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s1; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); + __h2_msg_hdr_name(s2, &h_name); + it->nm_num = h_name.nchunks; + it->nm_len = h_name.len; + it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s2; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); @@ -251,9 +277,17 @@ TEST(hpack, dec_table_dynamic_inc) if (entry) EXPECT_TRUE(tfw_strcmp(entry->hdr, s1) == 0); + __h2_msg_hdr_name(s3, &h_name); + it->nm_num = h_name.nchunks; + it->nm_len = h_name.len; + it->tag = TFW_TAG_HDR_CACHE_CONTROL; *it->parsed_hdr = *s3; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); + __h2_msg_hdr_name(s4, &h_name); + it->nm_num = h_name.nchunks; + it->nm_len = h_name.len; + it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s4; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); @@ -267,6 +301,10 @@ TEST(hpack, dec_table_dynamic_inc) if (entry) EXPECT_TRUE(tfw_strcmp(entry->hdr, s3) == 0); + __h2_msg_hdr_name(s5, &h_name); + it->nm_num = h_name.nchunks; + it->nm_len = h_name.len; + it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s5; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); @@ -342,11 +380,12 @@ TEST(hpack, dec_table_wrap) } if (i < end_idx) { + unsigned long shrink_sz = s->len + HPACK_ENTRY_OVERHEAD; /* * Evict first @shift entries, i.e shrink table to only * one existing entry. */ - EXPECT_OK(tfw_hpack_set_length(hp, s->len + 32)); + EXPECT_OK(tfw_hpack_set_length(hp, shrink_sz)); EXPECT_OK(tfw_hpack_set_length(hp, HPACK_TABLE_DEF_SIZE)); @@ -365,12 +404,15 @@ TEST(hpack, dec_table_wrap) */ entries = hp->dec_tbl.entries; for (i = 0; i < shift; ++i) { + TfwStr h_name; const TfwHPackEntry *l_entry = &last_entries[i]; const TfwHPackEntry *t_entry = &entries[i]; EXPECT_NOT_NULL(l_entry->hdr); + EXPECT_NOT_NULL(t_entry->hdr); if (l_entry->hdr) { - EXPECT_EQ(l_entry->hdr, t_entry->hdr); + __h2_msg_hdr_name(t_entry->hdr, &h_name); + EXPECT_TRUE(tfw_strcmp(&h_name, l_entry->hdr) == 0); } } @@ -397,20 +439,21 @@ TEST(hpack, dec_raw) const char *test_name1 = HDR_NAME_1; const char *test_value1 = HDR_VALUE_1; unsigned long hdr_len1 = 25; - char *hdr_data1 = + static char *hdr_data1 = "\x40" /* == With indexing == */ "\x0A" /* Literal name (len = 10) */ "\x63\x75\x73\x74\x6F" /* custom-key */ "\x6D\x2D\x6B\x65\x79" /* */ "\x0C" /* Literal value (len = 12) */ - "\x63\x75\x73\x74\x6F" /* custom-value */ + "\x63\x75\x73\x74\x6F" /* custom-value */ "\x6D\x2D\x76\x61\x6C" /* */ "\x75\x65"; /* */ + const char *test_name2 = HDR_NAME_2; const char *test_value2 = HDR_VALUE_2; unsigned long hdr_len2 = 37; - char *hdr_data2 = + static char *hdr_data2 = "\x00" /* == Without indexing == */ "\x0C" /* Literal name (len = 12) */ "\x78\x2D\x63\x75\x73" /* x-custom-hdr */ @@ -423,11 +466,12 @@ TEST(hpack, dec_raw) "\x65\x20\x76\x61\x6C" /* */ "\x75\x65"; /* */ + const char *test_name3 = HDR_NAME_3; const char *test_value3 = HDR_VALUE_3; unsigned long hdr_len3 = 41; - char *hdr_data3 = - "\x10" /* == Never indexing == */ + static char *hdr_data3 = + "\x10" /* == Never indexing == */ "\x0F" /* Literal name (len = 15) */ "\x78\x2D\x66\x6F\x72" /* x-forwarded-for */ "\x77\x61\x72\x64\x65" /* */ @@ -439,27 +483,48 @@ TEST(hpack, dec_raw) "\x6D\x70\x6C\x65\x2E" /* */ "\x63\x6F\x6D"; /* */ + hp = &ctx.hpack; + ht = test_req->h_tbl; + /* Set mandatory pseudo-headers to pass checks in @H2_MSG_VERIFY(). */ + ht->tbl[TFW_HTTP_HDR_H2_METHOD] = TFW_STR_STRING(HDR_METHOD_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_METHOD], HDR_METHOD_VAL, + SLEN(HDR_METHOD_VAL), TFW_STR_HDR_VALUE); + ht->tbl[TFW_HTTP_HDR_H2_SCHEME] = TFW_STR_STRING(HDR_SCHEME_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_SCHEME], HDR_SCHEME_VAL, + SLEN(HDR_SCHEME_VAL), TFW_STR_HDR_VALUE); + ht->tbl[TFW_HTTP_HDR_H2_PATH] = TFW_STR_STRING(HDR_PATH_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_PATH], HDR_PATH_VAL, + SLEN(HDR_PATH_VAL), TFW_STR_HDR_VALUE); + + parsed = 0; r = tfw_hpack_decode(hp, hdr_data1, hdr_len1, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len1); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data2, hdr_len2, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len2); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data3, hdr_len3, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len3); + /* + * Update pointer to headers table, since it had been enlarged and, + * as a result, relocated during decoding/parsing procedures. + */ + EXPECT_NE(ht, test_req->h_tbl); ht = test_req->h_tbl; __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW], &h_name); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(strlen(test_name1), h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, strlen(test_name1), 0)); @@ -472,7 +537,7 @@ TEST(hpack, dec_raw) __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(strlen(test_name2), h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, strlen(test_name2), 0)); @@ -485,7 +550,7 @@ TEST(hpack, dec_raw) __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(strlen(test_name3), h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name3, strlen(test_name3), 0)); @@ -515,14 +580,12 @@ TEST(hpack, dec_indexed) #define HDR_VALUE_1 " test.com, foo.com, example.com" #define HDR_NAME_2 "accept-encoding" #define HDR_VALUE_2 "gzip, deflate" -#define HDR_NAME_3 "accept-encoding" #define HDR_VALUE_3 "deflate, gzip;q=1.0, *;q=0.5" -#define HDR_NAME_4 "x-forwarded-for" #define HDR_VALUE_4 "127.0.0.1" #define HDR_NAME_5 "host" #define HDR_VALUE_5 "localhost" -#define HDR_NAME_6 "transfer-encoding" -#define HDR_VALUE_6 "chunked" +#define HDR_NAME_6 "referer" +#define HDR_VALUE_6 "http://www.example.org/overview.html" const char *test_name1 = HDR_NAME_1; const char *test_value1 = HDR_VALUE_1; @@ -600,51 +663,76 @@ TEST(hpack, dec_indexed) "\x6C\x6F\x63\x61\x6C" /* localhost */ "\x68\x6F\x73\x74"; /* */ - unsigned long hdr_len7 = 10; + unsigned long hdr_len7 = 39; char *hdr_data7 = - "\x0F\x2A" /* == Without indexing == */ - /* (name indexed - static: 57) */ + "\x0F\x24" /* == Without indexing == */ + /* (name indexed - static: 51) */ /* (multibyte integer encoding) */ /* */ - "\x07" /* Literal value (len = 7) */ - "\x63\x68\x75\x6E\x6B" /* chunked */ - "\x65\x64"; /* */ + "\x24" /* Literal value (len = 36) */ + "\x68\x74\x74\x70\x3A" /* http://www.example.org/... */ + "\x2F\x2F\x77\x77\x77" /* ...overview.html */ + "\x2E\x65\x78\x61\x6D" /* */ + "\x70\x6C\x65\x2E\x6F" /* */ + "\x72\x67\x2F\x6F\x76" /* */ + "\x65\x72\x76\x69\x65" /* */ + "\x77\x2E\x68\x74\x6D" /* */ + "\x6C"; /* */ hp = &ctx.hpack; + ht = test_req->h_tbl; + + /* Set mandatory pseudo-headers to pass checks in @H2_MSG_VERIFY(). */ + ht->tbl[TFW_HTTP_HDR_H2_METHOD] = TFW_STR_STRING(HDR_METHOD_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_METHOD], HDR_METHOD_VAL, + SLEN(HDR_METHOD_VAL), TFW_STR_HDR_VALUE); + ht->tbl[TFW_HTTP_HDR_H2_SCHEME] = TFW_STR_STRING(HDR_SCHEME_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_SCHEME], HDR_SCHEME_VAL, + SLEN(HDR_SCHEME_VAL), TFW_STR_HDR_VALUE); + ht->tbl[TFW_HTTP_HDR_H2_PATH] = TFW_STR_STRING(HDR_PATH_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_PATH], HDR_PATH_VAL, + SLEN(HDR_PATH_VAL), TFW_STR_HDR_VALUE); /* * Processing prepared HTTP/2 headers in HPACK decoding * procedure. */ + parsed = 0; r = tfw_hpack_decode(hp, hdr_data1, hdr_len1, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len1); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data2, hdr_len2, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len2); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data3, hdr_len3, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len3); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data4, hdr_len4, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len4); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data5, hdr_len5, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len5); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data6, hdr_len6, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len6); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data7, hdr_len7, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len7); - ht = test_req->h_tbl; + EXPECT_EQ(ht, test_req->h_tbl); /* * Verify that decoded headers had been correctly written into @@ -659,7 +747,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm1, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, test_len_nm1, 0)); @@ -672,7 +760,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm1, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, test_len_nm1, 0)); @@ -685,7 +773,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm1, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, test_len_nm1, 0)); @@ -704,7 +792,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm2, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, test_len_nm2, 0)); @@ -717,7 +805,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm2, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, test_len_nm2, 0)); @@ -732,7 +820,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm5, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name5, test_len_nm5, 0)); @@ -741,12 +829,12 @@ TEST(hpack, dec_indexed) test_len_val5, 0)); } - hdr = &ht->tbl[TFW_HTTP_HDR_TRANSFER_ENCODING]; + hdr = &ht->tbl[TFW_HTTP_HDR_REFERER]; __h2_msg_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm6, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name6, test_len_nm6, 0)); @@ -775,7 +863,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm1, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, test_len_nm1, 0)); @@ -796,7 +884,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm2, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, test_len_nm2, 0)); @@ -817,7 +905,7 @@ TEST(hpack, dec_indexed) __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm1, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, test_len_nm1, 0)); @@ -827,16 +915,14 @@ TEST(hpack, dec_indexed) } EXPECT_EQ(entry->name_len, test_len_nm1); EXPECT_EQ(entry->name_num, h_name.nchunks); - EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_X_FORWARDED_FOR); } #undef HDR_NAME_1 #undef HDR_VALUE_1 #undef HDR_NAME_2 #undef HDR_VALUE_2 -#undef HDR_NAME_3 #undef HDR_VALUE_3 -#undef HDR_NAME_4 #undef HDR_VALUE_4 #undef HDR_NAME_5 #undef HDR_VALUE_5 @@ -909,35 +995,73 @@ TEST(hpack, dec_huffman) "\xF4\xFF"; /* */ hp = &ctx.hpack; + ht = test_req->h_tbl; + + /* Set mandatory pseudo-headers to pass checks in @H2_MSG_VERIFY(). */ + ht->tbl[TFW_HTTP_HDR_H2_METHOD] = TFW_STR_STRING(HDR_METHOD_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_METHOD], HDR_METHOD_VAL, + SLEN(HDR_METHOD_VAL), TFW_STR_HDR_VALUE); + ht->tbl[TFW_HTTP_HDR_H2_SCHEME] = TFW_STR_STRING(HDR_SCHEME_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_SCHEME], HDR_SCHEME_VAL, + SLEN(HDR_SCHEME_VAL), TFW_STR_HDR_VALUE); + ht->tbl[TFW_HTTP_HDR_H2_PATH] = TFW_STR_STRING(HDR_PATH_NM); + collect_compound_str2(&ht->tbl[TFW_HTTP_HDR_H2_PATH], HDR_PATH_VAL, + SLEN(HDR_PATH_VAL), TFW_STR_HDR_VALUE); /* * Processing prepared Huffman-encoded HTTP/2 headers in HPACK * decoding procedure. + * + * NOTE: the ':authority' pseudo-header should be processed first + * (with mandatory pseudo-headers) to pass the verification in + * @H2_MSG_VERIFY(), according to RFC. */ + parsed = 0; + r = tfw_hpack_decode(hp, hdr_data3, hdr_len3, test_req, &parsed); + EXPECT_EQ(r, T_OK); + EXPECT_EQ(parsed, hdr_len3); + + parsed = 0; r = tfw_hpack_decode(hp, hdr_data1, hdr_len1, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len1); + parsed = 0; r = tfw_hpack_decode(hp, hdr_data2, hdr_len2, test_req, &parsed); EXPECT_EQ(r, T_OK); EXPECT_EQ(parsed, hdr_len2); - r = tfw_hpack_decode(hp, hdr_data3, hdr_len3, test_req, &parsed); - EXPECT_EQ(r, T_OK); - EXPECT_EQ(parsed, hdr_len3); - + /* + * Update pointer to headers table, since it had been enlarged and, + * as a result, relocated during decoding/parsing procedures. + */ + EXPECT_NE(ht, test_req->h_tbl); ht = test_req->h_tbl; /* * Verify that Huffman-decoded headers had been correctly written * into the headers table. */ + hdr = &ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm3, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name3, + test_len_nm3, 0)); + EXPECT_EQ(test_len_val3, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, + test_len_val3, 0)); + } + hdr = &ht->tbl[TFW_HTTP_HDR_RAW]; __h2_msg_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm1, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, test_len_nm1, 0)); @@ -951,7 +1075,7 @@ TEST(hpack, dec_huffman) __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm2, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, test_len_nm2, 0)); @@ -960,20 +1084,6 @@ TEST(hpack, dec_huffman) test_len_val2, 0)); } - hdr = &ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]; - __h2_msg_hdr_name(hdr, &h_name); - __h2_msg_hdr_val(hdr, &h_value); - EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); - EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { - EXPECT_EQ(test_len_nm3, h_name.len); - EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name3, - test_len_nm3, 0)); - EXPECT_EQ(test_len_val3, h_value.len); - EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, - test_len_val3, 0)); - } - /* * Verify that Huffman-decoded headers had been correctly placed into * decoder index table with appropriate indexes. @@ -986,7 +1096,28 @@ TEST(hpack, dec_huffman) __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { + EXPECT_EQ(test_len_nm3, h_name.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name3, + test_len_nm3, 0)); + EXPECT_EQ(test_len_val3, h_value.len); + EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, + test_len_val3, 0)); + } + EXPECT_EQ(entry->name_len, test_len_nm3); + EXPECT_EQ(entry->name_num, h_name.nchunks); + EXPECT_EQ(entry->tag, TFW_TAG_HDR_H2_AUTHORITY); + } + + entry = tfw_hpack_find_index(&hp->dec_tbl, 63); + EXPECT_NOT_NULL(entry); + if (entry) { + hdr = entry->hdr; + __h2_msg_hdr_name(hdr, &h_name); + __h2_msg_hdr_val(hdr, &h_value); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); + EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm1, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name1, test_len_nm1, 0)); @@ -999,7 +1130,7 @@ TEST(hpack, dec_huffman) EXPECT_EQ(entry->tag, TFW_TAG_HDR_RAW); } - entry = tfw_hpack_find_index(&hp->dec_tbl, 63); + entry = tfw_hpack_find_index(&hp->dec_tbl, 62); EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; @@ -1007,7 +1138,7 @@ TEST(hpack, dec_huffman) __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { + if (!TFW_STR_EMPTY(&h_name) && !TFW_STR_EMPTY(&h_value)) { EXPECT_EQ(test_len_nm2, h_name.len); EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name2, test_len_nm2, 0)); @@ -1020,27 +1151,6 @@ TEST(hpack, dec_huffman) EXPECT_EQ(entry->tag, TFW_TAG_HDR_CACHE_CONTROL); } - entry = tfw_hpack_find_index(&hp->dec_tbl, 62); - EXPECT_NOT_NULL(entry); - if (entry) { - hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); - __h2_msg_hdr_val(hdr, &h_value); - EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); - EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); - if (!TFW_STR_EMPTY(&h_name) || !TFW_STR_EMPTY(&h_value)) { - EXPECT_EQ(test_len_nm3, h_name.len); - EXPECT_TRUE(tfw_str_eq_cstr(&h_name, test_name3, - test_len_nm3, 0)); - EXPECT_EQ(test_len_val3, h_value.len); - EXPECT_TRUE(tfw_str_eq_cstr(&h_value, test_value3, - test_len_val3, 0)); - } - EXPECT_EQ(entry->name_len, test_len_nm3); - EXPECT_EQ(entry->name_num, h_name.nchunks); - EXPECT_EQ(entry->tag, TFW_TAG_HDR_H2_AUTHORITY); - } - #undef HDR_NAME_1 #undef HDR_VALUE_1 #undef HDR_NAME_2 @@ -1065,40 +1175,48 @@ TEST(hpack, enc_table_hdr_write) #define HDR_NAME_5 "custom-key" #define HDR_VALUE_5 "custom-example-value" - TFW_STR(s1, HDR_NAME_1 ": "); + TFW_STR(s1, HDR_NAME_1 ":"); + TFW_STR(s1_lws, " "); TFW_STR(s1_value, HDR_VALUE_1 " "); - unsigned long off1 = 4; const char *t_s1 = HDR_NAME_1 HDR_VALUE_1; + unsigned long off1 = s1_lws->len + 1; unsigned long t_s1_len = strlen(t_s1); TFW_STR(s2, HDR_NAME_2 ":"); TFW_STR(s2_value, HDR_VALUE_2); - unsigned long off2 = 1; const char *t_s2 = HDR_NAME_2 HDR_VALUE_2; + unsigned long off2 = 1; unsigned long t_s2_len = strlen(t_s2); - TFW_STR(s3, HDR_NAME_3 ":\t "); + TFW_STR(s3, HDR_NAME_3 ":"); + TFW_STR(s3_lws, "\t "); TFW_STR(s3_value, HDR_VALUE_3 " "); - unsigned long off3 = 4; const char *t_s3 = HDR_NAME_3 HDR_VALUE_3; + unsigned long off3 = s3_lws->len + 1; unsigned long t_s3_len = strlen(t_s3); - TFW_STR(s4, HDR_NAME_4 ": "); + TFW_STR(s4, HDR_NAME_4 ":"); + TFW_STR(s4_lws, " "); TFW_STR(s4_value, HDR_VALUE_4 "\t\t \t"); - unsigned long off4 = 6; const char *t_s4 = HDR_NAME_4 HDR_VALUE_4; + unsigned long off4 = s4_lws->len + 1; unsigned long t_s4_len = strlen(t_s4); - TFW_STR(s5, HDR_NAME_5 ":\t\t\t"); + TFW_STR(s5, HDR_NAME_5 ":"); + TFW_STR(s5_lws, "\t\t\t"); TFW_STR(s5_value, HDR_VALUE_5 "\t\t\t\t"); - unsigned long off5 = 4; const char *t_s5 = HDR_NAME_5 HDR_VALUE_5; + unsigned long off5 = s5_lws->len + 1; unsigned long t_s5_len = strlen(t_s5); + collect_compound_str(s1, s1_lws, 0); collect_compound_str(s1, s1_value, 0); collect_compound_str(s2, s2_value, 0); + collect_compound_str(s3, s3_lws, 0); collect_compound_str(s3, s3_value, 0); + collect_compound_str(s4, s4_lws, 0); collect_compound_str(s4, s4_value, 0); + collect_compound_str(s5, s5_lws, 0); collect_compound_str(s5, s5_value, 0); hdr_len = tfw_h2_msg_hdr_length(s1, &n_len, &v_off, &v_len); @@ -1178,7 +1296,8 @@ TEST(hpack, enc_table_index) #define HDR_NAME_3 "test-example-key" #define HDR_VALUE_3 "custom-example-value" - TFW_STR(s1, HDR_NAME_1 ": \t "); + TFW_STR(s1, HDR_NAME_1 ":"); + TFW_STR(s1_lws, " \t "); TFW_STR(s1_value, HDR_VALUE_1 " "); const char *t_s1 = HDR_NAME_1 HDR_VALUE_1; unsigned long t_s1_len = strlen(t_s1); @@ -1188,13 +1307,16 @@ TEST(hpack, enc_table_index) const char *t_s2 = HDR_NAME_2 HDR_VALUE_2; unsigned long t_s2_len = strlen(t_s2); - TFW_STR(s3, HDR_NAME_3 ":\t \t\t\t"); + TFW_STR(s3, HDR_NAME_3 ":"); + TFW_STR(s3_lws, "\t \t\t\t"); TFW_STR(s3_value, HDR_VALUE_3 "\t\t\t\t "); const char *t_s3 = HDR_NAME_3 HDR_VALUE_3; unsigned long t_s3_len = strlen(t_s3); + collect_compound_str(s1, s1_lws, 0); collect_compound_str(s1, s1_value, 0); collect_compound_str(s2, s2_value, 0); + collect_compound_str(s3, s3_lws, 0); collect_compound_str(s3, s3_value, 0); tbl = &ctx.hpack.enc_tbl; From e50d74de1c03bdd62ea67ad7f29c3637d6a088e8 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Tue, 12 Nov 2019 15:38:23 +0300 Subject: [PATCH 07/64] HTTP/2 Parser implementation: redesign of HTTP/1.1=>HTTP/2 transformation flow (#309). --- tempesta_fw/hpack.c | 464 ++++++++++++---------------------- tempesta_fw/hpack.h | 10 +- tempesta_fw/http.c | 558 ++++++++++++++++++++++++++++++++--------- tempesta_fw/http.h | 79 +++++- tempesta_fw/http_msg.c | 174 +++++++------ tempesta_fw/http_msg.h | 53 +++- tempesta_fw/msg.h | 11 - tempesta_fw/ss_skb.c | 94 +++++++ tempesta_fw/ss_skb.h | 7 + tempesta_fw/vhost.c | 3 - tempesta_fw/vhost.h | 3 + 11 files changed, 925 insertions(+), 531 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index 6225cf9230..6d8902e6fb 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -1041,7 +1041,7 @@ static const TfwHPackEntry static_table[] ____cacheline_aligned = { HP_ENTRY_NAME("www-authenticate", TFW_TAG_HDR_RAW) }; -#define HPACK_STATIC_ENTRIES (sizeof(static_table) / sizeof(TfwHPackEntry)) +#define HPACK_STATIC_ENTRIES (sizeof(static_table) / sizeof(static_table[0])) /* Limit for the HPACK variable-length integer. */ #define HPACK_LIMIT (1 << 20) @@ -3651,140 +3651,38 @@ write_int(unsigned long index, unsigned short max, unsigned short mask, res_idx->sz = size; } -static int -tfw_hpack_idx_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, - TfwHPackInt *__restrict idx) -{ - int ret; - TfwStr it = {}; - TfwMsgTransIter *mit = &resp->mit; - TfwStr idx_str = { - .data = idx->buf, - .len = idx->sz, - }; - - ret = ss_skb_get_room(resp->msg.skb_head, first->skb, first->data, - idx_str.len, &it); - if (ret) - return ret; - - if ((ret = tfw_strcpy(&it, &idx_str))) - return ret; - - mit->acc_len += idx_str.len; - - T_DBG3("%s: idx, acc_len=%lu, idx_str.len=%lu, idx_str.data='%.*s'\n", - __func__, mit->acc_len, idx_str.len, (int)idx_str.len, - idx_str.data); - - return 0; -} - -static inline int -tfw_hpack_idx_expand(TfwHttpResp *__restrict resp, TfwHPackInt *__restrict idx) -{ - int ret; - TfwMsgTransIter *mit = &resp->mit; - TfwStr idx_str = { - .data = idx->buf, - .len = idx->sz - }; - - ret = tfw_http_msg_expand_data(&mit->iter, &resp->msg.skb_head, - &idx_str); - if (ret) - return ret; - - mit->acc_len += idx_str.len; - - T_DBG3("%s: idx, acc_len=%lu, idx_str.len=%lu, idx_str.data='%.*s'\n", - __func__, mit->acc_len, idx_str.len, (int)idx_str.len, - idx_str.data); - - return 0; -} - /* - * Replace @str in the HTTP message by @idx as HPACK representation, with - * deletion of EOL at the end of @str. + * Add header @hdr in HTTP/2 HPACK format with metadata @idx into the + * response @resp. */ static int -tfw_hpack_idx_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict str, - TfwHPackInt *__restrict idx) +tfw_hpack_hdr_add(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, + TfwHPackInt *__restrict idx, bool name_indexed) { - int ret; - struct sk_buff *skb_head = resp->msg.skb_head; - TfwMsgTransIter *mit = &resp->mit; - TfwStr *last, it = {}; - TfwStr idx_str = { + int r; + TfwHPackInt vlen; + TfwStr *c, *end; + TfwHttpTransIter *mit = &resp->mit; + TfwStr s_val, s_vlen = {}; + const TfwStr s_idx = { .data = idx->buf, .len = idx->sz, }; - if (WARN_ON_ONCE(!str)) - return -EINVAL; - - if (str->len == idx_str.len) - goto copy; + T_DBG3("%s: s_idx->len=%lu, s_idx->data='%.*s'\n", + __func__, s_idx.len, (int)s_idx.len, s_idx.data); - if (str->len > idx_str.len) { - ret = ss_skb_cutoff_data(skb_head, str, idx_str.len, - tfw_str_eolen(str)); - if (ret) - return ret; - goto copy; - } - - last = TFW_STR_LAST(str); - ret = ss_skb_get_room(skb_head, last->skb, last->data + last->len, - idx_str.len - str->len, &it); - if (ret) - return ret; - - if ((ret = tfw_strcat(resp->pool, str, &it))) { - T_WARN("Cannot concatenate '%.*s' with '%.*s' (err=%d)\n", - PR_TFW_STR(str), PR_TFW_STR(&idx_str), ret); - return ret; - } - -copy: - if ((ret = tfw_strcpy(str, &idx_str))) - return ret; - - mit->acc_len += idx_str.len; - - T_DBG3("%s: idx, acc_len=%lu, idx_str.len=%lu, idx_str.data='%.*s'\n", - __func__, mit->acc_len, idx_str.len, (int)idx_str.len, - idx_str.data); - - return 0; -} + r = tfw_h2_msg_rewrite_data(mit, &s_idx, mit->bnd); + if (unlikely(r)) + return r; -/* - * Add header @hdr in HTTP/2 HPACK format into the response @resp, before - * the @first data. - */ -static int -tfw_hpack_hdr_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, - TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, - bool name_indexed) -{ - int ret; - TfwHPackInt vlen; - TfwStr *chunk, *end; - TfwMsgTransIter *mit = &resp->mit; - struct sk_buff *skb_head = resp->msg.skb_head; - TfwStr s_val, s_vlen = {}; - TfwStr it = {}; - TfwStr res_hdr = { - .data = idx->buf, - .len = idx->sz, - }; + if (!hdr) + return 0; - if (WARN_ON_ONCE(!hdr || TFW_STR_EMPTY(hdr))) + if (WARN_ON_ONCE(TFW_STR_PLAIN(hdr))) return -EINVAL; - if (!name_indexed) { + if (unlikely(!name_indexed)) { TfwHPackInt nlen; TfwStr *s_n = TFW_STR_CHUNK(hdr, 0); TfwStr s_name = { @@ -3797,41 +3695,44 @@ tfw_hpack_hdr_add(TfwHttpResp *__restrict resp, TfwStr *__restrict first, __TFW_STR_CH(&s_name, 0)->len = nlen.sz; __TFW_STR_CH(&s_name, 1)->data = s_n->data; __TFW_STR_CH(&s_name, 1)->len = s_n->len; + s_name.len = nlen.sz + s_n->len; - if ((ret = tfw_strcat(resp->pool, &res_hdr, &s_name))) - return ret; + r = tfw_h2_msg_rewrite_data(mit, &s_name, mit->bnd); + if (unlikely(r)) + return r; } /* - * The @hdr must have the HTTP/2-format chunk structure (without the - * colon and OWS), i.e.: { name, value1, value2, ... }. + * During headers addition into the message the source @hdr must have + * the following chunk structure (without the OWS): + * + * { name [S_DLM] value1 [value2 [value3 ...]] }. + * */ - chunk = TFW_STR_CHUNK(hdr, 1); + c = TFW_STR_CHUNK(hdr, 1); + if (WARN_ON_ONCE(!c)) + return -EINVAL; + + if (c->len == SLEN(S_DLM) && *(short *)c->data == *(short *)S_DLM) { + c = TFW_STR_CHUNK(hdr, 2); + if (WARN_ON_ONCE(!c)) + return -EINVAL; + } + end = hdr->chunks + hdr->nchunks; - tfw_str_collect_cmp(chunk, end, &s_val, NULL); + tfw_str_collect_cmp(c, end, &s_val, NULL); write_int(s_val.len, 0x7F, 0, &vlen); s_vlen.data = vlen.buf; s_vlen.len = vlen.sz; - if ((ret = tfw_strcat(resp->pool, &res_hdr, &s_vlen))) - return ret; - - if ((ret = tfw_strcat(resp->pool, &res_hdr, &s_val))) - return ret; - - ret = ss_skb_get_room(skb_head, first->skb, first->data, - res_hdr.len, &it); - if (ret) - return ret; - - if ((ret = tfw_strcpy(&it, &res_hdr))) - return ret; - - mit->acc_len += res_hdr.len; + r = tfw_h2_msg_rewrite_data(mit, &s_vlen, mit->bnd); + if (unlikely(r)) + return r; - T_DBG3("%s: idx, acc_len=%lu, it.len=%lu, it.data='%.*s'\n", __func__, - mit->acc_len, it.len, (int)it.len, it.data); + r = tfw_h2_msg_rewrite_data(mit, &s_val, mit->bnd); + if (unlikely(r)) + return r; return 0; } @@ -3847,8 +3748,8 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, { int ret; TfwHPackInt nlen, vlen; - TfwStr *chunk, *end, *s_name; - TfwMsgTransIter *mit = &resp->mit; + TfwStr *c, *end, *s_name; + TfwHttpTransIter *mit = &resp->mit; TfwMsgIter *iter = &mit->iter; struct sk_buff **skb_head = &resp->msg.skb_head; TfwStr s_val, s_nlen = {}, s_vlen = {}; @@ -3857,10 +3758,8 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, .len = idx->sz, }; - if (WARN_ON_ONCE(!hdr || TFW_STR_EMPTY(hdr))) - return -EINVAL; - - if ((ret = tfw_http_msg_expand_data(iter, skb_head, &idx_str))) + ret = tfw_http_msg_expand_data(iter, skb_head, &idx_str); + if (unlikely(ret)) return ret; mit->acc_len += idx_str.len; @@ -3869,13 +3768,20 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, __func__, mit->acc_len, idx_str.len, (int)idx_str.len, idx_str.data); - if (!name_indexed) { + if (!hdr) + return 0; + + if (WARN_ON_ONCE(TFW_STR_PLAIN(hdr))) + return -EINVAL; + + if (unlikely(!name_indexed)) { s_name = TFW_STR_CHUNK(hdr, 0); write_int(s_name->len, 0x7F, 0, &nlen); s_nlen.data = nlen.buf; s_nlen.len = nlen.sz; - if ((ret = tfw_http_msg_expand_data(iter, skb_head, &s_nlen))) + ret = tfw_http_msg_expand_data(iter, skb_head, &s_nlen); + if (unlikely(ret)) return ret; mit->acc_len += s_nlen.len; @@ -3884,7 +3790,8 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, "='%.*s'\n", __func__, mit->acc_len, s_nlen.len, (int)s_nlen.len, s_nlen.data); - if ((ret = tfw_http_msg_expand_data(iter, skb_head, s_name))) + ret = tfw_http_msg_expand_data(iter, skb_head, s_name); + if (unlikely(ret)) return ret; mit->acc_len += s_name->len; @@ -3893,19 +3800,33 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, "='%.*s'\n", __func__, mit->acc_len, s_name->len, (int)s_name->len, s_name->data); } + /* - * The @hdr must have the HTTP/2-format chunk structure (without the - * colon and OWS), i.e.: { name, value1, value2, ... }. + * During expanding the message the source @hdr must have the following + * chunk structure (without the OWS): + * + * { name [S_DLM] value1 [value2 [value3 ...]] }. + * */ - chunk = TFW_STR_CHUNK(hdr, 1); + c = TFW_STR_CHUNK(hdr, 1); + if (WARN_ON_ONCE(!c)) + return -EINVAL; + + if (c->len == SLEN(S_DLM) && *(short *)c->data == *(short *)S_DLM) { + c = TFW_STR_CHUNK(hdr, 2); + if (WARN_ON_ONCE(!c)) + return -EINVAL; + } + end = hdr->chunks + hdr->nchunks; - tfw_str_collect_cmp(chunk, end, &s_val, NULL); + tfw_str_collect_cmp(c, end, &s_val, NULL); write_int(s_val.len, 0x7F, 0, &vlen); s_vlen.data = vlen.buf; s_vlen.len = vlen.sz; - if ((ret = tfw_http_msg_expand_data(iter, skb_head, &s_vlen))) + ret = tfw_http_msg_expand_data(iter, skb_head, &s_vlen); + if (unlikely(ret)) return ret; mit->acc_len += s_vlen.len; @@ -3913,7 +3834,8 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, T_DBG3("%s: val len, acc_len=%lu, s_vlen.len=%lu, s_vlen.data='%.*s'\n", __func__, mit->acc_len, s_vlen.len, (int)s_vlen.len, s_vlen.data); - if ((ret = tfw_http_msg_expand_data(iter, skb_head, &s_val))) + ret = tfw_http_msg_expand_data(iter, skb_head, &s_val); + if (unlikely(ret)) return ret; mit->acc_len += s_val.len; @@ -3924,188 +3846,113 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return 0; } -/* - * Substitute the header @tgt with the new header @hdr in HTTP/2 HPACK format - * in the response @resp. - */ -static inline int -tfw_hpack_hdr_sub(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, - TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, - bool name_indexed) -{ - int ret; - TfwStr *fc; - struct sk_buff *skb_head = resp->msg.skb_head; - - if (WARN_ON_ONCE(!tgt || !hdr)) - return -EINVAL; - - fc = TFW_STR_CHUNK(tgt, 0); - - if ((ret = tfw_hpack_hdr_add(resp, fc, hdr, idx, name_indexed))) - return ret; - - if ((ret = ss_skb_cutoff_data(skb_head, tgt, 0, tfw_str_eolen(tgt)))) - return ret; - - return 0; -} - /* * Transform the HTTP/1.1 header @hdr in-place into HTTP/2 HPACK format in the * response @resp. */ static int tfw_hpack_hdr_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, - TfwHPackInt *__restrict idx, bool name_indexed) + TfwHPackInt *__restrict idx, bool name_indexed, + bool indexed) { int r; - TfwHPackInt nlen, vlen; - const TfwStr *c, *end, *h; - TfwStr it = {}; - bool val_found = false; - TfwMsgTransIter *mit = &resp->mit; - unsigned long hdr_len, nm_len, val_off, val_len; - struct sk_buff *skb_head = resp->msg.skb_head; - TfwStr s_nlen = {}, s_vlen = {}; - TfwStr s_idx = { + const char *bnd; + TfwHPackInt vlen; + TfwHttpTransIter *mit = &resp->mit; + TfwStr s_name = {}, s_val = {}, s_vlen = {}; + const TfwStr s_idx = { .data = idx->buf, .len = idx->sz, }; - if (WARN_ON_ONCE(!hdr)) - return -EINVAL; - - if ((r = tfw_http_msg_del_eol(skb_head, hdr))) - return r; + T_DBG3("%s: s_idx->len=%lu, s_idx->data='%.*s'\n", + __func__, s_idx.len, (int)s_idx.len, s_idx.data); - h = TFW_STR_CHUNK(hdr, 0); - if ((r = ss_skb_get_room(skb_head, h->skb, h->data, s_idx.len, &it))) - return r; + if (!hdr || WARN_ON_ONCE(TFW_STR_PLAIN(hdr) || TFW_STR_DUP(hdr))) + return -EINVAL; - if ((r = tfw_strcpy(&it, &s_idx))) + r = tfw_http_hdr_split(hdr, &s_name, &s_val); + if (unlikely(r)) return r; - mit->acc_len += s_idx.len; + if (unlikely(!name_indexed)) { + TfwHPackInt nlen; + TfwStr s_nlen = {}; - T_DBG3("%s: idx, acc_len=%lu, it.len=%lu, it.data='%.*s'\n", __func__, - mit->acc_len, it.len, (int)it.len, it.data); + bnd = __TFW_STR_CH(&s_name, 0)->data; - hdr_len = tfw_h2_msg_hdr_length(hdr, &nm_len, &val_off, &val_len); + r = tfw_h2_msg_rewrite_data(mit, &s_idx, bnd); + if (unlikely(r)) + return r; - if (!name_indexed) { - write_int(nm_len, 0x7F, 0, &nlen); + write_int(s_name.len, 0x7F, 0, &nlen); s_nlen.data = nlen.buf; s_nlen.len = nlen.sz; - r = ss_skb_get_room(skb_head, h->skb, h->data, s_nlen.len, &it); - if (r) + + r = tfw_h2_msg_rewrite_data(mit, &s_nlen, bnd); + if (unlikely(r)) return r; - if ((r = tfw_strcpy(&it, &s_nlen))) + + bnd = __TFW_STR_CH(&s_val, 0)->data; + + r = tfw_h2_msg_rewrite_data(mit, &s_name, bnd); + if (unlikely(r)) return r; + } else { + bnd = indexed + ? mit->bnd + : __TFW_STR_CH(&s_val, 0)->data; - mit->acc_len += s_nlen.len + nm_len; + r = tfw_h2_msg_rewrite_data(mit, &s_idx, bnd); + if (unlikely(r)) + return r; - T_DBG3("%s: name len, acc_len=%lu, it.len=%lu, it.data='%.*s'\n", - __func__, mit->acc_len, it.len, (int)it.len, it.data); + if (indexed) + return 0; } - write_int(val_len, 0x7F, 0, &vlen); + write_int(s_val.len, 0x7F, 0, &vlen); s_vlen.data = vlen.buf; s_vlen.len = vlen.sz; - mit->acc_len += s_vlen.len + val_len; - - TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { - unsigned long len; - - if (!c->len) - continue; - - len = 0; - if (nm_len) { - WARN_ON_ONCE(nm_len == c->len); - len = min(nm_len, c->len); - nm_len -= len; - if (name_indexed) { - r = skb_fragment(skb_head, c->skb, c->data, - -len, &it); - if (r < 0) - return r; - } - - if (len == c->len) - continue; - - WARN_ON_ONCE(!val_off); - } - - if (val_off) { - WARN_ON_ONCE(val_off < c->len - len); - val_off -= c->len - len; - r = skb_fragment(skb_head, c->skb, c->data + len, - -(c->len - len), &it); - if (r < 0) - return r; - - continue; - } - - if (val_len) { - len = min(val_len, c->len); - val_len -= len; - if (!val_found) { - val_found = true; - r = ss_skb_get_room(skb_head, c->skb, c->data, - s_vlen.len, &it); - if (r) - return r; - if ((r = tfw_strcpy(&it, &s_vlen))) - return r; - T_DBG3("%s: value len, acc_len=%lu, it.len=%lu," - " it.data='%.*s'\n", __func__, - mit->acc_len, it.len, - (int)it.len, it.data); - } - } + r = tfw_h2_msg_rewrite_data(mit, &s_vlen, bnd); + if (unlikely(r)) + return r; - if (len < c->len) { - r = skb_fragment(skb_head, c->skb, c->data + len, - -(c->len - len), &it); - if (r < 0) - return r; - } - } + r = tfw_h2_msg_rewrite_data(mit, &s_val, mit->bnd); + if (unlikely(r)) + return r; return 0; } /* - * Perform encoding of the header @hdr (or @tgt) into the HTTP/2 HPACK format. - * The four operation types can be executed here: addition, expansion, - * substitution and in-place transformation. In addition or expansion cases - * @tgt must be NULL, since the new header is inserted into the message and - * there is no target to replace. In case of substitution the header @tgt - * must be replaced by new header @hdr, so the both must be present. In-place - * transformation changes the header @tgt directly in the message, thus the - * @hdr is not needed and must be NULL. + * Perform encoding of the header @hdr into the HTTP/2 HPACK format. The four + * operation types can be executed here: addition, substitution, in-place + * transformation and expansion. In cases of addition, substitution and in-place + * operations the new headers overwrites the old data in the existing skb(s). + * In the expansion case new headers are added along with new skb(s) creation + * into the internally generated message. */ int -tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, - TfwStr *__restrict hdr, TfwH2TransOp op) +tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, + TfwH2TransOp op) { TfwHPackInt idx; + bool st_full_index; + unsigned short st_index, index = 0; TfwH2Ctx *ctx = tfw_h2_context(resp->req->conn); TfwHPackETbl *tbl = &ctx->hpack.enc_tbl; - TfwStr *h2i = op == TFW_H2_TRANS_INPLACE ? tgt : hdr; - TfwStr *fc = TFW_STR_CHUNK(&resp->crlf, 0); - unsigned short index = 0, st_index = TFW_STR_INDEX(h2i); - bool st_full_index = h2i->flags & TFW_STR_FULL_INDEX; int r = 0; - BUG_ON(tgt == hdr); + if (WARN_ON_ONCE(!hdr || TFW_STR_EMPTY(hdr))) + return -EINVAL; + + st_index = TFW_STR_INDEX(hdr); + st_full_index = hdr->flags & TFW_STR_FULL_INDEX; if (!st_full_index) { - r = tfw_hpack_encoder_index(tbl, h2i, &index, resp->flags); + r = tfw_hpack_encoder_index(tbl, hdr, &index, resp->flags); if (r < 0) return r; } @@ -4123,13 +3970,14 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, write_int(index, 0x7F, 0x80, &idx); switch (op) { + case TFW_H2_TRANS_SUB: case TFW_H2_TRANS_ADD: - return tfw_hpack_idx_add(resp, fc, &idx); + return tfw_hpack_hdr_add(resp, NULL, &idx, true); case TFW_H2_TRANS_EXPAND: - return tfw_hpack_idx_expand(resp, &idx); - case TFW_H2_TRANS_SUB: + return tfw_hpack_hdr_expand(resp, NULL, &idx, true); case TFW_H2_TRANS_INPLACE: - return tfw_hpack_idx_sub(resp, tgt, &idx); + return tfw_hpack_hdr_inplace(resp, hdr, &idx, true, + true); default: BUG(); } @@ -4152,14 +4000,14 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, write_int(index, 0xF, 0, &idx); switch (op) { + case TFW_H2_TRANS_SUB: case TFW_H2_TRANS_ADD: - return tfw_hpack_hdr_add(resp, fc, hdr, &idx, true); + return tfw_hpack_hdr_add(resp, hdr, &idx, true); case TFW_H2_TRANS_EXPAND: return tfw_hpack_hdr_expand(resp, hdr, &idx, true); - case TFW_H2_TRANS_SUB: - return tfw_hpack_hdr_sub(resp, tgt, hdr, &idx, true); case TFW_H2_TRANS_INPLACE: - return tfw_hpack_hdr_inplace(resp, tgt, &idx, true); + return tfw_hpack_hdr_inplace(resp, hdr, &idx, true, + false); default: BUG(); } @@ -4171,14 +4019,14 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, idx.buf[0] = (r & HPACK_IDX_FLAG_ADD) ? 0x40 : 0; switch (op) { + case TFW_H2_TRANS_SUB: case TFW_H2_TRANS_ADD: - return tfw_hpack_hdr_add(resp, fc, hdr, &idx, false); + return tfw_hpack_hdr_add(resp, hdr, &idx, false); case TFW_H2_TRANS_EXPAND: return tfw_hpack_hdr_expand(resp, hdr, &idx, false); - case TFW_H2_TRANS_SUB: - return tfw_hpack_hdr_sub(resp, tgt, hdr, &idx, false); case TFW_H2_TRANS_INPLACE: - return tfw_hpack_hdr_inplace(resp, tgt, &idx, false); + return tfw_hpack_hdr_inplace(resp, hdr, &idx, false, + false); default: BUG(); } diff --git a/tempesta_fw/hpack.h b/tempesta_fw/hpack.h index 436b103567..647dfb84f3 100644 --- a/tempesta_fw/hpack.h +++ b/tempesta_fw/hpack.h @@ -106,10 +106,10 @@ typedef struct { } TfwHPackETblIter; typedef enum { - TFW_H2_TRANS_ADD = 0, - TFW_H2_TRANS_EXPAND, + TFW_H2_TRANS_INPLACE = 0, TFW_H2_TRANS_SUB, - TFW_H2_TRANS_INPLACE + TFW_H2_TRANS_ADD, + TFW_H2_TRANS_EXPAND, } TfwH2TransOp; typedef enum { @@ -235,8 +235,8 @@ do { \ int tfw_hpack_init(TfwHPack *__restrict hp, unsigned int htbl_sz); void tfw_hpack_clean(TfwHPack *__restrict hp); -int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict tgt, - TfwStr *__restrict hdr, TfwH2TransOp op); +int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, + TfwH2TransOp op); void tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl, unsigned short new_size); int tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 4479a2fc61..265182dcc8 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -128,8 +128,8 @@ unsigned short tfw_blk_flags = TFW_CFG_BLK_DEF; /* Array of whitelist marks for request's skb. */ static struct { - unsigned int *mrks; - unsigned int sz; + unsigned int *mrks; + unsigned int sz; } tfw_wl_marks; #define S_CRLFCRLF "\r\n\r\n" @@ -719,7 +719,6 @@ tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) int ret; unsigned short index = tfw_h2_pseudo_index(resp->status); char buf[H2_STAT_VAL_LEN]; - TfwStr *s_line = NULL; TfwStr s_hdr = { .chunks = (TfwStr []){ { .data = S_H2_STAT, .len = SLEN(S_H2_STAT) }, @@ -736,14 +735,11 @@ tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) s_hdr.flags |= TFW_STR_FULL_INDEX; } - if (op == TFW_H2_TRANS_SUB) - s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; - if (!tfw_ultoa(resp->status, __TFW_STR_CH(&s_hdr, 1)->data, H2_STAT_VAL_LEN)) return -E2BIG; - if ((ret = tfw_hpack_encode(resp, s_line, &s_hdr, op))) + if ((ret = tfw_hpack_encode(resp, &s_hdr, op))) return ret; return 0; @@ -791,7 +787,7 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) }; TfwStr f_hdr = { .data = buf, - .len = FRAME_HEADER_SIZE + .len = sizeof(buf) }; if (!(stream_id = tfw_h2_stream_id_close(req))) @@ -854,7 +850,7 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) __TFW_STR_CH(&hdr, 1)->len = len_dt_val; hdr.len = len_dt_nm + len_dt_val; TFW_STR_INDEX_SET(&hdr, 33); - if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; __TFW_STR_CH(&hdr, 0)->data = pos_cl_nm; @@ -863,7 +859,7 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) __TFW_STR_CH(&hdr, 1)->len = len_cl_val; hdr.len = len_cl_nm + len_cl_val; TFW_STR_INDEX_SET(&hdr, 28); - if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; __TFW_STR_CH(&hdr, 0)->data = pos_srv_nm; @@ -872,7 +868,7 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) __TFW_STR_CH(&hdr, 1)->len = len_srv_val; hdr.len = len_srv_nm + len_srv_val; TFW_STR_INDEX_SET(&hdr, 54); - if (tfw_hpack_encode(resp, NULL, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; if (body->data) { @@ -2231,6 +2227,20 @@ tfw_http_conn_msg_alloc(TfwConn *conn, TfwStream *stream) if (unlikely(tfw_http_resp_pair(hm))) goto clean; + if (TFW_MSG_H2(hm->req)) { + size_t sz = TFW_HDR_MAP_SZ(TFW_HDR_MAP_INIT_CNT); + TfwHttpTransIter *mit = &((TfwHttpResp *)hm)->mit; + + mit->map = tfw_pool_alloc(hm->pool, sz); + if (unlikely(!mit->map)) { + T_WARN("HTTP/2: unable to allocate memory for" + " response header map\n"); + goto clean; + } + mit->map->size = TFW_HDR_MAP_INIT_CNT; + mit->map->count = 0; + } + TFW_INC_STAT_BH(serv.rx_messages); } @@ -2758,8 +2768,7 @@ __h2_hdrs_dup_decrease(TfwHttpReq *req, const TfwStr *hdr) } static int -__h2_set_req_loc_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, - bool append) +__h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) { TfwStr *orig_hdr; TfwHttpMsg *hm = (TfwHttpMsg *)req; @@ -2857,8 +2866,9 @@ __h2_set_req_loc_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, return 0; } + static int -tfw_h2_set_req_loc_hdrs(TfwHttpReq *req) +tfw_h2_req_set_loc_hdrs(TfwHttpReq *req) { int i; TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, @@ -2867,15 +2877,14 @@ tfw_h2_set_req_loc_hdrs(TfwHttpReq *req) return 0; for (i = 0; i < h_mods->sz; ++i) { + int r; TfwHdrModsDesc *d = &h_mods->hdrs[i]; - int r = __h2_set_req_loc_hdrs(req, d->hdr, d->hid, d->append); - if (r) { + if ((r = __h2_req_hdrs(req, d->hdr, d->hid, d->append))) { T_ERR("HTTP/2: can't update location-specific header in" - " request [%p]\n", req); + " the request [%p]\n", req); return r; } - T_DBG3("%s: hdr updated, req=[%p]\n", __func__, req); } return 0; @@ -2998,7 +3007,7 @@ do { \ * Conditional substitution/addition/deletion or appending of statically * configured headers. */ - if ((r = tfw_h2_set_req_loc_hdrs(req))) + if ((r = tfw_h2_req_set_loc_hdrs(req))) return r; /* @@ -3335,71 +3344,46 @@ tfw_http_resp_fwd(TfwHttpResp *resp) } } -static int -tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, - unsigned long h_len) +int +tfw_h2_hdr_map(TfwHttpResp *resp, const TfwStr *hdr, unsigned int id) { - int ret; - TfwFrameHdr frame_hdr; - unsigned char buf[FRAME_HEADER_SIZE]; - TfwStr *s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; - unsigned long b_len = resp->body.len; - TfwStr *first, it = {}; - TfwStr s_hdr = { - .data = buf, - .len = FRAME_HEADER_SIZE - }; + TfwHdrIndex *index; + TfwHttpHdrMap *map = resp->mit.map; - frame_hdr.stream_id = stream_id; + if (id >= (1 << TFW_IDX_BITS) || hdr->nchunks > (1 << TFW_D_IDX_BITS)) { + T_WARN("HTTP/2: too many headers (duplicates) in" + " HTTP/1.1-response (header id: %u, header dups: %u\n", + id, hdr->nchunks); + return -EINVAL; + } - if (b_len) { - if (b_len > FRAME_MAX_LENGTH) { - T_WARN("Unable to make HTTP/2 DATA frame: too big" - " message body (%lu)\n", b_len); - return -E2BIG; - } + T_DBG3("%s: id=%u, hdr->nchunks=%u, map->size=%u, map->count=%u\n", + __func__, id, hdr->nchunks, map->size, map->count); - /* - * Set frame header for DATA, if body part of HTTP/1.1 - * response exists. - */ - frame_hdr.length = b_len; - frame_hdr.type = HTTP2_DATA; - frame_hdr.flags = HTTP2_F_END_STREAM; - tfw_h2_pack_frame_header(buf, &frame_hdr); + if (unlikely(map->count == map->size)) { + unsigned int new_size = map->size << 1; - first = TFW_STR_CHUNK(&resp->crlf, 0); - ret = ss_skb_get_room(resp->msg.skb_head, first->skb, - first->data, s_hdr.len, &it); - if (ret) - return ret; + map = tfw_pool_realloc(resp->pool, map, + TFW_HDR_MAP_SZ(map->size), + TFW_HDR_MAP_SZ(new_size)); + if (!map) { + T_WARN("HTTP/2: unable to reallocate memory for" + " response header map\n"); + return -ENOMEM; + } - if ((ret = tfw_strcpy(&it, &s_hdr))) - return ret; - } + map->size = new_size; + resp->mit.map = map; - if (h_len > FRAME_MAX_LENGTH) { - T_WARN("Unable to make HTTP/2 HEADERS frame: too big header block" - " fragment (%lu)\n", h_len); - return -E2BIG; + BUG_ON(map->count >= map->size); + T_DBG3("%s: expanded map, map->size=%u, map->count=%u\n", + __func__, map->size, map->count); } - /* Set frame header for HEADERS. */ - frame_hdr.length = h_len; - frame_hdr.type = HTTP2_HEADERS; - frame_hdr.flags = HTTP2_F_END_HEADERS; - if (!b_len) - frame_hdr.flags |= HTTP2_F_END_STREAM; - - tfw_h2_pack_frame_header(buf, &frame_hdr); - first = TFW_STR_CHUNK(s_line, 0); - ret = ss_skb_get_room(resp->msg.skb_head, first->skb, first->data, - s_hdr.len, &it); - if (ret) - return ret; - - if ((ret = tfw_strcpy(&it, &s_hdr))) - return ret; + index = &map->index[map->count]; + index->idx = id; + index->d_idx = TFW_STR_PLAIN(hdr) ? 0 : hdr->nchunks - 1; + ++map->count; return 0; } @@ -3430,6 +3414,7 @@ tfw_h2_add_hdr_via(TfwHttpResp *resp) g_vhost->hdr_via_len); TFW_STR_INDEX_SET(&via, 60); + r = __hdr_h2_add(resp, &via); if (unlikely(r)) T_ERR("HTTP/2: unable to add 'via' header (resp=[%p])\n", resp); @@ -3445,13 +3430,13 @@ tfw_h2_add_hdr_via(TfwHttpResp *resp) * transformation. */ static int -tfw_h2_set_hdr_date(TfwHttpResp *resp) +tfw_h2_add_hdr_date(TfwHttpResp *resp) { int r; char *s_date = *this_cpu_ptr(&g_buf); tfw_http_prep_date_from(s_date, resp->date); - r = tfw_h2_msg_hdr_sub((TfwHttpMsg *)resp, "date", SLEN("date"), s_date, + r = tfw_h2_msg_hdr_add(resp, "date", SLEN("date"), s_date, SLEN(S_V_DATE), TFW_HTTP_HDR_RAW, 33); if (unlikely(r)) T_ERR("HTTP/2: unable to add 'date' header to response" @@ -3485,20 +3470,348 @@ tfw_h2_set_stale_warn(TfwHttpResp *resp) #undef WARN_VAL } +/* + * Split header in two parts: name and value, evicting ':' and OWS. + * + * NOTE: use this function with caution since it changes the underlying + * @TfwStr descriptors of @hdr; thus, this procedure should be used + * only in cases when the headers descriptors will not be needed any more + * (e.g. during message final transformation/adjusting, just before + * forwarding), or when the descriptors has been copied previously (e.g. + * via @tfw_strcpy_desc() function). + */ +int +tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) +{ + bool name_found = false, val_found = false; + unsigned long tail, last_tail = 0, hdr_tail = 0; + TfwStr *chunk, *end, *last_chunk = NULL; + + BUG_ON(!TFW_STR_EMPTY(name_out) || !TFW_STR_EMPTY(val_out)); + + name_out->chunks = hdr->chunks; + + TFW_STR_FOR_EACH_CHUNK(chunk, hdr, end) { + unsigned long idx; + + if (!chunk->len) + continue; + + if (!name_found) { + ++name_out->nchunks; + name_out->len += chunk->len; + if (chunk->data[chunk->len - 1] == ':') { + --name_out->len; + TFW_STR_LAST(name_out)->len -= 1; + name_found = true; + } + continue; + } + + /* + * LWS is always in the separate chunks between the name and + * value; thus, we can skip length of the entire (LWS) chunks. + */ + if (!val_found) { + if (unlikely(chunk->data[0] == ' ' + || chunk->data[0] == '\t')) + continue; + + val_out->chunks = chunk; + val_found = true; + } + + val_out->len += chunk->len; + + /* + * Skip OWS after the header value (RWS); accumulate the length + * in @tail for RWS cutting off (if this is not the end chunk, + * @tail will be reset). + */ + tail = 0; + idx = chunk->len - 1; + while (chunk->data[idx] == ' ' + || chunk->data[idx] == '\t') + { + ++tail; + if (unlikely(!idx)) + break; + --idx; + } + + if (unlikely(tail == chunk->len)) { + hdr_tail += tail; + } else { + last_chunk = chunk; + last_tail = hdr_tail = tail; + } + } + + T_DBG3("%s: hdr_tail=%lu, val_out->len=%lu, last_tail=%lu," + " last_chunk->len=%lu, last_chunk->data='%.*s'\n", __func__, + hdr_tail, val_out->len, last_tail, last_chunk->len, + (int)last_chunk->len, last_chunk->data); + + if (WARN_ON_ONCE(!last_chunk)) + return -EINVAL; + + val_out->nchunks = chunk - val_out->chunks + 1; + val_out->len -= hdr_tail; + last_chunk->len -= last_tail; + + return 0; +} + +static int +tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods) +{ + int r; + unsigned int i; + TfwHttpTransIter *mit = &resp->mit; + + for (i = 0; i < h_mods->sz; ++i) { + const TfwHdrModsDesc *desc = &h_mods->hdrs[i]; + + if (test_bit(i, mit->found) || !TFW_STR_CHUNK(desc->hdr, 2)) + continue; + + r = __hdr_h2_add(resp, desc->hdr); + if (unlikely(r)) + return r; + } + + return 0; +} + +/* + * Get next header from the @mit->map. Procedure designed to be called from the + * outer cycle with changing of @mit iterator (including @mit->curr index of + * current header in the indirection map). Note, for optimization purposes, on + * each iteration function produces the boundary pointer @mit->bnd for current + * iteration and the operation instance @mit->next - for the next iteration + * (including source header @mit->next.s_hdr). + */ +static int +tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) +{ + int r; + unsigned int i; + TfwHttpTransIter *mit = &resp->mit; + TfwHttpHdrMap *map = mit->map; + TfwNextHdrOp *next = &mit->next; + TfwHttpHdrTbl *ht = resp->h_tbl; + + mit->bnd = NULL; + + for (i = mit->curr; i < map->count; ++i) { + int k; + unsigned short hid = map->index[i].idx; + unsigned short d_num = map->index[i].d_idx; + TfwStr *tgt = &ht->tbl[hid]; + TfwStr *first = TFW_STR_CHUNK(tgt, 0); + TfwHdrModsDesc *f_desc = NULL; + + tgt = TFW_STR_CHUNK(tgt, d_num); + if (WARN_ON_ONCE(!tgt + || TFW_STR_EMPTY(tgt) + || TFW_STR_DUP(tgt))) + return -EINVAL; + + if (!h_mods) + goto def; + + for (k = 0; k < h_mods->sz; ++k) { + TfwHdrModsDesc *desc = &h_mods->hdrs[k]; + + if ((hid < TFW_HTTP_HDR_RAW && hid == desc->hid) + || (hid >= TFW_HTTP_HDR_RAW + && !__hdr_name_cmp(tgt, + TFW_STR_CHUNK(desc->hdr, 0)))) + { + f_desc = desc; + break; + } + } + + if (f_desc) { + const TfwStr *val = TFW_STR_CHUNK(f_desc->hdr, 2); + /* + * If this is a duplicate of already processed header, + * leave this duplicate as is (for transformation + * in-place) in case of appending operation, and remove + * it (by skipping) in case of substitution or deletion + * operations. + */ + if (test_bit(k, mit->found)) { + if (!val || !f_desc->append) + continue; + + mit->bnd = first->data; + next->s_hdr = *tgt; + next->op = TFW_H2_TRANS_INPLACE; + + break; + } + + __set_bit(k, mit->found); + + /* + * If header configured with empty value, it should be + * removed from the response; so, just skip such header. + */ + if (!val) + continue; + + mit->bnd = first->data; + + /* + * If the header configured for value appending, + * concatenate it with the target header in skb for + * subsequent in-place rewriting. + */ + if (f_desc->append) { + TfwStr h_app = { + .chunks = (TfwStr []){ + { .data = ", ", .len = 2 }, + { .data = val->data, + .len = val->len } + }, + .len = val->len + 2, + .nchunks = 2 + }; + + r = tfw_strcat(resp->pool, tgt, &h_app); + if (unlikely(r)) + return r; + + next->s_hdr = *tgt; + next->op = TFW_H2_TRANS_INPLACE; + + break; + } + + next->s_hdr = *f_desc->hdr; + next->op = TFW_H2_TRANS_SUB; + + break; + } +def: + /* + * Remove 'Connection', 'Keep-Alive' headers and all hop-by-hop + * headers from the HTTP/2 response. + */ + if (hid == TFW_HTTP_HDR_KEEP_ALIVE + || hid == TFW_HTTP_HDR_CONNECTION + || tgt->flags & TFW_STR_HBH_HDR) + continue; + + /* + * 'Server' and 'Date' headers must be replaced; thus, remove + * the original headers (and all their duplicates) skipping them + * here; the new headers will be written later, during new + * headers' addition stage. + */ + if (hid == TFW_HTTP_HDR_SERVER) + continue; + + if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { + DEFINE_TFW_STR(n_date, "date"); + if (__hdr_name_cmp(tgt, &n_date)) + continue; + } + + /* + * In general case the header should be transformed in-place + * from its original HTTP/1.1-representation in skb. + */ + mit->bnd = first->data; + next->s_hdr = *tgt; + next->op = TFW_H2_TRANS_INPLACE; + + break; + } + + mit->curr = i + 1; + + return 0; + +} + +static int +tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, + unsigned long h_len) +{ + int r; + char *head_ptr; + TfwFrameHdr frame_hdr; + unsigned char buf[FRAME_HEADER_SIZE]; + TfwHttpTransIter *mit = &resp->mit; + unsigned long b_len = resp->body.len; + + frame_hdr.stream_id = stream_id; + + if (b_len) { + TfwStr s_hdr = {}; + + if (b_len > FRAME_MAX_LENGTH) { + T_WARN("Unable to make HTTP/2 DATA frame: too big" + " message body (%lu)\n", b_len); + return -E2BIG; + } + + /* + * Set frame header for DATA, if body part of HTTP/1.1 + * response exists. + */ + frame_hdr.length = b_len; + frame_hdr.type = HTTP2_DATA; + frame_hdr.flags = HTTP2_F_END_STREAM; + + tfw_h2_pack_frame_header(buf, &frame_hdr); + + s_hdr.data = buf; + s_hdr.len = sizeof(buf); + + r = tfw_h2_msg_rewrite_data(mit, &s_hdr, mit->bnd); + if (unlikely(r)) + return r; + } + + if (h_len > FRAME_MAX_LENGTH) { + T_WARN("Unable to make HTTP/2 HEADERS frame: too big header" + " block fragment (%lu)\n", h_len); + return -E2BIG; + } + + /* Set frame header for HEADERS. */ + frame_hdr.length = h_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + if (!b_len) + frame_hdr.flags |= HTTP2_F_END_STREAM; + + tfw_h2_pack_frame_header(buf, &frame_hdr); + + head_ptr = ss_skb_data(resp->msg.skb_head); + memcpy_fast(head_ptr, buf, sizeof(buf)); + + return 0; +} + static void tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) { int r; unsigned int stream_id; - TfwStr *field, *hdrs_end, *hdr, *dup_end; - TfwHttpMsg *hmresp = (TfwHttpMsg *)resp; + bool hdrs_end = false; TfwHttpReq *req = resp->req; TfwH2Ctx *ctx = tfw_h2_context(req->conn); - TfwStr *s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; - TfwMsgTransIter *mit = &resp->mit; - - WARN_ON_ONCE(mit->acc_len); - + TfwHttpTransIter *mit = &resp->mit; + TfwNextHdrOp *next = &mit->next; + TfwMsgIter *iter = &mit->iter; + const TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, + req->vhost, + TFW_VHOST_HDRMOD_RESP); /* * Get ID of corresponding stream and unlink request from the * stream. @@ -3506,20 +3819,53 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) if (!(stream_id = tfw_h2_stream_id_close(req))) goto out; - /* Adjust HTTP/1.1 headers with transformation to HTTP/2 form. */ - r = tfw_http_sess_resp_process(resp); + /* + * Transform HTTP/1.1 headers into HTTP/2 form, in parallel with + * adjusting of particular headers. + */ + WARN_ON_ONCE(mit->acc_len || mit->curr); + + tfw_h2_msg_transform_setup(mit, resp->msg.skb_head, true); + + r = tfw_h2_resp_next_hdr(resp, h_mods); if (unlikely(r)) goto clean; - r = tfw_http_msg_del_hbh_hdrs(hmresp); + r = tfw_h2_pseudo_write(resp, TFW_H2_TRANS_SUB); if (unlikely(r)) goto clean; - r = TFW_H2_MSG_HDR_DEL(hmresp, "keep-alive", TFW_HTTP_HDR_KEEP_ALIVE); - if (unlikely(r)) + if (WARN_ON_ONCE(!mit->bnd)) goto clean; - r = TFW_H2_MSG_HDR_DEL(hmresp, "connection", TFW_HTTP_HDR_CONNECTION); + do { + TfwStr *last; + TfwStr hdr = next->s_hdr; + TfwH2TransOp op = next->op; + + r = tfw_h2_resp_next_hdr(resp, h_mods); + if (unlikely(r)) + goto clean; + + if (!mit->bnd) { + last = TFW_STR_LAST(&resp->crlf); + mit->bnd = last->data + last->len; + hdrs_end = true; + } + + r = tfw_hpack_encode(resp, &hdr, op); + if (unlikely(r)) + goto clean; + + } while (!hdrs_end); + + /* + * Write additional headers in HTTP/2 format in the end of the + * headers block, including configured headers which haven't been + * processed above and which have non-empty value (i.e. configured + * not for deletion). + */ + r = tfw_http_sess_resp_process(resp); if (unlikely(r)) goto clean; @@ -3532,35 +3878,17 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) goto clean; if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { - r = tfw_h2_set_hdr_date(resp); + r = tfw_h2_add_hdr_date(resp); if (unlikely(r)) goto clean; } - r = TFW_H2_MSG_HDR_SUB(hmresp, "server", TFW_NAME "/" TFW_VERSION, + r = TFW_H2_MSG_HDR_ADD(resp, "server", TFW_NAME "/" TFW_VERSION, TFW_HTTP_HDR_SERVER, 54); if (unlikely(r)) goto clean; - /* Transform HTTP/1.1 headers to HTTP/2 form. */ - FOR_EACH_HDR_FIELD_FROM(field, hdrs_end, resp, TFW_HTTP_HDR_REGULAR) { - if (TFW_STR_EMPTY(field)) - continue; - TFW_STR_FOR_EACH_DUP(hdr, field, dup_end) { - r = tfw_hpack_encode(resp, hdr, NULL, - TFW_H2_TRANS_INPLACE); - if (unlikely(r)) - goto clean; - } - TFW_STR_INIT(field); - } - resp->h_tbl->off = TFW_HTTP_HDR_RAW; - - /* - * Transform the whole response from HTTP/1.1 to HTTP/2 form and - * forward it to the client. - */ - r = tfw_h2_pseudo_write(resp, TFW_H2_TRANS_SUB); + r = tfw_h2_resp_add_loc_hdrs(resp, h_mods); if (unlikely(r)) goto clean; @@ -3568,18 +3896,16 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) if (unlikely(r)) goto clean; - r = tfw_http_msg_del_str(hmresp, &resp->crlf); + r = ss_skb_cut_extra_data(iter->skb_head, iter->skb, iter->frag, + mit->curr_ptr, mit->bnd); if (unlikely(r)) goto clean; - TFW_STR_INIT(s_line); - TFW_STR_INIT(&resp->body); - tfw_h2_resp_fwd(resp); return; clean: - tfw_http_conn_msg_free(hmresp); + tfw_http_conn_msg_free((TfwHttpMsg *)resp); tfw_http_send_resp(req, 500, "response dropped: processing error"); tfw_hpack_enc_release(&ctx->hpack, resp->flags); diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index 60a195a68f..a257fa488c 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -453,6 +453,77 @@ struct tfw_http_req_t { #define TFW_HTTP_REQ_STR_START(r) __MSG_STR_START(r) #define TFW_HTTP_REQ_STR_END(r) ((&(r)->uri_path) + 1) +#define TFW_IDX_BITS 12 +#define TFW_D_IDX_BITS 4 + +/** + * Representation of operation with the next header (in order of headers in the + * message) during HTTP/1.1=>HTTP/2 transformation process. + * + * @s_hdr - source header for transformation; + * @off - offset of not copied data from last processed @chunk; + * @chunk - last chunk to be processed from @s_hdr; + * @op - transformation operation which should be executed. + */ +typedef struct { + TfwStr s_hdr; + unsigned long off; + unsigned int chunk; + TfwH2TransOp op; +} TfwNextHdrOp; + +/** + * The indirection map entry. + * + * @idx - header index in @h_tbl; + * @d_idx - header's order in the array of duplicates of particular + * @h_tbl record. + */ +typedef struct { + unsigned short idx : TFW_IDX_BITS; + unsigned short d_idx : TFW_D_IDX_BITS; +} TfwHdrIndex; + +/** + * Indirection map which links the header's order with its index in @h_tbl. + * + * @count - the actual count of headers in the map (equal to the amount + * of all headers in the message); + * @size - the size of the map (in entries); + * @index - array of the indexes (which are located in the order of + * corresponding headers' appearance in the message). + */ +typedef struct { + unsigned int size; + unsigned int count; + TfwHdrIndex index[0]; +} TfwHttpHdrMap; + +/** + * Iterator for message HTTP/2 transformation process. + * + * @map - indirection map for tracking headers order in skb; + * @curr - current header index in the @map; + * @next - operation (with necessary attributes) which should be executed + * with next header; + * @found - bit mask of configured headers found in the message. + * @curr_ptr - pointer in the skb to write the current header; + * @bnd - pointer to the boundary data (which should not be + * overwritten); + * @iter - skb creation/writing iterator; + * @acc_len - accumulated length of transformed message. + */ +typedef struct { + TfwHttpHdrMap *map; + unsigned int curr; + TfwNextHdrOp next; + DECLARE_BITMAP (found, TFW_USRHDRS_ARRAY_SZ); + char *curr_ptr; + char *bnd; + TfwMsgIter iter; + unsigned long acc_len; +} TfwHttpTransIter; + /** * HTTP Response. * TfwStr members must be the first for efficient scanning. @@ -467,9 +538,13 @@ struct tfw_http_resp_t { time_t date; time_t last_modified; unsigned long jrxtstamp; - TfwMsgTransIter mit; + TfwHttpTransIter mit; }; +#define TFW_HDR_MAP_INIT_CNT 32 +#define TFW_HDR_MAP_SZ(cnt) (sizeof(TfwHttpHdrMap) \ + + sizeof(TfwHdrIndex) * (cnt)) + #define TFW_HTTP_RESP_STR_START(r) __MSG_STR_START(r) #define TFW_HTTP_RESP_STR_END(r) ((&(r)->body) + 1) @@ -546,6 +621,7 @@ void tfw_http_resp_fwd(TfwHttpResp *resp); void tfw_http_resp_build_error(TfwHttpReq *req); int tfw_cfgop_parse_http_status(const char *status, int *out); void tfw_http_hm_srv_send(TfwServer *srv, char *data, unsigned long len); +int tfw_h2_hdr_map(TfwHttpResp *resp, const TfwStr *hdr, unsigned int id); /* * Functions to send an HTTP error response to a client. @@ -559,5 +635,6 @@ void tfw_http_send_resp(TfwHttpReq *req, int status, const char *reason); /* Helper functions */ char *tfw_http_msg_body_dup(const char *filename, size_t *len); +int tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out); #endif /* __TFW_HTTP_H__ */ diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index ff314c6ba0..65345052e2 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -376,7 +376,7 @@ __hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) * Special procedure comparing specified name against the header in HTTP/2 * or HTTP/1.1 format. */ -static int +int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name) { long n; @@ -496,7 +496,7 @@ tfw_http_msg_hdr_open(TfwHttpMsg *hm, unsigned char *hdr_start) int tfw_http_msg_hdr_close(TfwHttpMsg *hm) { - TfwStr *h; + TfwStr *hdr, *h; TfwHttpHdrTbl *ht = hm->h_tbl; TfwHttpParser *parser = &hm->stream->parser; unsigned int id = parser->_hdr_tag; @@ -509,8 +509,8 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) /* Quick path for special headers. */ if (likely(id < TFW_HTTP_HDR_RAW)) { - h = &ht->tbl[id]; - if (TFW_STR_EMPTY(h)) + hdr = h = &ht->tbl[id]; + if (TFW_STR_EMPTY(hdr)) /* Just store the special header in empty slot. */ goto done; /* @@ -553,20 +553,23 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) ht = hm->h_tbl; } - h = &ht->tbl[id]; + hdr = h = &ht->tbl[id]; - if (TFW_STR_EMPTY(h)) + if (TFW_STR_EMPTY(hdr)) /* Add the new header. */ goto done; duplicate: - h = tfw_str_add_duplicate(hm->pool, h); + h = tfw_str_add_duplicate(hm->pool, hdr); if (unlikely(!h)) { T_WARN("Cannot close header %p id=%d\n", &parser->hdr, id); return TFW_BLOCK; } done: + if (TFW_RESP_TO_H2(hm) && tfw_h2_hdr_map((TfwHttpResp *)hm, hdr, id)) + return TFW_BLOCK; + *h = parser->hdr; TFW_STR_INIT(&parser->hdr); @@ -676,7 +679,7 @@ __hdr_add(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid) int __hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr) { - return tfw_hpack_encode(resp, NULL, hdr, TFW_H2_TRANS_ADD); + return tfw_hpack_encode(resp, hdr, TFW_H2_TRANS_ADD); } /** @@ -787,35 +790,6 @@ __hdr_sub(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid) return TFW_PASS; } -static int -__hdr_h2_sub(TfwHttpResp *resp, TfwStr *hdr, unsigned int hid) -{ - int ret; - TfwHttpHdrTbl *ht = resp->h_tbl; - TfwStr *tgt, *dup, *end, *orig_hdr = &ht->tbl[hid]; - - tgt = TFW_STR_DUP(orig_hdr) ? __TFW_STR_CH(orig_hdr, 0) : orig_hdr; - - TFW_STR_FOR_EACH_DUP(dup, orig_hdr, end) { - if (dup == tgt) - continue; - if ((ret = ss_skb_cutoff_data(resp->msg.skb_head, dup, 0, - tfw_str_eolen(dup)))) - return ret; - } - - ret = tfw_hpack_encode(resp, tgt, hdr, TFW_H2_TRANS_SUB); - if (ret) - return ret; - /* - * Exclude header from the subsequent transformation - * processing. - */ - TFW_STR_INIT(orig_hdr); - - return 0; -} - /** * Transform HTTP message @hm header with identifier @hid. * @hdr must be compound string and contain two or three parts: @@ -1502,57 +1476,99 @@ tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, } int -tfw_h2_msg_hdr_sub(TfwHttpMsg *hm, char *name, size_t nlen, char *val, - size_t vlen, unsigned int hid, unsigned short idx) +tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, + const char *stop) { - TfwHttpHdrTbl *ht = hm->h_tbl; - TfwStr *orig_hdr = NULL; - TfwStr hdr = { - .chunks = (TfwStr []){ - { .data = name, .len = nlen }, - { .data = val, .len = vlen }, - }, - .len = nlen + vlen, - .nchunks = 2 - }; + const TfwStr *c, *end; + TfwMsgIter *it = &mit->iter; + TfwNextHdrOp *next = &mit->next; - WARN_ON_ONCE(!ht); + BUG_ON(!stop); + BUG_ON(TFW_STR_DUP(str)); - if (hid < TFW_HTTP_HDR_RAW) { - orig_hdr = &ht->tbl[hid]; - } - else { - hid = __h2_hdr_lookup(hm, TFW_STR_CHUNK(&hdr, 0)); - if (hid < ht->off) - orig_hdr = &ht->tbl[hid]; - } + TFW_STR_FOR_EACH_CHUNK(c, str, end) { + const char *addr, *next_ptr; + long offset, stop_offset; + unsigned int c_off = 0, f_size, c_size, f_room, n_copy; +this_chunk: + c_size = c->len - c_off; + if (it->frag >= 0) { + skb_frag_t *frag = &skb_shinfo(it->skb)->frags[it->frag]; - TFW_STR_INDEX_SET(&hdr, idx); + f_size = skb_frag_size(frag); + addr = skb_frag_address(frag); + } else { + f_size = skb_headlen(it->skb); + addr = ss_skb_data(it->skb); + } - if (!orig_hdr || TFW_STR_EMPTY(orig_hdr)) - return __hdr_h2_add((TfwHttpResp *)hm, &hdr); + offset = mit->curr_ptr - addr; + if (WARN_ON_ONCE(offset < 0 || offset >= f_size)) + return -EINVAL; - return __hdr_h2_sub((TfwHttpResp *)hm, &hdr, hid); -} + f_room = f_size - offset; + n_copy = min(c_size, f_room); + next_ptr = mit->curr_ptr + n_copy; -int -tfw_h2_msg_hdr_del(TfwHttpMsg *hm, char *name, size_t nlen, unsigned int hid) -{ - TfwHttpHdrTbl *ht = hm->h_tbl; - TfwStr h_name = { - .data = name, - .len = nlen - }; + /* + * If the stop mark is met, we should verify that all the data + * will be copied from @str; if not, save offsets for not copied + * data from current @str (for subsequent copying after more + * space allocation) and exit with corresponding error code. + */ + stop_offset = stop - addr; + if (stop_offset >= 0 && stop_offset < f_size + && (next_ptr > stop + || (next_ptr == stop + && (c_size > f_room || c + 1 < end)))) + { + WARN_ON_ONCE(next->chunk || next->off); + next->chunk = TFW_STR_PLAIN(str) ? 0 : c - str->chunks; + next->off = c_off; - WARN_ON_ONCE(!ht); + T_WARN("Unable to transform HTTP/1.1 data into HTTP/2" + " format: free space exhausted (accumulated" + " length: %lu\n", mit->acc_len); - if (hid < TFW_HTTP_HDR_RAW) { - if (TFW_STR_EMPTY(&ht->tbl[hid])) - return 0; - } else { - if ((hid = __h2_hdr_lookup(hm, &h_name)) == ht->off) - return 0; + return -E2BIG; + } + + memcpy_fast(mit->curr_ptr, c->data + c_off, n_copy); + + T_DBG3("%s: acc_len=%lu, n_copy=%u, mit->curr_ptr='%.*s'\n", + __func__, mit->acc_len, n_copy, n_copy, mit->curr_ptr); + + mit->curr_ptr += n_copy; + mit->acc_len += n_copy; + + /* + * The fragment is exhausted. Move to the next fragment (or skb) + * and reset the @mit->curr_ptr pointer. + */ + if (c_size >= f_room) { + if (skb_shinfo(it->skb)->nr_frags > it->frag + 1) { + skb_frag_t *frag; + + ++it->frag; + frag = &skb_shinfo(it->skb)->frags[it->frag]; + mit->curr_ptr = skb_frag_address(frag); + } + else { + if (WARN_ON_ONCE(it->skb_head == it->skb->next && + (c_size != f_room + || c + 1 < end))) + return -EINVAL; + + tfw_h2_msg_transform_setup(mit, it->skb->next, + false); + } + + if (c_size != f_room) { + c_off += n_copy; + goto this_chunk; + } + } } - return __hdr_del(hm, hid); + return 0; } diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index a839e0f258..7681d5d651 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -50,6 +50,7 @@ __tfw_http_msg_set_str_data(TfwStr *str, void *data, struct sk_buff *skb) __tfw_http_msg_set_str_data(str, data, \ ss_skb_peek_tail(&hm->msg.skb_head)) +int __hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr); void __h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name); void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val); void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client); @@ -101,6 +102,45 @@ tfw_http_msg_alloc_resp_light(TfwHttpReq *req) return __tfw_http_msg_alloc_resp(req, false); } +static inline void +tfw_h2_msg_transform_setup(TfwHttpTransIter *mit, struct sk_buff *skb, + bool init) +{ + TfwMsgIter *iter = &mit->iter; + + BUILD_BUG_ON(HTTP2_MAX_OFFSET <= FRAME_HEADER_SIZE); + BUG_ON(!skb); + + iter->frag = -1; + iter->skb = skb; + if (!iter->skb_head) + iter->skb_head = skb; + + skb_push(skb, HTTP2_MAX_OFFSET); + mit->curr_ptr = ss_skb_data(skb); + + if (init) + mit->curr_ptr += FRAME_HEADER_SIZE; +} + +static inline int +tfw_h2_msg_hdr_add(TfwHttpResp *resp, char *name, size_t nlen, char *val, + size_t vlen, unsigned int hid, unsigned short idx) +{ + TfwStr hdr = { + .chunks = (TfwStr []){ + { .data = name, .len = nlen }, + { .data = val, .len = vlen }, + }, + .len = nlen + vlen, + .nchunks = 2 + }; + + TFW_STR_INDEX_SET(&hdr, idx); + + return __hdr_h2_add(resp, &hdr); +} + int __tfw_http_msg_add_str_data(TfwHttpMsg *hm, TfwStr *str, void *data, size_t len, struct sk_buff *skb); #define tfw_http_msg_add_str_data(hm, str, data, len) \ @@ -135,6 +175,7 @@ int tfw_http_msg_grow_hdr_tbl(TfwHttpMsg *hm); void tfw_http_msg_free(TfwHttpMsg *m); int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, const TfwStr *src); +int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name); int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name); unsigned long tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, unsigned long *val_off, @@ -142,15 +183,11 @@ unsigned long tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, void tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, unsigned long val_off, unsigned long val_len, char *out_buf); -int __hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr); -int tfw_h2_msg_hdr_sub(TfwHttpMsg *hm, char *name, size_t nlen, char *val, - size_t vlen, unsigned int hid, unsigned short idx); -int tfw_h2_msg_hdr_del(TfwHttpMsg *hm, char *name, size_t nlen, unsigned int hid); +int tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, + const char *stop); -#define TFW_H2_MSG_HDR_SUB(hm, name, val, hid, idx) \ - tfw_h2_msg_hdr_sub(hm, name, sizeof(name) - 1, val, \ +#define TFW_H2_MSG_HDR_ADD(hm, name, val, hid, idx) \ + tfw_h2_msg_hdr_add(hm, name, sizeof(name) - 1, val, \ sizeof(val) - 1, hid, idx) -#define TFW_H2_MSG_HDR_DEL(hm, name, hid) \ - tfw_h2_msg_hdr_del(hm, name, sizeof(name) - 1, hid) #endif /* __TFW_HTTP_MSG_H__ */ diff --git a/tempesta_fw/msg.h b/tempesta_fw/msg.h index 7e9c8ce560..13df931e45 100644 --- a/tempesta_fw/msg.h +++ b/tempesta_fw/msg.h @@ -97,17 +97,6 @@ typedef struct { unsigned int tag; } TfwMsgParseIter; -/** - * Iterator for message HTTP/2 transformation process. - * - * @it - skb creation/writing iterator; - * @acc_len - accumulated length of transformed message; - */ -typedef struct { - TfwMsgIter iter; - unsigned long acc_len; -} TfwMsgTransIter; - int tfw_msg_write(TfwMsgIter *it, const TfwStr *data); int tfw_msg_iter_setup(TfwMsgIter *it, struct sk_buff **skb_head, size_t data_len, unsigned int tx_flags); diff --git a/tempesta_fw/ss_skb.c b/tempesta_fw/ss_skb.c index a0637bf9c2..6473ac1199 100644 --- a/tempesta_fw/ss_skb.c +++ b/tempesta_fw/ss_skb.c @@ -1627,3 +1627,97 @@ ss_skb_replace_page(struct sk_buff **skb_head, struct page *pg, return 0; } +/** + * Evict extra data between @curr and @stop pointers, beginning from the @skb + * and the @frag_num fragment. + */ +int +ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, + int frag_num, char *curr, const char *stop) +{ + TfwStr it; + long offset; + int size, ret; + const char *addr; + + if (frag_num >= 0) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[frag_num]; + + size = skb_frag_size(frag); + addr = skb_frag_address(frag); + } else { + size = skb_headlen(skb); + addr = ss_skb_data(skb); + } + + offset = curr - addr; + if (WARN_ON_ONCE(offset < 0 || offset >= size)) + return -EINVAL; + + for (;;) { + int len, tail; + long stop_offset = stop - addr; + + len = tail = size - offset; + + /* + * We found the stop pointer; evict the delta between @curr and + * @stop, and exit. + */ + if (stop_offset >= 0 && stop_offset <= size) + { + if (WARN_ON_ONCE(curr > stop)) + return -EINVAL; + + if (curr == stop) + return 0; + + len -= size - stop_offset; + } + + T_DBG3("%s: frag_num=%d, size=%d, offset=%ld, stop_offset=%ld," + " len=%d, tail=%d\n", __func__, frag_num, size, offset, + stop_offset, len, tail); + + if (frag_num >= 0) + ret = __split_pgfrag_del(skb_head, skb, frag_num, + offset, len, &it); + else + ret = __split_linear_data(skb_head, skb, curr, -len, + &it); + if (unlikely(ret)) + return ret; + + if (len != tail) + return 0; + + /* + * The extra space is evicted from current fragment (or from skb + * head space), but the stop pointer is not reached yet. Move to + * the next fragment (or skb). + */ + if (skb_shinfo(skb)->nr_frags > frag_num + 1) { + skb_frag_t *frag; + + ++frag_num; + frag = &skb_shinfo(skb)->frags[frag_num]; + size = skb_frag_size(frag); + addr = curr = skb_frag_address(frag); + } + else { + if (WARN_ON_ONCE(skb_head == skb->next)) + return -EINVAL; + + frag_num = -1; + skb = skb->next; + size = skb_headlen(skb); + addr = curr = ss_skb_data(skb); + } + + offset = 0; + } + + return 0; +} + + diff --git a/tempesta_fw/ss_skb.h b/tempesta_fw/ss_skb.h index e68f572602..413e255cfc 100644 --- a/tempesta_fw/ss_skb.h +++ b/tempesta_fw/ss_skb.h @@ -164,6 +164,11 @@ ss_skb_alloc(size_t n) return skb; } +static inline char * +ss_skb_data(struct sk_buff *skb) +{ + return skb->data; +} #define SS_SKB_MAX_DATA_LEN (SKB_MAX_HEADER + MAX_SKB_FRAGS * PAGE_SIZE) @@ -194,5 +199,7 @@ int ss_skb_to_sgvec_with_new_pages(struct sk_buff *skb, struct scatterlist *sgl, struct page ***old_pages); int ss_skb_replace_page(struct sk_buff **skb_head, struct page *pg, unsigned long pg_len, unsigned long offset); +int ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, + int frag_num, char *curr, const char *stop); #endif /* __TFW_SS_SKB_H__ */ diff --git a/tempesta_fw/vhost.c b/tempesta_fw/vhost.c index 59681b5e57..acb14ff235 100644 --- a/tempesta_fw/vhost.c +++ b/tempesta_fw/vhost.c @@ -92,9 +92,6 @@ static const TfwCfgEnum tfw_method_enum[] = { */ #define TFW_NIPDEF_ARRAY_SZ (64) -/* Max number of headers allowed for end user to modify. */ -#define TFW_USRHDRS_ARRAY_SZ (64) - /* * All 'location' directives are put into a fixed size array. * Duplicate directives are not allowed. diff --git a/tempesta_fw/vhost.h b/tempesta_fw/vhost.h index 8de0a8759b..18ab49f4c4 100644 --- a/tempesta_fw/vhost.h +++ b/tempesta_fw/vhost.h @@ -134,6 +134,9 @@ enum { TFW_VHOST_B_REMOVED = 0, }; +/* Max number of headers allowed for end user to modify. */ +#define TFW_USRHDRS_ARRAY_SZ 64 + /** * Virtual host defined by directives and policies. * From c9354841f5f700a95d5105181640cb9652c6aa27 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Fri, 22 Nov 2019 01:23:54 +0300 Subject: [PATCH 08/64] HTTP/2 Parser implementation: corrections according to debugging results (#309). --- tempesta_fw/hpack.c | 51 ++++-- tempesta_fw/hpack.h | 4 +- tempesta_fw/http.c | 315 +++++++++++++++++--------------- tempesta_fw/http_frame.c | 119 +++++++----- tempesta_fw/http_frame.h | 6 +- tempesta_fw/http_msg.c | 71 +++++-- tempesta_fw/http_msg.h | 5 +- tempesta_fw/http_parser.c | 1 + tempesta_fw/http_stream.c | 184 +++++++++++-------- tempesta_fw/http_stream.h | 18 +- tempesta_fw/ss_skb.c | 12 +- tempesta_fw/t/unit/test_hpack.c | 35 ++-- 12 files changed, 484 insertions(+), 337 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index 6d8902e6fb..bf27309119 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -2140,8 +2140,8 @@ tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, unsigned long n, TfwHttpReq *__restrict req, unsigned int *__restrict parsed) { + unsigned int state; int r = T_POSTPONE; - unsigned int state = hp->state; TfwMsgParseIter *it = &req->pit; const unsigned char *last = src + n; @@ -2150,8 +2150,11 @@ tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, WARN_ON_ONCE(!n); *parsed += n; do { + state = hp->state; + T_DBG3("%s: header processing, n=%lu, to_parse=%lu, state=%d\n", __func__, n, last - src, state); + switch (state & HPACK_STATE_MASK) { case HPACK_STATE_READY: { @@ -2388,6 +2391,7 @@ tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, get_all_indexed: T_DBG3("%s: get entire header by index: %lu\n", __func__, hp->index); + WARN_ON_ONCE(!(state & HPACK_FLAGS_NO_VALUE)); WARN_ON_ONCE(!hp->index); @@ -3414,7 +3418,7 @@ tfw_hpack_rbuf_commit(TfwHPackETbl *__restrict tbl, */ static int tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, - TfwHPackNodeIter *__restrict place) + TfwHPackNodeIter *__restrict place, TfwH2TransOp op) { unsigned long node_size, hdr_len; unsigned short new_size, node_len; @@ -3423,7 +3427,7 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, TfwHPackNode *del_list[HPACK_MAX_ENC_EVICTION] = {}; TfwHPackETblIter it = {}; - hdr_len = tfw_h2_msg_hdr_length(hdr, &nm_len, &val_off, &val_len); + hdr_len = tfw_h2_msg_hdr_length(hdr, &nm_len, &val_off, &val_len, op); WARN_ON_ONCE(cur_size > window || window > HPACK_ENC_TABLE_MAX_SIZE); if ((node_size = hdr_len + HPACK_ENTRY_OVERHEAD) > window) { @@ -3536,7 +3540,8 @@ static TfwHPackETblRes tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, unsigned short *__restrict out_index, - unsigned long *__restrict flags) + unsigned long *__restrict flags, + TfwH2TransOp op) { TfwHPackNodeIter place = {}; const TfwHPackNode *node = NULL; @@ -3571,7 +3576,7 @@ tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, if (!test_bit(TFW_HTTP_B_H2_TRANS_ENTERED, flags)) { if(res != HPACK_IDX_ST_FOUND && !atomic64_read(&tbl->guard) - && !tfw_hpack_add_node(tbl, hdr, &place)) + && !tfw_hpack_add_node(tbl, hdr, &place, op)) { res |= HPACK_IDX_FLAG_ADD; atomic64_set(&tbl->guard, -1); @@ -3596,7 +3601,7 @@ tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, WARN_ON_ONCE(!atomic64_read(&tbl->guard)); if (res != HPACK_IDX_ST_FOUND && atomic64_read(&tbl->guard) <= 1 - && !tfw_hpack_add_node(tbl, hdr, &place)) + && !tfw_hpack_add_node(tbl, hdr, &place, op)) { res |= HPACK_IDX_FLAG_ADD; atomic64_set(&tbl->guard, -1); @@ -3630,24 +3635,23 @@ static void write_int(unsigned long index, unsigned short max, unsigned short mask, TfwHPackInt *__restrict res_idx) { - unsigned int size = 0; + unsigned int size = 1; unsigned char *dst = res_idx->buf; if (likely(index < max)) { index |= mask; } else { - index -= max; + ++size; *dst++ = max | mask; + index -= max; while (index > 0x7F) { ++size; *dst++ = (index & 0x7F) | 0x80; index >>= 7; } } - ++size; *dst = index; - res_idx->sz = size; } @@ -3737,7 +3741,6 @@ tfw_hpack_hdr_add(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return 0; } - /* * Expand the response @resp with the new @hdr in HTTP/2 HPACK format, via * extending of skb/frags chain. @@ -3872,6 +3875,7 @@ tfw_hpack_hdr_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return -EINVAL; r = tfw_http_hdr_split(hdr, &s_name, &s_val); + if (unlikely(r)) return r; @@ -3951,8 +3955,11 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, st_index = TFW_STR_INDEX(hdr); st_full_index = hdr->flags & TFW_STR_FULL_INDEX; + T_DBG3("%s: op=%d, st_index=%hu, st_full_index=%d\n", __func__, op, + st_index, st_full_index); + if (!st_full_index) { - r = tfw_hpack_encoder_index(tbl, hdr, &index, resp->flags); + r = tfw_hpack_encoder_index(tbl, hdr, &index, resp->flags, op); if (r < 0) return r; } @@ -4035,19 +4042,23 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, void tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl, unsigned short new_size) { - WARN_ON_ONCE(new_size > HPACK_ENC_TABLE_MAX_SIZE); + if (WARN_ON_ONCE(new_size > HPACK_ENC_TABLE_MAX_SIZE)) + return; spin_lock(&tbl->lock); - T_DBG3("%s: tbl->rb_len=%hu, tbl->size=%hu, new_size=%hu\n", __func__, - tbl->rb_len, tbl->size, new_size); + T_DBG3("%s: tbl->rb_len=%hu, tbl->size=%hu, tbl->window=%hu," + " new_size=%hu\n", __func__, tbl->rb_len, tbl->size, + tbl->window, new_size); - if (tbl->size > new_size) - tfw_hpack_rbuf_calc(tbl, new_size, NULL, - (TfwHPackETblIter *)tbl); - WARN_ON_ONCE(tbl->rb_len > tbl->size); + if (tbl->window > new_size) { + if (tbl->size > new_size) + tfw_hpack_rbuf_calc(tbl, new_size, NULL, + (TfwHPackETblIter *)tbl); + WARN_ON_ONCE(tbl->rb_len > tbl->size); - tbl->window = new_size; + tbl->window = new_size; + } spin_unlock(&tbl->lock); } diff --git a/tempesta_fw/hpack.h b/tempesta_fw/hpack.h index 647dfb84f3..7b7b296426 100644 --- a/tempesta_fw/hpack.h +++ b/tempesta_fw/hpack.h @@ -186,7 +186,7 @@ typedef struct { * @enc_tbl - table for headers compression; * @dec_tbl - table for headers decompression; * @length - remaining length of decoded string; - * @max_window - maximum allowed dynamic table size; + * @max_window - maximum allowed size for the decoder dynamic table; * @curr - current shift in Huffman decoding context; * @hctx - current Huffman decoding context; * @__off - offset to reinitialize processing context; @@ -214,7 +214,7 @@ typedef struct { * encoded from 64-bit unsigned long integer: one byte for each 7-bit part of * source long integer plus on byte for initial prefix. */ -#define HPACK_MAX_INT \ +#define HPACK_MAX_INT \ (DIV_ROUND_UP(sizeof(unsigned long), 7) + 1) typedef struct { diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 265182dcc8..e9e4b4661d 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -639,7 +639,7 @@ static inline void tfw_http_conn_req_clean(TfwHttpReq *req) { if (TFW_MSG_H2(req)) { - tfw_h2_stream_id_close(req); + tfw_h2_stream_id_close(req, _HTTP2_UNDEFINED, 0); } else { spin_lock_bh(&((TfwCliConn *)req->conn)->seq_qlock); if (likely(!list_empty(&req->msg.seq_list))) @@ -790,10 +790,15 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) .len = sizeof(buf) }; - if (!(stream_id = tfw_h2_stream_id_close(req))) - goto err; + stream_id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!stream_id)) { + tfw_http_conn_msg_free((TfwHttpMsg *)req); + return; + } - if (!(resp = tfw_http_msg_alloc_resp_light(req))) + resp = tfw_http_msg_alloc_resp_light(req); + if (unlikely(!resp)) goto err; skb_head = &resp->msg.skb_head; @@ -2334,7 +2339,7 @@ tfw_http_conn_release(TfwConn *conn) list_for_each_entry_safe(req, tmp, &zap_queue, fwd_list) { list_del_init(&req->fwd_list); if (TFW_MSG_H2(req)) { - tfw_h2_stream_id_close(req); + tfw_h2_stream_id_close(req, _HTTP2_UNDEFINED, 0); } else if (unlikely(!list_empty_careful(&req->msg.seq_list))) { spin_lock_bh(&((TfwCliConn *)req->conn)->seq_qlock); @@ -2909,14 +2914,15 @@ tfw_h2_adjust_req(TfwHttpReq *req) bool auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); bool host = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_HOST]); TfwGlobal *g_vhost = tfw_vhost_get_global(); - char *xff_buf = *this_cpu_ptr(&g_buf); - char *xff_ptr = ss_skb_fmt_src_addr(req->msg.skb_head, xff_buf); + char *buf = *this_cpu_ptr(&g_buf); + char *xff_ptr = ss_skb_fmt_src_addr(req->msg.skb_head, buf); TfwStr h_xff = { .chunks = (TfwStr []){ { .data = S_XFF, .len = SLEN(S_XFF) }, - { .data = xff_buf, .len = xff_ptr - xff_buf } + { .data = buf, .len = xff_ptr - buf, + .flags = TFW_STR_HDR_VALUE } }, - .len = SLEN(S_XFF) + xff_ptr - xff_buf, + .len = SLEN(S_XFF) + xff_ptr - buf, .nchunks = 2 }; TfwStr h_ct_new = { @@ -2932,8 +2938,7 @@ tfw_h2_adjust_req(TfwHttpReq *req) .chunks = (TfwStr []) { { .data = S_F_VIA, .len = SLEN(S_F_VIA) }, { .data = "1.1 ", .len = 4 }, - { .data = *this_cpu_ptr(&g_buf), - .len = g_vhost->hdr_via_len }, + { .data = buf, .len = g_vhost->hdr_via_len }, { .data = S_CRLF, .len = SLEN(S_CRLF) } }, .len = SLEN(S_F_VIA) + 4 + g_vhost->hdr_via_len + SLEN(S_CRLF), @@ -3038,7 +3043,7 @@ do { \ * count and length is correcting by three (or four) pseudo-headers * count and length. */ - it->hdrs_cnt -= 3; + it->hdrs_cnt -= 4; it->hdrs_len += 2 + SLEN(S_VERSION11) + SLEN(S_CRLF) - ht->tbl[TFW_HTTP_HDR_H2_SCHEME].len - SLEN(S_H2_METHOD) @@ -3049,7 +3054,8 @@ do { \ it->hdrs_len += SLEN(S_HTTP) - SLEN(S_H2_AUTH) + SLEN(S_F_HOST) - + req->host.len; + + req->host.len + + SLEN(S_CRLF); if (host) { --it->hdrs_cnt; it->hdrs_len -= ht->tbl[TFW_HTTP_HDR_HOST].len; @@ -3057,8 +3063,8 @@ do { \ } it->hdrs_len += it->hdrs_cnt * (SLEN(S_DLM) + SLEN(S_CRLF)); - it->hdrs_len += SLEN(S_CRLF); it->hdrs_len += h_conn.len + h_via.len; + it->hdrs_len += SLEN(S_CRLF); /* * Create special buffer to write headers block into the target HTTP/1.1 @@ -3085,11 +3091,13 @@ do { \ WRITE_LIT(dst, S_CRLF); /* Add 'host' as the first header in request (RFC 7230 section 5.4). */ + WRITE_LIT(dst, S_F_HOST); if (auth) { - WRITE_LIT(dst, S_F_HOST); WRITE_STR(dst, &req->host); } else if (host) { - WRITE_HDR(dst, &ht->tbl[TFW_HTTP_HDR_HOST]); + BUG_ON(!TFW_STR_EMPTY(&req->host)); + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_HOST], &req->host); + WRITE_HDR(dst, &req->host); } WRITE_LIT(dst, S_CRLF); @@ -3110,13 +3118,16 @@ do { \ * Headers, which should be added unconditionally, inserted in * the end. */ + memcpy_fast(__TFW_STR_CH(&h_via, 2)->data, g_vhost->hdr_via, + g_vhost->hdr_via_len); WRITE_STR(dst, &h_via); WRITE_STR(dst, &h_conn); WRITE_LIT(dst, S_CRLF); - T_DBG3("%s: req adjusted, it->hdrs_len=%lu, it->hdrs_cnt=%u, dst=[%p]," - " pg=[%p]\n", __func__, it->hdrs_len, it->hdrs_cnt, dst, - (char *)page_address(pg)); + T_DBG3("%s: req adjusted, it->hb_len=%lu, it->hdrs_len=%lu," + " it->hdrs_cnt=%u, dst=[%p], pg=[%p]\n", __func__, it->hb_len, + it->hdrs_len, it->hdrs_cnt, dst, (char *)page_address(pg)); + WARN_ON_ONCE(dst - (char *)page_address(pg) != it->hdrs_len); r = ss_skb_replace_page(&req->msg.skb_head, pg, it->hdrs_len, @@ -3382,7 +3393,7 @@ tfw_h2_hdr_map(TfwHttpResp *resp, const TfwStr *hdr, unsigned int id) index = &map->index[map->count]; index->idx = id; - index->d_idx = TFW_STR_PLAIN(hdr) ? 0 : hdr->nchunks - 1; + index->d_idx = TFW_STR_DUP(hdr) ? hdr->nchunks - 1 : 0; ++map->count; return 0; @@ -3555,7 +3566,7 @@ tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) if (WARN_ON_ONCE(!last_chunk)) return -EINVAL; - val_out->nchunks = chunk - val_out->chunks + 1; + val_out->nchunks = chunk - val_out->chunks; val_out->len -= hdr_tail; last_chunk->len -= last_tail; @@ -3611,12 +3622,17 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) TfwStr *first = TFW_STR_CHUNK(tgt, 0); TfwHdrModsDesc *f_desc = NULL; - tgt = TFW_STR_CHUNK(tgt, d_num); + if (TFW_STR_DUP(tgt)) + tgt = TFW_STR_CHUNK(tgt, d_num); + if (WARN_ON_ONCE(!tgt || TFW_STR_EMPTY(tgt) || TFW_STR_DUP(tgt))) return -EINVAL; + T_DBG3("%s: hid=%hu, d_num=%hu, nchunks=%u, h_mods->sz=%lu\n", + __func__, hid, d_num, ht->tbl[hid].nchunks, h_mods->sz); + if (!h_mods) goto def; @@ -3706,20 +3722,14 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) continue; /* - * 'Server' and 'Date' headers must be replaced; thus, remove - * the original headers (and all their duplicates) skipping them - * here; the new headers will be written later, during new - * headers' addition stage. + * 'Server' header must be replaced; thus, remove the original + * header (and all its duplicates) skipping it here; the new + * header will be written later, during new headers' addition + * stage. */ if (hid == TFW_HTTP_HDR_SERVER) continue; - if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { - DEFINE_TFW_STR(n_date, "date"); - if (__hdr_name_cmp(tgt, &n_date)) - continue; - } - /* * In general case the header should be transformed in-place * from its original HTTP/1.1-representation in skb. @@ -3798,124 +3808,6 @@ tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, return 0; } -static void -tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) -{ - int r; - unsigned int stream_id; - bool hdrs_end = false; - TfwHttpReq *req = resp->req; - TfwH2Ctx *ctx = tfw_h2_context(req->conn); - TfwHttpTransIter *mit = &resp->mit; - TfwNextHdrOp *next = &mit->next; - TfwMsgIter *iter = &mit->iter; - const TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, - req->vhost, - TFW_VHOST_HDRMOD_RESP); - /* - * Get ID of corresponding stream and unlink request from the - * stream. - */ - if (!(stream_id = tfw_h2_stream_id_close(req))) - goto out; - - /* - * Transform HTTP/1.1 headers into HTTP/2 form, in parallel with - * adjusting of particular headers. - */ - WARN_ON_ONCE(mit->acc_len || mit->curr); - - tfw_h2_msg_transform_setup(mit, resp->msg.skb_head, true); - - r = tfw_h2_resp_next_hdr(resp, h_mods); - if (unlikely(r)) - goto clean; - - r = tfw_h2_pseudo_write(resp, TFW_H2_TRANS_SUB); - if (unlikely(r)) - goto clean; - - if (WARN_ON_ONCE(!mit->bnd)) - goto clean; - - do { - TfwStr *last; - TfwStr hdr = next->s_hdr; - TfwH2TransOp op = next->op; - - r = tfw_h2_resp_next_hdr(resp, h_mods); - if (unlikely(r)) - goto clean; - - if (!mit->bnd) { - last = TFW_STR_LAST(&resp->crlf); - mit->bnd = last->data + last->len; - hdrs_end = true; - } - - r = tfw_hpack_encode(resp, &hdr, op); - if (unlikely(r)) - goto clean; - - } while (!hdrs_end); - - /* - * Write additional headers in HTTP/2 format in the end of the - * headers block, including configured headers which haven't been - * processed above and which have non-empty value (i.e. configured - * not for deletion). - */ - r = tfw_http_sess_resp_process(resp); - if (unlikely(r)) - goto clean; - - r = tfw_h2_add_hdr_via(resp); - if (unlikely(r)) - goto clean; - - r = tfw_h2_set_stale_warn(resp); - if (unlikely(r)) - goto clean; - - if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { - r = tfw_h2_add_hdr_date(resp); - if (unlikely(r)) - goto clean; - } - - r = TFW_H2_MSG_HDR_ADD(resp, "server", TFW_NAME "/" TFW_VERSION, - TFW_HTTP_HDR_SERVER, 54); - if (unlikely(r)) - goto clean; - - r = tfw_h2_resp_add_loc_hdrs(resp, h_mods); - if (unlikely(r)) - goto clean; - - r = tfw_h2_make_frames(resp, stream_id, mit->acc_len); - if (unlikely(r)) - goto clean; - - r = ss_skb_cut_extra_data(iter->skb_head, iter->skb, iter->frag, - mit->curr_ptr, mit->bnd); - if (unlikely(r)) - goto clean; - - tfw_h2_resp_fwd(resp); - - return; -clean: - tfw_http_conn_msg_free((TfwHttpMsg *)resp); - tfw_http_send_resp(req, 500, - "response dropped: processing error"); - tfw_hpack_enc_release(&ctx->hpack, resp->flags); - TFW_INC_STAT_BH(serv.msgs_otherr); - - return; -out: - tfw_http_resp_pair_free(req); -} - static void tfw_h1_resp_adjust_fwd(TfwHttpResp *resp) { @@ -4034,7 +3926,7 @@ tfw_h2_error_resp(TfwHttpReq *req, int status, bool reply, bool attack, * remote peer (via RST_STREAM frame), that the stream has entered * into closed state (RFC 7540 section 6.4). */ - stream_id = tfw_h2_stream_id_close(req); + stream_id = tfw_h2_stream_id_close(req, HTTP2_RST_STREAM, 0); if (stream_id && !attack) tfw_h2_send_rst_stream(ctx, stream_id, HTTP2_ECODE_CANCEL); @@ -4191,6 +4083,127 @@ tfw_http_req_block(TfwHttpReq *req, int status, const char *msg) tfw_http_cli_error_resp_and_log(req, status, msg, true, false); } + +static void +tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) +{ + int r; + unsigned int stream_id; + bool hdrs_end = false; + TfwHttpReq *req = resp->req; + TfwH2Ctx *ctx = tfw_h2_context(req->conn); + TfwHttpTransIter *mit = &resp->mit; + TfwNextHdrOp *next = &mit->next; + TfwMsgIter *iter = &mit->iter; + const TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, + req->vhost, + TFW_VHOST_HDRMOD_RESP); + /* + * Get ID of corresponding stream to prepare/send HTTP/2 response, and + * unlink request from the stream. + */ + stream_id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!stream_id)) + goto out; + + /* + * Transform HTTP/1.1 headers into HTTP/2 form, in parallel with + * adjusting of particular headers. + */ + WARN_ON_ONCE(mit->acc_len || mit->curr); + + tfw_h2_msg_transform_setup(mit, resp->msg.skb_head, true); + + r = tfw_h2_resp_next_hdr(resp, h_mods); + if (unlikely(r)) + goto clean; + + r = tfw_h2_pseudo_write(resp, TFW_H2_TRANS_SUB); + if (unlikely(r)) + goto clean; + + if (WARN_ON_ONCE(!mit->bnd)) + goto clean; + + do { + TfwStr *last; + TfwStr hdr = next->s_hdr; + TfwH2TransOp op = next->op; + + r = tfw_h2_resp_next_hdr(resp, h_mods); + if (unlikely(r)) + goto clean; + + if (!mit->bnd) { + last = TFW_STR_LAST(&resp->crlf); + mit->bnd = last->data + last->len; + hdrs_end = true; + } + + r = tfw_hpack_encode(resp, &hdr, op); + if (unlikely(r)) + goto clean; + + } while (!hdrs_end); + + /* + * Write additional headers in HTTP/2 format in the end of the + * headers block, including configured headers which haven't been + * processed above and which have non-empty value (i.e. configured + * not for deletion). + */ + r = tfw_http_sess_resp_process(resp); + if (unlikely(r)) + goto clean; + + r = tfw_h2_add_hdr_via(resp); + if (unlikely(r)) + goto clean; + + r = tfw_h2_set_stale_warn(resp); + if (unlikely(r)) + goto clean; + + if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { + r = tfw_h2_add_hdr_date(resp); + if (unlikely(r)) + goto clean; + } + + r = TFW_H2_MSG_HDR_ADD(resp, "server", TFW_NAME "/" TFW_VERSION, + TFW_HTTP_HDR_SERVER, 54); + if (unlikely(r)) + goto clean; + + r = tfw_h2_resp_add_loc_hdrs(resp, h_mods); + if (unlikely(r)) + goto clean; + + r = tfw_h2_make_frames(resp, stream_id, mit->acc_len); + if (unlikely(r)) + goto clean; + + r = ss_skb_cut_extra_data(iter->skb_head, iter->skb, iter->frag, + mit->curr_ptr, mit->bnd); + if (unlikely(r)) + goto clean; + + tfw_h2_resp_fwd(resp); + + return; +clean: + tfw_http_conn_msg_free((TfwHttpMsg *)resp); + tfw_http_send_resp(req, 500, + "response dropped: processing error"); + tfw_hpack_enc_release(&ctx->hpack, resp->flags); + TFW_INC_STAT_BH(serv.msgs_otherr); + + return; +out: + tfw_http_resp_pair_free(req); +} + /** * The request is serviced from cache. * Send the response as is and unrefer its data. diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index 824153d2ab..01643412fd 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -166,7 +166,7 @@ do { \ TfwH2Err err = HTTP2_ECODE_NO_ERROR; \ BUG_ON(!(ctx)->cur_stream); \ if ((res = tfw_h2_stream_fsm((ctx)->cur_stream, (hdr)->type, \ - (hdr)->flags, &err))) \ + (hdr)->flags, false, &err))) \ { \ T_DBG3("stream recv processed: result=%d, state=%d, id=%u," \ " err=%d\n", res, (ctx)->cur_stream->state, \ @@ -694,14 +694,18 @@ tfw_h2_stream_close(TfwH2Ctx *ctx, unsigned int id, TfwStream **stream, } /* - * Unlink request from corresponding stream (if linked) and move the stream - * to the queue of closed streams (if it is not contained there yet). + * Get stream ID for upper layer to prepare and send frame (of type specified + * in @type and with flags set in @flags) with response to client. This + * procedure also unlinks request from corresponding stream (if linked) and + * moves the stream to the queue of closed streams (if it is not contained + * there yet). */ unsigned int -tfw_h2_stream_id_close(TfwHttpReq *req) +tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type, + unsigned char flags) { TfwStream *stream; - unsigned int id; + unsigned int id = 0; TfwH2Ctx *ctx = tfw_h2_context(req->conn); spin_lock(&ctx->lock); @@ -712,7 +716,12 @@ tfw_h2_stream_id_close(TfwHttpReq *req) return 0; } - id = stream->id; + if (type < _HTTP2_UNDEFINED && + !STREAM_SEND_PROCESS(stream, type, flags)) + { + id = stream->id; + } + stream->msg = NULL; __tfw_h2_stream_add_closed(&ctx->hclosed_streams, stream); @@ -798,9 +807,11 @@ tfw_h2_headers_process(TfwH2Ctx *ctx) ctx->state = HTTP2_IGNORE_FRAME_DATA; - return tfw_h2_stream_close(ctx, hdr->stream_id, - &ctx->cur_stream, - HTTP2_ECODE_PROTO); + if (!STREAM_SEND_PROCESS(ctx->cur_stream, HTTP2_RST_STREAM, 0)) + return tfw_h2_stream_close(ctx, hdr->stream_id, + &ctx->cur_stream, + HTTP2_ECODE_PROTO); + return T_OK; } if (!ctx->cur_stream) { @@ -826,19 +837,24 @@ tfw_h2_wnd_update_process(TfwH2Ctx *ctx) TfwFrameHdr *hdr = &ctx->hdr; wnd_incr = ntohl(*(unsigned int *)ctx->rbuf) & ((1U << 31) - 1); - if (!wnd_incr) { - if (ctx->cur_stream) - return tfw_h2_stream_close(ctx, hdr->stream_id, - &ctx->cur_stream, - HTTP2_ECODE_PROTO); + if (wnd_incr) { + /* + * TODO #498: apply new window size for entire connection or + * particular stream. + */ + return T_OK; + } + + if (!ctx->cur_stream) { tfw_h2_conn_terminate(ctx, HTTP2_ECODE_PROTO); return T_DROP; } - /* - * TODO: apply new window size for entire connection or - * particular stream; ignore until #498. - */ - return T_OK; + + if (STREAM_SEND_PROCESS(ctx->cur_stream, HTTP2_RST_STREAM, 0)) + return T_OK; + + return tfw_h2_stream_close(ctx, hdr->stream_id, &ctx->cur_stream, + HTTP2_ECODE_PROTO); } static inline int @@ -849,26 +865,29 @@ tfw_h2_priority_process(TfwH2Ctx *ctx) tfw_h2_unpack_priority(pri, ctx->rbuf); + if (pri->stream_id != hdr->stream_id) { + T_DBG3("%s: parsed, stream_id=%u, dep_stream_id=%u, weight=%hu," + " excl=%hhu\n", __func__, hdr->stream_id, pri->stream_id, + pri->weight, pri->exclusive); + + tfw_h2_change_stream_dep(&ctx->sched, hdr->stream_id, + pri->stream_id, pri->weight, + pri->exclusive); + return T_OK; + } + /* * Stream cannot depend on itself (see RFC 7540 section 5.1.2 for * details). */ - if (pri->stream_id == hdr->stream_id) { - T_DBG("Invalid dependency: new stream with %u depends on" + T_DBG("Invalid dependency: new stream with %u depends on" " itself\n", hdr->stream_id); - return tfw_h2_stream_close(ctx, hdr->stream_id, - &ctx->cur_stream, - HTTP2_ECODE_PROTO); - } - - T_DBG3("%s: parsed, stream_id=%u, dep_stream_id=%u, weight=%hu," - " excl=%hhu\n", __func__, hdr->stream_id, pri->stream_id, - pri->weight, pri->exclusive); + if (STREAM_SEND_PROCESS(ctx->cur_stream, HTTP2_RST_STREAM, 0)) + return T_OK; - tfw_h2_change_stream_dep(&ctx->sched, hdr->stream_id, pri->stream_id, - pri->weight, pri->exclusive); - return T_OK; + return tfw_h2_stream_close(ctx, hdr->stream_id, &ctx->cur_stream, + HTTP2_ECODE_PROTO); } static inline void @@ -890,8 +909,9 @@ tfw_h2_apply_settings_entry(TfwH2Ctx *ctx, unsigned short id, switch (id) { case HTTP2_SETTINGS_TABLE_SIZE: - dest->hdr_tbl_sz = val; - tfw_hpack_set_rbuf_size(&ctx->hpack.enc_tbl, val); + dest->hdr_tbl_sz = min_t(unsigned int, + val, HPACK_ENC_TABLE_MAX_SIZE); + tfw_hpack_set_rbuf_size(&ctx->hpack.enc_tbl, dest->hdr_tbl_sz); break; case HTTP2_SETTINGS_ENABLE_PUSH: @@ -1079,17 +1099,22 @@ tfw_h2_flow_control(TfwH2Ctx *ctx) stream->loc_wnd -= hdr->length; ctx->loc_wnd -= hdr->length; - if (stream->loc_wnd <= lset->wnd_sz / 2 - && tfw_h2_send_wnd_update(ctx, stream->id, - lset->wnd_sz - stream->loc_wnd)) - { - return T_DROP; + if (stream->loc_wnd <= lset->wnd_sz / 2) { + if( tfw_h2_send_wnd_update(ctx, stream->id, + lset->wnd_sz - stream->loc_wnd)) + { + return T_DROP; + } + stream->loc_wnd = lset->wnd_sz; } - if (ctx->loc_wnd <= MAX_WND_SIZE / 2 - && tfw_h2_send_wnd_update(ctx, 0, MAX_WND_SIZE - ctx->loc_wnd)) - { - return T_DROP; + + if (ctx->loc_wnd <= MAX_WND_SIZE / 2) { + if (tfw_h2_send_wnd_update(ctx, 0, MAX_WND_SIZE - ctx->loc_wnd)) + { + return T_DROP; + } + ctx->loc_wnd = MAX_WND_SIZE; } return T_OK; @@ -1245,12 +1270,8 @@ tfw_h2_frame_type_process(TfwH2Ctx *ctx) ctx->cur_stream = tfw_h2_find_stream(&ctx->sched, hdr->stream_id); - if (hdr->length != FRAME_PRIORITY_SIZE) { - SET_TO_READ(ctx); - return tfw_h2_stream_close(ctx, hdr->stream_id, - &ctx->cur_stream, - HTTP2_ECODE_SIZE); - } + if (hdr->length != FRAME_PRIORITY_SIZE) + goto conn_term; if (ctx->cur_stream) STREAM_RECV_PROCESS(ctx, hdr); diff --git a/tempesta_fw/http_frame.h b/tempesta_fw/http_frame.h index d4c054e8f2..029ed36dbb 100644 --- a/tempesta_fw/http_frame.h +++ b/tempesta_fw/http_frame.h @@ -42,7 +42,8 @@ typedef enum { HTTP2_PING, HTTP2_GOAWAY, HTTP2_WINDOW_UPDATE, - HTTP2_CONTINUATION + HTTP2_CONTINUATION, + _HTTP2_UNDEFINED } TfwFrameType; /** @@ -203,7 +204,8 @@ int tfw_h2_context_init(TfwH2Ctx *ctx); void tfw_h2_context_clear(TfwH2Ctx *ctx); int tfw_h2_frame_process(void *c, TfwFsmData *data); void tfw_h2_conn_streams_cleanup(TfwH2Ctx *ctx); -unsigned int tfw_h2_stream_id_close(TfwHttpReq *req); +unsigned int tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type, + unsigned char flags); void tfw_h2_conn_terminate_close(TfwH2Ctx *ctx, TfwH2Err err_code, bool close); int tfw_h2_send_rst_stream(TfwH2Ctx *ctx, unsigned int id, TfwH2Err err_code); diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 65345052e2..8b732b2003 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -520,12 +520,18 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) * headers must be blocked as early as possible, * just when parser reads them. */ - BUG_ON(id < TFW_HTTP_HDR_NONSINGULAR); - /* - * RFC 7230 3.2.2: duplicate of non-singular special - * header - leave the decision to classification layer. - */ - __set_bit(TFW_HTTP_B_FIELD_DUPENTRY, hm->flags); + if (id < TFW_HTTP_HDR_NONSINGULAR) { + if (WARN_ON_ONCE(!TFW_MSG_H2(hm) + || id != TFW_HTTP_HDR_COOKIE)) + return TFW_BLOCK; + } else { + /* + * RFC 7230 3.2.2: duplicate of non-singular special + * header - leave the decision to classification layer. + */ + __set_bit(TFW_HTTP_B_FIELD_DUPENTRY, hm->flags); + } + goto duplicate; } @@ -567,7 +573,15 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) } done: - if (TFW_RESP_TO_H2(hm) && tfw_h2_hdr_map((TfwHttpResp *)hm, hdr, id)) + /* + * During response HTTP/1.1=>HTTP/2 transformation we need only regular + * headers to be transformed, and status-line must not be present in the + * resulting HTTP/2 response at all; thus, we do not need status-line in + * the indirection map. + */ + if (TFW_RESP_TO_H2(hm) + && id > TFW_HTTP_STATUS_LINE + && tfw_h2_hdr_map((TfwHttpResp *)hm, hdr, id)) return TFW_BLOCK; *h = parser->hdr; @@ -1274,7 +1288,8 @@ __tfw_http_msg_alloc(int type, bool full) */ unsigned long tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, - unsigned long *val_off, unsigned long *val_len) + unsigned long *val_off, unsigned long *val_len, + TfwH2TransOp op) { const TfwStr *chunk, *end; unsigned long tail, hdr_tail = 0, hdr_len = 0; @@ -1282,6 +1297,32 @@ tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, *name_len = *val_off = *val_len = 0; + if (op != TFW_H2_TRANS_INPLACE) { + /* + * During headers addition (or message expansion) the the source + * @hdr must have the following chunk structure (without the + * OWS): + * + * { name [S_DLM] value1 [value2 [value3 ...]] }. + * + */ + chunk = TFW_STR_CHUNK(hdr, 1); + if (WARN_ON_ONCE(!chunk)) + return 0; + + if (chunk->len == SLEN(S_DLM) + && *(short *)chunk->data == *(short *)S_DLM) + { + *val_off = SLEN(S_DLM); + } + + hdr_len = hdr->len; + *name_len = TFW_STR_CHUNK(hdr, 0)->len; + *val_len = hdr_len - *name_len - *val_off; + + return hdr_len - *val_off; + } + TFW_STR_FOR_EACH_CHUNK(chunk, hdr, end) { unsigned long idx; @@ -1366,8 +1407,10 @@ tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, { const TfwStr *c, *end; + T_DBG3("%s: enter, nm_len=%lu, val_off=%lu, val_len=%lu\n", __func__, + nm_len, val_off, val_len); + BUG_ON(!nm_len); - BUG_ON(!val_off); TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { unsigned long len; @@ -1385,13 +1428,10 @@ tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, WARN_ON_ONCE(val_off < c->len - len); val_off -= c->len - len; } - else if (val_len) { + else if (val_len && !len) { len = min(val_len, c->len); val_len -= len; } - else { - break; - } } if (!len) @@ -1535,8 +1575,9 @@ tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, memcpy_fast(mit->curr_ptr, c->data + c_off, n_copy); - T_DBG3("%s: acc_len=%lu, n_copy=%u, mit->curr_ptr='%.*s'\n", - __func__, mit->acc_len, n_copy, n_copy, mit->curr_ptr); + T_DBG3("%s: acc_len=%lu, n_copy=%u, mit->curr_ptr='%.*s'," + " ptr_diff=%ld\n", __func__, mit->acc_len, n_copy, + n_copy, mit->curr_ptr, stop - mit->curr_ptr); mit->curr_ptr += n_copy; mit->acc_len += n_copy; diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 7681d5d651..9b8f4740fe 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -104,7 +104,7 @@ tfw_http_msg_alloc_resp_light(TfwHttpReq *req) static inline void tfw_h2_msg_transform_setup(TfwHttpTransIter *mit, struct sk_buff *skb, - bool init) + bool init) { TfwMsgIter *iter = &mit->iter; @@ -179,7 +179,8 @@ int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name); int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name); unsigned long tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, unsigned long *val_off, - unsigned long *val_len); + unsigned long *val_len, + TfwH2TransOp op); void tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, unsigned long val_off, unsigned long val_len, char *out_buf); diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index a79242c879..8c0ac957b1 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -4213,6 +4213,7 @@ do { \ bool ret = true; \ TfwStr *tbl = msg->h_tbl->tbl; \ if (unlikely(hid < TFW_HTTP_HDR_NONSINGULAR \ + && hid != TFW_HTTP_HDR_COOKIE \ && !TFW_STR_EMPTY(&tbl[hid]))) \ { \ ret = false; \ diff --git a/tempesta_fw/http_stream.c b/tempesta_fw/http_stream.c index 028e01b2ed..54456fe3e9 100644 --- a/tempesta_fw/http_stream.c +++ b/tempesta_fw/http_stream.c @@ -52,12 +52,39 @@ tfw_h2_stream_cache_destroy(void) */ TfwStreamFsmRes tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, - TfwH2Err *err) + bool send, TfwH2Err *err) { + TfwStreamFsmRes res = STREAM_FSM_RES_OK; + + if (unlikely(!stream)) + return STREAM_FSM_RES_IGNORE; + + spin_lock(&stream->st_lock); + T_DBG3("enter %s: stream->state=%d, stream->id=%u, type=%hhu," " flags=0x%hhx\n", __func__, stream->state, stream->id, type, flags); + if (send) { + /* + * In the sending flow this FSM procedure intended only for + * HEADERS, DATA and RST_STREAM frames processing. + */ + BUG_ON(!(flags & HTTP2_F_END_STREAM) + && type != HTTP2_RST_STREAM); + /* + * We can send HEADERS or DATA frames to the client only + * when HTTP2_STREAM_REM_HALF_CLOSED state is passed (RFC + * 7540 section 5.1). + */ + if (WARN_ON_ONCE(stream->state < HTTP2_STREAM_REM_HALF_CLOSED + && flags & HTTP2_F_END_STREAM)) + { + res = STREAM_FSM_RES_IGNORE; + goto done; + } + } + switch (stream->state) { case HTTP2_STREAM_LOC_RESERVED: case HTTP2_STREAM_REM_RESERVED: @@ -97,10 +124,13 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, } /* * Received RST_STREAM frame immediately moves stream into the - * final 'closed' state. + * final 'closed' state, while the the sent RST_STREAM moves stream + * into the intermediate 'locally closed' state. */ else if (type == HTTP2_RST_STREAM) { - stream->state = HTTP2_STREAM_CLOSED; + stream->state = send + ? HTTP2_STREAM_LOC_CLOSED + : HTTP2_STREAM_CLOSED; } else if (type == HTTP2_CONTINUATION) { /* @@ -109,11 +139,16 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, * (RFC 7540 section 6.10). */ *err = HTTP2_ECODE_PROTO; - return STREAM_FSM_RES_TERM_CONN; + res = STREAM_FSM_RES_TERM_CONN; } + break; case HTTP2_STREAM_CONT: + if (send && type == HTTP2_RST_STREAM) { + stream->state = HTTP2_STREAM_LOC_CLOSED; + break; + } /* * Only CONTINUATION frames are allowed (after HEADERS or * CONTINUATION frames) until frame with END_HEADERS flag will @@ -121,7 +156,8 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, */ if (type != HTTP2_CONTINUATION) { *err = HTTP2_ECODE_PROTO; - return STREAM_FSM_RES_TERM_CONN; + res = STREAM_FSM_RES_TERM_CONN; + break; } /* * Once END_HEADERS flag is received, move stream into standard @@ -129,12 +165,18 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, */ if (flags & HTTP2_F_END_HEADERS) stream->state = HTTP2_STREAM_OPENED; + break; case HTTP2_STREAM_CONT_CLOSED: + if (send && type == HTTP2_RST_STREAM) { + stream->state = HTTP2_STREAM_LOC_CLOSED; + break; + } if (type != HTTP2_CONTINUATION) { *err = HTTP2_ECODE_PROTO; - return STREAM_FSM_RES_TERM_CONN; + res = STREAM_FSM_RES_TERM_CONN; + break; } /* * If END_HEADERS flag arrived in this state, this means that @@ -144,23 +186,30 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, */ if (flags & HTTP2_F_END_HEADERS) stream->state = HTTP2_STREAM_REM_HALF_CLOSED; + break; - case HTTP2_STREAM_REM_CLOSED: - /* - * RST_STREAM and WINDOW_UPDATE frames must be ignored in this - * state. - */ - if (type == HTTP2_WINDOW_UPDATE - || type == HTTP2_RST_STREAM) - { - return STREAM_FSM_RES_IGNORE; + case HTTP2_STREAM_LOC_CLOSED: + /* All types of frames are allowed in this state. */ + if (type == HTTP2_RST_STREAM) { + if (send) { + res = STREAM_FSM_RES_IGNORE; + break; + } + stream->state = HTTP2_STREAM_CLOSED; } - /* Fall through. */ + + break; case HTTP2_STREAM_REM_HALF_CLOSED: + if (send && (type == HTTP2_RST_STREAM + || flags & HTTP2_F_END_STREAM)) + { + stream->state = HTTP2_STREAM_REM_CLOSED; + break; + } /* - * The only allowed stream-related frames in 'half-closed + * The only allowed received stream-related frames in 'half-closed * (remote)' state are PRIORITY, RST_STREAM and WINDOW_UPDATE. * If RST_STREAM frame is received in this state, the stream * will be removed from stream's storage (i.e. moved into final @@ -168,92 +217,80 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, */ if (type == HTTP2_CONTINUATION) { *err = HTTP2_ECODE_PROTO; - return STREAM_FSM_RES_TERM_CONN; + res = STREAM_FSM_RES_TERM_CONN; + break; } if (type == HTTP2_RST_STREAM) { stream->state = HTTP2_STREAM_CLOSED; } - else if (type != HTTP2_PRIORITY || type != HTTP2_WINDOW_UPDATE) + else if (type != HTTP2_PRIORITY && type != HTTP2_WINDOW_UPDATE) { + /* + * We always send RST_STREAM to the peer in this case; + * thus, the stream should be switched to the + * 'closed (remote)' state. + */ + stream->state = HTTP2_STREAM_REM_CLOSED; *err = HTTP2_ECODE_CLOSED; - return STREAM_FSM_RES_TERM_STREAM; + res = STREAM_FSM_RES_TERM_STREAM; } break; - case HTTP2_STREAM_CONT_HC: - if (type != HTTP2_CONTINUATION) { - *err = HTTP2_ECODE_PROTO; - return STREAM_FSM_RES_TERM_CONN; - } + case HTTP2_STREAM_REM_CLOSED: /* - * If END_HEADERS flag is received in this state, move stream - * into 'half-closed (local)' state (see RFC 7540 section 5.1 - * and section 6.2 for details). + * Sending of HEADERS/DATA and RST_STREAM frames must be ignored + * in this state, since the state is in 'closed (remote)' state, + * i.e. either it had already been reset from our side, or the + * HEADERS/DATA with END_STREAM flag had already been sent from + * us. Receiving of RST_STREAM and WINDOW_UPDATE frames should + * also be ignored according to RFC 7540, section 5.1 ('closed' + * paragraph). Receiving of all other frames should be ignored + * as well (except PRIORITY frames which should be processed + * even for closed streams), since there is no sense to respond + * into already closed stream. Receiving of CONTINUATION frames + * is forbidden (RFC 7540 section 6.10 stated that connection + * must be closed in such case). */ - if (flags & HTTP2_F_END_HEADERS) - stream->state = HTTP2_STREAM_LOC_HALF_CLOSED; - break; + if (type == HTTP2_PRIORITY) + break; - case HTTP2_STREAM_CONT_HC_CLOSED: - if (type != HTTP2_CONTINUATION) { + if (type == HTTP2_CONTINUATION) { *err = HTTP2_ECODE_PROTO; - return STREAM_FSM_RES_TERM_CONN; + res = STREAM_FSM_RES_TERM_CONN; + break; } - /* - * If END_HEADERS flag arrived in this state, this means that - * END_STREAM flag had been already received earlier - in - * half-closed (local) state, and now we must close the stream, - * i.e. move it to final 'closed' state. - */ - if (flags & HTTP2_F_END_HEADERS) - stream->state = HTTP2_STREAM_CLOSED; - break; - case HTTP2_STREAM_LOC_HALF_CLOSED: - /* - * According to section 5.1 of RFC 7540 any frame type can be - * received and will be valid in 'half-closed (local)' state. - */ - if (type == HTTP2_HEADERS) - { - if (flags & (HTTP2_F_END_STREAM | HTTP2_F_END_HEADERS)) { - stream->state = HTTP2_STREAM_CLOSED; - } - else if (flags & HTTP2_F_END_STREAM) { - stream->state = HTTP2_STREAM_CONT_HC_CLOSED; - } - else if (!(flags & HTTP2_F_END_HEADERS)) { - stream->state = HTTP2_STREAM_CONT_HC; - } - } - else if ((type == HTTP2_DATA && (flags & HTTP2_F_END_STREAM)) - || type == HTTP2_RST_STREAM) - { - stream->state = HTTP2_STREAM_CLOSED; - } - else if (type == HTTP2_CONTINUATION) - { - *err = HTTP2_ECODE_PROTO; - return STREAM_FSM_RES_TERM_CONN; - } + res = STREAM_FSM_RES_IGNORE; + break; case HTTP2_STREAM_CLOSED: + T_DBG3("%s, stream fully closed: stream->id=%u, type=%hhu," + " flags=0x%hhx\n", __func__, stream->id, type, flags); + if (send) { + res = STREAM_FSM_RES_IGNORE; + break; + } /* * In moment when the final 'closed' state is achieved, stream * actually must be removed from stream's storage (and from - * memory), thus the execution flow must not reach this point. + * memory), thus the receive execution flow must not reach this + * point. */ default: BUG(); } - T_DBG3("exit %s: stream->state=%d\n", __func__, stream->state); +done: + T_DBG3("exit %s: stream->state=%d, res=%d\n", __func__, + stream->state, res); + + spin_unlock(&stream->st_lock); - return STREAM_FSM_RES_OK; + return res; } static inline void @@ -262,6 +299,7 @@ tfw_h2_init_stream(TfwStream *stream, unsigned int id, unsigned short weight, { RB_CLEAR_NODE(&stream->node); INIT_LIST_HEAD(&stream->hcl_node); + spin_lock_init(&stream->st_lock); stream->id = id; stream->state = HTTP2_STREAM_OPENED; stream->loc_wnd = wnd; diff --git a/tempesta_fw/http_stream.h b/tempesta_fw/http_stream.h index dbb8c36647..788d3e7371 100644 --- a/tempesta_fw/http_stream.h +++ b/tempesta_fw/http_stream.h @@ -42,11 +42,8 @@ typedef enum { HTTP2_STREAM_OPENED, HTTP2_STREAM_CONT, HTTP2_STREAM_CONT_CLOSED, - HTTP2_STREAM_CONT_HC, - HTTP2_STREAM_CONT_HC_CLOSED, - HTTP2_STREAM_LOC_HALF_CLOSED, - HTTP2_STREAM_REM_HALF_CLOSED, HTTP2_STREAM_LOC_CLOSED, + HTTP2_STREAM_REM_HALF_CLOSED, HTTP2_STREAM_REM_CLOSED, HTTP2_STREAM_CLOSED } TfwStreamState; @@ -90,6 +87,7 @@ typedef enum { * @hcl_node - entry in queue of half-closed streams; * @id - stream ID; * @state - stream's current state; + * @st_lock - spinlock to synchronize concurrent access to stream FSM; * @loc_wnd - stream's current flow controlled window; * @weight - stream's priority weight; * @msg - message that is currently being processed; @@ -100,6 +98,7 @@ typedef struct { struct list_head hcl_node; unsigned int id; int state; + spinlock_t st_lock; unsigned int loc_wnd; unsigned short weight; TfwMsg *msg; @@ -121,7 +120,8 @@ typedef struct { int tfw_h2_stream_cache_create(void); void tfw_h2_stream_cache_destroy(void); TfwStreamFsmRes tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, - unsigned char flags, TfwH2Err *err); + unsigned char flags, bool send, + TfwH2Err *err); TfwStream *tfw_h2_find_stream(TfwStreamSched *sched, unsigned int id); TfwStream *tfw_h2_add_stream(TfwStreamSched *sched, unsigned int id, unsigned short weight, unsigned int wnd); @@ -147,4 +147,12 @@ tfw_h2_stream_is_closed(TfwStream *stream) return stream->state == HTTP2_STREAM_CLOSED; } +static inline TfwStreamFsmRes +STREAM_SEND_PROCESS(TfwStream *stream, unsigned char type, unsigned char flags) +{ + TfwH2Err err; + + return tfw_h2_stream_fsm(stream, type, flags, true, &err); +} + #endif /* __HTTP_STREAM__ */ diff --git a/tempesta_fw/ss_skb.c b/tempesta_fw/ss_skb.c index 6473ac1199..adadf70acf 100644 --- a/tempesta_fw/ss_skb.c +++ b/tempesta_fw/ss_skb.c @@ -1651,14 +1651,15 @@ ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, } offset = curr - addr; - if (WARN_ON_ONCE(offset < 0 || offset >= size)) - return -EINVAL; for (;;) { - int len, tail; + int len, tail = 0; long stop_offset = stop - addr; - len = tail = size - offset; + if (WARN_ON_ONCE(offset < 0 || offset >= size)) + return -EINVAL; + + len = size - offset; /* * We found the stop pointer; evict the delta between @curr and @@ -1672,6 +1673,7 @@ ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, if (curr == stop) return 0; + tail = len; len -= size - stop_offset; } @@ -1688,7 +1690,7 @@ ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, if (unlikely(ret)) return ret; - if (len != tail) + if (tail) return 0; /* diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index 32873bc6f2..116919aee6 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -1219,7 +1219,8 @@ TEST(hpack, enc_table_hdr_write) collect_compound_str(s5, s5_lws, 0); collect_compound_str(s5, s5_value, 0); - hdr_len = tfw_h2_msg_hdr_length(s1, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s1, &n_len, &v_off, &v_len, + TFW_H2_TRANS_INPLACE); EXPECT_EQ(n_len, strlen(HDR_NAME_1)); EXPECT_EQ(v_len, strlen(HDR_VALUE_1)); EXPECT_EQ(v_off, off1); @@ -1229,7 +1230,8 @@ TEST(hpack, enc_table_hdr_write) tfw_h2_msg_hdr_write(s1, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s1, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s2, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s2, &n_len, &v_off, &v_len, + TFW_H2_TRANS_INPLACE); EXPECT_EQ(n_len, strlen(HDR_NAME_2)); EXPECT_EQ(v_len, strlen(HDR_VALUE_2)); EXPECT_EQ(v_off, off2); @@ -1239,7 +1241,8 @@ TEST(hpack, enc_table_hdr_write) tfw_h2_msg_hdr_write(s2, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s2, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s3, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s3, &n_len, &v_off, &v_len, + TFW_H2_TRANS_INPLACE); EXPECT_EQ(n_len, strlen(HDR_NAME_3)); EXPECT_EQ(v_len, strlen(HDR_VALUE_3)); EXPECT_EQ(v_off, off3); @@ -1249,7 +1252,8 @@ TEST(hpack, enc_table_hdr_write) tfw_h2_msg_hdr_write(s3, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s3, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s4, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s4, &n_len, &v_off, &v_len, + TFW_H2_TRANS_INPLACE); EXPECT_EQ(n_len, strlen(HDR_NAME_4)); EXPECT_EQ(v_len, strlen(HDR_VALUE_4)); EXPECT_EQ(v_off, off4); @@ -1259,7 +1263,8 @@ TEST(hpack, enc_table_hdr_write) tfw_h2_msg_hdr_write(s4, n_len, v_off, v_len, buf); EXPECT_OK(memcmp_fast(t_s4, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s5, &n_len, &v_off, &v_len); + hdr_len = tfw_h2_msg_hdr_length(s5, &n_len, &v_off, &v_len, + TFW_H2_TRANS_INPLACE); EXPECT_EQ(n_len, strlen(HDR_NAME_5)); EXPECT_EQ(v_len, strlen(HDR_VALUE_5)); EXPECT_EQ(v_off, off5); @@ -1328,7 +1333,7 @@ TEST(hpack, enc_table_index) res = tfw_hpack_rbtree_find(tbl, s1, &node, &pl); EXPECT_EQ(res, HPACK_IDX_ST_NOT_FOUND); EXPECT_NULL(node); - EXPECT_OK(tfw_hpack_add_node(tbl, s1, &pl)); + EXPECT_OK(tfw_hpack_add_node(tbl, s1, &pl, TFW_H2_TRANS_INPLACE)); node = NULL; bzero_fast(&pl, sizeof(pl)); @@ -1337,7 +1342,8 @@ TEST(hpack, enc_table_index) EXPECT_NULL(node); EXPECT_NOT_NULL(pl.parent); if (pl.parent) - EXPECT_OK(tfw_hpack_add_node(tbl, s2, &pl)); + EXPECT_OK(tfw_hpack_add_node(tbl, s2, &pl, + TFW_H2_TRANS_INPLACE)); node = NULL; bzero_fast(&pl, sizeof(pl)); @@ -1346,7 +1352,8 @@ TEST(hpack, enc_table_index) EXPECT_NULL(node); EXPECT_NOT_NULL(pl.parent); if (pl.parent) - EXPECT_OK(tfw_hpack_add_node(tbl, s3, &pl)); + EXPECT_OK(tfw_hpack_add_node(tbl, s3, &pl, + TFW_H2_TRANS_INPLACE)); /* * Verify that headers had been correctly added into encoder dynamic @@ -1459,7 +1466,7 @@ TEST(hpack, enc_table_rbtree) res = tfw_hpack_rbtree_find(tbl, s1, &node, &pl); EXPECT_EQ(res, HPACK_IDX_ST_NOT_FOUND); EXPECT_NULL(node); - EXPECT_OK(tfw_hpack_add_node(tbl, s1, &pl)); + EXPECT_OK(tfw_hpack_add_node(tbl, s1, &pl, TFW_H2_TRANS_INPLACE)); bzero_fast(&pl, sizeof(pl)); res = tfw_hpack_rbtree_find(tbl, s1, &n1, &pl); EXPECT_EQ(res, HPACK_IDX_ST_FOUND); @@ -1483,7 +1490,7 @@ TEST(hpack, enc_table_rbtree) */ EXPECT_EQ(pl.parent, n1); EXPECT_EQ(pl.poff, &n1->right); - EXPECT_OK(tfw_hpack_add_node(tbl, s2, &pl)); + EXPECT_OK(tfw_hpack_add_node(tbl, s2, &pl, TFW_H2_TRANS_INPLACE)); } bzero_fast(&pl, sizeof(pl)); res = tfw_hpack_rbtree_find(tbl, s2, &n2, &pl); @@ -1504,7 +1511,7 @@ TEST(hpack, enc_table_rbtree) */ EXPECT_EQ(pl.parent, n2); EXPECT_EQ(pl.poff, &n2->left); - EXPECT_OK(tfw_hpack_add_node(tbl, s3, &pl)); + EXPECT_OK(tfw_hpack_add_node(tbl, s3, &pl, TFW_H2_TRANS_INPLACE)); } bzero_fast(&pl, sizeof(pl)); res = tfw_hpack_rbtree_find(tbl, s3, &n3, &pl); @@ -1568,7 +1575,8 @@ TEST(hpack, enc_table_rbtree) */ EXPECT_EQ(pl.parent, n1); EXPECT_EQ(pl.poff, &n1->left); - EXPECT_OK(tfw_hpack_add_node(tbl, s4, &pl)); + EXPECT_OK(tfw_hpack_add_node(tbl, s4, &pl, + TFW_H2_TRANS_INPLACE)); } bzero_fast(&pl, sizeof(pl)); res = tfw_hpack_rbtree_find(tbl, s4, &n4, &pl); @@ -1617,7 +1625,8 @@ TEST(hpack, enc_table_rbtree) */ EXPECT_EQ(pl.parent, n1); EXPECT_EQ(pl.poff, &n1->right); - EXPECT_OK(tfw_hpack_add_node(tbl, s5, &pl)); + EXPECT_OK(tfw_hpack_add_node(tbl, s5, &pl, + TFW_H2_TRANS_INPLACE)); } bzero_fast(&pl, sizeof(pl)); res = tfw_hpack_rbtree_find(tbl, s5, &n5, &pl); From f2e01bd570496c3c76a56e75f763397e57355260 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Fri, 22 Nov 2019 16:35:09 +0300 Subject: [PATCH 09/64] HTTP/2 Parser implementation: corrections in internal response generation (#309). --- tempesta_fw/http.c | 242 ++++++++++++++++++++++++--------------------- tempesta_fw/http.h | 2 +- 2 files changed, 131 insertions(+), 113 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index e9e4b4661d..5303625731 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -649,7 +649,6 @@ tfw_http_conn_req_clean(TfwHttpReq *req) tfw_http_conn_msg_free((TfwHttpMsg *)req); } - /* * Free the request and the paired response. */ @@ -686,6 +685,33 @@ tfw_http_resp_build_error(TfwHttpReq *req) TFW_INC_STAT_BH(clnt.msgs_otherr); } +static inline resp_code_t +tfw_http_enum_resp_code(int status) +{ + switch(status) { + case 200: + return RESP_200; + case 400: + return RESP_400; + case 403: + return RESP_403; + case 404: + return RESP_404; + case 412: + return RESP_412; + case 500: + return RESP_500; + case 502: + return RESP_502; + case 503: + return RESP_503; + case 504: + return RESP_504; + default: + return RESP_NUM; + } +} + /* * Static index determination for response ':status' pseudo-header (see RFC * 7541 Appendix A for details). @@ -730,8 +756,12 @@ tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) WARN_ON_ONCE(op != TFW_H2_TRANS_EXPAND && op != TFW_H2_TRANS_SUB); + /* + * If the status code is not in the static table, set the default + * static index just for the ':status' name. + */ + TFW_STR_INDEX_SET(&s_hdr, index ? index : 8 ); if (index) { - TFW_STR_INDEX_SET(&s_hdr, index); s_hdr.flags |= TFW_STR_FULL_INDEX; } @@ -766,21 +796,21 @@ tfw_h2_resp_fwd(TfwHttpResp *resp) } static void -tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) +tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) { + TfwStr *msg; + resp_code_t code; TfwHttpResp *resp; struct sk_buff **skb_head; TfwFrameHdr frame_hdr; - unsigned int stream_id; - TfwStr *start, *date, *clen, *srv, *body; - char *pos_dt_nm, *pos_cl_nm, *pos_srv_nm; - char *pos_dt_val, *pos_cl_val, *pos_srv_val; - unsigned long len_dt_nm, len_cl_nm, len_srv_nm; - unsigned long len_dt_val, len_cl_val, len_srv_val; - unsigned long len = SLEN(S_H2_STAT) + H2_STAT_VAL_LEN; + TfwHttpTransIter *mit; + char *date_val, *data_ptr; + unsigned int len_be; + unsigned long nlen, vlen; + unsigned char *dst_len_p, *src_len_p; unsigned char buf[FRAME_HEADER_SIZE]; + TfwStr *start, *date, *clen, *srv, *body; TfwH2Ctx *ctx = tfw_h2_context(req->conn); - TfwStr *msg = &http_predef_resps[code]; TfwStr hdr = { .chunks = (TfwStr []){ {}, {} }, .nchunks = 2 @@ -790,94 +820,110 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) .len = sizeof(buf) }; - stream_id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, - HTTP2_F_END_STREAM); - if (unlikely(!stream_id)) { - tfw_http_conn_msg_free((TfwHttpMsg *)req); - return; + if (!stream_id) { + stream_id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!stream_id)) { + tfw_http_conn_msg_free((TfwHttpMsg *)req); + return; + } } + code = tfw_http_enum_resp_code(status); + if (code == RESP_NUM) { + T_WARN("Unexpected response error code: [%d]\n", status); + code = RESP_500; + } + msg = &http_predef_resps[code]; + resp = tfw_http_msg_alloc_resp_light(req); if (unlikely(!resp)) goto err; + mit = &resp->mit; skb_head = &resp->msg.skb_head; - - /* - * Form HTTP/2 response headers excluding "\r\n", ':' separators - * and OWS. - */ - start = TFW_STR_START_CH(msg); - date = TFW_STR_DATE_CH(msg); - pos_dt_nm = start->data + start->len - SLEN(S_F_DATE); - len_dt_nm = SLEN(S_F_DATE) - 2; - pos_dt_val = *this_cpu_ptr(&g_buf); - len_dt_val = date->len; - len += len_dt_nm + len_dt_val; - - clen = TFW_STR_CLEN_CH(msg); - pos_cl_nm = clen->data + SLEN(S_CRLF); - len_cl_nm = SLEN(S_F_CONTENT_LENGTH) - 2; - pos_cl_val = pos_cl_nm + len_cl_nm + 2; - len_cl_val = clen->data + clen->len - pos_cl_val - SLEN(S_CRLF); - len += len_cl_nm + len_cl_val; - - srv = TFW_STR_SRV_CH(msg); - pos_srv_nm = srv->data; - len_srv_nm = SLEN(S_F_SERVER) - 2; - pos_srv_val = pos_srv_nm + len_srv_nm + 2; - len_srv_val = srv->data + srv->len - pos_srv_val - SLEN(S_CRLF); - len += len_srv_nm + len_srv_val; - body = TFW_STR_BODY_CH(msg); - /* Create frame header for HEADERS. */ + /* Create frame header for HEADERS. Note, that we leave the length of + * HEADERS frame unset, to be filled later (below) after all headers + * will be processed, indexed (if needed) and written into the target + * skbs. */ frame_hdr.stream_id = stream_id; - frame_hdr.length = len; + frame_hdr.length = 0; frame_hdr.type = HTTP2_HEADERS; frame_hdr.flags = HTTP2_F_END_HEADERS; if (!body->data) frame_hdr.flags |= HTTP2_F_END_STREAM; tfw_h2_pack_frame_header(buf, &frame_hdr); - /* Set frame header and payload for HEADERS. */ + /* Set frame header and HTTP/2 ':status' pseudo-header. */ if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr)) goto err_setup; - resp->status = status; + resp->status = (unsigned short)status; if (tfw_h2_pseudo_write(resp, TFW_H2_TRANS_EXPAND)) goto err_setup; - __TFW_STR_CH(&hdr, 0)->data = pos_dt_nm; - __TFW_STR_CH(&hdr, 0)->len = len_dt_nm; - tfw_http_prep_date(pos_dt_val); - __TFW_STR_CH(&hdr, 1)->data = pos_dt_val; - __TFW_STR_CH(&hdr, 1)->len = len_dt_val; - hdr.len = len_dt_nm + len_dt_val; + /* + * Form and write HTTP/2 response headers excluding "\r\n", ':' + * separators and OWS. + */ + start = TFW_STR_START_CH(msg); + date = TFW_STR_DATE_CH(msg); + __TFW_STR_CH(&hdr, 0)->data = start->data + start->len - SLEN(S_F_DATE); + __TFW_STR_CH(&hdr, 0)->len = nlen = SLEN(S_F_DATE) - 2; + date_val = *this_cpu_ptr(&g_buf); + tfw_http_prep_date(date_val); + __TFW_STR_CH(&hdr, 1)->data = date_val; + __TFW_STR_CH(&hdr, 1)->len = vlen = date->len; + hdr.len = nlen + vlen; TFW_STR_INDEX_SET(&hdr, 33); if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; - __TFW_STR_CH(&hdr, 0)->data = pos_cl_nm; - __TFW_STR_CH(&hdr, 0)->len = len_cl_nm; - __TFW_STR_CH(&hdr, 1)->data = pos_cl_val; - __TFW_STR_CH(&hdr, 1)->len = len_cl_val; - hdr.len = len_cl_nm + len_cl_val; + clen = TFW_STR_CLEN_CH(msg); + __TFW_STR_CH(&hdr, 0)->data = data_ptr = clen->data + SLEN(S_CRLF); + __TFW_STR_CH(&hdr, 0)->len = nlen = SLEN(S_F_CONTENT_LENGTH) - 2; + __TFW_STR_CH(&hdr, 1)->data = data_ptr = data_ptr + nlen + 2; + __TFW_STR_CH(&hdr, 1)->len = vlen = clen->data + clen->len + - data_ptr - SLEN(S_CRLF); + hdr.len = nlen + vlen; TFW_STR_INDEX_SET(&hdr, 28); if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; - __TFW_STR_CH(&hdr, 0)->data = pos_srv_nm; - __TFW_STR_CH(&hdr, 0)->len = len_srv_nm; - __TFW_STR_CH(&hdr, 1)->data = pos_srv_val; - __TFW_STR_CH(&hdr, 1)->len = len_srv_val; - hdr.len = len_srv_nm + len_srv_val; + srv = TFW_STR_SRV_CH(msg); + __TFW_STR_CH(&hdr, 0)->data = data_ptr = srv->data; + __TFW_STR_CH(&hdr, 0)->len = nlen = SLEN(S_F_SERVER) - 2; + __TFW_STR_CH(&hdr, 1)->data = data_ptr = data_ptr + nlen + 2; + __TFW_STR_CH(&hdr, 1)->len = vlen = srv->data + srv->len + - data_ptr - SLEN(S_CRLF); + hdr.len = nlen + vlen; TFW_STR_INDEX_SET(&hdr, 54); if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; + if (WARN_ON_ONCE(!mit->acc_len)) + goto err_setup; + + /* + * Get the pointer to head skb data to write the frame's accumulated + * length. We can do this here, since we always allocate new skb in + * @tfw_http_msg_expand_data() with SKB_MAX_HEADER length for the + * head part of skb (also, see description of frame header format in + * RFC 7540 section 4.1). + */ + len_be = htonl((unsigned int)mit->acc_len); + src_len_p = (unsigned char *)&len_be; + dst_len_p = (*skb_head)->data; + + *(unsigned short *)dst_len_p = *(unsigned short *)++src_len_p; + src_len_p += 2; + dst_len_p += 2; + *dst_len_p = *src_len_p; + + /* Create and set frame header and set payload for DATA. */ if (body->data) { - /* Create and set frame header and set payload for DATA. */ frame_hdr.length = body->len; frame_hdr.type = HTTP2_DATA; frame_hdr.flags = HTTP2_F_END_STREAM; @@ -916,9 +962,10 @@ tfw_h2_send_resp(TfwHttpReq *req, resp_code_t code, unsigned short status) * the fourth chunk must be CRLF. */ static void -tfw_h1_send_resp(TfwHttpReq *req, resp_code_t code) +tfw_h1_send_resp(TfwHttpReq *req, int status) { TfwMsgIter it; + resp_code_t code; TfwHttpResp *resp; TfwStr *date, *crlf, *body; TfwStr msg = { @@ -927,6 +974,12 @@ tfw_h1_send_resp(TfwHttpReq *req, resp_code_t code) .nchunks = 6 }; + code = tfw_http_enum_resp_code(status); + if (code == RESP_NUM) { + T_WARN("Unexpected response error code: [%d]\n", status); + code = RESP_500; + } + if (tfw_strcpy_desc(&msg, &http_predef_resps[code])) goto err; @@ -1200,48 +1253,6 @@ tfw_http_nip_req_resched_err(TfwSrvConn *srv_conn, TfwHttpReq *req, " re-forwarded or re-scheduled"); } -static inline resp_code_t -tfw_http_enum_resp_code(int status) -{ - switch(status) { - case 200: - return RESP_200; - case 400: - return RESP_400; - case 403: - return RESP_403; - case 404: - return RESP_404; - case 412: - return RESP_412; - case 500: - return RESP_500; - case 502: - return RESP_502; - case 503: - return RESP_503; - case 504: - return RESP_504; - default: - return RESP_NUM; - } -} - -static inline void -tfw_http_error_resp_switch(TfwHttpReq *req, int status) -{ - resp_code_t code = tfw_http_enum_resp_code(status); - if (code == RESP_NUM) { - T_WARN("Unexpected response error code: [%d]\n", status); - code = RESP_500; - } - - if (TFW_MSG_H2(req)) - tfw_h2_send_resp(req, code, (unsigned short)status); - else - tfw_h1_send_resp(req, code); -} - /* Common interface for sending error responses. */ void tfw_http_send_resp(TfwHttpReq *req, int status, const char *reason) @@ -1250,7 +1261,11 @@ tfw_http_send_resp(TfwHttpReq *req, int status, const char *reason) T_WARN_ADDR_STATUS(reason, &req->conn->peer->addr, TFW_WITH_PORT, status); } - tfw_http_error_resp_switch(req, status); + + if (TFW_MSG_H2(req)) + tfw_h2_send_resp(req, status, 0); + else + tfw_h1_send_resp(req, status); } static bool @@ -3913,7 +3928,7 @@ tfw_h2_error_resp(TfwHttpReq *req, int status, bool reply, bool attack, * (RFC 7540 section 6.8) after error response. */ if (reply) { - tfw_http_error_resp_switch(req, status); + tfw_h2_send_resp(req, status, 0); if (attack) tfw_h2_conn_terminate_close(ctx, HTTP2_ECODE_PROTO, !on_req_recv_event); @@ -3975,7 +3990,7 @@ tfw_h1_error_resp(TfwHttpReq *req, int status, bool reply, bool attack, */ if (on_req_recv_event || attack) tfw_http_req_set_conn_close(req); - tfw_http_error_resp_switch(req, status); + tfw_h1_send_resp(req, status); } /* * Serve all pending requests if not under attack, close immediately @@ -4194,8 +4209,11 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) return; clean: tfw_http_conn_msg_free((TfwHttpMsg *)resp); - tfw_http_send_resp(req, 500, - "response dropped: processing error"); + if (!(tfw_blk_flags & TFW_BLK_ERR_NOLOG)) + T_WARN_ADDR_STATUS("response dropped: processing error", + &req->conn->peer->addr, + TFW_WITH_PORT, 500); + tfw_h2_send_resp(req, 500, stream_id); tfw_hpack_enc_release(&ctx->hpack, resp->flags); TFW_INC_STAT_BH(serv.msgs_otherr); diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index a257fa488c..c7e4ba6cf3 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -510,7 +510,7 @@ typedef struct { * @curr_ptr - pointer in the skb to write the current header; * @bnd - pointer to the boundary data (which should not be * overwritten); - * @iter - skb creation/writing iterator; + * @iter - skb expansion iterator; * @acc_len - accumulated length of transformed message. */ typedef struct { From c692adb2ca17a5984d4f09f48fe8220c3f45c575 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 4 Dec 2019 19:00:16 +0500 Subject: [PATCH 10/64] bump copyrights --- tempesta_fw/cache.c | 2 +- tempesta_fw/http_match.c | 2 +- tempesta_fw/http_msg.c | 2 +- tempesta_fw/http_msg.h | 2 +- tempesta_fw/http_sess.c | 4 ++-- tempesta_fw/msg.h | 2 +- tempesta_fw/pool.c | 2 +- tempesta_fw/pool.h | 2 +- tempesta_fw/ss_skb.h | 2 +- tempesta_fw/t/unit/tfw_str_helper.c | 2 +- tempesta_fw/t/unit/tfw_str_helper.h | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tempesta_fw/cache.c b/tempesta_fw/cache.c index dfa8c3a44b..2183245ee9 100644 --- a/tempesta_fw/cache.c +++ b/tempesta_fw/cache.c @@ -4,7 +4,7 @@ * HTTP cache (RFC 7234). * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/http_match.c b/tempesta_fw/http_match.c index a43729006b..eab1541bf7 100644 --- a/tempesta_fw/http_match.c +++ b/tempesta_fw/http_match.c @@ -51,7 +51,7 @@ * - Case-sensitive matching for headers when required by RFC. * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 8b732b2003..974aa13d08 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -4,7 +4,7 @@ * HTTP message manipulation helpers for the protocol processing. * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 9b8f4740fe..9425666144 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -2,7 +2,7 @@ * Tempesta FW * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/http_sess.c b/tempesta_fw/http_sess.c index 10256d18f9..73b92822cb 100644 --- a/tempesta_fw/http_sess.c +++ b/tempesta_fw/http_sess.c @@ -1,4 +1,4 @@ -/* +/** * Tempesta FW * * HTTP session management. @@ -17,7 +17,7 @@ * value can be used to cope the non anonymous forward proxy problem and * identify real clients. * - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/msg.h b/tempesta_fw/msg.h index 13df931e45..936c255e3e 100644 --- a/tempesta_fw/msg.h +++ b/tempesta_fw/msg.h @@ -4,7 +4,7 @@ * Generic protocol message. * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/pool.c b/tempesta_fw/pool.c index fac99f6427..7167486895 100644 --- a/tempesta_fw/pool.c +++ b/tempesta_fw/pool.c @@ -25,7 +25,7 @@ * be immediately freed to keep stack-like memory management. * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/pool.h b/tempesta_fw/pool.h index 550551a0d6..a8186cfb14 100644 --- a/tempesta_fw/pool.h +++ b/tempesta_fw/pool.h @@ -4,7 +4,7 @@ * Memory pool. * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/ss_skb.h b/tempesta_fw/ss_skb.h index 413e255cfc..9fa7da6cfe 100644 --- a/tempesta_fw/ss_skb.h +++ b/tempesta_fw/ss_skb.h @@ -3,7 +3,7 @@ * * Synchronous Sockets API for Linux socket buffers manipulation. * - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/t/unit/tfw_str_helper.c b/tempesta_fw/t/unit/tfw_str_helper.c index 05e720bd19..67ba025f97 100644 --- a/tempesta_fw/t/unit/tfw_str_helper.c +++ b/tempesta_fw/t/unit/tfw_str_helper.c @@ -1,7 +1,7 @@ /** * Tempesta FW * - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/t/unit/tfw_str_helper.h b/tempesta_fw/t/unit/tfw_str_helper.h index bd955c4c6c..be27b808b2 100644 --- a/tempesta_fw/t/unit/tfw_str_helper.h +++ b/tempesta_fw/t/unit/tfw_str_helper.h @@ -1,7 +1,7 @@ /** * Tempesta FW * - * Copyright (C) 2015 Tempesta Technologies, Inc. + * Copyright (C) 2015-2019 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by From dae477a2f977db58e71ce2c4e6f805250514f255 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 4 Dec 2019 19:00:20 +0500 Subject: [PATCH 11/64] Divide TfwStr flags and HPACK index members TfwStrs often go in arrays, and there is a 1-byte alignment gap between TfwStr neighbours. `eolen` is always [0,2], 14-bits are enough for hpack index. --- tempesta_fw/hpack.c | 2 +- tempesta_fw/http.c | 12 ++++++------ tempesta_fw/http_msg.h | 5 ++--- tempesta_fw/http_parser.c | 2 +- tempesta_fw/http_sess.c | 2 +- tempesta_fw/str.h | 17 +++++------------ tempesta_fw/t/unit/test_tfw_str.c | 6 +++--- 7 files changed, 19 insertions(+), 27 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index bf27309119..cf897653a9 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -3952,7 +3952,7 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, if (WARN_ON_ONCE(!hdr || TFW_STR_EMPTY(hdr))) return -EINVAL; - st_index = TFW_STR_INDEX(hdr); + st_index = hdr->hpack_idx; st_full_index = hdr->flags & TFW_STR_FULL_INDEX; T_DBG3("%s: op=%d, st_index=%hu, st_full_index=%d\n", __func__, op, diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 5303625731..1547f3143c 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -751,7 +751,8 @@ tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) { .data = buf, .len = H2_STAT_VAL_LEN } }, .len = SLEN(S_H2_STAT) + H2_STAT_VAL_LEN, - .nchunks = 2 + .nchunks = 2, + .hpack_idx = index ? index : 8 }; WARN_ON_ONCE(op != TFW_H2_TRANS_EXPAND && op != TFW_H2_TRANS_SUB); @@ -760,7 +761,6 @@ tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) * If the status code is not in the static table, set the default * static index just for the ':status' name. */ - TFW_STR_INDEX_SET(&s_hdr, index ? index : 8 ); if (index) { s_hdr.flags |= TFW_STR_FULL_INDEX; } @@ -877,7 +877,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) __TFW_STR_CH(&hdr, 1)->data = date_val; __TFW_STR_CH(&hdr, 1)->len = vlen = date->len; hdr.len = nlen + vlen; - TFW_STR_INDEX_SET(&hdr, 33); + hdr.hpack_idx = 33; if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; @@ -888,7 +888,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) __TFW_STR_CH(&hdr, 1)->len = vlen = clen->data + clen->len - data_ptr - SLEN(S_CRLF); hdr.len = nlen + vlen; - TFW_STR_INDEX_SET(&hdr, 28); + hdr.hpack_idx = 28; if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; @@ -899,7 +899,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) __TFW_STR_CH(&hdr, 1)->len = vlen = srv->data + srv->len - data_ptr - SLEN(S_CRLF); hdr.len = nlen + vlen; - TFW_STR_INDEX_SET(&hdr, 54); + hdr.hpack_idx = 54; if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) goto err_setup; @@ -3439,7 +3439,7 @@ tfw_h2_add_hdr_via(TfwHttpResp *resp) memcpy_fast(__TFW_STR_CH(&via, 2)->data, g_vhost->hdr_via, g_vhost->hdr_via_len); - TFW_STR_INDEX_SET(&via, 60); + via.hpack_idx = 60; r = __hdr_h2_add(resp, &via); if (unlikely(r)) diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 9425666144..1c92be5fa5 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -133,11 +133,10 @@ tfw_h2_msg_hdr_add(TfwHttpResp *resp, char *name, size_t nlen, char *val, { .data = val, .len = vlen }, }, .len = nlen + vlen, - .nchunks = 2 + .nchunks = 2, + .hpack_idx = idx }; - TFW_STR_INDEX_SET(&hdr, idx); - return __hdr_h2_add(resp, &hdr); } diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 8c0ac957b1..f164297dc6 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -87,7 +87,7 @@ do { \ tfw_http_msg_add_str_data(msg, &msg->stream->parser.hdr, data, len) #define __msg_hdr_set_hpack_index(idx) \ - TFW_STR_INDEX_SET(&parser->hdr, idx); + parser->hdr.hpack_idx = idx; /** * GCC 4.8 (CentOS 7) does a poor work on memory reusage of automatic local diff --git a/tempesta_fw/http_sess.c b/tempesta_fw/http_sess.c index 73b92822cb..138fbf54d7 100644 --- a/tempesta_fw/http_sess.c +++ b/tempesta_fw/http_sess.c @@ -457,7 +457,7 @@ tfw_http_sticky_add(TfwHttpResp *resp) PR_TFW_STR(&tfw_cfg_sticky.name), len, buf); if (to_h2) { - TFW_STR_INDEX_SET(&set_cookie, 55); + set_cookie.hpack_idx = 55; r = __hdr_h2_add(resp, &set_cookie); } else { diff --git a/tempesta_fw/str.h b/tempesta_fw/str.h index a0aac746f7..405bc9c847 100644 --- a/tempesta_fw/str.h +++ b/tempesta_fw/str.h @@ -186,8 +186,6 @@ size_t tfw_ultohex(unsigned long ai, char *buf, unsigned int len); * casually from sleepable context, e.g. on configuration phase. * ------------------------------------------------------------------------ */ -#define TFW_STR_FBITS 10 -#define TFW_STR_FMASK ((1U << TFW_STR_FBITS) - 1) #define __TFW_STR_CN_MAX UINT_MAX /* * Str consists from compound or plain strings. @@ -226,11 +224,10 @@ size_t tfw_ultohex(unsigned long ai, char *buf, unsigned int len); * it should be 0 if the string has no EOL at all, 1 for LF and * 2 for CRLF); * @nchunks - number of chunks of compound string; - * @flags - double-byte field for flags; the least significant 10 bits are - * used for common flags' bits, and the most significant 6 bits - * are intended for HPACK static index (in cases when the HTTP + * @flags - double-byte field for flags; + * @hpack_idx - HPACK static index (in cases when the HTTP * header represented in @TfwStr is found in corresponding HPACK - * static table). + * static table). */ typedef struct tfwstr_t { union { @@ -241,7 +238,8 @@ typedef struct tfwstr_t { unsigned long len; unsigned int nchunks; unsigned short flags; - unsigned char eolen; + unsigned short hpack_idx:14; + unsigned short eolen:2; } TfwStr; #define TFW_STR_STRING(val) ((TfwStr){.data = (val), NULL, \ @@ -255,11 +253,6 @@ typedef struct tfwstr_t { /* Use this with "%.*s" in printing calls. */ #define PR_TFW_STR(s) (int)min(20UL, (s)->len), (s)->data -/* Get/set HPACK static index from/into @s. */ -#define TFW_STR_INDEX(s) ((s)->flags >> TFW_STR_FBITS) -#define TFW_STR_INDEX_SET(s, i) ((s)->flags = ((s)->flags & TFW_STR_FMASK) \ - | ((i) << TFW_STR_FBITS)) - #define TFW_STR_INIT(s) memset((s), 0, sizeof(TfwStr)) #define TFW_STR_EMPTY(s) (!((s)->nchunks | (s)->len)) diff --git a/tempesta_fw/t/unit/test_tfw_str.c b/tempesta_fw/t/unit/test_tfw_str.c index 02b2ea4f0f..8267cf341d 100644 --- a/tempesta_fw/t/unit/test_tfw_str.c +++ b/tempesta_fw/t/unit/test_tfw_str.c @@ -1252,7 +1252,7 @@ TEST(tfw_str_collect_cmp, collect_chunks) TfwStr *chunks = in.chunks; TfwStr out = { .data = (void *)123, .skb = (void *)456, .len = 789, - .eolen = 10, .flags = 111, .nchunks = 123}; + .eolen = 3, .flags = 111, .nchunks = 123}; tfw_str_collect_cmp(chunks, chunks + 5, &out, NULL); EXPECT_TRUE(tfw_str_eq_cstr(&out, "abcdefghijklmnopqrstuvwxyz", 26, 0)); @@ -1262,8 +1262,8 @@ TEST(tfw_str_collect_cmp, collect_chunks) * tfw_str_collect_cmp() is expected to clear previous values from all * other fields of the output TfwStr. */ - EXPECT_EQ(out.eolen, 0); - EXPECT_EQ(out.flags & TFW_STR_FMASK, 0); + EXPECT_TRUE(out.eolen == 0); + EXPECT_EQ(out.flags, 0); /* * Try to start at other chunks too. From 8849e8752278c1a19d46153080b9ee19c1fabe0f Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 4 Dec 2019 19:00:21 +0500 Subject: [PATCH 12/64] Fix compile-time issues with 'stack frame non-standard' warnings --- tempesta_fw/http_parser.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index f164297dc6..6204008d75 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -5567,7 +5567,7 @@ __h2_req_parse_if_nmatch(TfwHttpMsg *hm, unsigned char *data, size_t len, done: return r; } -STACK_FRAME_NON_STANDARD(__h2_req_parse_if_none_match); +STACK_FRAME_NON_STANDARD(__h2_req_parse_if_nmatch); static int __h2_req_parse_host(TfwHttpReq *req, unsigned char *data, size_t len, bool fin) @@ -7158,6 +7158,7 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, out: return ret; } +STACK_FRAME_NON_STANDARD(tfw_h2_parse_req_hdr); static int tfw_h2_parse_body(char *data, unsigned long len, TfwHttpReq *req, From ae7ce4065b7f09de2a20c0b166be4849bd12333a Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 4 Dec 2019 19:00:21 +0500 Subject: [PATCH 13/64] Update code comments --- tempesta_fw/http.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 1547f3143c..1c1d4ee6f6 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -739,6 +739,11 @@ tfw_h2_pseudo_index(unsigned short status) } } +/** + * Write HTTP/2 Pseudo-header fields. Only ':status' is defined as response + * pseudo-header and all HTTP/2 responses must contain that. + * https://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.4 + */ static int tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) { @@ -2993,7 +2998,7 @@ do { \ WRITE_LIT(buf, S_DLM); \ val_found = true; \ } else if (*c->data == ':') { \ - /* For statically configured adjustments. */ \ + /* For statically configured adjustments. */\ WARN_ON_ONCE(c->len != SLEN(S_DLM)); \ WRITE_LIT(buf, S_DLM); \ val_found = true; \ From e9bf4864ba156b967b69f0141ccc0d5edec775c7 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 4 Dec 2019 19:00:22 +0500 Subject: [PATCH 14/64] Update Linux kernel patch --- linux-4.14.32.patch | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/linux-4.14.32.patch b/linux-4.14.32.patch index a138730b68..9c02a3d9c6 100644 --- a/linux-4.14.32.patch +++ b/linux-4.14.32.patch @@ -515,10 +515,10 @@ index caeb159a..274c6cc0 100644 struct kvec; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h -index 46bf7cc7..b698c645 100644 +index 46bf7cc7..cc55a4f5 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h -@@ -148,11 +148,22 @@ static inline bool dev_xmit_complete(int rc) +@@ -148,11 +148,32 @@ static inline bool dev_xmit_complete(int rc) # define LL_MAX_HEADER 32 #endif @@ -529,17 +529,27 @@ index 46bf7cc7..b698c645 100644 + * movement on tcp_write_xmit(). Not all skbs have TLS headers - not a big deal + * to allocate 16 more bytes (5 - TLS header, 8 - IV, 3 - alignment). + */ -+#define TLS_MAX_HDR 16 ++#define TLS_MAX_HDR 16 ++/* ++ * For fast transformation of HTTP/1.1 responses into HTTP/2 format, Tempesta ++ * uses zero-copy in-place rewriting of the response data, right in original ++ * skb. HTTP/2 data is almost always smaller of its source HTTP/1.1 data, but ++ * for the sake of robustness we use 32-byte initial offset in front of skb ++ * data. Thus, in order to guarantee the stack headers to fit, we should ++ * increase the total space for them. ++ */ ++#define HTTP2_MAX_OFFSET 32 +#else -+#define TLS_MAX_HDR 0 ++#define TLS_MAX_HDR 0 ++#define HTTP2_MAX_OFFSET 0 +#endif #if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \ !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL) -#define MAX_HEADER LL_MAX_HEADER -+#define MAX_HEADER (LL_MAX_HEADER + TLS_MAX_HDR) ++#define MAX_HEADER (LL_MAX_HEADER + TLS_MAX_HDR + HTTP2_MAX_OFFSET) #else -#define MAX_HEADER (LL_MAX_HEADER + 48) -+#define MAX_HEADER (LL_MAX_HEADER + 48 + TLS_MAX_HDR) ++#define MAX_HEADER (LL_MAX_HEADER + 48 + TLS_MAX_HDR + HTTP2_MAX_OFFSET) #endif /* @@ -892,7 +902,7 @@ index e89c3b0c..7715cad7 100644 restart: /* Reset the pending bitmask before enabling irqs */ set_softirq_pending(0); -@@ -305,6 +310,9 @@ restart: +@@ -305,6 +310,9 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) wakeup_softirqd(); } @@ -1300,7 +1310,7 @@ index 564beb7e..92e9a635 100644 /* * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells * the caller if emergency pfmemalloc reserves are being used. If it is and -@@ -150,6 +153,219 @@ out: +@@ -150,6 +153,219 @@ static void *__kmalloc_reserve(size_t size, gfp_t flags, int node, return obj; } @@ -1520,7 +1530,7 @@ index 564beb7e..92e9a635 100644 /* Allocate a new skbuff. We do this ourselves so we can fill in a few * 'private' fields and also do memory statistics to find all the -@@ -174,11 +390,11 @@ out: +@@ -174,11 +390,11 @@ static void *__kmalloc_reserve(size_t size, gfp_t flags, int node, * Buffers may only be allocated from interrupts using a @gfp_mask of * %GFP_ATOMIC. */ @@ -1573,7 +1583,7 @@ index 564beb7e..92e9a635 100644 out: return skb; nodata: -@@ -251,6 +436,42 @@ nodata: +@@ -251,6 +436,42 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, skb = NULL; goto out; } @@ -1950,7 +1960,7 @@ diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index e7d15fb0..09a21845 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c -@@ -616,7 +616,8 @@ other_parity_scan: +@@ -616,7 +616,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, goto ok; next_port: spin_unlock_bh(&head->lock); @@ -2089,7 +2099,7 @@ diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 14474ace..73325b48 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c -@@ -654,6 +654,7 @@ new_measure: +@@ -654,6 +654,7 @@ void tcp_rcv_space_adjust(struct sock *sk) tp->rcvq_space.seq = tp->copied_seq; tp->rcvq_space.time = tp->tcp_mstamp; } From e8b1f87779ef864a0add472f6177c96f12368cc7 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 4 Dec 2019 19:00:23 +0500 Subject: [PATCH 15/64] Remove unneeded ss_skb_data() function --- tempesta_fw/http.c | 2 +- tempesta_fw/http_msg.c | 2 +- tempesta_fw/http_msg.h | 2 +- tempesta_fw/ss_skb.c | 4 ++-- tempesta_fw/ss_skb.h | 5 ----- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 1c1d4ee6f6..5754dd8af6 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3822,7 +3822,7 @@ tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, tfw_h2_pack_frame_header(buf, &frame_hdr); - head_ptr = ss_skb_data(resp->msg.skb_head); + head_ptr = resp->msg.skb_head->data; memcpy_fast(head_ptr, buf, sizeof(buf)); return 0; diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 974aa13d08..e1a73e46d0 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -1539,7 +1539,7 @@ tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, addr = skb_frag_address(frag); } else { f_size = skb_headlen(it->skb); - addr = ss_skb_data(it->skb); + addr = it->skb->data; } offset = mit->curr_ptr - addr; diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 1c92be5fa5..92b00d7c81 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -117,7 +117,7 @@ tfw_h2_msg_transform_setup(TfwHttpTransIter *mit, struct sk_buff *skb, iter->skb_head = skb; skb_push(skb, HTTP2_MAX_OFFSET); - mit->curr_ptr = ss_skb_data(skb); + mit->curr_ptr = skb->data; if (init) mit->curr_ptr += FRAME_HEADER_SIZE; diff --git a/tempesta_fw/ss_skb.c b/tempesta_fw/ss_skb.c index adadf70acf..b38cdba2d8 100644 --- a/tempesta_fw/ss_skb.c +++ b/tempesta_fw/ss_skb.c @@ -1647,7 +1647,7 @@ ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, addr = skb_frag_address(frag); } else { size = skb_headlen(skb); - addr = ss_skb_data(skb); + addr = skb->data; } offset = curr - addr; @@ -1713,7 +1713,7 @@ ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, frag_num = -1; skb = skb->next; size = skb_headlen(skb); - addr = curr = ss_skb_data(skb); + addr = curr = skb->data; } offset = 0; diff --git a/tempesta_fw/ss_skb.h b/tempesta_fw/ss_skb.h index 9fa7da6cfe..d284aff89c 100644 --- a/tempesta_fw/ss_skb.h +++ b/tempesta_fw/ss_skb.h @@ -164,11 +164,6 @@ ss_skb_alloc(size_t n) return skb; } -static inline char * -ss_skb_data(struct sk_buff *skb) -{ - return skb->data; -} #define SS_SKB_MAX_DATA_LEN (SKB_MAX_HEADER + MAX_SKB_FRAGS * PAGE_SIZE) From 266581d2f281efc85f0585354242c87d42e42aec Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 4 Dec 2019 19:00:24 +0500 Subject: [PATCH 16/64] Minor style fixups --- tempesta_fw/ss_skb.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tempesta_fw/ss_skb.c b/tempesta_fw/ss_skb.c index b38cdba2d8..a2c98cf7d5 100644 --- a/tempesta_fw/ss_skb.c +++ b/tempesta_fw/ss_skb.c @@ -1665,8 +1665,7 @@ ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, * We found the stop pointer; evict the delta between @curr and * @stop, and exit. */ - if (stop_offset >= 0 && stop_offset <= size) - { + if (stop_offset >= 0 && stop_offset <= size) { if (WARN_ON_ONCE(curr > stop)) return -EINVAL; From 45c9ebab42885baa2354ab611a1caeb95626a47b Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 4 Dec 2019 19:00:25 +0500 Subject: [PATCH 17/64] Generate HPACK header on compiletime --- Makefile | 5 +- tempesta_fw/.gitignore | 1 + tempesta_fw/fill_hpack_hdr.pl | 963 ++++++++++++++++++++++++++++++++++ tempesta_fw/hpack.c | 914 +------------------------------- tempesta_fw/hpack_tbl.h.tpl | 60 +++ 5 files changed, 1029 insertions(+), 914 deletions(-) create mode 100644 tempesta_fw/.gitignore create mode 100755 tempesta_fw/fill_hpack_hdr.pl create mode 100644 tempesta_fw/hpack_tbl.h.tpl diff --git a/Makefile b/Makefile index bfb770f2b4..754183a8c6 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,10 @@ obj-m += lib/ tempesta_db/core/ tempesta_fw/ tls/ all: build -build: +generate_headers: + tempesta_fw/fill_hpack_hdr.pl tempesta_fw/hpack_tbl.h.tpl + +build: generate_headers ifdef ERROR $(error $(ERROR)) endif diff --git a/tempesta_fw/.gitignore b/tempesta_fw/.gitignore new file mode 100644 index 0000000000..edd04ab3be --- /dev/null +++ b/tempesta_fw/.gitignore @@ -0,0 +1 @@ +hpack_tbl.h diff --git a/tempesta_fw/fill_hpack_hdr.pl b/tempesta_fw/fill_hpack_hdr.pl new file mode 100755 index 0000000000..4309441d57 --- /dev/null +++ b/tempesta_fw/fill_hpack_hdr.pl @@ -0,0 +1,963 @@ +#!/usr/bin/env perl +# +# Copyright (C) 2019 Tempesta Technologies, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, +# or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. +use 5.16.0; +use strict; +use warnings; +use File::Basename; +use POSIX; +use Template; +use Cwd 'abs_path'; + +#my $table_size; +my ($template) = @ARGV; +if (!$template || $template !~ '.h.tpl$' ) +{ + die "Bad args!\n" +} + +sub generate_table +{ + say "Generate HPACK table"; + + my $table_size = 3392; + my $table = << 'ARRAY_END'; + /* --- [TABLE-128: offset = 0] --- */ + {5, 48}, /* 5: '0' (48) */ + {5, 48}, /* 5: '0' (48) */ + {5, 48}, /* 5: '0' (48) */ + {5, 48}, /* 5: '0' (48) */ + {5, 49}, /* 5: '1' (49) */ + {5, 49}, /* 5: '1' (49) */ + {5, 49}, /* 5: '1' (49) */ + {5, 49}, /* 5: '1' (49) */ + {5, 50}, /* 5: '2' (50) */ + {5, 50}, /* 5: '2' (50) */ + {5, 50}, /* 5: '2' (50) */ + {5, 50}, /* 5: '2' (50) */ + {5, 97}, /* 5: 'a' (97) */ + {5, 97}, /* 5: 'a' (97) */ + {5, 97}, /* 5: 'a' (97) */ + {5, 97}, /* 5: 'a' (97) */ + {5, 99}, /* 5: 'c' (99) */ + {5, 99}, /* 5: 'c' (99) */ + {5, 99}, /* 5: 'c' (99) */ + {5, 99}, /* 5: 'c' (99) */ + {5, 101}, /* 5: 'e' (101) */ + {5, 101}, /* 5: 'e' (101) */ + {5, 101}, /* 5: 'e' (101) */ + {5, 101}, /* 5: 'e' (101) */ + {5, 105}, /* 5: 'i' (105) */ + {5, 105}, /* 5: 'i' (105) */ + {5, 105}, /* 5: 'i' (105) */ + {5, 105}, /* 5: 'i' (105) */ + {5, 111}, /* 5: 'o' (111) */ + {5, 111}, /* 5: 'o' (111) */ + {5, 111}, /* 5: 'o' (111) */ + {5, 111}, /* 5: 'o' (111) */ + {5, 115}, /* 5: 's' (115) */ + {5, 115}, /* 5: 's' (115) */ + {5, 115}, /* 5: 's' (115) */ + {5, 115}, /* 5: 's' (115) */ + {5, 116}, /* 5: 't' (116) */ + {5, 116}, /* 5: 't' (116) */ + {5, 116}, /* 5: 't' (116) */ + {5, 116}, /* 5: 't' (116) */ + {6, 32}, /* 6: ' ' (32) */ + {6, 32}, /* 6: ' ' (32) */ + {6, 37}, /* 6: '%' (37) */ + {6, 37}, /* 6: '%' (37) */ + {6, 45}, /* 6: '-' (45) */ + {6, 45}, /* 6: '-' (45) */ + {6, 46}, /* 6: '.' (46) */ + {6, 46}, /* 6: '.' (46) */ + {6, 47}, /* 6: '/' (47) */ + {6, 47}, /* 6: '/' (47) */ + {6, 51}, /* 6: '3' (51) */ + {6, 51}, /* 6: '3' (51) */ + {6, 52}, /* 6: '4' (52) */ + {6, 52}, /* 6: '4' (52) */ + {6, 53}, /* 6: '5' (53) */ + {6, 53}, /* 6: '5' (53) */ + {6, 54}, /* 6: '6' (54) */ + {6, 54}, /* 6: '6' (54) */ + {6, 55}, /* 6: '7' (55) */ + {6, 55}, /* 6: '7' (55) */ + {6, 56}, /* 6: '8' (56) */ + {6, 56}, /* 6: '8' (56) */ + {6, 57}, /* 6: '9' (57) */ + {6, 57}, /* 6: '9' (57) */ + {6, 61}, /* 6: '=' (61) */ + {6, 61}, /* 6: '=' (61) */ + {6, 65}, /* 6: 'A' (65) */ + {6, 65}, /* 6: 'A' (65) */ + {6, 95}, /* 6: '_' (95) */ + {6, 95}, /* 6: '_' (95) */ + {6, 98}, /* 6: 'b' (98) */ + {6, 98}, /* 6: 'b' (98) */ + {6, 100}, /* 6: 'd' (100) */ + {6, 100}, /* 6: 'd' (100) */ + {6, 102}, /* 6: 'f' (102) */ + {6, 102}, /* 6: 'f' (102) */ + {6, 103}, /* 6: 'g' (103) */ + {6, 103}, /* 6: 'g' (103) */ + {6, 104}, /* 6: 'h' (104) */ + {6, 104}, /* 6: 'h' (104) */ + {6, 108}, /* 6: 'l' (108) */ + {6, 108}, /* 6: 'l' (108) */ + {6, 109}, /* 6: 'm' (109) */ + {6, 109}, /* 6: 'm' (109) */ + {6, 110}, /* 6: 'n' (110) */ + {6, 110}, /* 6: 'n' (110) */ + {6, 112}, /* 6: 'p' (112) */ + {6, 112}, /* 6: 'p' (112) */ + {6, 114}, /* 6: 'r' (114) */ + {6, 114}, /* 6: 'r' (114) */ + {6, 117}, /* 6: 'u' (117) */ + {6, 117}, /* 6: 'u' (117) */ + {7, 58}, /* 7: ':' (58) */ + {7, 66}, /* 7: 'B' (66) */ + {7, 67}, /* 7: 'C' (67) */ + {7, 68}, /* 7: 'D' (68) */ + {7, 69}, /* 7: 'E' (69) */ + {7, 70}, /* 7: 'F' (70) */ + {7, 71}, /* 7: 'G' (71) */ + {7, 72}, /* 7: 'H' (72) */ + {7, 73}, /* 7: 'I' (73) */ + {7, 74}, /* 7: 'J' (74) */ + {7, 75}, /* 7: 'K' (75) */ + {7, 76}, /* 7: 'L' (76) */ + {7, 77}, /* 7: 'M' (77) */ + {7, 78}, /* 7: 'N' (78) */ + {7, 79}, /* 7: 'O' (79) */ + {7, 80}, /* 7: 'P' (80) */ + {7, 81}, /* 7: 'Q' (81) */ + {7, 82}, /* 7: 'R' (82) */ + {7, 83}, /* 7: 'S' (83) */ + {7, 84}, /* 7: 'T' (84) */ + {7, 85}, /* 7: 'U' (85) */ + {7, 86}, /* 7: 'V' (86) */ + {7, 87}, /* 7: 'W' (87) */ + {7, 89}, /* 7: 'Y' (89) */ + {7, 106}, /* 7: 'j' (106) */ + {7, 107}, /* 7: 'k' (107) */ + {7, 113}, /* 7: 'q' (113) */ + {7, 118}, /* 7: 'v' (118) */ + {7, 119}, /* 7: 'w' (119) */ + {7, 120}, /* 7: 'x' (120) */ + {7, 121}, /* 7: 'y' (121) */ + {7, 122}, /* 7: 'z' (122) */ + {-7, 640}, /* 7: ---> TABLE 640 */ + {-7, 648}, /* 7: ---> TABLE 648 */ + {-7, 656}, /* 7: ---> TABLE 656 */ + {-7, 128}, /* 7: ---> TABLE 128 */ +/* --- [TABLE-128: offset = 128] --- */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {5, 35}, /* 5: '#' (35) */ + {5, 35}, /* 5: '#' (35) */ + {5, 35}, /* 5: '#' (35) */ + {5, 35}, /* 5: '#' (35) */ + {5, 62}, /* 5: '>' (62) */ + {5, 62}, /* 5: '>' (62) */ + {5, 62}, /* 5: '>' (62) */ + {5, 62}, /* 5: '>' (62) */ + {6, 0}, /* 6: '\x00' (0) */ + {6, 0}, /* 6: '\x00' (0) */ + {6, 36}, /* 6: '$' (36) */ + {6, 36}, /* 6: '$' (36) */ + {6, 64}, /* 6: '@' (64) */ + {6, 64}, /* 6: '@' (64) */ + {6, 91}, /* 6: '[' (91) */ + {6, 91}, /* 6: '[' (91) */ + {6, 93}, /* 6: ']' (93) */ + {6, 93}, /* 6: ']' (93) */ + {6, 126}, /* 6: '~' (126) */ + {6, 126}, /* 6: '~' (126) */ + {7, 94}, /* 7: '^' (94) */ + {7, 125}, /* 7: '}' (125) */ + {-7, 664}, /* 7: ---> TABLE 664 */ + {-7, 256}, /* 7: ---> TABLE 256 */ +/* --- [TABLE-128: offset = 256] --- */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {5, 92}, /* 5: '\\' (92) */ + {5, 92}, /* 5: '\\' (92) */ + {5, 92}, /* 5: '\\' (92) */ + {5, 92}, /* 5: '\\' (92) */ + {5, -61}, /* 5: '\xC3' (195) */ + {5, -61}, /* 5: '\xC3' (195) */ + {5, -61}, /* 5: '\xC3' (195) */ + {5, -61}, /* 5: '\xC3' (195) */ + {5, -48}, /* 5: '\xD0' (208) */ + {5, -48}, /* 5: '\xD0' (208) */ + {5, -48}, /* 5: '\xD0' (208) */ + {5, -48}, /* 5: '\xD0' (208) */ + {6, -128}, /* 6: '\x80' (128) */ + {6, -128}, /* 6: '\x80' (128) */ + {6, -126}, /* 6: '\x82' (130) */ + {6, -126}, /* 6: '\x82' (130) */ + {6, -125}, /* 6: '\x83' (131) */ + {6, -125}, /* 6: '\x83' (131) */ + {6, -94}, /* 6: '\xA2' (162) */ + {6, -94}, /* 6: '\xA2' (162) */ + {6, -72}, /* 6: '\xB8' (184) */ + {6, -72}, /* 6: '\xB8' (184) */ + {6, -62}, /* 6: '\xC2' (194) */ + {6, -62}, /* 6: '\xC2' (194) */ + {6, -32}, /* 6: '\xE0' (224) */ + {6, -32}, /* 6: '\xE0' (224) */ + {6, -30}, /* 6: '\xE2' (226) */ + {6, -30}, /* 6: '\xE2' (226) */ + {7, -103}, /* 7: '\x99' (153) */ + {7, -95}, /* 7: '\xA1' (161) */ + {7, -89}, /* 7: '\xA7' (167) */ + {7, -84}, /* 7: '\xAC' (172) */ + {7, -80}, /* 7: '\xB0' (176) */ + {7, -79}, /* 7: '\xB1' (177) */ + {7, -77}, /* 7: '\xB3' (179) */ + {7, -47}, /* 7: '\xD1' (209) */ + {7, -40}, /* 7: '\xD8' (216) */ + {7, -39}, /* 7: '\xD9' (217) */ + {7, -29}, /* 7: '\xE3' (227) */ + {7, -27}, /* 7: '\xE5' (229) */ + {7, -26}, /* 7: '\xE6' (230) */ + {-7, 672}, /* 7: ---> TABLE 672 */ + {-7, 680}, /* 7: ---> TABLE 680 */ + {-7, 688}, /* 7: ---> TABLE 688 */ + {-7, 696}, /* 7: ---> TABLE 696 */ + {-7, 704}, /* 7: ---> TABLE 704 */ + {-7, 712}, /* 7: ---> TABLE 712 */ + {-7, 720}, /* 7: ---> TABLE 720 */ + {-7, 728}, /* 7: ---> TABLE 728 */ + {-7, 736}, /* 7: ---> TABLE 736 */ + {-7, 744}, /* 7: ---> TABLE 744 */ + {-7, 752}, /* 7: ---> TABLE 752 */ + {-7, 760}, /* 7: ---> TABLE 760 */ + {-7, 768}, /* 7: ---> TABLE 768 */ + {-7, 776}, /* 7: ---> TABLE 776 */ + {-7, 784}, /* 7: ---> TABLE 784 */ + {-7, 792}, /* 7: ---> TABLE 792 */ + {-7, 800}, /* 7: ---> TABLE 800 */ + {-7, 808}, /* 7: ---> TABLE 808 */ + {-7, 816}, /* 7: ---> TABLE 816 */ + {-7, 824}, /* 7: ---> TABLE 824 */ + {-7, 832}, /* 7: ---> TABLE 832 */ + {-7, 384}, /* 7: ---> TABLE 384 */ + {-7, 512}, /* 7: ---> TABLE 512 */ +/* --- [TABLE-128: offset = 384] --- */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ +/* --- [TABLE-128: offset = 512] --- */ + {5, -64}, /* 5: '\xC0' (192) */ + {5, -64}, /* 5: '\xC0' (192) */ + {5, -64}, /* 5: '\xC0' (192) */ + {5, -64}, /* 5: '\xC0' (192) */ + {5, -63}, /* 5: '\xC1' (193) */ + {5, -63}, /* 5: '\xC1' (193) */ + {5, -63}, /* 5: '\xC1' (193) */ + {5, -63}, /* 5: '\xC1' (193) */ + {5, -56}, /* 5: '\xC8' (200) */ + {5, -56}, /* 5: '\xC8' (200) */ + {5, -56}, /* 5: '\xC8' (200) */ + {5, -56}, /* 5: '\xC8' (200) */ + {5, -55}, /* 5: '\xC9' (201) */ + {5, -55}, /* 5: '\xC9' (201) */ + {5, -55}, /* 5: '\xC9' (201) */ + {5, -55}, /* 5: '\xC9' (201) */ + {5, -54}, /* 5: '\xCA' (202) */ + {5, -54}, /* 5: '\xCA' (202) */ + {5, -54}, /* 5: '\xCA' (202) */ + {5, -54}, /* 5: '\xCA' (202) */ + {5, -51}, /* 5: '\xCD' (205) */ + {5, -51}, /* 5: '\xCD' (205) */ + {5, -51}, /* 5: '\xCD' (205) */ + {5, -51}, /* 5: '\xCD' (205) */ + {5, -46}, /* 5: '\xD2' (210) */ + {5, -46}, /* 5: '\xD2' (210) */ + {5, -46}, /* 5: '\xD2' (210) */ + {5, -46}, /* 5: '\xD2' (210) */ + {5, -43}, /* 5: '\xD5' (213) */ + {5, -43}, /* 5: '\xD5' (213) */ + {5, -43}, /* 5: '\xD5' (213) */ + {5, -43}, /* 5: '\xD5' (213) */ + {5, -38}, /* 5: '\xDA' (218) */ + {5, -38}, /* 5: '\xDA' (218) */ + {5, -38}, /* 5: '\xDA' (218) */ + {5, -38}, /* 5: '\xDA' (218) */ + {5, -37}, /* 5: '\xDB' (219) */ + {5, -37}, /* 5: '\xDB' (219) */ + {5, -37}, /* 5: '\xDB' (219) */ + {5, -37}, /* 5: '\xDB' (219) */ + {5, -18}, /* 5: '\xEE' (238) */ + {5, -18}, /* 5: '\xEE' (238) */ + {5, -18}, /* 5: '\xEE' (238) */ + {5, -18}, /* 5: '\xEE' (238) */ + {5, -16}, /* 5: '\xF0' (240) */ + {5, -16}, /* 5: '\xF0' (240) */ + {5, -16}, /* 5: '\xF0' (240) */ + {5, -16}, /* 5: '\xF0' (240) */ + {5, -14}, /* 5: '\xF2' (242) */ + {5, -14}, /* 5: '\xF2' (242) */ + {5, -14}, /* 5: '\xF2' (242) */ + {5, -14}, /* 5: '\xF2' (242) */ + {5, -13}, /* 5: '\xF3' (243) */ + {5, -13}, /* 5: '\xF3' (243) */ + {5, -13}, /* 5: '\xF3' (243) */ + {5, -13}, /* 5: '\xF3' (243) */ + {5, -1}, /* 5: '\xFF' (255) */ + {5, -1}, /* 5: '\xFF' (255) */ + {5, -1}, /* 5: '\xFF' (255) */ + {5, -1}, /* 5: '\xFF' (255) */ + {6, -53}, /* 6: '\xCB' (203) */ + {6, -53}, /* 6: '\xCB' (203) */ + {6, -52}, /* 6: '\xCC' (204) */ + {6, -52}, /* 6: '\xCC' (204) */ + {6, -45}, /* 6: '\xD3' (211) */ + {6, -45}, /* 6: '\xD3' (211) */ + {6, -44}, /* 6: '\xD4' (212) */ + {6, -44}, /* 6: '\xD4' (212) */ + {6, -42}, /* 6: '\xD6' (214) */ + {6, -42}, /* 6: '\xD6' (214) */ + {6, -35}, /* 6: '\xDD' (221) */ + {6, -35}, /* 6: '\xDD' (221) */ + {6, -34}, /* 6: '\xDE' (222) */ + {6, -34}, /* 6: '\xDE' (222) */ + {6, -33}, /* 6: '\xDF' (223) */ + {6, -33}, /* 6: '\xDF' (223) */ + {6, -15}, /* 6: '\xF1' (241) */ + {6, -15}, /* 6: '\xF1' (241) */ + {6, -12}, /* 6: '\xF4' (244) */ + {6, -12}, /* 6: '\xF4' (244) */ + {6, -11}, /* 6: '\xF5' (245) */ + {6, -11}, /* 6: '\xF5' (245) */ + {6, -10}, /* 6: '\xF6' (246) */ + {6, -10}, /* 6: '\xF6' (246) */ + {6, -9}, /* 6: '\xF7' (247) */ + {6, -9}, /* 6: '\xF7' (247) */ + {6, -8}, /* 6: '\xF8' (248) */ + {6, -8}, /* 6: '\xF8' (248) */ + {6, -6}, /* 6: '\xFA' (250) */ + {6, -6}, /* 6: '\xFA' (250) */ + {6, -5}, /* 6: '\xFB' (251) */ + {6, -5}, /* 6: '\xFB' (251) */ + {6, -4}, /* 6: '\xFC' (252) */ + {6, -4}, /* 6: '\xFC' (252) */ + {6, -3}, /* 6: '\xFD' (253) */ + {6, -3}, /* 6: '\xFD' (253) */ + {6, -2}, /* 6: '\xFE' (254) */ + {6, -2}, /* 6: '\xFE' (254) */ + {7, 2}, /* 7: '\x02' (2) */ + {7, 3}, /* 7: '\x03' (3) */ + {7, 4}, /* 7: '\x04' (4) */ + {7, 5}, /* 7: '\x05' (5) */ + {7, 6}, /* 7: '\x06' (6) */ + {7, 7}, /* 7: '\x07' (7) */ + {7, 8}, /* 7: '\x08' (8) */ + {7, 11}, /* 7: '\x0B' (11) */ + {7, 12}, /* 7: '\x0C' (12) */ + {7, 14}, /* 7: '\x0E' (14) */ + {7, 15}, /* 7: '\x0F' (15) */ + {7, 16}, /* 7: '\x10' (16) */ + {7, 17}, /* 7: '\x11' (17) */ + {7, 18}, /* 7: '\x12' (18) */ + {7, 19}, /* 7: '\x13' (19) */ + {7, 20}, /* 7: '\x14' (20) */ + {7, 21}, /* 7: '\x15' (21) */ + {7, 23}, /* 7: '\x17' (23) */ + {7, 24}, /* 7: '\x18' (24) */ + {7, 25}, /* 7: '\x19' (25) */ + {7, 26}, /* 7: '\x1A' (26) */ + {7, 27}, /* 7: '\x1B' (27) */ + {7, 28}, /* 7: '\x1C' (28) */ + {7, 29}, /* 7: '\x1D' (29) */ + {7, 30}, /* 7: '\x1E' (30) */ + {7, 31}, /* 7: '\x1F' (31) */ + {7, 127}, /* 7: '\x7F' (127) */ + {7, -36}, /* 7: '\xDC' (220) */ + {7, -7}, /* 7: '\xF9' (249) */ + {-7, 840}, /* 7: ---> TABLE 840 */ +/* --- [TABLE-8: offset = 640] --- */ + {5, 38}, /* 1: '&' (38) */ + {5, 38}, /* 1: '&' (38) */ + {5, 38}, /* 1: '&' (38) */ + {5, 38}, /* 1: '&' (38) */ + {5, 42}, /* 1: '*' (42) */ + {5, 42}, /* 1: '*' (42) */ + {5, 42}, /* 1: '*' (42) */ + {5, 42}, /* 1: '*' (42) */ +/* --- [TABLE-8: offset = 648] --- */ + {5, 44}, /* 1: ',' (44) */ + {5, 44}, /* 1: ',' (44) */ + {5, 44}, /* 1: ',' (44) */ + {5, 44}, /* 1: ',' (44) */ + {5, 59}, /* 1: ';' (59) */ + {5, 59}, /* 1: ';' (59) */ + {5, 59}, /* 1: ';' (59) */ + {5, 59}, /* 1: ';' (59) */ +/* --- [TABLE-8: offset = 656] --- */ + {5, 88}, /* 1: 'X' (88) */ + {5, 88}, /* 1: 'X' (88) */ + {5, 88}, /* 1: 'X' (88) */ + {5, 88}, /* 1: 'X' (88) */ + {5, 90}, /* 1: 'Z' (90) */ + {5, 90}, /* 1: 'Z' (90) */ + {5, 90}, /* 1: 'Z' (90) */ + {5, 90}, /* 1: 'Z' (90) */ +/* --- [TABLE-8: offset = 664] --- */ + {5, 60}, /* 1: '<' (60) */ + {5, 60}, /* 1: '<' (60) */ + {5, 60}, /* 1: '<' (60) */ + {5, 60}, /* 1: '<' (60) */ + {5, 96}, /* 1: '`' (96) */ + {5, 96}, /* 1: '`' (96) */ + {5, 96}, /* 1: '`' (96) */ + {5, 96}, /* 1: '`' (96) */ +/* --- [TABLE-8: offset = 672] --- */ + {5, -127}, /* 1: '\x81' (129) */ + {5, -127}, /* 1: '\x81' (129) */ + {5, -127}, /* 1: '\x81' (129) */ + {5, -127}, /* 1: '\x81' (129) */ + {5, -124}, /* 1: '\x84' (132) */ + {5, -124}, /* 1: '\x84' (132) */ + {5, -124}, /* 1: '\x84' (132) */ + {5, -124}, /* 1: '\x84' (132) */ +/* --- [TABLE-8: offset = 680] --- */ + {5, -123}, /* 1: '\x85' (133) */ + {5, -123}, /* 1: '\x85' (133) */ + {5, -123}, /* 1: '\x85' (133) */ + {5, -123}, /* 1: '\x85' (133) */ + {5, -122}, /* 1: '\x86' (134) */ + {5, -122}, /* 1: '\x86' (134) */ + {5, -122}, /* 1: '\x86' (134) */ + {5, -122}, /* 1: '\x86' (134) */ +/* --- [TABLE-8: offset = 688] --- */ + {5, -120}, /* 1: '\x88' (136) */ + {5, -120}, /* 1: '\x88' (136) */ + {5, -120}, /* 1: '\x88' (136) */ + {5, -120}, /* 1: '\x88' (136) */ + {5, -110}, /* 1: '\x92' (146) */ + {5, -110}, /* 1: '\x92' (146) */ + {5, -110}, /* 1: '\x92' (146) */ + {5, -110}, /* 1: '\x92' (146) */ +/* --- [TABLE-8: offset = 696] --- */ + {5, -102}, /* 1: '\x9A' (154) */ + {5, -102}, /* 1: '\x9A' (154) */ + {5, -102}, /* 1: '\x9A' (154) */ + {5, -102}, /* 1: '\x9A' (154) */ + {5, -100}, /* 1: '\x9C' (156) */ + {5, -100}, /* 1: '\x9C' (156) */ + {5, -100}, /* 1: '\x9C' (156) */ + {5, -100}, /* 1: '\x9C' (156) */ +/* --- [TABLE-8: offset = 704] --- */ + {5, -96}, /* 1: '\xA0' (160) */ + {5, -96}, /* 1: '\xA0' (160) */ + {5, -96}, /* 1: '\xA0' (160) */ + {5, -96}, /* 1: '\xA0' (160) */ + {5, -93}, /* 1: '\xA3' (163) */ + {5, -93}, /* 1: '\xA3' (163) */ + {5, -93}, /* 1: '\xA3' (163) */ + {5, -93}, /* 1: '\xA3' (163) */ +/* --- [TABLE-8: offset = 712] --- */ + {5, -92}, /* 1: '\xA4' (164) */ + {5, -92}, /* 1: '\xA4' (164) */ + {5, -92}, /* 1: '\xA4' (164) */ + {5, -92}, /* 1: '\xA4' (164) */ + {5, -87}, /* 1: '\xA9' (169) */ + {5, -87}, /* 1: '\xA9' (169) */ + {5, -87}, /* 1: '\xA9' (169) */ + {5, -87}, /* 1: '\xA9' (169) */ +/* --- [TABLE-8: offset = 720] --- */ + {5, -86}, /* 1: '\xAA' (170) */ + {5, -86}, /* 1: '\xAA' (170) */ + {5, -86}, /* 1: '\xAA' (170) */ + {5, -86}, /* 1: '\xAA' (170) */ + {5, -83}, /* 1: '\xAD' (173) */ + {5, -83}, /* 1: '\xAD' (173) */ + {5, -83}, /* 1: '\xAD' (173) */ + {5, -83}, /* 1: '\xAD' (173) */ +/* --- [TABLE-8: offset = 728] --- */ + {5, -78}, /* 1: '\xB2' (178) */ + {5, -78}, /* 1: '\xB2' (178) */ + {5, -78}, /* 1: '\xB2' (178) */ + {5, -78}, /* 1: '\xB2' (178) */ + {5, -75}, /* 1: '\xB5' (181) */ + {5, -75}, /* 1: '\xB5' (181) */ + {5, -75}, /* 1: '\xB5' (181) */ + {5, -75}, /* 1: '\xB5' (181) */ +/* --- [TABLE-8: offset = 736] --- */ + {5, -71}, /* 1: '\xB9' (185) */ + {5, -71}, /* 1: '\xB9' (185) */ + {5, -71}, /* 1: '\xB9' (185) */ + {5, -71}, /* 1: '\xB9' (185) */ + {5, -70}, /* 1: '\xBA' (186) */ + {5, -70}, /* 1: '\xBA' (186) */ + {5, -70}, /* 1: '\xBA' (186) */ + {5, -70}, /* 1: '\xBA' (186) */ +/* --- [TABLE-8: offset = 744] --- */ + {5, -69}, /* 1: '\xBB' (187) */ + {5, -69}, /* 1: '\xBB' (187) */ + {5, -69}, /* 1: '\xBB' (187) */ + {5, -69}, /* 1: '\xBB' (187) */ + {5, -67}, /* 1: '\xBD' (189) */ + {5, -67}, /* 1: '\xBD' (189) */ + {5, -67}, /* 1: '\xBD' (189) */ + {5, -67}, /* 1: '\xBD' (189) */ +/* --- [TABLE-8: offset = 752] --- */ + {5, -66}, /* 1: '\xBE' (190) */ + {5, -66}, /* 1: '\xBE' (190) */ + {5, -66}, /* 1: '\xBE' (190) */ + {5, -66}, /* 1: '\xBE' (190) */ + {5, -60}, /* 1: '\xC4' (196) */ + {5, -60}, /* 1: '\xC4' (196) */ + {5, -60}, /* 1: '\xC4' (196) */ + {5, -60}, /* 1: '\xC4' (196) */ +/* --- [TABLE-8: offset = 760] --- */ + {5, -58}, /* 1: '\xC6' (198) */ + {5, -58}, /* 1: '\xC6' (198) */ + {5, -58}, /* 1: '\xC6' (198) */ + {5, -58}, /* 1: '\xC6' (198) */ + {5, -28}, /* 1: '\xE4' (228) */ + {5, -28}, /* 1: '\xE4' (228) */ + {5, -28}, /* 1: '\xE4' (228) */ + {5, -28}, /* 1: '\xE4' (228) */ +/* --- [TABLE-8: offset = 768] --- */ + {5, -24}, /* 1: '\xE8' (232) */ + {5, -24}, /* 1: '\xE8' (232) */ + {5, -24}, /* 1: '\xE8' (232) */ + {5, -24}, /* 1: '\xE8' (232) */ + {5, -23}, /* 1: '\xE9' (233) */ + {5, -23}, /* 1: '\xE9' (233) */ + {5, -23}, /* 1: '\xE9' (233) */ + {5, -23}, /* 1: '\xE9' (233) */ +/* --- [TABLE-8: offset = 776] --- */ + {6, 1}, /* 2: '\x01' (1) */ + {6, 1}, /* 2: '\x01' (1) */ + {6, -121}, /* 2: '\x87' (135) */ + {6, -121}, /* 2: '\x87' (135) */ + {6, -119}, /* 2: '\x89' (137) */ + {6, -119}, /* 2: '\x89' (137) */ + {6, -118}, /* 2: '\x8A' (138) */ + {6, -118}, /* 2: '\x8A' (138) */ +/* --- [TABLE-8: offset = 784] --- */ + {6, -117}, /* 2: '\x8B' (139) */ + {6, -117}, /* 2: '\x8B' (139) */ + {6, -116}, /* 2: '\x8C' (140) */ + {6, -116}, /* 2: '\x8C' (140) */ + {6, -115}, /* 2: '\x8D' (141) */ + {6, -115}, /* 2: '\x8D' (141) */ + {6, -113}, /* 2: '\x8F' (143) */ + {6, -113}, /* 2: '\x8F' (143) */ +/* --- [TABLE-8: offset = 792] --- */ + {6, -109}, /* 2: '\x93' (147) */ + {6, -109}, /* 2: '\x93' (147) */ + {6, -107}, /* 2: '\x95' (149) */ + {6, -107}, /* 2: '\x95' (149) */ + {6, -106}, /* 2: '\x96' (150) */ + {6, -106}, /* 2: '\x96' (150) */ + {6, -105}, /* 2: '\x97' (151) */ + {6, -105}, /* 2: '\x97' (151) */ +/* --- [TABLE-8: offset = 800] --- */ + {6, -104}, /* 2: '\x98' (152) */ + {6, -104}, /* 2: '\x98' (152) */ + {6, -101}, /* 2: '\x9B' (155) */ + {6, -101}, /* 2: '\x9B' (155) */ + {6, -99}, /* 2: '\x9D' (157) */ + {6, -99}, /* 2: '\x9D' (157) */ + {6, -98}, /* 2: '\x9E' (158) */ + {6, -98}, /* 2: '\x9E' (158) */ +/* --- [TABLE-8: offset = 808] --- */ + {6, -91}, /* 2: '\xA5' (165) */ + {6, -91}, /* 2: '\xA5' (165) */ + {6, -90}, /* 2: '\xA6' (166) */ + {6, -90}, /* 2: '\xA6' (166) */ + {6, -88}, /* 2: '\xA8' (168) */ + {6, -88}, /* 2: '\xA8' (168) */ + {6, -82}, /* 2: '\xAE' (174) */ + {6, -82}, /* 2: '\xAE' (174) */ +/* --- [TABLE-8: offset = 816] --- */ + {6, -81}, /* 2: '\xAF' (175) */ + {6, -81}, /* 2: '\xAF' (175) */ + {6, -76}, /* 2: '\xB4' (180) */ + {6, -76}, /* 2: '\xB4' (180) */ + {6, -74}, /* 2: '\xB6' (182) */ + {6, -74}, /* 2: '\xB6' (182) */ + {6, -73}, /* 2: '\xB7' (183) */ + {6, -73}, /* 2: '\xB7' (183) */ +/* --- [TABLE-8: offset = 824] --- */ + {6, -68}, /* 2: '\xBC' (188) */ + {6, -68}, /* 2: '\xBC' (188) */ + {6, -65}, /* 2: '\xBF' (191) */ + {6, -65}, /* 2: '\xBF' (191) */ + {6, -59}, /* 2: '\xC5' (197) */ + {6, -59}, /* 2: '\xC5' (197) */ + {6, -25}, /* 2: '\xE7' (231) */ + {6, -25}, /* 2: '\xE7' (231) */ +/* --- [TABLE-8: offset = 832] --- */ + {6, -17}, /* 2: '\xEF' (239) */ + {6, -17}, /* 2: '\xEF' (239) */ + {7, 9}, /* 3: '\x09' (9) */ + {7, -114}, /* 3: '\x8E' (142) */ + {7, -112}, /* 3: '\x90' (144) */ + {7, -111}, /* 3: '\x91' (145) */ + {7, -108}, /* 3: '\x94' (148) */ + {7, -97}, /* 3: '\x9F' (159) */ +/* --- [TABLE-8: offset = 840] --- */ + {6, 10}, /* 2: '\x0A' (10) */ + {6, 10}, /* 2: '\x0A' (10) */ + {6, 13}, /* 2: '\x0D' (13) */ + {6, 13}, /* 2: '\x0D' (13) */ + {6, 22}, /* 2: '\x16' (22) */ + {6, 22}, /* 2: '\x16' (22) */ + {6, 0}, /* 2: EOS */ + {6, 0}, /* 2: EOS */ +ARRAY_END + + return($table, $table_size); +} + +# Assemble HTML templates and minify resulting files. +sub assemble +{ + my ($src) = @_; + my $dir = dirname($src); + $src = basename($src); + (my $dest = $src) =~ s/.tpl$//; + my $t = Template->new({ + INCLUDE_PATH => "$dir", + OUTPUT_PATH => "$dir", + }); + + say "Assemble template $src -> $dest"; + + my ($table, $table_size) = generate_table(); + my $t_vals = { + TABLE_SIZE => $table_size, + TABLE => $table + }; + $t->process($src, $t_vals, $dest) + || die $t->error(), "\n"; +} + + +assemble(abs_path($template)); + +__END__ + +=head1 NAME + +HPACK decoder state machine generator for TempestaFW. + +=head1 SYNOPSIS + +./fill_hpack_hdr.pl FILE + +FILE - Path to template file for the header. File extension must + be '.h.tpl'. + + +=cut diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index cf897653a9..a9e6309e42 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -28,919 +28,7 @@ #include "http_msg.h" #include "hpack.h" -/** - * Huffman decoder state machine: - * |shift| = modulus of the "shift" field always contains - * number of bits in the Huffman-encoded representation, - * which must be taken by decoder on this step. - * NB! for short tables (f.e. 8-entries tables) difference - * in the prefix length (f.e. 7 bits for the big tables - * minus 3 bits for short tables = 4 bits) was pre-added - * to the value of "shift" field (just to speedup the decoder), - * thus true value of the Huffman-encoded prefix, which must - * be taken by decoder on this step is equal to the "shift" - * minus three. - * shift > 0 ---> normal symbol: - * offset = signed char representation of the decoded symbol. - * shift < 0 && offset == 0 ---> EOS. - * |shift| = number of bits in the truncated path. - * shift < 0 && offset > 0 ---> jump to next table: - * offset = offset of the next table. - */ -typedef struct { - short shift; - short offset; -} HTState; - -#define HT_NBITS 7 -#define HT_MBITS 3 -#define HT_NMASK 127 -#define HT_MMASK 7 -#define HT_SMALL 640 -#define HT_EOS_HIGH 0xFF - -/* TODO #309: @ht_decode table should be generated on compile stage .*/ -static const HTState ht_decode[] ____cacheline_aligned = { -/* --- [TABLE-128: offset = 0] --- */ - {5, 48}, /* 5: '0' (48) */ - {5, 48}, /* 5: '0' (48) */ - {5, 48}, /* 5: '0' (48) */ - {5, 48}, /* 5: '0' (48) */ - {5, 49}, /* 5: '1' (49) */ - {5, 49}, /* 5: '1' (49) */ - {5, 49}, /* 5: '1' (49) */ - {5, 49}, /* 5: '1' (49) */ - {5, 50}, /* 5: '2' (50) */ - {5, 50}, /* 5: '2' (50) */ - {5, 50}, /* 5: '2' (50) */ - {5, 50}, /* 5: '2' (50) */ - {5, 97}, /* 5: 'a' (97) */ - {5, 97}, /* 5: 'a' (97) */ - {5, 97}, /* 5: 'a' (97) */ - {5, 97}, /* 5: 'a' (97) */ - {5, 99}, /* 5: 'c' (99) */ - {5, 99}, /* 5: 'c' (99) */ - {5, 99}, /* 5: 'c' (99) */ - {5, 99}, /* 5: 'c' (99) */ - {5, 101}, /* 5: 'e' (101) */ - {5, 101}, /* 5: 'e' (101) */ - {5, 101}, /* 5: 'e' (101) */ - {5, 101}, /* 5: 'e' (101) */ - {5, 105}, /* 5: 'i' (105) */ - {5, 105}, /* 5: 'i' (105) */ - {5, 105}, /* 5: 'i' (105) */ - {5, 105}, /* 5: 'i' (105) */ - {5, 111}, /* 5: 'o' (111) */ - {5, 111}, /* 5: 'o' (111) */ - {5, 111}, /* 5: 'o' (111) */ - {5, 111}, /* 5: 'o' (111) */ - {5, 115}, /* 5: 's' (115) */ - {5, 115}, /* 5: 's' (115) */ - {5, 115}, /* 5: 's' (115) */ - {5, 115}, /* 5: 's' (115) */ - {5, 116}, /* 5: 't' (116) */ - {5, 116}, /* 5: 't' (116) */ - {5, 116}, /* 5: 't' (116) */ - {5, 116}, /* 5: 't' (116) */ - {6, 32}, /* 6: ' ' (32) */ - {6, 32}, /* 6: ' ' (32) */ - {6, 37}, /* 6: '%' (37) */ - {6, 37}, /* 6: '%' (37) */ - {6, 45}, /* 6: '-' (45) */ - {6, 45}, /* 6: '-' (45) */ - {6, 46}, /* 6: '.' (46) */ - {6, 46}, /* 6: '.' (46) */ - {6, 47}, /* 6: '/' (47) */ - {6, 47}, /* 6: '/' (47) */ - {6, 51}, /* 6: '3' (51) */ - {6, 51}, /* 6: '3' (51) */ - {6, 52}, /* 6: '4' (52) */ - {6, 52}, /* 6: '4' (52) */ - {6, 53}, /* 6: '5' (53) */ - {6, 53}, /* 6: '5' (53) */ - {6, 54}, /* 6: '6' (54) */ - {6, 54}, /* 6: '6' (54) */ - {6, 55}, /* 6: '7' (55) */ - {6, 55}, /* 6: '7' (55) */ - {6, 56}, /* 6: '8' (56) */ - {6, 56}, /* 6: '8' (56) */ - {6, 57}, /* 6: '9' (57) */ - {6, 57}, /* 6: '9' (57) */ - {6, 61}, /* 6: '=' (61) */ - {6, 61}, /* 6: '=' (61) */ - {6, 65}, /* 6: 'A' (65) */ - {6, 65}, /* 6: 'A' (65) */ - {6, 95}, /* 6: '_' (95) */ - {6, 95}, /* 6: '_' (95) */ - {6, 98}, /* 6: 'b' (98) */ - {6, 98}, /* 6: 'b' (98) */ - {6, 100}, /* 6: 'd' (100) */ - {6, 100}, /* 6: 'd' (100) */ - {6, 102}, /* 6: 'f' (102) */ - {6, 102}, /* 6: 'f' (102) */ - {6, 103}, /* 6: 'g' (103) */ - {6, 103}, /* 6: 'g' (103) */ - {6, 104}, /* 6: 'h' (104) */ - {6, 104}, /* 6: 'h' (104) */ - {6, 108}, /* 6: 'l' (108) */ - {6, 108}, /* 6: 'l' (108) */ - {6, 109}, /* 6: 'm' (109) */ - {6, 109}, /* 6: 'm' (109) */ - {6, 110}, /* 6: 'n' (110) */ - {6, 110}, /* 6: 'n' (110) */ - {6, 112}, /* 6: 'p' (112) */ - {6, 112}, /* 6: 'p' (112) */ - {6, 114}, /* 6: 'r' (114) */ - {6, 114}, /* 6: 'r' (114) */ - {6, 117}, /* 6: 'u' (117) */ - {6, 117}, /* 6: 'u' (117) */ - {7, 58}, /* 7: ':' (58) */ - {7, 66}, /* 7: 'B' (66) */ - {7, 67}, /* 7: 'C' (67) */ - {7, 68}, /* 7: 'D' (68) */ - {7, 69}, /* 7: 'E' (69) */ - {7, 70}, /* 7: 'F' (70) */ - {7, 71}, /* 7: 'G' (71) */ - {7, 72}, /* 7: 'H' (72) */ - {7, 73}, /* 7: 'I' (73) */ - {7, 74}, /* 7: 'J' (74) */ - {7, 75}, /* 7: 'K' (75) */ - {7, 76}, /* 7: 'L' (76) */ - {7, 77}, /* 7: 'M' (77) */ - {7, 78}, /* 7: 'N' (78) */ - {7, 79}, /* 7: 'O' (79) */ - {7, 80}, /* 7: 'P' (80) */ - {7, 81}, /* 7: 'Q' (81) */ - {7, 82}, /* 7: 'R' (82) */ - {7, 83}, /* 7: 'S' (83) */ - {7, 84}, /* 7: 'T' (84) */ - {7, 85}, /* 7: 'U' (85) */ - {7, 86}, /* 7: 'V' (86) */ - {7, 87}, /* 7: 'W' (87) */ - {7, 89}, /* 7: 'Y' (89) */ - {7, 106}, /* 7: 'j' (106) */ - {7, 107}, /* 7: 'k' (107) */ - {7, 113}, /* 7: 'q' (113) */ - {7, 118}, /* 7: 'v' (118) */ - {7, 119}, /* 7: 'w' (119) */ - {7, 120}, /* 7: 'x' (120) */ - {7, 121}, /* 7: 'y' (121) */ - {7, 122}, /* 7: 'z' (122) */ - {-7, 640}, /* 7: ---> TABLE 640 */ - {-7, 648}, /* 7: ---> TABLE 648 */ - {-7, 656}, /* 7: ---> TABLE 656 */ - {-7, 128}, /* 7: ---> TABLE 128 */ -/* --- [TABLE-128: offset = 128] --- */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {5, 35}, /* 5: '#' (35) */ - {5, 35}, /* 5: '#' (35) */ - {5, 35}, /* 5: '#' (35) */ - {5, 35}, /* 5: '#' (35) */ - {5, 62}, /* 5: '>' (62) */ - {5, 62}, /* 5: '>' (62) */ - {5, 62}, /* 5: '>' (62) */ - {5, 62}, /* 5: '>' (62) */ - {6, 0}, /* 6: '\x00' (0) */ - {6, 0}, /* 6: '\x00' (0) */ - {6, 36}, /* 6: '$' (36) */ - {6, 36}, /* 6: '$' (36) */ - {6, 64}, /* 6: '@' (64) */ - {6, 64}, /* 6: '@' (64) */ - {6, 91}, /* 6: '[' (91) */ - {6, 91}, /* 6: '[' (91) */ - {6, 93}, /* 6: ']' (93) */ - {6, 93}, /* 6: ']' (93) */ - {6, 126}, /* 6: '~' (126) */ - {6, 126}, /* 6: '~' (126) */ - {7, 94}, /* 7: '^' (94) */ - {7, 125}, /* 7: '}' (125) */ - {-7, 664}, /* 7: ---> TABLE 664 */ - {-7, 256}, /* 7: ---> TABLE 256 */ -/* --- [TABLE-128: offset = 256] --- */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {5, 92}, /* 5: '\\' (92) */ - {5, 92}, /* 5: '\\' (92) */ - {5, 92}, /* 5: '\\' (92) */ - {5, 92}, /* 5: '\\' (92) */ - {5, -61}, /* 5: '\xC3' (195) */ - {5, -61}, /* 5: '\xC3' (195) */ - {5, -61}, /* 5: '\xC3' (195) */ - {5, -61}, /* 5: '\xC3' (195) */ - {5, -48}, /* 5: '\xD0' (208) */ - {5, -48}, /* 5: '\xD0' (208) */ - {5, -48}, /* 5: '\xD0' (208) */ - {5, -48}, /* 5: '\xD0' (208) */ - {6, -128}, /* 6: '\x80' (128) */ - {6, -128}, /* 6: '\x80' (128) */ - {6, -126}, /* 6: '\x82' (130) */ - {6, -126}, /* 6: '\x82' (130) */ - {6, -125}, /* 6: '\x83' (131) */ - {6, -125}, /* 6: '\x83' (131) */ - {6, -94}, /* 6: '\xA2' (162) */ - {6, -94}, /* 6: '\xA2' (162) */ - {6, -72}, /* 6: '\xB8' (184) */ - {6, -72}, /* 6: '\xB8' (184) */ - {6, -62}, /* 6: '\xC2' (194) */ - {6, -62}, /* 6: '\xC2' (194) */ - {6, -32}, /* 6: '\xE0' (224) */ - {6, -32}, /* 6: '\xE0' (224) */ - {6, -30}, /* 6: '\xE2' (226) */ - {6, -30}, /* 6: '\xE2' (226) */ - {7, -103}, /* 7: '\x99' (153) */ - {7, -95}, /* 7: '\xA1' (161) */ - {7, -89}, /* 7: '\xA7' (167) */ - {7, -84}, /* 7: '\xAC' (172) */ - {7, -80}, /* 7: '\xB0' (176) */ - {7, -79}, /* 7: '\xB1' (177) */ - {7, -77}, /* 7: '\xB3' (179) */ - {7, -47}, /* 7: '\xD1' (209) */ - {7, -40}, /* 7: '\xD8' (216) */ - {7, -39}, /* 7: '\xD9' (217) */ - {7, -29}, /* 7: '\xE3' (227) */ - {7, -27}, /* 7: '\xE5' (229) */ - {7, -26}, /* 7: '\xE6' (230) */ - {-7, 672}, /* 7: ---> TABLE 672 */ - {-7, 680}, /* 7: ---> TABLE 680 */ - {-7, 688}, /* 7: ---> TABLE 688 */ - {-7, 696}, /* 7: ---> TABLE 696 */ - {-7, 704}, /* 7: ---> TABLE 704 */ - {-7, 712}, /* 7: ---> TABLE 712 */ - {-7, 720}, /* 7: ---> TABLE 720 */ - {-7, 728}, /* 7: ---> TABLE 728 */ - {-7, 736}, /* 7: ---> TABLE 736 */ - {-7, 744}, /* 7: ---> TABLE 744 */ - {-7, 752}, /* 7: ---> TABLE 752 */ - {-7, 760}, /* 7: ---> TABLE 760 */ - {-7, 768}, /* 7: ---> TABLE 768 */ - {-7, 776}, /* 7: ---> TABLE 776 */ - {-7, 784}, /* 7: ---> TABLE 784 */ - {-7, 792}, /* 7: ---> TABLE 792 */ - {-7, 800}, /* 7: ---> TABLE 800 */ - {-7, 808}, /* 7: ---> TABLE 808 */ - {-7, 816}, /* 7: ---> TABLE 816 */ - {-7, 824}, /* 7: ---> TABLE 824 */ - {-7, 832}, /* 7: ---> TABLE 832 */ - {-7, 384}, /* 7: ---> TABLE 384 */ - {-7, 512}, /* 7: ---> TABLE 512 */ -/* --- [TABLE-128: offset = 384] --- */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ -/* --- [TABLE-128: offset = 512] --- */ - {5, -64}, /* 5: '\xC0' (192) */ - {5, -64}, /* 5: '\xC0' (192) */ - {5, -64}, /* 5: '\xC0' (192) */ - {5, -64}, /* 5: '\xC0' (192) */ - {5, -63}, /* 5: '\xC1' (193) */ - {5, -63}, /* 5: '\xC1' (193) */ - {5, -63}, /* 5: '\xC1' (193) */ - {5, -63}, /* 5: '\xC1' (193) */ - {5, -56}, /* 5: '\xC8' (200) */ - {5, -56}, /* 5: '\xC8' (200) */ - {5, -56}, /* 5: '\xC8' (200) */ - {5, -56}, /* 5: '\xC8' (200) */ - {5, -55}, /* 5: '\xC9' (201) */ - {5, -55}, /* 5: '\xC9' (201) */ - {5, -55}, /* 5: '\xC9' (201) */ - {5, -55}, /* 5: '\xC9' (201) */ - {5, -54}, /* 5: '\xCA' (202) */ - {5, -54}, /* 5: '\xCA' (202) */ - {5, -54}, /* 5: '\xCA' (202) */ - {5, -54}, /* 5: '\xCA' (202) */ - {5, -51}, /* 5: '\xCD' (205) */ - {5, -51}, /* 5: '\xCD' (205) */ - {5, -51}, /* 5: '\xCD' (205) */ - {5, -51}, /* 5: '\xCD' (205) */ - {5, -46}, /* 5: '\xD2' (210) */ - {5, -46}, /* 5: '\xD2' (210) */ - {5, -46}, /* 5: '\xD2' (210) */ - {5, -46}, /* 5: '\xD2' (210) */ - {5, -43}, /* 5: '\xD5' (213) */ - {5, -43}, /* 5: '\xD5' (213) */ - {5, -43}, /* 5: '\xD5' (213) */ - {5, -43}, /* 5: '\xD5' (213) */ - {5, -38}, /* 5: '\xDA' (218) */ - {5, -38}, /* 5: '\xDA' (218) */ - {5, -38}, /* 5: '\xDA' (218) */ - {5, -38}, /* 5: '\xDA' (218) */ - {5, -37}, /* 5: '\xDB' (219) */ - {5, -37}, /* 5: '\xDB' (219) */ - {5, -37}, /* 5: '\xDB' (219) */ - {5, -37}, /* 5: '\xDB' (219) */ - {5, -18}, /* 5: '\xEE' (238) */ - {5, -18}, /* 5: '\xEE' (238) */ - {5, -18}, /* 5: '\xEE' (238) */ - {5, -18}, /* 5: '\xEE' (238) */ - {5, -16}, /* 5: '\xF0' (240) */ - {5, -16}, /* 5: '\xF0' (240) */ - {5, -16}, /* 5: '\xF0' (240) */ - {5, -16}, /* 5: '\xF0' (240) */ - {5, -14}, /* 5: '\xF2' (242) */ - {5, -14}, /* 5: '\xF2' (242) */ - {5, -14}, /* 5: '\xF2' (242) */ - {5, -14}, /* 5: '\xF2' (242) */ - {5, -13}, /* 5: '\xF3' (243) */ - {5, -13}, /* 5: '\xF3' (243) */ - {5, -13}, /* 5: '\xF3' (243) */ - {5, -13}, /* 5: '\xF3' (243) */ - {5, -1}, /* 5: '\xFF' (255) */ - {5, -1}, /* 5: '\xFF' (255) */ - {5, -1}, /* 5: '\xFF' (255) */ - {5, -1}, /* 5: '\xFF' (255) */ - {6, -53}, /* 6: '\xCB' (203) */ - {6, -53}, /* 6: '\xCB' (203) */ - {6, -52}, /* 6: '\xCC' (204) */ - {6, -52}, /* 6: '\xCC' (204) */ - {6, -45}, /* 6: '\xD3' (211) */ - {6, -45}, /* 6: '\xD3' (211) */ - {6, -44}, /* 6: '\xD4' (212) */ - {6, -44}, /* 6: '\xD4' (212) */ - {6, -42}, /* 6: '\xD6' (214) */ - {6, -42}, /* 6: '\xD6' (214) */ - {6, -35}, /* 6: '\xDD' (221) */ - {6, -35}, /* 6: '\xDD' (221) */ - {6, -34}, /* 6: '\xDE' (222) */ - {6, -34}, /* 6: '\xDE' (222) */ - {6, -33}, /* 6: '\xDF' (223) */ - {6, -33}, /* 6: '\xDF' (223) */ - {6, -15}, /* 6: '\xF1' (241) */ - {6, -15}, /* 6: '\xF1' (241) */ - {6, -12}, /* 6: '\xF4' (244) */ - {6, -12}, /* 6: '\xF4' (244) */ - {6, -11}, /* 6: '\xF5' (245) */ - {6, -11}, /* 6: '\xF5' (245) */ - {6, -10}, /* 6: '\xF6' (246) */ - {6, -10}, /* 6: '\xF6' (246) */ - {6, -9}, /* 6: '\xF7' (247) */ - {6, -9}, /* 6: '\xF7' (247) */ - {6, -8}, /* 6: '\xF8' (248) */ - {6, -8}, /* 6: '\xF8' (248) */ - {6, -6}, /* 6: '\xFA' (250) */ - {6, -6}, /* 6: '\xFA' (250) */ - {6, -5}, /* 6: '\xFB' (251) */ - {6, -5}, /* 6: '\xFB' (251) */ - {6, -4}, /* 6: '\xFC' (252) */ - {6, -4}, /* 6: '\xFC' (252) */ - {6, -3}, /* 6: '\xFD' (253) */ - {6, -3}, /* 6: '\xFD' (253) */ - {6, -2}, /* 6: '\xFE' (254) */ - {6, -2}, /* 6: '\xFE' (254) */ - {7, 2}, /* 7: '\x02' (2) */ - {7, 3}, /* 7: '\x03' (3) */ - {7, 4}, /* 7: '\x04' (4) */ - {7, 5}, /* 7: '\x05' (5) */ - {7, 6}, /* 7: '\x06' (6) */ - {7, 7}, /* 7: '\x07' (7) */ - {7, 8}, /* 7: '\x08' (8) */ - {7, 11}, /* 7: '\x0B' (11) */ - {7, 12}, /* 7: '\x0C' (12) */ - {7, 14}, /* 7: '\x0E' (14) */ - {7, 15}, /* 7: '\x0F' (15) */ - {7, 16}, /* 7: '\x10' (16) */ - {7, 17}, /* 7: '\x11' (17) */ - {7, 18}, /* 7: '\x12' (18) */ - {7, 19}, /* 7: '\x13' (19) */ - {7, 20}, /* 7: '\x14' (20) */ - {7, 21}, /* 7: '\x15' (21) */ - {7, 23}, /* 7: '\x17' (23) */ - {7, 24}, /* 7: '\x18' (24) */ - {7, 25}, /* 7: '\x19' (25) */ - {7, 26}, /* 7: '\x1A' (26) */ - {7, 27}, /* 7: '\x1B' (27) */ - {7, 28}, /* 7: '\x1C' (28) */ - {7, 29}, /* 7: '\x1D' (29) */ - {7, 30}, /* 7: '\x1E' (30) */ - {7, 31}, /* 7: '\x1F' (31) */ - {7, 127}, /* 7: '\x7F' (127) */ - {7, -36}, /* 7: '\xDC' (220) */ - {7, -7}, /* 7: '\xF9' (249) */ - {-7, 840}, /* 7: ---> TABLE 840 */ -/* --- [TABLE-8: offset = 640] --- */ - {5, 38}, /* 1: '&' (38) */ - {5, 38}, /* 1: '&' (38) */ - {5, 38}, /* 1: '&' (38) */ - {5, 38}, /* 1: '&' (38) */ - {5, 42}, /* 1: '*' (42) */ - {5, 42}, /* 1: '*' (42) */ - {5, 42}, /* 1: '*' (42) */ - {5, 42}, /* 1: '*' (42) */ -/* --- [TABLE-8: offset = 648] --- */ - {5, 44}, /* 1: ',' (44) */ - {5, 44}, /* 1: ',' (44) */ - {5, 44}, /* 1: ',' (44) */ - {5, 44}, /* 1: ',' (44) */ - {5, 59}, /* 1: ';' (59) */ - {5, 59}, /* 1: ';' (59) */ - {5, 59}, /* 1: ';' (59) */ - {5, 59}, /* 1: ';' (59) */ -/* --- [TABLE-8: offset = 656] --- */ - {5, 88}, /* 1: 'X' (88) */ - {5, 88}, /* 1: 'X' (88) */ - {5, 88}, /* 1: 'X' (88) */ - {5, 88}, /* 1: 'X' (88) */ - {5, 90}, /* 1: 'Z' (90) */ - {5, 90}, /* 1: 'Z' (90) */ - {5, 90}, /* 1: 'Z' (90) */ - {5, 90}, /* 1: 'Z' (90) */ -/* --- [TABLE-8: offset = 664] --- */ - {5, 60}, /* 1: '<' (60) */ - {5, 60}, /* 1: '<' (60) */ - {5, 60}, /* 1: '<' (60) */ - {5, 60}, /* 1: '<' (60) */ - {5, 96}, /* 1: '`' (96) */ - {5, 96}, /* 1: '`' (96) */ - {5, 96}, /* 1: '`' (96) */ - {5, 96}, /* 1: '`' (96) */ -/* --- [TABLE-8: offset = 672] --- */ - {5, -127}, /* 1: '\x81' (129) */ - {5, -127}, /* 1: '\x81' (129) */ - {5, -127}, /* 1: '\x81' (129) */ - {5, -127}, /* 1: '\x81' (129) */ - {5, -124}, /* 1: '\x84' (132) */ - {5, -124}, /* 1: '\x84' (132) */ - {5, -124}, /* 1: '\x84' (132) */ - {5, -124}, /* 1: '\x84' (132) */ -/* --- [TABLE-8: offset = 680] --- */ - {5, -123}, /* 1: '\x85' (133) */ - {5, -123}, /* 1: '\x85' (133) */ - {5, -123}, /* 1: '\x85' (133) */ - {5, -123}, /* 1: '\x85' (133) */ - {5, -122}, /* 1: '\x86' (134) */ - {5, -122}, /* 1: '\x86' (134) */ - {5, -122}, /* 1: '\x86' (134) */ - {5, -122}, /* 1: '\x86' (134) */ -/* --- [TABLE-8: offset = 688] --- */ - {5, -120}, /* 1: '\x88' (136) */ - {5, -120}, /* 1: '\x88' (136) */ - {5, -120}, /* 1: '\x88' (136) */ - {5, -120}, /* 1: '\x88' (136) */ - {5, -110}, /* 1: '\x92' (146) */ - {5, -110}, /* 1: '\x92' (146) */ - {5, -110}, /* 1: '\x92' (146) */ - {5, -110}, /* 1: '\x92' (146) */ -/* --- [TABLE-8: offset = 696] --- */ - {5, -102}, /* 1: '\x9A' (154) */ - {5, -102}, /* 1: '\x9A' (154) */ - {5, -102}, /* 1: '\x9A' (154) */ - {5, -102}, /* 1: '\x9A' (154) */ - {5, -100}, /* 1: '\x9C' (156) */ - {5, -100}, /* 1: '\x9C' (156) */ - {5, -100}, /* 1: '\x9C' (156) */ - {5, -100}, /* 1: '\x9C' (156) */ -/* --- [TABLE-8: offset = 704] --- */ - {5, -96}, /* 1: '\xA0' (160) */ - {5, -96}, /* 1: '\xA0' (160) */ - {5, -96}, /* 1: '\xA0' (160) */ - {5, -96}, /* 1: '\xA0' (160) */ - {5, -93}, /* 1: '\xA3' (163) */ - {5, -93}, /* 1: '\xA3' (163) */ - {5, -93}, /* 1: '\xA3' (163) */ - {5, -93}, /* 1: '\xA3' (163) */ -/* --- [TABLE-8: offset = 712] --- */ - {5, -92}, /* 1: '\xA4' (164) */ - {5, -92}, /* 1: '\xA4' (164) */ - {5, -92}, /* 1: '\xA4' (164) */ - {5, -92}, /* 1: '\xA4' (164) */ - {5, -87}, /* 1: '\xA9' (169) */ - {5, -87}, /* 1: '\xA9' (169) */ - {5, -87}, /* 1: '\xA9' (169) */ - {5, -87}, /* 1: '\xA9' (169) */ -/* --- [TABLE-8: offset = 720] --- */ - {5, -86}, /* 1: '\xAA' (170) */ - {5, -86}, /* 1: '\xAA' (170) */ - {5, -86}, /* 1: '\xAA' (170) */ - {5, -86}, /* 1: '\xAA' (170) */ - {5, -83}, /* 1: '\xAD' (173) */ - {5, -83}, /* 1: '\xAD' (173) */ - {5, -83}, /* 1: '\xAD' (173) */ - {5, -83}, /* 1: '\xAD' (173) */ -/* --- [TABLE-8: offset = 728] --- */ - {5, -78}, /* 1: '\xB2' (178) */ - {5, -78}, /* 1: '\xB2' (178) */ - {5, -78}, /* 1: '\xB2' (178) */ - {5, -78}, /* 1: '\xB2' (178) */ - {5, -75}, /* 1: '\xB5' (181) */ - {5, -75}, /* 1: '\xB5' (181) */ - {5, -75}, /* 1: '\xB5' (181) */ - {5, -75}, /* 1: '\xB5' (181) */ -/* --- [TABLE-8: offset = 736] --- */ - {5, -71}, /* 1: '\xB9' (185) */ - {5, -71}, /* 1: '\xB9' (185) */ - {5, -71}, /* 1: '\xB9' (185) */ - {5, -71}, /* 1: '\xB9' (185) */ - {5, -70}, /* 1: '\xBA' (186) */ - {5, -70}, /* 1: '\xBA' (186) */ - {5, -70}, /* 1: '\xBA' (186) */ - {5, -70}, /* 1: '\xBA' (186) */ -/* --- [TABLE-8: offset = 744] --- */ - {5, -69}, /* 1: '\xBB' (187) */ - {5, -69}, /* 1: '\xBB' (187) */ - {5, -69}, /* 1: '\xBB' (187) */ - {5, -69}, /* 1: '\xBB' (187) */ - {5, -67}, /* 1: '\xBD' (189) */ - {5, -67}, /* 1: '\xBD' (189) */ - {5, -67}, /* 1: '\xBD' (189) */ - {5, -67}, /* 1: '\xBD' (189) */ -/* --- [TABLE-8: offset = 752] --- */ - {5, -66}, /* 1: '\xBE' (190) */ - {5, -66}, /* 1: '\xBE' (190) */ - {5, -66}, /* 1: '\xBE' (190) */ - {5, -66}, /* 1: '\xBE' (190) */ - {5, -60}, /* 1: '\xC4' (196) */ - {5, -60}, /* 1: '\xC4' (196) */ - {5, -60}, /* 1: '\xC4' (196) */ - {5, -60}, /* 1: '\xC4' (196) */ -/* --- [TABLE-8: offset = 760] --- */ - {5, -58}, /* 1: '\xC6' (198) */ - {5, -58}, /* 1: '\xC6' (198) */ - {5, -58}, /* 1: '\xC6' (198) */ - {5, -58}, /* 1: '\xC6' (198) */ - {5, -28}, /* 1: '\xE4' (228) */ - {5, -28}, /* 1: '\xE4' (228) */ - {5, -28}, /* 1: '\xE4' (228) */ - {5, -28}, /* 1: '\xE4' (228) */ -/* --- [TABLE-8: offset = 768] --- */ - {5, -24}, /* 1: '\xE8' (232) */ - {5, -24}, /* 1: '\xE8' (232) */ - {5, -24}, /* 1: '\xE8' (232) */ - {5, -24}, /* 1: '\xE8' (232) */ - {5, -23}, /* 1: '\xE9' (233) */ - {5, -23}, /* 1: '\xE9' (233) */ - {5, -23}, /* 1: '\xE9' (233) */ - {5, -23}, /* 1: '\xE9' (233) */ -/* --- [TABLE-8: offset = 776] --- */ - {6, 1}, /* 2: '\x01' (1) */ - {6, 1}, /* 2: '\x01' (1) */ - {6, -121}, /* 2: '\x87' (135) */ - {6, -121}, /* 2: '\x87' (135) */ - {6, -119}, /* 2: '\x89' (137) */ - {6, -119}, /* 2: '\x89' (137) */ - {6, -118}, /* 2: '\x8A' (138) */ - {6, -118}, /* 2: '\x8A' (138) */ -/* --- [TABLE-8: offset = 784] --- */ - {6, -117}, /* 2: '\x8B' (139) */ - {6, -117}, /* 2: '\x8B' (139) */ - {6, -116}, /* 2: '\x8C' (140) */ - {6, -116}, /* 2: '\x8C' (140) */ - {6, -115}, /* 2: '\x8D' (141) */ - {6, -115}, /* 2: '\x8D' (141) */ - {6, -113}, /* 2: '\x8F' (143) */ - {6, -113}, /* 2: '\x8F' (143) */ -/* --- [TABLE-8: offset = 792] --- */ - {6, -109}, /* 2: '\x93' (147) */ - {6, -109}, /* 2: '\x93' (147) */ - {6, -107}, /* 2: '\x95' (149) */ - {6, -107}, /* 2: '\x95' (149) */ - {6, -106}, /* 2: '\x96' (150) */ - {6, -106}, /* 2: '\x96' (150) */ - {6, -105}, /* 2: '\x97' (151) */ - {6, -105}, /* 2: '\x97' (151) */ -/* --- [TABLE-8: offset = 800] --- */ - {6, -104}, /* 2: '\x98' (152) */ - {6, -104}, /* 2: '\x98' (152) */ - {6, -101}, /* 2: '\x9B' (155) */ - {6, -101}, /* 2: '\x9B' (155) */ - {6, -99}, /* 2: '\x9D' (157) */ - {6, -99}, /* 2: '\x9D' (157) */ - {6, -98}, /* 2: '\x9E' (158) */ - {6, -98}, /* 2: '\x9E' (158) */ -/* --- [TABLE-8: offset = 808] --- */ - {6, -91}, /* 2: '\xA5' (165) */ - {6, -91}, /* 2: '\xA5' (165) */ - {6, -90}, /* 2: '\xA6' (166) */ - {6, -90}, /* 2: '\xA6' (166) */ - {6, -88}, /* 2: '\xA8' (168) */ - {6, -88}, /* 2: '\xA8' (168) */ - {6, -82}, /* 2: '\xAE' (174) */ - {6, -82}, /* 2: '\xAE' (174) */ -/* --- [TABLE-8: offset = 816] --- */ - {6, -81}, /* 2: '\xAF' (175) */ - {6, -81}, /* 2: '\xAF' (175) */ - {6, -76}, /* 2: '\xB4' (180) */ - {6, -76}, /* 2: '\xB4' (180) */ - {6, -74}, /* 2: '\xB6' (182) */ - {6, -74}, /* 2: '\xB6' (182) */ - {6, -73}, /* 2: '\xB7' (183) */ - {6, -73}, /* 2: '\xB7' (183) */ -/* --- [TABLE-8: offset = 824] --- */ - {6, -68}, /* 2: '\xBC' (188) */ - {6, -68}, /* 2: '\xBC' (188) */ - {6, -65}, /* 2: '\xBF' (191) */ - {6, -65}, /* 2: '\xBF' (191) */ - {6, -59}, /* 2: '\xC5' (197) */ - {6, -59}, /* 2: '\xC5' (197) */ - {6, -25}, /* 2: '\xE7' (231) */ - {6, -25}, /* 2: '\xE7' (231) */ -/* --- [TABLE-8: offset = 832] --- */ - {6, -17}, /* 2: '\xEF' (239) */ - {6, -17}, /* 2: '\xEF' (239) */ - {7, 9}, /* 3: '\x09' (9) */ - {7, -114}, /* 3: '\x8E' (142) */ - {7, -112}, /* 3: '\x90' (144) */ - {7, -111}, /* 3: '\x91' (145) */ - {7, -108}, /* 3: '\x94' (148) */ - {7, -97}, /* 3: '\x9F' (159) */ -/* --- [TABLE-8: offset = 840] --- */ - {6, 10}, /* 2: '\x0A' (10) */ - {6, 10}, /* 2: '\x0A' (10) */ - {6, 13}, /* 2: '\x0D' (13) */ - {6, 13}, /* 2: '\x0D' (13) */ - {6, 22}, /* 2: '\x16' (22) */ - {6, 22}, /* 2: '\x16' (22) */ - {6, 0}, /* 2: EOS */ - {6, 0}, /* 2: EOS */ -}; +#include "hpack_tbl.h" #define HP_HDR_NAME(name) \ (&(TfwStr){ \ diff --git a/tempesta_fw/hpack_tbl.h.tpl b/tempesta_fw/hpack_tbl.h.tpl new file mode 100644 index 0000000000..4ac4079150 --- /dev/null +++ b/tempesta_fw/hpack_tbl.h.tpl @@ -0,0 +1,60 @@ +/** + * Tempesta FW + * + * Copyright (C) 2019 Tempesta Technologies, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifndef __TFW_HTTP_HPACK_TBL_H__ +#define __TFW_HTTP_HPACK_TBL_H__ + +/* This file is auto generated, please do not edit it... */ + +/** + * Huffman decoder state machine: + * |shift| = modulus of the "shift" field always contains + * number of bits in the Huffman-encoded representation, + * which must be taken by decoder on this step. + * NB! for short tables (f.e. 8-entries tables) difference + * in the prefix length (f.e. 7 bits for the big tables + * minus 3 bits for short tables = 4 bits) was pre-added + * to the value of "shift" field (just to speedup the decoder), + * thus true value of the Huffman-encoded prefix, which must + * be taken by decoder on this step is equal to the "shift" + * minus three. + * shift > 0 ---> normal symbol: + * offset = signed char representation of the decoded symbol. + * shift < 0 && offset == 0 ---> EOS. + * |shift| = number of bits in the truncated path. + * shift < 0 && offset > 0 ---> jump to next table: + * offset = offset of the next table. + */ +typedef struct { + short shift; + short offset; +} HTState; + +#define HT_NBITS 7 +#define HT_MBITS 3 +#define HT_NMASK 127 +#define HT_MMASK 7 +#define HT_SMALL 640 +#define HT_EOS_HIGH 0xFF + +static const HTState ht_decode[ [% TABLE_SIZE %] ] ____cacheline_aligned = { + [% TABLE %] +}; + +#endif /* __TFW_HTTP_HPACK_TBL_H__ */ From fd48c6e283c2276a0158f25032a7f2574bc36d2e Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 18/64] Resolve merge issues --- tempesta_fw/http_parser.c | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index e96d7600f0..c7b97be9c6 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -3761,7 +3761,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && *(p + 10) == ':')) { parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_OWS, 14); + __FSM_MOVE_n(RGen_LWS, 14); } if (likely(__data_available(p, 23) && *(p + 1) == '-' @@ -3776,7 +3776,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, 'i', 'd', 'e', ':'))) { parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_OWS, 23); + __FSM_MOVE_n(RGen_LWS, 23); } if (likely(__data_available(p, 18) @@ -3789,7 +3789,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, 'i', 'd', 'e', ':'))) { parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_OWS, 18); + __FSM_MOVE_n(RGen_LWS, 18); } __FSM_MOVE(Req_HdrX); case 'u': @@ -4418,7 +4418,7 @@ Req_Method_1CharStep: __attribute__((cold)) case 'm': __FSM_MOVE(Req_HdrX_M); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -4475,9 +4475,9 @@ Req_Method_1CharStep: __attribute__((cold)) __FSM_MOVE(Req_HdrX_Http_Method_); case ':': parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE(RGen_OWS); + __FSM_MOVE(RGen_LWS); default: - __FSM_JMP(RGen_HdrOther); + __FSM_JMP(RGen_HdrOtherN); } } @@ -6381,7 +6381,7 @@ STACK_FRAME_NON_STANDARD(__h2_req_parse_x_forwarded_for); static int __h2_req_parse_mark(TfwHttpReq *req, unsigned char *data, size_t len, bool fin) { - TfwStr *str; + const TfwStr *str; int r = CSTR_NEQ; __FSM_DECLARE_VARS(req); @@ -8421,7 +8421,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && *(p + 10) == ':')) { parser->_i_st = &&Resp_HdrSet_CookieV; - __FSM_MOVE_n(RGen_OWS, 11); + __FSM_MOVE_n(RGen_LWS, 11); } __FSM_MOVE(Resp_HdrS); case 't': @@ -9043,17 +9043,7 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __FSM_TX_AF(Resp_HdrSet_Coo, 'k', Resp_HdrSet_Cook); __FSM_TX_AF(Resp_HdrSet_Cook, 'i', Resp_HdrSet_Cooki); __FSM_TX_AF(Resp_HdrSet_Cooki, 'e', Resp_HdrSet_Cookie); - __FSM_TX_AF_OWS_HP(Resp_HdrSet_Cookie, RGen_HdrOtherV, 55); - - /* Set-Cookie header processing. */ - __FSM_TX_AF(Resp_HdrSet, '-', Resp_HdrSet_); - __FSM_TX_AF(Resp_HdrSet_, 'c', Resp_HdrSet_C); - __FSM_TX_AF(Resp_HdrSet_C, 'o', Resp_HdrSet_Co); - __FSM_TX_AF(Resp_HdrSet_Co, 'o', Resp_HdrSet_Coo); - __FSM_TX_AF(Resp_HdrSet_Coo, 'k', Resp_HdrSet_Cook); - __FSM_TX_AF(Resp_HdrSet_Cook, 'i', Resp_HdrSet_Cooki); - __FSM_TX_AF(Resp_HdrSet_Cooki, 'e', Resp_HdrSet_Cookie); - __FSM_TX_AF_OWS(Resp_HdrSet_Cookie, Resp_HdrSet_CookieV); + __FSM_TX_AF_OWS_HP(Resp_HdrSet_Cookie, Resp_HdrSet_CookieV, 55); /* Transfer-Encoding header processing. */ __FSM_TX_AF(Resp_HdrT, 'r', Resp_HdrTr); From 80d2d4a136ed26043eed0dca59210852f2a0b312 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 19/64] Fail test if parsed string was composed incorrectly, don't crash --- tempesta_fw/t/unit/test_http_parser.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tempesta_fw/t/unit/test_http_parser.c b/tempesta_fw/t/unit/test_http_parser.c index 9ee7ec058a..de102cdac9 100644 --- a/tempesta_fw/t/unit/test_http_parser.c +++ b/tempesta_fw/t/unit/test_http_parser.c @@ -241,7 +241,10 @@ test_string_split(const TfwStr *expected, const TfwStr *parsed) { TfwStr *end_p, *end_e, *c_p, *c_e; - BUG_ON(TFW_STR_PLAIN(parsed) || TFW_STR_PLAIN(expected)); + BUG_ON(TFW_STR_PLAIN(expected)); + EXPECT_FALSE(TFW_STR_PLAIN(parsed)); + if (TFW_STR_PLAIN(parsed)) + return; EXPECT_GE(parsed->nchunks, expected->nchunks); EXPECT_EQ(parsed->len, expected->len); From b1e6a1149c2d106234644e035a18906a3ae15cf1 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 20/64] port changes from avb-1202 to http2 --- tempesta_fw/http_parser.c | 385 ++++++++++++++++++++------------------ 1 file changed, 206 insertions(+), 179 deletions(-) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index c7b97be9c6..4d42ff9452 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -4921,6 +4921,21 @@ do { \ __FSM_EXIT(CSTR_POSTPONE); \ } while (0) +#define __FSM_H2_I_MOVE_BY_REF_NEQ_LAMBDA_n(to, n, lambda) \ +do { \ + parser->_i_st = to; \ + p += n; \ + if (__data_off(p) < len) \ + goto *to; \ + if (likely(fin)) { \ + lambda; \ + __FSM_EXIT(CSTR_NEQ); \ + } \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + __FSM_EXIT(CSTR_POSTPONE); \ +} while (0) + #define __FSM_H2_I_MOVE_n(to, n) \ __FSM_H2_I_MOVE_LAMBDA_n(to, n, {}) @@ -4929,6 +4944,9 @@ do { \ #define __FSM_H2_I_MOVE_NEQ(to, n) \ __FSM_H2_I_MOVE_NEQ_LAMBDA_n(to, n, {}) +#define __FSM_H2_I_MOVE_BY_REF_NEQ(to, n) \ + __FSM_H2_I_MOVE_BY_REF_NEQ_LAMBDA_n(to, n, {}) + #define __FSM_H2_I_MATCH(alphabet) \ do { \ __fsm_n = __data_remain(p); \ @@ -5070,6 +5088,32 @@ do { \ __FSM_EXIT(CSTR_EQ); \ }, curr_st, next_st) +#define H2_TRY_STR_LAMBDA_BY_REF_finish(str, lambda1, lambda2, finish, state)\ + if (!chunk->data) \ + chunk->data = p; \ + __fsm_n = __try_str(&parser->hdr, chunk, p, __data_remain(p), \ + str, sizeof(str) - 1); \ + if (__fsm_n > 0) { \ + if (chunk->len == sizeof(str) - 1) { \ + lambda1; \ + TRY_STR_INIT(); \ + __FSM_H2_I_MOVE_BY_REF_NEQ_LAMBDA_n(state, __fsm_n, lambda2); \ + } \ + if (likely(fin)) \ + return CSTR_NEQ; \ + __msg_hdr_chunk_fixup(data, len); \ + __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); \ + finish; \ + __FSM_EXIT(CSTR_POSTPONE); \ + } + +#define H2_TRY_STR_BY_REF(str, curr_st, next_st) \ + H2_TRY_STR_LAMBDA_BY_REF_finish(str, { }, { \ + __FSM_EXIT(CSTR_NEQ); \ + }, { \ + parser->_i_st = curr_st; \ + }, next_st) + /** * The same as @H2_TRY_STR_2LAMBDA(), but with explicit chunks control; * besides, @str must be of plain @TfwStr{} type and variable @fld is @@ -5990,246 +6034,229 @@ STACK_FRAME_NON_STANDARD(__h2_req_parse_referer); static int __h2_parse_http_date(TfwHttpMsg *hm, unsigned char *data, size_t len, bool fin) { - static const unsigned long colon_a[] ____cacheline_aligned = { - /* ':' (0x3a)(58) Colon */ - 0x0400000000000000UL, 0, 0, 0 + static const void *st[][23] ____cacheline_aligned = { + [RFC_822] = { + &&I_Day, &&I_Day, &&I_SP, + &&I_MonthBeg, &&I_Month, &&I_Month, &&I_SP, + &&I_Year, &&I_Year, &&I_Year, &&I_Year, &&I_SP, + &&I_Hour, &&I_Hour, &&I_SC, + &&I_Min, &&I_Min, &&I_SC, + &&I_Sec, &&I_Sec, &&I_SP, + &&I_GMT, &&I_Res + }, + [RFC_850] = { + &&I_Day, &&I_Day, &&I_Minus, + &&I_MonthBeg, &&I_Month, &&I_Month, &&I_Minus, + &&I_Year, &&I_Year, &&I_SP, + &&I_Hour, &&I_Hour, &&I_SC, + &&I_Min, &&I_Min, &&I_SC, + &&I_Sec, &&I_Sec, &&I_SP, + &&I_GMT, &&I_Res + }, + [ISOC] = { + &&I_MonthBeg, &&I_Month, &&I_Month, &&I_SP, + &&I_SpaceOrDay, &&I_Day, &&I_SP, + &&I_Hour, &&I_Hour, &&I_SC, + &&I_Min, &&I_Min, &&I_SC, + &&I_Sec, &&I_Sec, &&I_SP, + &&I_Year, &&I_Year, &&I_Year, &&I_Year, + &&I_Res + } }; int r = CSTR_NEQ; __FSM_DECLARE_VARS(hm); -#define H2_TRY_STR_NEQ_LAMBDA(str, lambda, curr_st, next_st) \ - H2_TRY_STR_2LAMBDA(str, lambda, { \ - __FSM_EXIT(CSTR_NEQ); \ - }, curr_st, next_st) - __FSM_START_ALT(parser->_i_st); - __FSM_STATE(I_Date) { - /* Skip redundant weekday with comma (e.g. "Sun,"). */ - __fsm_sz = __data_remain(p); - __fsm_n = __skip_weekday(p, __fsm_sz); - if (unlikely(__fsm_sz == __fsm_n)) { - if (likely(fin)) - __FSM_EXIT(CSTR_NEQ); - parser->_i_st = &&I_Date; - __msg_hdr_chunk_fixup(data, len); - __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); - __FSM_EXIT(CSTR_POSTPONE); - } - if (unlikely(__fsm_n == CSTR_NEQ)) - return CSTR_NEQ; - p += __fsm_n + 1; - __FSM_I_JMP(I_DateDay); + /* + * Skip a weekday with comma (e.g. "Sun,") as redundant + * information. + */ + __FSM_STATE(I_WDate1) { + if (likely('A' <= c && c <= 'Z')) + __FSM_H2_I_MOVE_NEQ(I_WDate2, 1); + return CSTR_NEQ; } - __FSM_STATE(I_DateDay) { - __fsm_sz = __data_remain(p); - /* Parse a 2-digit day. */ - __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); - if (unlikely(__fsm_n == CSTR_POSTPONE)) { - if (likely(fin)) - return CSTR_NEQ; - parser->_i_st = &&I_DateDay; - __msg_hdr_chunk_fixup(data, len); - __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); - } - if (__fsm_n < 0) - return __fsm_n; - if (parser->_acc < 1 || parser->_acc > 31) - return CSTR_BADLEN; - /* Add seconds in full passed days. */ - parser->_date = (parser->_acc - 1) * SEC24H; - parser->_acc = 0; - p += __fsm_n; - __FSM_I_JMP(I_DateMonthSP); + __FSM_STATE(I_WDate2) { + if (likely('a' <= c && c <= 'z')) + __FSM_H2_I_MOVE_NEQ(I_WDate3, 1); + return CSTR_NEQ; } - __FSM_STATE(I_DateMonthSP) { - if (likely(c == ' ')) - __FSM_H2_I_MOVE_NEQ(I_DateMonth, 1); + __FSM_STATE(I_WDate3) { + if (likely('a' <= c && c <= 'z')) + __FSM_H2_I_MOVE_NEQ(I_WDate4, 1); return CSTR_NEQ; } - __FSM_STATE(I_DateMonth) { - switch (TFW_LC(c)) { - case 'a': - __FSM_I_JMP(I_DateMonth_A); - case 'j': - __FSM_I_JMP(I_DateMonth_J); - case 'm': - __FSM_I_JMP(I_DateMonth_M); + __FSM_STATE(I_WDate4) { + parser->_acc = 0; + parser->month_int = ((size_t)' ') << 24; + if (likely(c == ',')) { + parser->date.type = RFC_822; + __FSM_H2_I_MOVE_NEQ(I_WDaySP, 1); + } + if ('a' <= c && c <= 'z') { + parser->date.type = RFC_850; + __FSM_H2_I_MOVE_NEQ(I_WDate5, 1); + } + if (c == ' ') { + parser->date.type = ISOC; + __FSM_H2_I_MOVE_BY_REF_NEQ( + st[parser->date.type][parser->date.pos], 1); } - __FSM_I_JMP(I_DateMonth_Other); + return CSTR_NEQ; } - __FSM_STATE(I_DateMonth_A) { - H2_TRY_STR_NEQ_LAMBDA("apr ", { - parser->_date += SB_APR; - }, I_DateMonth_A, I_DateYear); - H2_TRY_STR_NEQ_LAMBDA("aug ", { - parser->_date += SB_AUG; - }, I_DateMonth_A, I_DateYear); - TRY_STR_INIT(); + __FSM_STATE(I_WDate5) { + if ('a' <= c && c <= 'z') + __FSM_H2_I_MOVE_NEQ(I_WDate5, 1); + if (c == ',') + __FSM_H2_I_MOVE_NEQ(I_WDaySP, 1); return CSTR_NEQ; } - __FSM_STATE(I_DateMonth_J) { - H2_TRY_STR_NEQ_LAMBDA("jan ", { }, I_DateMonth_J, I_DateYear); - H2_TRY_STR_NEQ_LAMBDA("jun ", { - parser->_date += SB_JUN; - }, I_DateMonth_J, I_DateYear); - H2_TRY_STR_NEQ_LAMBDA("jul ", { - parser->_date += SB_JUL; - }, I_DateMonth_J, I_DateYear); - TRY_STR_INIT(); + __FSM_STATE(I_WDaySP) { + if (likely(c == ' ')) + __FSM_H2_I_MOVE_BY_REF_NEQ( + st[parser->date.type][parser->date.pos], 1); return CSTR_NEQ; } - __FSM_STATE(I_DateMonth_M) { - H2_TRY_STR_NEQ_LAMBDA("mar ", { - /* Add SEC24H for leap year on year parsing. */ - parser->_date += SB_MAR; - }, I_DateMonth_M, I_DateYear); - H2_TRY_STR_NEQ_LAMBDA("may ", { - parser->_date += SB_MAY; - }, I_DateMonth_M, I_DateYear); - TRY_STR_INIT(); +#define __NEXT_TEMPL_STATE() \ +do { \ + ++parser->date.pos; \ + __FSM_H2_I_MOVE_BY_REF_NEQ(st[parser->date.type][parser->date.pos], 1);\ +} while (0) + + __FSM_STATE(I_SP) { + if (likely(c == ' ')) + __NEXT_TEMPL_STATE(); return CSTR_NEQ; } - __FSM_STATE(I_DateMonth_Other) { - H2_TRY_STR_NEQ_LAMBDA("feb ", { - parser->_date += SB_FEB; - }, I_DateMonth_Other, I_DateYear); - H2_TRY_STR_NEQ_LAMBDA("sep ", { - parser->_date += SB_SEP; - }, I_DateMonth_Other, I_DateYear); - H2_TRY_STR_NEQ_LAMBDA("oct ", { - parser->_date += SB_OCT; - }, I_DateMonth_Other, I_DateYear); - H2_TRY_STR_NEQ_LAMBDA("nov ", { - parser->_date += SB_NOV; - }, I_DateMonth_Other, I_DateYear); - H2_TRY_STR_NEQ_LAMBDA("dec ", { - parser->_date += SB_DEC; - }, I_DateMonth_Other, I_DateYear); - TRY_STR_INIT(); + __FSM_STATE(I_Minus) { + if (likely(c == '-')) + __NEXT_TEMPL_STATE(); return CSTR_NEQ; } - /* 4-digit year. */ - __FSM_STATE(I_DateYear) { - __fsm_sz = __data_remain(p); - __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); - if (unlikely(__fsm_n == CSTR_POSTPONE)) { - if (likely(fin)) - return CSTR_NEQ; - parser->_i_st = &&I_DateYear; - __msg_hdr_chunk_fixup(data, len); - __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + __FSM_STATE(I_SC) { + if (likely(c == ':')) + __NEXT_TEMPL_STATE(); + return CSTR_NEQ; + } + + __FSM_STATE(I_SpaceOrDay) { + if (c == ' ') + __NEXT_TEMPL_STATE(); + if ('0' <= c && c <= '9') { + parser->date.day = parser->date.day * 10 + (c - '0'); + __NEXT_TEMPL_STATE(); } - if (__fsm_n < 0) - return __fsm_n; - parser->_date = __year_day_secs(parser->_acc, parser->_date); - if (parser->_date < 0) - return CSTR_NEQ; - parser->_acc = 0; - __FSM_H2_I_MOVE_NEQ(I_DateHourSP, __fsm_n); + return CSTR_NEQ; } - __FSM_STATE(I_DateHourSP) { - if (likely(c == ' ')) - __FSM_H2_I_MOVE_NEQ(I_DateHour, 1); + __FSM_STATE(I_Day) { + if ('0' <= c && c <= '9') { + parser->date.day = parser->date.day * 10 + (c - '0'); + __NEXT_TEMPL_STATE(); + } return CSTR_NEQ; } - __FSM_STATE(I_DateHour) { - __fsm_sz = __data_remain(p); - __fsm_n = parse_int_a(p, __fsm_sz, colon_a, &parser->_acc); - if (unlikely(__fsm_n == CSTR_POSTPONE)) { - if (likely(fin)) - return CSTR_NEQ; - parser->_i_st = &&I_DateHour; - __msg_hdr_chunk_fixup(data, len); - __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + __FSM_STATE(I_MonthBeg) { + if ('A' <= c && c <= 'Z') { + parser->month_int = + ((size_t)c) << 24 | (parser->month_int >> 8); + __NEXT_TEMPL_STATE(); } - if (__fsm_n < 0) - return __fsm_n; - parser->_date += parser->_acc * 3600; - parser->_acc = 0; - __FSM_H2_I_MOVE_NEQ(I_DateMinCln, __fsm_n); + return CSTR_NEQ; } - __FSM_STATE(I_DateMinCln) { - if (likely(c == ':')) - __FSM_H2_I_MOVE_NEQ(I_DateMin, 1); + __FSM_STATE(I_Month) { + if ('a' <= c && c <= 'z') { + parser->month_int = + ((size_t)c) << 24 | (parser->month_int >> 8); + __NEXT_TEMPL_STATE(); + } return CSTR_NEQ; } - __FSM_STATE(I_DateMin) { - __fsm_sz = __data_remain(p); - __fsm_n = parse_int_a(p, __fsm_sz, colon_a, &parser->_acc); - if (unlikely(__fsm_n == CSTR_POSTPONE)) { - if (likely(fin)) - return CSTR_NEQ; - parser->_i_st = &&I_DateMin; - __msg_hdr_chunk_fixup(data, len); - __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + __FSM_STATE(I_Year) { + if ('0' <= c && c <= '9') { + parser->date.year = parser->date.year * 10 + (c - '0'); + __NEXT_TEMPL_STATE(); } - if (__fsm_n < 0) - return __fsm_n; - parser->_date += parser->_acc * 60; - parser->_acc = 0; - __FSM_H2_I_MOVE_NEQ(I_DateSecCln, __fsm_n); + return CSTR_NEQ; } - __FSM_STATE(I_DateSecCln) { - if (likely(c == ':')) - __FSM_H2_I_MOVE_NEQ(I_DateSec, 1); + __FSM_STATE(I_Hour) { + if ('0' <= c && c <= '9') { + parser->date.hour = parser->date.hour * 10 + (c - '0'); + __NEXT_TEMPL_STATE(); + } return CSTR_NEQ; } - __FSM_STATE(I_DateSec) { - __fsm_sz = __data_remain(p); - __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); - if (unlikely(__fsm_n == CSTR_POSTPONE)) { - if (likely(fin)) - return CSTR_NEQ; - parser->_i_st = &&I_DateSec; - __msg_hdr_chunk_fixup(data, len); - __FSM_I_chunk_flags(TFW_STR_HDR_VALUE); + __FSM_STATE(I_Min) { + if ('0' <= c && c <= '9') { + parser->date.min = parser->date.min * 10 + (c - '0'); + __NEXT_TEMPL_STATE(); } - if (__fsm_n < 0) - return __fsm_n; - parser->_date += parser->_acc; - parser->_acc = 0; - __FSM_H2_I_MOVE_NEQ(I_DateSecSP, __fsm_n); + return CSTR_NEQ; } - __FSM_STATE(I_DateSecSP) { - if (likely(c == ' ')) - __FSM_H2_I_MOVE_NEQ(I_DateZone, 1); + __FSM_STATE(I_Sec) { + if ('0' <= c && c <= '9') { + parser->date.sec = parser->date.sec * 10 + (c - '0'); + __NEXT_TEMPL_STATE(); + } return CSTR_NEQ; } +#undef __NEXT_TEMPL_STATE - __FSM_STATE(I_DateZone) { - H2_TRY_STR("gmt", I_DateZone, I_EoL); + __FSM_STATE(I_GMT) { + H2_TRY_STR_BY_REF("gmt", + &&I_GMT, st[parser->date.type][parser->date.pos + 1]); TRY_STR_INIT(); return CSTR_NEQ; } + __FSM_STATE(I_Res) { + unsigned int month; + time_t date; + + month = __parse_month(parser->month_int); + if (month < 0) + return CSTR_NEQ; + + if (parser->date.year < 100) + parser->date.year += (parser->date.year < 70) ? 2000 + : 1900; + + date = __date_secs(parser->date.year, month, + parser->date.day, parser->date.hour, + parser->date.min, parser->date.sec); + if (date < 0) + return CSTR_NEQ; + parser->_date = date; + __FSM_JMP(I_EoL); + } + __FSM_STATE(I_EoL) { + parser->_acc = 0; /* Skip the rest of the line. */ - __FSM_H2_I_MATCH_MOVE_LAMBDA(nctl, I_EoL, { - T_DBG3("%s: parsed date %lu", __func__, parser->_date); - }); - return CSTR_NEQ; + __FSM_H2_I_MATCH_MOVE(nctl, I_EoL); + if (!IS_CRLF(*(p + __fsm_sz))) + return CSTR_NEQ; + T_DBG3("%s: parsed date %lu", __func__, parser->_date); + return __data_off(p + __fsm_sz); } done: return r; - -#undef H2_TRY_STR_NEQ_LAMBDA } STACK_FRAME_NON_STANDARD(__h2_parse_http_date); @@ -6252,7 +6279,7 @@ __h2_req_parse_if_msince(TfwHttpMsg *msg, unsigned char *data, size_t len, r = __h2_parse_http_date(msg, data, len, fin); } - if (r >= 0) { + if (r >= 0 && parser->_date != 0) { req->cond.m_date = parser->_date; req->cond.flags |= TFW_HTTP_COND_IF_MSINCE; From 062034959bb8007cda8a243f00eea7499aa55b98 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 21/64] Set-cookie was added in two branches with different processing, resolve duplication conflicts --- tempesta_fw/http_parser.c | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 4d42ff9452..d5ea2b14b3 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -339,7 +339,7 @@ __FSM_STATE(st) { \ /* * Automaton transition with alphabet checking and fallback state. - * Improbble states only, so cold label. + * Improbable states only, so cold label. */ #define __FSM_TX_AF(st, ch, st_next) \ __FSM_STATE(st, cold) { \ @@ -8422,10 +8422,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, if (likely(__data_available(p, 11) && C8_INT_LCM(p + 1, 'e', 't', '-', 'c', 'o', 'o', 'k', 'i') + && *(p + 3) == '-' && TFW_LC(*(p + 9)) == 'e' && *(p + 10) == ':')) { - parser->_i_st = &&RGen_HdrOtherV; + parser->_i_st = &&Resp_HdrSet_CookieV; __msg_hdr_set_hpack_index(55); __FSM_MOVE_n(RGen_LWS, 11); } @@ -8438,18 +8439,6 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, __msg_hdr_set_hpack_index(54); __FSM_MOVE_n(RGen_LWS, 7); } - if (likely(__data_available(p, 11) - && TFW_LC(*(p + 1)) == 'e' - && TFW_LC(*(p + 2)) == 't' - && *(p + 3) == '-' - && C4_INT_LCM(p + 4, 'c', 'o', 'o', 'k') - && TFW_LC(*(p + 8)) == 'i' - && TFW_LC(*(p + 9)) == 'e' - && *(p + 10) == ':')) - { - parser->_i_st = &&Resp_HdrSet_CookieV; - __FSM_MOVE_n(RGen_LWS, 11); - } __FSM_MOVE(Resp_HdrS); case 't': if (likely(__data_available(p, 18) From d4c682dcf49e54a52b11809129be3be41eaa517e Mon Sep 17 00:00:00 2001 From: Alexander K Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 22/64] Add HPACK generator and generated file --- tempesta_fw/hgen.c | 592 ++++++++++++++++++++++ tempesta_fw/hpack_tbl.h | 1033 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 1625 insertions(+) create mode 100644 tempesta_fw/hgen.c create mode 100644 tempesta_fw/hpack_tbl.h diff --git a/tempesta_fw/hgen.c b/tempesta_fw/hgen.c new file mode 100644 index 0000000000..34dab6c20c --- /dev/null +++ b/tempesta_fw/hgen.c @@ -0,0 +1,592 @@ +/** + * Tempesta FW + * + * HTTP/2 Huffman state machine generator. + * + * Copyright (C) 2017-2019 Tempesta Technologies, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#include +#include +#include +#include + +typedef struct { + int16_t symbol; + uint32_t code; + uint8_t length; +} HCode; + +/* + * Huffman code RFC 7541, Apendix B. + * + */ +static HCode source[] = { + {0, 0x1ff8, 13}, + {1, 0x7fffd8, 23}, + {2, 0xfffffe2, 28}, + {3, 0xfffffe3, 28}, + {4, 0xfffffe4, 28}, + {5, 0xfffffe5, 28}, + {6, 0xfffffe6, 28}, + {7, 0xfffffe7, 28}, + {8, 0xfffffe8, 28}, + {9, 0xffffea, 24}, + {10, 0x3ffffffc, 30}, + {11, 0xfffffe9, 28}, + {12, 0xfffffea, 28}, + {13, 0x3ffffffd, 30}, + {14, 0xfffffeb, 28}, + {15, 0xfffffec, 28}, + {16, 0xfffffed, 28}, + {17, 0xfffffee, 28}, + {18, 0xfffffef, 28}, + {19, 0xffffff0, 28}, + {20, 0xffffff1, 28}, + {21, 0xffffff2, 28}, + {22, 0x3ffffffe, 30}, + {23, 0xffffff3, 28}, + {24, 0xffffff4, 28}, + {25, 0xffffff5, 28}, + {26, 0xffffff6, 28}, + {27, 0xffffff7, 28}, + {28, 0xffffff8, 28}, + {29, 0xffffff9, 28}, + {30, 0xffffffa, 28}, + {31, 0xffffffb, 28}, + {32, 0x14, 6}, + {33, 0x3f8, 10}, + {34, 0x3f9, 10}, + {35, 0xffa, 12}, + {36, 0x1ff9, 13}, + {37, 0x15, 6}, + {38, 0xf8, 8}, + {39, 0x7fa, 11}, + {40, 0x3fa, 10}, + {41, 0x3fb, 10}, + {42, 0xf9, 8}, + {43, 0x7fb, 11}, + {44, 0xfa, 8}, + {45, 0x16, 6}, + {46, 0x17, 6}, + {47, 0x18, 6}, + {48, 0x0, 5}, + {49, 0x1, 5}, + {50, 0x2, 5}, + {51, 0x19, 6}, + {52, 0x1a, 6}, + {53, 0x1b, 6}, + {54, 0x1c, 6}, + {55, 0x1d, 6}, + {56, 0x1e, 6}, + {57, 0x1f, 6}, + {58, 0x5c, 7}, + {59, 0xfb, 8}, + {60, 0x7ffc, 15}, + {61, 0x20, 6}, + {62, 0xffb, 12}, + {63, 0x3fc, 10}, + {64, 0x1ffa, 13}, + {65, 0x21, 6}, + {66, 0x5d, 7}, + {67, 0x5e, 7}, + {68, 0x5f, 7}, + {69, 0x60, 7}, + {70, 0x61, 7}, + {71, 0x62, 7}, + {72, 0x63, 7}, + {73, 0x64, 7}, + {74, 0x65, 7}, + {75, 0x66, 7}, + {76, 0x67, 7}, + {77, 0x68, 7}, + {78, 0x69, 7}, + {79, 0x6a, 7}, + {80, 0x6b, 7}, + {81, 0x6c, 7}, + {82, 0x6d, 7}, + {83, 0x6e, 7}, + {84, 0x6f, 7}, + {85, 0x70, 7}, + {86, 0x71, 7}, + {87, 0x72, 7}, + {88, 0xfc, 8}, + {89, 0x73, 7}, + {90, 0xfd, 8}, + {91, 0x1ffb, 13}, + {92, 0x7fff0, 19}, + {93, 0x1ffc, 13}, + {94, 0x3ffc, 14}, + {95, 0x22, 6}, + {96, 0x7ffd, 15}, + {97, 0x3, 5}, + {98, 0x23, 6}, + {99, 0x4, 5}, + {100, 0x24, 6}, + {101, 0x5, 5}, + {102, 0x25, 6}, + {103, 0x26, 6}, + {104, 0x27, 6}, + {105, 0x6, 5}, + {106, 0x74, 7}, + {107, 0x75, 7}, + {108, 0x28, 6}, + {109, 0x29, 6}, + {110, 0x2a, 6}, + {111, 0x7, 5}, + {112, 0x2b, 6}, + {113, 0x76, 7}, + {114, 0x2c, 6}, + {115, 0x8, 5}, + {116, 0x9, 5}, + {117, 0x2d, 6}, + {118, 0x77, 7}, + {119, 0x78, 7}, + {120, 0x79, 7}, + {121, 0x7a, 7}, + {122, 0x7b, 7}, + {123, 0x7ffe, 15}, + {124, 0x7fc, 11}, + {125, 0x3ffd, 14}, + {126, 0x1ffd, 13}, + {127, 0xffffffc, 28}, + {128, 0xfffe6, 20}, + {129, 0x3fffd2, 22}, + {130, 0xfffe7, 20}, + {131, 0xfffe8, 20}, + {132, 0x3fffd3, 22}, + {133, 0x3fffd4, 22}, + {134, 0x3fffd5, 22}, + {135, 0x7fffd9, 23}, + {136, 0x3fffd6, 22}, + {137, 0x7fffda, 23}, + {138, 0x7fffdb, 23}, + {139, 0x7fffdc, 23}, + {140, 0x7fffdd, 23}, + {141, 0x7fffde, 23}, + {142, 0xffffeb, 24}, + {143, 0x7fffdf, 23}, + {144, 0xffffec, 24}, + {145, 0xffffed, 24}, + {146, 0x3fffd7, 22}, + {147, 0x7fffe0, 23}, + {148, 0xffffee, 24}, + {149, 0x7fffe1, 23}, + {150, 0x7fffe2, 23}, + {151, 0x7fffe3, 23}, + {152, 0x7fffe4, 23}, + {153, 0x1fffdc, 21}, + {154, 0x3fffd8, 22}, + {155, 0x7fffe5, 23}, + {156, 0x3fffd9, 22}, + {157, 0x7fffe6, 23}, + {158, 0x7fffe7, 23}, + {159, 0xffffef, 24}, + {160, 0x3fffda, 22}, + {161, 0x1fffdd, 21}, + {162, 0xfffe9, 20}, + {163, 0x3fffdb, 22}, + {164, 0x3fffdc, 22}, + {165, 0x7fffe8, 23}, + {166, 0x7fffe9, 23}, + {167, 0x1fffde, 21}, + {168, 0x7fffea, 23}, + {169, 0x3fffdd, 22}, + {170, 0x3fffde, 22}, + {171, 0xfffff0, 24}, + {172, 0x1fffdf, 21}, + {173, 0x3fffdf, 22}, + {174, 0x7fffeb, 23}, + {175, 0x7fffec, 23}, + {176, 0x1fffe0, 21}, + {177, 0x1fffe1, 21}, + {178, 0x3fffe0, 22}, + {179, 0x1fffe2, 21}, + {180, 0x7fffed, 23}, + {181, 0x3fffe1, 22}, + {182, 0x7fffee, 23}, + {183, 0x7fffef, 23}, + {184, 0xfffea, 20}, + {185, 0x3fffe2, 22}, + {186, 0x3fffe3, 22}, + {187, 0x3fffe4, 22}, + {188, 0x7ffff0, 23}, + {189, 0x3fffe5, 22}, + {190, 0x3fffe6, 22}, + {191, 0x7ffff1, 23}, + {192, 0x3ffffe0, 26}, + {193, 0x3ffffe1, 26}, + {194, 0xfffeb, 20}, + {195, 0x7fff1, 19}, + {196, 0x3fffe7, 22}, + {197, 0x7ffff2, 23}, + {198, 0x3fffe8, 22}, + {199, 0x1ffffec, 25}, + {200, 0x3ffffe2, 26}, + {201, 0x3ffffe3, 26}, + {202, 0x3ffffe4, 26}, + {203, 0x7ffffde, 27}, + {204, 0x7ffffdf, 27}, + {205, 0x3ffffe5, 26}, + {206, 0xfffff1, 24}, + {207, 0x1ffffed, 25}, + {208, 0x7fff2, 19}, + {209, 0x1fffe3, 21}, + {210, 0x3ffffe6, 26}, + {211, 0x7ffffe0, 27}, + {212, 0x7ffffe1, 27}, + {213, 0x3ffffe7, 26}, + {214, 0x7ffffe2, 27}, + {215, 0xfffff2, 24}, + {216, 0x1fffe4, 21}, + {217, 0x1fffe5, 21}, + {218, 0x3ffffe8, 26}, + {219, 0x3ffffe9, 26}, + {220, 0xffffffd, 28}, + {221, 0x7ffffe3, 27}, + {222, 0x7ffffe4, 27}, + {223, 0x7ffffe5, 27}, + {224, 0xfffec, 20}, + {225, 0xfffff3, 24}, + {226, 0xfffed, 20}, + {227, 0x1fffe6, 21}, + {228, 0x3fffe9, 22}, + {229, 0x1fffe7, 21}, + {230, 0x1fffe8, 21}, + {231, 0x7ffff3, 23}, + {232, 0x3fffea, 22}, + {233, 0x3fffeb, 22}, + {234, 0x1ffffee, 25}, + {235, 0x1ffffef, 25}, + {236, 0xfffff4, 24}, + {237, 0xfffff5, 24}, + {238, 0x3ffffea, 26}, + {239, 0x7ffff4, 23}, + {240, 0x3ffffeb, 26}, + {241, 0x7ffffe6, 27}, + {242, 0x3ffffec, 26}, + {243, 0x3ffffed, 26}, + {244, 0x7ffffe7, 27}, + {245, 0x7ffffe8, 27}, + {246, 0x7ffffe9, 27}, + {247, 0x7ffffea, 27}, + {248, 0x7ffffeb, 27}, + {249, 0xffffffe, 28}, + {250, 0x7ffffec, 27}, + {251, 0x7ffffed, 27}, + {252, 0x7ffffee, 27}, + {253, 0x7ffffef, 27}, + {254, 0x7fffff0, 27}, + {255, 0x3ffffee, 26}, + {-1, 0x3fffffff, 30} /* EOS */ +}; + +#define HF_SYMBOLS (sizeof(source) / sizeof(HCode)) + +#define NBITS 7 +#define MBITS 3 +#define BIG (1 << NBITS) +#define SMALL (1 << MBITS) +#define STEP (1 << (NBITS - MBITS)) + +#define BAD_SYMBOL (-2) + +static uint32_t codes[257]; +static uint8_t codes_n[257]; + +typedef struct htree_t { + int16_t symbol; + uint8_t shift; + uint8_t count; + uint8_t max; + uint16_t offset; + struct htree_t *down; +} HTree; + +static HTree root[BIG]; + +static void +ht_add(HTree * __restrict base, uint32_t code, uint8_t length, int16_t symbol) +{ + unsigned int j, n, index = code >> (32 - NBITS); + int remain = length - NBITS; + + if (remain <= 0) { + base[index].symbol = symbol; + base[index].shift = length; + if (remain < 0) { + n = 1 << (unsigned int)-remain; + for (j = 1; j < n; j++) { + base[index + j].symbol = symbol; + base[index + j].shift = length; + } + } + } else { + HTree *hb = base + index; + HTree *hp; + + hp = hb->down; + if (!hp) { + if (!(hp = calloc(BIG * sizeof(HTree), 1))) { + puts("Memory allocation error..."); + exit(1); + } + hb->down = hp; + for (j = 0; j < BIG; j++) + hp[j].symbol = BAD_SYMBOL; + } + hb->count++; + if (hb->max < remain) + hb->max = remain; + + ht_add(hp, code << NBITS, remain, symbol); + } +} + +#ifdef DEBUG +static void +ht_print(HTree *base) +{ + unsigned int i; + + for (i = 0; i < BIG; i++) { + if (!base[i].down) { + printf("%3u --> %3d, %2u\n", i, + base[i].symbol, base[i].shift); + } else { + printf("%3u --> (%u, max: %u)\n", i, + base[i].count, base[i].max); + } + } + puts("---"); + for (i = 0; i < BIG; i++) + if (base[i].down) + ht_print(base[i].down); +} +#endif + +static unsigned int +ht_gen(HTree *base, unsigned int offset) +{ + unsigned int i; + + offset += BIG; + for (i = 0; i < BIG; i++) { + if (base[i].down && base[i].max > MBITS) { + base[i].offset = offset; + offset = ht_gen(base[i].down, offset); + } + } + + return offset; +} + +static unsigned int +ht_gen16(HTree *base, unsigned int offset) +{ + unsigned int i; + + for (i = 0; i < BIG; i++) { + if (!base[i].down) + continue; + if (base[i].max <= MBITS) { + base[i].offset = offset; + offset += SMALL; + } else { + offset = ht_gen16(base[i].down, offset); + } + } + + return offset; +} + +static unsigned int +ht_out(const HTree *base, unsigned int offset, const unsigned int last) +{ + unsigned int i, shift; + int symbol; + + printf("/* --- [TABLE-%u: offset = %u] --- */\n", BIG, offset); + offset += BIG; + for (i = 0; i < BIG; i++) { + char comma = (i == BIG - 1 && offset == last) ? ' ' : ','; + + if (base[i].down) { + printf("\t{-%u, %4u}%c /* %u: ---> TABLE %u */\n", + NBITS, base[i].offset, comma, NBITS, + base[i].offset); + continue; + } + + shift = base[i].shift; + symbol = base[i].symbol; + assert(symbol != BAD_SYMBOL); + + if (symbol == -1) { + printf("\t{-%u, %4d}%c /* %u: EOS */\n", + shift, 0, comma, shift); + } + else if (symbol == '\\') { + printf("\t{%u, %4d}%c /* %u: '\\\\' (%d) */\n", + shift, (signed char)symbol, comma, shift, + symbol); + } + else if (symbol == '\'') { + printf("\t{%u, %4d}%c /* %u: '\\'' (%d) */\n", + shift, (signed char)symbol, comma, shift, + symbol); + } + else if (symbol >= 32 && symbol < 127) { + printf("\t{%u, %4d}%c /* %u: '%c' (%d) */\n", + shift, (signed char)symbol, comma, shift, + symbol, symbol); + } + else { + printf("\t{%u, %4d}%c /* %u: '\\x%02X' (%d) */\n", + shift, (signed char)symbol, comma, shift, + symbol, symbol); + } + } + for (i = 0; i < BIG; i++) + if (base[i].down && base[i].max > MBITS) + offset = ht_out(base[i].down, offset, last); + + return offset; +} + +static unsigned int +ht_out16(const HTree *base, unsigned int offset, const unsigned int last) +{ + unsigned int i, j, shift, shift2; + int symbol; + char comma; + + for (i = 0; i < BIG; i++) { + HTree *hp = base[i].down; + if (!hp) + continue; + + if (base[i].max > MBITS) { + offset = ht_out16(hp, offset, last); + continue; + } + + printf("/* --- [TABLE-%u: offset = %u] --- */\n", + SMALL, offset); + + offset += SMALL; + for (j = 0; j < BIG; j += STEP) { + shift = hp[j].shift; + shift2 = shift + NBITS - MBITS; + symbol = hp[j].symbol; + comma = (i == BIG - STEP && offset == last) ? ' ' : ','; + + assert(symbol != BAD_SYMBOL); + + if (symbol == -1) { + printf("\t{-%u, %4d}%c /* %u: EOS */\n", + shift2, 0, comma, shift); + } + else if (symbol == '\\') { + printf("\t{%u, %4d}%c /* %u: '\\\\' (%d) */\n", + shift2, (signed char)symbol, comma, + shift, symbol); + } + else if (symbol == '\'') { + printf("\t{%u, %4d}%c /* %u: '\\'' (%d) */\n", + shift2, (signed char)symbol, comma, + shift, symbol); + } + else if (symbol >= 32 && symbol < 127) { + printf("\t{%u, %4d}%c /* %u: '%c' (%d) */\n", + shift2, (signed char)symbol, comma, + shift, symbol, symbol); + } else { + printf("\t{%u, %4d}%c /* %u: '\\x%02X' (%d) */" + "\n", shift2, (signed char)symbol, comma, + shift, symbol, symbol); + } + } + } + + return offset; +} + +int +main(int argc, char *argv[]) +{ + unsigned int i, j, offset, offset16, code, length, index; + int symbol; + + for (i = 0; i < BIG; i++) + root[i].symbol = BAD_SYMBOL; + + for (i = 0; i < HF_SYMBOLS; i++) { + code = source[i].code; + length = source[i].length; + symbol = source[i].symbol; + index = symbol >= 0 ? symbol : 256; + + codes[index] = code; + codes_n[index] = length; + code <<= 32 - length; + ht_add(root, code, length, symbol); + } +#ifdef DEBUG + ht_print(root); +#endif + offset = ht_gen(root, 0); + offset16 = ht_gen16(root, offset); + + printf("#define HT_NBITS\t%u\n", NBITS); + printf("#define HT_MBITS\t%u\n", MBITS); + printf("#define HT_NMASK\t%u\n", BIG - 1); + printf("#define HT_MMASK\t%u\n", SMALL - 1); + printf("#define HT_SMALL\t%u\n", offset); + + code = codes[256]; + length = codes_n[256]; + printf("#define HT_EOS_HIGH\t0x%02X\n\n", code >> (length - 8)); + + printf("static const unsigned int ht_encode[] __page_aligned_data" + " = {\n\t"); + for (i = 0; i < 256; i += 4) { + for (j = 0; j < 3; j++) + printf("0x%08X, ", codes[i + j]); + if (i != 256 - 4) + printf("0x%08X,\n\t", codes[i + j]); + else + printf("0x%08X\n};\n\n", codes[i + j]); + } + + printf("static const unsigned char ht_length[] ____cacheline_aligned" + " = {\n\t"); + for (i = 0; i < 256; i += 16) { + for (j = 0; j < 15; j++) + printf("%2u, ", codes_n[i + j]); + if (i != 240) + printf("%2u,\n\t", codes_n[i + j]); + else + printf("%2u\n};\n\n", codes_n[i + j]); + } + + puts("static const HTState ht_decode[] __page_aligned_data = {"); + ht_out(root, 0, offset16); + ht_out16(root, offset, offset16); + puts("};\n\n#endif /* __TFW_HTTP_HPACK_TBL_H__ */"); + + return 0; +} diff --git a/tempesta_fw/hpack_tbl.h b/tempesta_fw/hpack_tbl.h new file mode 100644 index 0000000000..d44a806dc2 --- /dev/null +++ b/tempesta_fw/hpack_tbl.h @@ -0,0 +1,1033 @@ +/** + * Tempesta FW + * + * Copyright (C) 2019 Tempesta Technologies, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifndef __TFW_HTTP_HPACK_TBL_H__ +#define __TFW_HTTP_HPACK_TBL_H__ + +/** + * Huffman decoder state machine: + * + * |shift| - modulus of the "shift" field always contains number of bits in + * the Huffman-encoded representation, which must be taken by + * decoder on this step. + * + * For short tables (f.e. 8-entries tables) difference in the prefix length + * (e.g. 7 bits for the big tables minus 3 bits for short tables = 4 bits) was + * pre-added to the value of "shift" field (just to speedup the decoder), thus + * real value of the Huffman-encoded prefix, which must be taken by decoder on + * this step is equal to the "shift" minus three. + * + * @shift > 0 - normal symbol, @offset in this case is a signed char + * representation of the decoded symbol; + * + * @shift < 0 && offset == 0 - end of sequence (EOS), |shift| is the number of + * bits in the truncated path. + * + * @shift < 0 && offset > 0 - jump to next table, @offset is offset of the next + * table. + */ +typedef struct { + short shift; + short offset; +} __attribute__((packed)) HTState; + +/* + * All the below is generated by hgen.c. DO NOT EDIT IT BY HANDS! + * If you need to change the constants, then update the genertion program, + * remove all the below the comment and run: + * + * $ gcc ./hgen.c && ./a.out >> hpack_tbl.h + */ + +#define HT_NBITS 7 +#define HT_MBITS 3 +#define HT_NMASK 127 +#define HT_MMASK 7 +#define HT_SMALL 640 +#define HT_EOS_HIGH 0xFF + +static const unsigned int ht_encode[] __page_aligned_data = { + 0x00001FF8, 0x007FFFD8, 0x0FFFFFE2, 0x0FFFFFE3, + 0x0FFFFFE4, 0x0FFFFFE5, 0x0FFFFFE6, 0x0FFFFFE7, + 0x0FFFFFE8, 0x00FFFFEA, 0x3FFFFFFC, 0x0FFFFFE9, + 0x0FFFFFEA, 0x3FFFFFFD, 0x0FFFFFEB, 0x0FFFFFEC, + 0x0FFFFFED, 0x0FFFFFEE, 0x0FFFFFEF, 0x0FFFFFF0, + 0x0FFFFFF1, 0x0FFFFFF2, 0x3FFFFFFE, 0x0FFFFFF3, + 0x0FFFFFF4, 0x0FFFFFF5, 0x0FFFFFF6, 0x0FFFFFF7, + 0x0FFFFFF8, 0x0FFFFFF9, 0x0FFFFFFA, 0x0FFFFFFB, + 0x00000014, 0x000003F8, 0x000003F9, 0x00000FFA, + 0x00001FF9, 0x00000015, 0x000000F8, 0x000007FA, + 0x000003FA, 0x000003FB, 0x000000F9, 0x000007FB, + 0x000000FA, 0x00000016, 0x00000017, 0x00000018, + 0x00000000, 0x00000001, 0x00000002, 0x00000019, + 0x0000001A, 0x0000001B, 0x0000001C, 0x0000001D, + 0x0000001E, 0x0000001F, 0x0000005C, 0x000000FB, + 0x00007FFC, 0x00000020, 0x00000FFB, 0x000003FC, + 0x00001FFA, 0x00000021, 0x0000005D, 0x0000005E, + 0x0000005F, 0x00000060, 0x00000061, 0x00000062, + 0x00000063, 0x00000064, 0x00000065, 0x00000066, + 0x00000067, 0x00000068, 0x00000069, 0x0000006A, + 0x0000006B, 0x0000006C, 0x0000006D, 0x0000006E, + 0x0000006F, 0x00000070, 0x00000071, 0x00000072, + 0x000000FC, 0x00000073, 0x000000FD, 0x00001FFB, + 0x0007FFF0, 0x00001FFC, 0x00003FFC, 0x00000022, + 0x00007FFD, 0x00000003, 0x00000023, 0x00000004, + 0x00000024, 0x00000005, 0x00000025, 0x00000026, + 0x00000027, 0x00000006, 0x00000074, 0x00000075, + 0x00000028, 0x00000029, 0x0000002A, 0x00000007, + 0x0000002B, 0x00000076, 0x0000002C, 0x00000008, + 0x00000009, 0x0000002D, 0x00000077, 0x00000078, + 0x00000079, 0x0000007A, 0x0000007B, 0x00007FFE, + 0x000007FC, 0x00003FFD, 0x00001FFD, 0x0FFFFFFC, + 0x000FFFE6, 0x003FFFD2, 0x000FFFE7, 0x000FFFE8, + 0x003FFFD3, 0x003FFFD4, 0x003FFFD5, 0x007FFFD9, + 0x003FFFD6, 0x007FFFDA, 0x007FFFDB, 0x007FFFDC, + 0x007FFFDD, 0x007FFFDE, 0x00FFFFEB, 0x007FFFDF, + 0x00FFFFEC, 0x00FFFFED, 0x003FFFD7, 0x007FFFE0, + 0x00FFFFEE, 0x007FFFE1, 0x007FFFE2, 0x007FFFE3, + 0x007FFFE4, 0x001FFFDC, 0x003FFFD8, 0x007FFFE5, + 0x003FFFD9, 0x007FFFE6, 0x007FFFE7, 0x00FFFFEF, + 0x003FFFDA, 0x001FFFDD, 0x000FFFE9, 0x003FFFDB, + 0x003FFFDC, 0x007FFFE8, 0x007FFFE9, 0x001FFFDE, + 0x007FFFEA, 0x003FFFDD, 0x003FFFDE, 0x00FFFFF0, + 0x001FFFDF, 0x003FFFDF, 0x007FFFEB, 0x007FFFEC, + 0x001FFFE0, 0x001FFFE1, 0x003FFFE0, 0x001FFFE2, + 0x007FFFED, 0x003FFFE1, 0x007FFFEE, 0x007FFFEF, + 0x000FFFEA, 0x003FFFE2, 0x003FFFE3, 0x003FFFE4, + 0x007FFFF0, 0x003FFFE5, 0x003FFFE6, 0x007FFFF1, + 0x03FFFFE0, 0x03FFFFE1, 0x000FFFEB, 0x0007FFF1, + 0x003FFFE7, 0x007FFFF2, 0x003FFFE8, 0x01FFFFEC, + 0x03FFFFE2, 0x03FFFFE3, 0x03FFFFE4, 0x07FFFFDE, + 0x07FFFFDF, 0x03FFFFE5, 0x00FFFFF1, 0x01FFFFED, + 0x0007FFF2, 0x001FFFE3, 0x03FFFFE6, 0x07FFFFE0, + 0x07FFFFE1, 0x03FFFFE7, 0x07FFFFE2, 0x00FFFFF2, + 0x001FFFE4, 0x001FFFE5, 0x03FFFFE8, 0x03FFFFE9, + 0x0FFFFFFD, 0x07FFFFE3, 0x07FFFFE4, 0x07FFFFE5, + 0x000FFFEC, 0x00FFFFF3, 0x000FFFED, 0x001FFFE6, + 0x003FFFE9, 0x001FFFE7, 0x001FFFE8, 0x007FFFF3, + 0x003FFFEA, 0x003FFFEB, 0x01FFFFEE, 0x01FFFFEF, + 0x00FFFFF4, 0x00FFFFF5, 0x03FFFFEA, 0x007FFFF4, + 0x03FFFFEB, 0x07FFFFE6, 0x03FFFFEC, 0x03FFFFED, + 0x07FFFFE7, 0x07FFFFE8, 0x07FFFFE9, 0x07FFFFEA, + 0x07FFFFEB, 0x0FFFFFFE, 0x07FFFFEC, 0x07FFFFED, + 0x07FFFFEE, 0x07FFFFEF, 0x07FFFFF0, 0x03FFFFEE +}; + +static const unsigned char ht_length[] ____cacheline_aligned = { + 13, 23, 28, 28, 28, 28, 28, 28, 28, 24, 30, 28, 28, 30, 28, 28, + 28, 28, 28, 28, 28, 28, 30, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 6, 10, 10, 12, 13, 6, 8, 11, 10, 10, 8, 11, 8, 6, 6, 6, + 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 8, 15, 6, 12, 10, + 13, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 8, 7, 8, 13, 19, 13, 14, 6, + 15, 5, 6, 5, 6, 5, 6, 6, 6, 5, 7, 7, 6, 6, 6, 5, + 6, 7, 6, 5, 5, 6, 7, 7, 7, 7, 7, 15, 11, 14, 13, 28, + 20, 22, 20, 20, 22, 22, 22, 23, 22, 23, 23, 23, 23, 23, 24, 23, + 24, 24, 22, 23, 24, 23, 23, 23, 23, 21, 22, 23, 22, 23, 23, 24, + 22, 21, 20, 22, 22, 23, 23, 21, 23, 22, 22, 24, 21, 22, 23, 23, + 21, 21, 22, 21, 23, 22, 23, 23, 20, 22, 22, 22, 23, 22, 22, 23, + 26, 26, 20, 19, 22, 23, 22, 25, 26, 26, 26, 27, 27, 26, 24, 25, + 19, 21, 26, 27, 27, 26, 27, 24, 21, 21, 26, 26, 28, 27, 27, 27, + 20, 24, 20, 21, 22, 21, 21, 23, 22, 22, 25, 25, 24, 24, 26, 23, + 26, 27, 26, 26, 27, 27, 27, 27, 27, 28, 27, 27, 27, 27, 27, 26 +}; + +static const HTState ht_decode[] __page_aligned_data = { +/* --- [TABLE-128: offset = 0] --- */ + {5, 48}, /* 5: '0' (48) */ + {5, 48}, /* 5: '0' (48) */ + {5, 48}, /* 5: '0' (48) */ + {5, 48}, /* 5: '0' (48) */ + {5, 49}, /* 5: '1' (49) */ + {5, 49}, /* 5: '1' (49) */ + {5, 49}, /* 5: '1' (49) */ + {5, 49}, /* 5: '1' (49) */ + {5, 50}, /* 5: '2' (50) */ + {5, 50}, /* 5: '2' (50) */ + {5, 50}, /* 5: '2' (50) */ + {5, 50}, /* 5: '2' (50) */ + {5, 97}, /* 5: 'a' (97) */ + {5, 97}, /* 5: 'a' (97) */ + {5, 97}, /* 5: 'a' (97) */ + {5, 97}, /* 5: 'a' (97) */ + {5, 99}, /* 5: 'c' (99) */ + {5, 99}, /* 5: 'c' (99) */ + {5, 99}, /* 5: 'c' (99) */ + {5, 99}, /* 5: 'c' (99) */ + {5, 101}, /* 5: 'e' (101) */ + {5, 101}, /* 5: 'e' (101) */ + {5, 101}, /* 5: 'e' (101) */ + {5, 101}, /* 5: 'e' (101) */ + {5, 105}, /* 5: 'i' (105) */ + {5, 105}, /* 5: 'i' (105) */ + {5, 105}, /* 5: 'i' (105) */ + {5, 105}, /* 5: 'i' (105) */ + {5, 111}, /* 5: 'o' (111) */ + {5, 111}, /* 5: 'o' (111) */ + {5, 111}, /* 5: 'o' (111) */ + {5, 111}, /* 5: 'o' (111) */ + {5, 115}, /* 5: 's' (115) */ + {5, 115}, /* 5: 's' (115) */ + {5, 115}, /* 5: 's' (115) */ + {5, 115}, /* 5: 's' (115) */ + {5, 116}, /* 5: 't' (116) */ + {5, 116}, /* 5: 't' (116) */ + {5, 116}, /* 5: 't' (116) */ + {5, 116}, /* 5: 't' (116) */ + {6, 32}, /* 6: ' ' (32) */ + {6, 32}, /* 6: ' ' (32) */ + {6, 37}, /* 6: '%' (37) */ + {6, 37}, /* 6: '%' (37) */ + {6, 45}, /* 6: '-' (45) */ + {6, 45}, /* 6: '-' (45) */ + {6, 46}, /* 6: '.' (46) */ + {6, 46}, /* 6: '.' (46) */ + {6, 47}, /* 6: '/' (47) */ + {6, 47}, /* 6: '/' (47) */ + {6, 51}, /* 6: '3' (51) */ + {6, 51}, /* 6: '3' (51) */ + {6, 52}, /* 6: '4' (52) */ + {6, 52}, /* 6: '4' (52) */ + {6, 53}, /* 6: '5' (53) */ + {6, 53}, /* 6: '5' (53) */ + {6, 54}, /* 6: '6' (54) */ + {6, 54}, /* 6: '6' (54) */ + {6, 55}, /* 6: '7' (55) */ + {6, 55}, /* 6: '7' (55) */ + {6, 56}, /* 6: '8' (56) */ + {6, 56}, /* 6: '8' (56) */ + {6, 57}, /* 6: '9' (57) */ + {6, 57}, /* 6: '9' (57) */ + {6, 61}, /* 6: '=' (61) */ + {6, 61}, /* 6: '=' (61) */ + {6, 65}, /* 6: 'A' (65) */ + {6, 65}, /* 6: 'A' (65) */ + {6, 95}, /* 6: '_' (95) */ + {6, 95}, /* 6: '_' (95) */ + {6, 98}, /* 6: 'b' (98) */ + {6, 98}, /* 6: 'b' (98) */ + {6, 100}, /* 6: 'd' (100) */ + {6, 100}, /* 6: 'd' (100) */ + {6, 102}, /* 6: 'f' (102) */ + {6, 102}, /* 6: 'f' (102) */ + {6, 103}, /* 6: 'g' (103) */ + {6, 103}, /* 6: 'g' (103) */ + {6, 104}, /* 6: 'h' (104) */ + {6, 104}, /* 6: 'h' (104) */ + {6, 108}, /* 6: 'l' (108) */ + {6, 108}, /* 6: 'l' (108) */ + {6, 109}, /* 6: 'm' (109) */ + {6, 109}, /* 6: 'm' (109) */ + {6, 110}, /* 6: 'n' (110) */ + {6, 110}, /* 6: 'n' (110) */ + {6, 112}, /* 6: 'p' (112) */ + {6, 112}, /* 6: 'p' (112) */ + {6, 114}, /* 6: 'r' (114) */ + {6, 114}, /* 6: 'r' (114) */ + {6, 117}, /* 6: 'u' (117) */ + {6, 117}, /* 6: 'u' (117) */ + {7, 58}, /* 7: ':' (58) */ + {7, 66}, /* 7: 'B' (66) */ + {7, 67}, /* 7: 'C' (67) */ + {7, 68}, /* 7: 'D' (68) */ + {7, 69}, /* 7: 'E' (69) */ + {7, 70}, /* 7: 'F' (70) */ + {7, 71}, /* 7: 'G' (71) */ + {7, 72}, /* 7: 'H' (72) */ + {7, 73}, /* 7: 'I' (73) */ + {7, 74}, /* 7: 'J' (74) */ + {7, 75}, /* 7: 'K' (75) */ + {7, 76}, /* 7: 'L' (76) */ + {7, 77}, /* 7: 'M' (77) */ + {7, 78}, /* 7: 'N' (78) */ + {7, 79}, /* 7: 'O' (79) */ + {7, 80}, /* 7: 'P' (80) */ + {7, 81}, /* 7: 'Q' (81) */ + {7, 82}, /* 7: 'R' (82) */ + {7, 83}, /* 7: 'S' (83) */ + {7, 84}, /* 7: 'T' (84) */ + {7, 85}, /* 7: 'U' (85) */ + {7, 86}, /* 7: 'V' (86) */ + {7, 87}, /* 7: 'W' (87) */ + {7, 89}, /* 7: 'Y' (89) */ + {7, 106}, /* 7: 'j' (106) */ + {7, 107}, /* 7: 'k' (107) */ + {7, 113}, /* 7: 'q' (113) */ + {7, 118}, /* 7: 'v' (118) */ + {7, 119}, /* 7: 'w' (119) */ + {7, 120}, /* 7: 'x' (120) */ + {7, 121}, /* 7: 'y' (121) */ + {7, 122}, /* 7: 'z' (122) */ + {-7, 640}, /* 7: ---> TABLE 640 */ + {-7, 648}, /* 7: ---> TABLE 648 */ + {-7, 656}, /* 7: ---> TABLE 656 */ + {-7, 128}, /* 7: ---> TABLE 128 */ +/* --- [TABLE-128: offset = 128] --- */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 33}, /* 3: '!' (33) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 34}, /* 3: '"' (34) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 40}, /* 3: '(' (40) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 41}, /* 3: ')' (41) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {3, 63}, /* 3: '?' (63) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 39}, /* 4: '\'' (39) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 43}, /* 4: '+' (43) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {4, 124}, /* 4: '|' (124) */ + {5, 35}, /* 5: '#' (35) */ + {5, 35}, /* 5: '#' (35) */ + {5, 35}, /* 5: '#' (35) */ + {5, 35}, /* 5: '#' (35) */ + {5, 62}, /* 5: '>' (62) */ + {5, 62}, /* 5: '>' (62) */ + {5, 62}, /* 5: '>' (62) */ + {5, 62}, /* 5: '>' (62) */ + {6, 0}, /* 6: '\x00' (0) */ + {6, 0}, /* 6: '\x00' (0) */ + {6, 36}, /* 6: '$' (36) */ + {6, 36}, /* 6: '$' (36) */ + {6, 64}, /* 6: '@' (64) */ + {6, 64}, /* 6: '@' (64) */ + {6, 91}, /* 6: '[' (91) */ + {6, 91}, /* 6: '[' (91) */ + {6, 93}, /* 6: ']' (93) */ + {6, 93}, /* 6: ']' (93) */ + {6, 126}, /* 6: '~' (126) */ + {6, 126}, /* 6: '~' (126) */ + {7, 94}, /* 7: '^' (94) */ + {7, 125}, /* 7: '}' (125) */ + {-7, 664}, /* 7: ---> TABLE 664 */ + {-7, 256}, /* 7: ---> TABLE 256 */ +/* --- [TABLE-128: offset = 256] --- */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {1, 123}, /* 1: '{' (123) */ + {5, 92}, /* 5: '\\' (92) */ + {5, 92}, /* 5: '\\' (92) */ + {5, 92}, /* 5: '\\' (92) */ + {5, 92}, /* 5: '\\' (92) */ + {5, -61}, /* 5: '\xC3' (195) */ + {5, -61}, /* 5: '\xC3' (195) */ + {5, -61}, /* 5: '\xC3' (195) */ + {5, -61}, /* 5: '\xC3' (195) */ + {5, -48}, /* 5: '\xD0' (208) */ + {5, -48}, /* 5: '\xD0' (208) */ + {5, -48}, /* 5: '\xD0' (208) */ + {5, -48}, /* 5: '\xD0' (208) */ + {6, -128}, /* 6: '\x80' (128) */ + {6, -128}, /* 6: '\x80' (128) */ + {6, -126}, /* 6: '\x82' (130) */ + {6, -126}, /* 6: '\x82' (130) */ + {6, -125}, /* 6: '\x83' (131) */ + {6, -125}, /* 6: '\x83' (131) */ + {6, -94}, /* 6: '\xA2' (162) */ + {6, -94}, /* 6: '\xA2' (162) */ + {6, -72}, /* 6: '\xB8' (184) */ + {6, -72}, /* 6: '\xB8' (184) */ + {6, -62}, /* 6: '\xC2' (194) */ + {6, -62}, /* 6: '\xC2' (194) */ + {6, -32}, /* 6: '\xE0' (224) */ + {6, -32}, /* 6: '\xE0' (224) */ + {6, -30}, /* 6: '\xE2' (226) */ + {6, -30}, /* 6: '\xE2' (226) */ + {7, -103}, /* 7: '\x99' (153) */ + {7, -95}, /* 7: '\xA1' (161) */ + {7, -89}, /* 7: '\xA7' (167) */ + {7, -84}, /* 7: '\xAC' (172) */ + {7, -80}, /* 7: '\xB0' (176) */ + {7, -79}, /* 7: '\xB1' (177) */ + {7, -77}, /* 7: '\xB3' (179) */ + {7, -47}, /* 7: '\xD1' (209) */ + {7, -40}, /* 7: '\xD8' (216) */ + {7, -39}, /* 7: '\xD9' (217) */ + {7, -29}, /* 7: '\xE3' (227) */ + {7, -27}, /* 7: '\xE5' (229) */ + {7, -26}, /* 7: '\xE6' (230) */ + {-7, 672}, /* 7: ---> TABLE 672 */ + {-7, 680}, /* 7: ---> TABLE 680 */ + {-7, 688}, /* 7: ---> TABLE 688 */ + {-7, 696}, /* 7: ---> TABLE 696 */ + {-7, 704}, /* 7: ---> TABLE 704 */ + {-7, 712}, /* 7: ---> TABLE 712 */ + {-7, 720}, /* 7: ---> TABLE 720 */ + {-7, 728}, /* 7: ---> TABLE 728 */ + {-7, 736}, /* 7: ---> TABLE 736 */ + {-7, 744}, /* 7: ---> TABLE 744 */ + {-7, 752}, /* 7: ---> TABLE 752 */ + {-7, 760}, /* 7: ---> TABLE 760 */ + {-7, 768}, /* 7: ---> TABLE 768 */ + {-7, 776}, /* 7: ---> TABLE 776 */ + {-7, 784}, /* 7: ---> TABLE 784 */ + {-7, 792}, /* 7: ---> TABLE 792 */ + {-7, 800}, /* 7: ---> TABLE 800 */ + {-7, 808}, /* 7: ---> TABLE 808 */ + {-7, 816}, /* 7: ---> TABLE 816 */ + {-7, 824}, /* 7: ---> TABLE 824 */ + {-7, 832}, /* 7: ---> TABLE 832 */ + {-7, 384}, /* 7: ---> TABLE 384 */ + {-7, 512}, /* 7: ---> TABLE 512 */ +/* --- [TABLE-128: offset = 384] --- */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -85}, /* 3: '\xAB' (171) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -50}, /* 3: '\xCE' (206) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -41}, /* 3: '\xD7' (215) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -31}, /* 3: '\xE1' (225) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -20}, /* 3: '\xEC' (236) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {3, -19}, /* 3: '\xED' (237) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -57}, /* 4: '\xC7' (199) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -49}, /* 4: '\xCF' (207) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -22}, /* 4: '\xEA' (234) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ + {4, -21}, /* 4: '\xEB' (235) */ +/* --- [TABLE-128: offset = 512] --- */ + {5, -64}, /* 5: '\xC0' (192) */ + {5, -64}, /* 5: '\xC0' (192) */ + {5, -64}, /* 5: '\xC0' (192) */ + {5, -64}, /* 5: '\xC0' (192) */ + {5, -63}, /* 5: '\xC1' (193) */ + {5, -63}, /* 5: '\xC1' (193) */ + {5, -63}, /* 5: '\xC1' (193) */ + {5, -63}, /* 5: '\xC1' (193) */ + {5, -56}, /* 5: '\xC8' (200) */ + {5, -56}, /* 5: '\xC8' (200) */ + {5, -56}, /* 5: '\xC8' (200) */ + {5, -56}, /* 5: '\xC8' (200) */ + {5, -55}, /* 5: '\xC9' (201) */ + {5, -55}, /* 5: '\xC9' (201) */ + {5, -55}, /* 5: '\xC9' (201) */ + {5, -55}, /* 5: '\xC9' (201) */ + {5, -54}, /* 5: '\xCA' (202) */ + {5, -54}, /* 5: '\xCA' (202) */ + {5, -54}, /* 5: '\xCA' (202) */ + {5, -54}, /* 5: '\xCA' (202) */ + {5, -51}, /* 5: '\xCD' (205) */ + {5, -51}, /* 5: '\xCD' (205) */ + {5, -51}, /* 5: '\xCD' (205) */ + {5, -51}, /* 5: '\xCD' (205) */ + {5, -46}, /* 5: '\xD2' (210) */ + {5, -46}, /* 5: '\xD2' (210) */ + {5, -46}, /* 5: '\xD2' (210) */ + {5, -46}, /* 5: '\xD2' (210) */ + {5, -43}, /* 5: '\xD5' (213) */ + {5, -43}, /* 5: '\xD5' (213) */ + {5, -43}, /* 5: '\xD5' (213) */ + {5, -43}, /* 5: '\xD5' (213) */ + {5, -38}, /* 5: '\xDA' (218) */ + {5, -38}, /* 5: '\xDA' (218) */ + {5, -38}, /* 5: '\xDA' (218) */ + {5, -38}, /* 5: '\xDA' (218) */ + {5, -37}, /* 5: '\xDB' (219) */ + {5, -37}, /* 5: '\xDB' (219) */ + {5, -37}, /* 5: '\xDB' (219) */ + {5, -37}, /* 5: '\xDB' (219) */ + {5, -18}, /* 5: '\xEE' (238) */ + {5, -18}, /* 5: '\xEE' (238) */ + {5, -18}, /* 5: '\xEE' (238) */ + {5, -18}, /* 5: '\xEE' (238) */ + {5, -16}, /* 5: '\xF0' (240) */ + {5, -16}, /* 5: '\xF0' (240) */ + {5, -16}, /* 5: '\xF0' (240) */ + {5, -16}, /* 5: '\xF0' (240) */ + {5, -14}, /* 5: '\xF2' (242) */ + {5, -14}, /* 5: '\xF2' (242) */ + {5, -14}, /* 5: '\xF2' (242) */ + {5, -14}, /* 5: '\xF2' (242) */ + {5, -13}, /* 5: '\xF3' (243) */ + {5, -13}, /* 5: '\xF3' (243) */ + {5, -13}, /* 5: '\xF3' (243) */ + {5, -13}, /* 5: '\xF3' (243) */ + {5, -1}, /* 5: '\xFF' (255) */ + {5, -1}, /* 5: '\xFF' (255) */ + {5, -1}, /* 5: '\xFF' (255) */ + {5, -1}, /* 5: '\xFF' (255) */ + {6, -53}, /* 6: '\xCB' (203) */ + {6, -53}, /* 6: '\xCB' (203) */ + {6, -52}, /* 6: '\xCC' (204) */ + {6, -52}, /* 6: '\xCC' (204) */ + {6, -45}, /* 6: '\xD3' (211) */ + {6, -45}, /* 6: '\xD3' (211) */ + {6, -44}, /* 6: '\xD4' (212) */ + {6, -44}, /* 6: '\xD4' (212) */ + {6, -42}, /* 6: '\xD6' (214) */ + {6, -42}, /* 6: '\xD6' (214) */ + {6, -35}, /* 6: '\xDD' (221) */ + {6, -35}, /* 6: '\xDD' (221) */ + {6, -34}, /* 6: '\xDE' (222) */ + {6, -34}, /* 6: '\xDE' (222) */ + {6, -33}, /* 6: '\xDF' (223) */ + {6, -33}, /* 6: '\xDF' (223) */ + {6, -15}, /* 6: '\xF1' (241) */ + {6, -15}, /* 6: '\xF1' (241) */ + {6, -12}, /* 6: '\xF4' (244) */ + {6, -12}, /* 6: '\xF4' (244) */ + {6, -11}, /* 6: '\xF5' (245) */ + {6, -11}, /* 6: '\xF5' (245) */ + {6, -10}, /* 6: '\xF6' (246) */ + {6, -10}, /* 6: '\xF6' (246) */ + {6, -9}, /* 6: '\xF7' (247) */ + {6, -9}, /* 6: '\xF7' (247) */ + {6, -8}, /* 6: '\xF8' (248) */ + {6, -8}, /* 6: '\xF8' (248) */ + {6, -6}, /* 6: '\xFA' (250) */ + {6, -6}, /* 6: '\xFA' (250) */ + {6, -5}, /* 6: '\xFB' (251) */ + {6, -5}, /* 6: '\xFB' (251) */ + {6, -4}, /* 6: '\xFC' (252) */ + {6, -4}, /* 6: '\xFC' (252) */ + {6, -3}, /* 6: '\xFD' (253) */ + {6, -3}, /* 6: '\xFD' (253) */ + {6, -2}, /* 6: '\xFE' (254) */ + {6, -2}, /* 6: '\xFE' (254) */ + {7, 2}, /* 7: '\x02' (2) */ + {7, 3}, /* 7: '\x03' (3) */ + {7, 4}, /* 7: '\x04' (4) */ + {7, 5}, /* 7: '\x05' (5) */ + {7, 6}, /* 7: '\x06' (6) */ + {7, 7}, /* 7: '\x07' (7) */ + {7, 8}, /* 7: '\x08' (8) */ + {7, 11}, /* 7: '\x0B' (11) */ + {7, 12}, /* 7: '\x0C' (12) */ + {7, 14}, /* 7: '\x0E' (14) */ + {7, 15}, /* 7: '\x0F' (15) */ + {7, 16}, /* 7: '\x10' (16) */ + {7, 17}, /* 7: '\x11' (17) */ + {7, 18}, /* 7: '\x12' (18) */ + {7, 19}, /* 7: '\x13' (19) */ + {7, 20}, /* 7: '\x14' (20) */ + {7, 21}, /* 7: '\x15' (21) */ + {7, 23}, /* 7: '\x17' (23) */ + {7, 24}, /* 7: '\x18' (24) */ + {7, 25}, /* 7: '\x19' (25) */ + {7, 26}, /* 7: '\x1A' (26) */ + {7, 27}, /* 7: '\x1B' (27) */ + {7, 28}, /* 7: '\x1C' (28) */ + {7, 29}, /* 7: '\x1D' (29) */ + {7, 30}, /* 7: '\x1E' (30) */ + {7, 31}, /* 7: '\x1F' (31) */ + {7, 127}, /* 7: '\x7F' (127) */ + {7, -36}, /* 7: '\xDC' (220) */ + {7, -7}, /* 7: '\xF9' (249) */ + {-7, 840}, /* 7: ---> TABLE 840 */ +/* --- [TABLE-8: offset = 640] --- */ + {5, 38}, /* 1: '&' (38) */ + {5, 38}, /* 1: '&' (38) */ + {5, 38}, /* 1: '&' (38) */ + {5, 38}, /* 1: '&' (38) */ + {5, 42}, /* 1: '*' (42) */ + {5, 42}, /* 1: '*' (42) */ + {5, 42}, /* 1: '*' (42) */ + {5, 42}, /* 1: '*' (42) */ +/* --- [TABLE-8: offset = 648] --- */ + {5, 44}, /* 1: ',' (44) */ + {5, 44}, /* 1: ',' (44) */ + {5, 44}, /* 1: ',' (44) */ + {5, 44}, /* 1: ',' (44) */ + {5, 59}, /* 1: ';' (59) */ + {5, 59}, /* 1: ';' (59) */ + {5, 59}, /* 1: ';' (59) */ + {5, 59}, /* 1: ';' (59) */ +/* --- [TABLE-8: offset = 656] --- */ + {5, 88}, /* 1: 'X' (88) */ + {5, 88}, /* 1: 'X' (88) */ + {5, 88}, /* 1: 'X' (88) */ + {5, 88}, /* 1: 'X' (88) */ + {5, 90}, /* 1: 'Z' (90) */ + {5, 90}, /* 1: 'Z' (90) */ + {5, 90}, /* 1: 'Z' (90) */ + {5, 90}, /* 1: 'Z' (90) */ +/* --- [TABLE-8: offset = 664] --- */ + {5, 60}, /* 1: '<' (60) */ + {5, 60}, /* 1: '<' (60) */ + {5, 60}, /* 1: '<' (60) */ + {5, 60}, /* 1: '<' (60) */ + {5, 96}, /* 1: '`' (96) */ + {5, 96}, /* 1: '`' (96) */ + {5, 96}, /* 1: '`' (96) */ + {5, 96}, /* 1: '`' (96) */ +/* --- [TABLE-8: offset = 672] --- */ + {5, -127}, /* 1: '\x81' (129) */ + {5, -127}, /* 1: '\x81' (129) */ + {5, -127}, /* 1: '\x81' (129) */ + {5, -127}, /* 1: '\x81' (129) */ + {5, -124}, /* 1: '\x84' (132) */ + {5, -124}, /* 1: '\x84' (132) */ + {5, -124}, /* 1: '\x84' (132) */ + {5, -124}, /* 1: '\x84' (132) */ +/* --- [TABLE-8: offset = 680] --- */ + {5, -123}, /* 1: '\x85' (133) */ + {5, -123}, /* 1: '\x85' (133) */ + {5, -123}, /* 1: '\x85' (133) */ + {5, -123}, /* 1: '\x85' (133) */ + {5, -122}, /* 1: '\x86' (134) */ + {5, -122}, /* 1: '\x86' (134) */ + {5, -122}, /* 1: '\x86' (134) */ + {5, -122}, /* 1: '\x86' (134) */ +/* --- [TABLE-8: offset = 688] --- */ + {5, -120}, /* 1: '\x88' (136) */ + {5, -120}, /* 1: '\x88' (136) */ + {5, -120}, /* 1: '\x88' (136) */ + {5, -120}, /* 1: '\x88' (136) */ + {5, -110}, /* 1: '\x92' (146) */ + {5, -110}, /* 1: '\x92' (146) */ + {5, -110}, /* 1: '\x92' (146) */ + {5, -110}, /* 1: '\x92' (146) */ +/* --- [TABLE-8: offset = 696] --- */ + {5, -102}, /* 1: '\x9A' (154) */ + {5, -102}, /* 1: '\x9A' (154) */ + {5, -102}, /* 1: '\x9A' (154) */ + {5, -102}, /* 1: '\x9A' (154) */ + {5, -100}, /* 1: '\x9C' (156) */ + {5, -100}, /* 1: '\x9C' (156) */ + {5, -100}, /* 1: '\x9C' (156) */ + {5, -100}, /* 1: '\x9C' (156) */ +/* --- [TABLE-8: offset = 704] --- */ + {5, -96}, /* 1: '\xA0' (160) */ + {5, -96}, /* 1: '\xA0' (160) */ + {5, -96}, /* 1: '\xA0' (160) */ + {5, -96}, /* 1: '\xA0' (160) */ + {5, -93}, /* 1: '\xA3' (163) */ + {5, -93}, /* 1: '\xA3' (163) */ + {5, -93}, /* 1: '\xA3' (163) */ + {5, -93}, /* 1: '\xA3' (163) */ +/* --- [TABLE-8: offset = 712] --- */ + {5, -92}, /* 1: '\xA4' (164) */ + {5, -92}, /* 1: '\xA4' (164) */ + {5, -92}, /* 1: '\xA4' (164) */ + {5, -92}, /* 1: '\xA4' (164) */ + {5, -87}, /* 1: '\xA9' (169) */ + {5, -87}, /* 1: '\xA9' (169) */ + {5, -87}, /* 1: '\xA9' (169) */ + {5, -87}, /* 1: '\xA9' (169) */ +/* --- [TABLE-8: offset = 720] --- */ + {5, -86}, /* 1: '\xAA' (170) */ + {5, -86}, /* 1: '\xAA' (170) */ + {5, -86}, /* 1: '\xAA' (170) */ + {5, -86}, /* 1: '\xAA' (170) */ + {5, -83}, /* 1: '\xAD' (173) */ + {5, -83}, /* 1: '\xAD' (173) */ + {5, -83}, /* 1: '\xAD' (173) */ + {5, -83}, /* 1: '\xAD' (173) */ +/* --- [TABLE-8: offset = 728] --- */ + {5, -78}, /* 1: '\xB2' (178) */ + {5, -78}, /* 1: '\xB2' (178) */ + {5, -78}, /* 1: '\xB2' (178) */ + {5, -78}, /* 1: '\xB2' (178) */ + {5, -75}, /* 1: '\xB5' (181) */ + {5, -75}, /* 1: '\xB5' (181) */ + {5, -75}, /* 1: '\xB5' (181) */ + {5, -75}, /* 1: '\xB5' (181) */ +/* --- [TABLE-8: offset = 736] --- */ + {5, -71}, /* 1: '\xB9' (185) */ + {5, -71}, /* 1: '\xB9' (185) */ + {5, -71}, /* 1: '\xB9' (185) */ + {5, -71}, /* 1: '\xB9' (185) */ + {5, -70}, /* 1: '\xBA' (186) */ + {5, -70}, /* 1: '\xBA' (186) */ + {5, -70}, /* 1: '\xBA' (186) */ + {5, -70}, /* 1: '\xBA' (186) */ +/* --- [TABLE-8: offset = 744] --- */ + {5, -69}, /* 1: '\xBB' (187) */ + {5, -69}, /* 1: '\xBB' (187) */ + {5, -69}, /* 1: '\xBB' (187) */ + {5, -69}, /* 1: '\xBB' (187) */ + {5, -67}, /* 1: '\xBD' (189) */ + {5, -67}, /* 1: '\xBD' (189) */ + {5, -67}, /* 1: '\xBD' (189) */ + {5, -67}, /* 1: '\xBD' (189) */ +/* --- [TABLE-8: offset = 752] --- */ + {5, -66}, /* 1: '\xBE' (190) */ + {5, -66}, /* 1: '\xBE' (190) */ + {5, -66}, /* 1: '\xBE' (190) */ + {5, -66}, /* 1: '\xBE' (190) */ + {5, -60}, /* 1: '\xC4' (196) */ + {5, -60}, /* 1: '\xC4' (196) */ + {5, -60}, /* 1: '\xC4' (196) */ + {5, -60}, /* 1: '\xC4' (196) */ +/* --- [TABLE-8: offset = 760] --- */ + {5, -58}, /* 1: '\xC6' (198) */ + {5, -58}, /* 1: '\xC6' (198) */ + {5, -58}, /* 1: '\xC6' (198) */ + {5, -58}, /* 1: '\xC6' (198) */ + {5, -28}, /* 1: '\xE4' (228) */ + {5, -28}, /* 1: '\xE4' (228) */ + {5, -28}, /* 1: '\xE4' (228) */ + {5, -28}, /* 1: '\xE4' (228) */ +/* --- [TABLE-8: offset = 768] --- */ + {5, -24}, /* 1: '\xE8' (232) */ + {5, -24}, /* 1: '\xE8' (232) */ + {5, -24}, /* 1: '\xE8' (232) */ + {5, -24}, /* 1: '\xE8' (232) */ + {5, -23}, /* 1: '\xE9' (233) */ + {5, -23}, /* 1: '\xE9' (233) */ + {5, -23}, /* 1: '\xE9' (233) */ + {5, -23}, /* 1: '\xE9' (233) */ +/* --- [TABLE-8: offset = 776] --- */ + {6, 1}, /* 2: '\x01' (1) */ + {6, 1}, /* 2: '\x01' (1) */ + {6, -121}, /* 2: '\x87' (135) */ + {6, -121}, /* 2: '\x87' (135) */ + {6, -119}, /* 2: '\x89' (137) */ + {6, -119}, /* 2: '\x89' (137) */ + {6, -118}, /* 2: '\x8A' (138) */ + {6, -118}, /* 2: '\x8A' (138) */ +/* --- [TABLE-8: offset = 784] --- */ + {6, -117}, /* 2: '\x8B' (139) */ + {6, -117}, /* 2: '\x8B' (139) */ + {6, -116}, /* 2: '\x8C' (140) */ + {6, -116}, /* 2: '\x8C' (140) */ + {6, -115}, /* 2: '\x8D' (141) */ + {6, -115}, /* 2: '\x8D' (141) */ + {6, -113}, /* 2: '\x8F' (143) */ + {6, -113}, /* 2: '\x8F' (143) */ +/* --- [TABLE-8: offset = 792] --- */ + {6, -109}, /* 2: '\x93' (147) */ + {6, -109}, /* 2: '\x93' (147) */ + {6, -107}, /* 2: '\x95' (149) */ + {6, -107}, /* 2: '\x95' (149) */ + {6, -106}, /* 2: '\x96' (150) */ + {6, -106}, /* 2: '\x96' (150) */ + {6, -105}, /* 2: '\x97' (151) */ + {6, -105}, /* 2: '\x97' (151) */ +/* --- [TABLE-8: offset = 800] --- */ + {6, -104}, /* 2: '\x98' (152) */ + {6, -104}, /* 2: '\x98' (152) */ + {6, -101}, /* 2: '\x9B' (155) */ + {6, -101}, /* 2: '\x9B' (155) */ + {6, -99}, /* 2: '\x9D' (157) */ + {6, -99}, /* 2: '\x9D' (157) */ + {6, -98}, /* 2: '\x9E' (158) */ + {6, -98}, /* 2: '\x9E' (158) */ +/* --- [TABLE-8: offset = 808] --- */ + {6, -91}, /* 2: '\xA5' (165) */ + {6, -91}, /* 2: '\xA5' (165) */ + {6, -90}, /* 2: '\xA6' (166) */ + {6, -90}, /* 2: '\xA6' (166) */ + {6, -88}, /* 2: '\xA8' (168) */ + {6, -88}, /* 2: '\xA8' (168) */ + {6, -82}, /* 2: '\xAE' (174) */ + {6, -82}, /* 2: '\xAE' (174) */ +/* --- [TABLE-8: offset = 816] --- */ + {6, -81}, /* 2: '\xAF' (175) */ + {6, -81}, /* 2: '\xAF' (175) */ + {6, -76}, /* 2: '\xB4' (180) */ + {6, -76}, /* 2: '\xB4' (180) */ + {6, -74}, /* 2: '\xB6' (182) */ + {6, -74}, /* 2: '\xB6' (182) */ + {6, -73}, /* 2: '\xB7' (183) */ + {6, -73}, /* 2: '\xB7' (183) */ +/* --- [TABLE-8: offset = 824] --- */ + {6, -68}, /* 2: '\xBC' (188) */ + {6, -68}, /* 2: '\xBC' (188) */ + {6, -65}, /* 2: '\xBF' (191) */ + {6, -65}, /* 2: '\xBF' (191) */ + {6, -59}, /* 2: '\xC5' (197) */ + {6, -59}, /* 2: '\xC5' (197) */ + {6, -25}, /* 2: '\xE7' (231) */ + {6, -25}, /* 2: '\xE7' (231) */ +/* --- [TABLE-8: offset = 832] --- */ + {6, -17}, /* 2: '\xEF' (239) */ + {6, -17}, /* 2: '\xEF' (239) */ + {7, 9}, /* 3: '\x09' (9) */ + {7, -114}, /* 3: '\x8E' (142) */ + {7, -112}, /* 3: '\x90' (144) */ + {7, -111}, /* 3: '\x91' (145) */ + {7, -108}, /* 3: '\x94' (148) */ + {7, -97}, /* 3: '\x9F' (159) */ +/* --- [TABLE-8: offset = 840] --- */ + {6, 10}, /* 2: '\x0A' (10) */ + {6, 10}, /* 2: '\x0A' (10) */ + {6, 13}, /* 2: '\x0D' (13) */ + {6, 13}, /* 2: '\x0D' (13) */ + {6, 22}, /* 2: '\x16' (22) */ + {6, 22}, /* 2: '\x16' (22) */ + {-6, 0}, /* 2: EOS */ + {-6, 0}, /* 2: EOS */ +}; + +#endif /* __TFW_HTTP_HPACK_TBL_H__ */ From c67a387140d85954ae430b72a524217ad8ffd10b Mon Sep 17 00:00:00 2001 From: Alexander K Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 23/64] Fix some small comments --- tempesta_fw/hpack.c | 119 ++++++++++++++++++++++---------------------- tempesta_fw/hpack.h | 3 +- tempesta_fw/http.c | 113 +++++++++++++++++++++-------------------- 3 files changed, 117 insertions(+), 118 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index a9e6309e42..839b465f6d 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -288,6 +288,64 @@ do { \ } \ } while (0) +static inline int +__hpack_process_hdr_name(TfwHttpReq *req) +{ + const TfwStr *c, *end; + TfwMsgParseIter *it = &req->pit; + const TfwStr *hdr = &it->hdr, *next = it->next; + int ret = T_BAD; + + WARN_ON_ONCE(next != hdr); + TFW_STR_FOR_EACH_CHUNK(c, next, end) { + bool last = c + 1 == end; + + WARN_ON_ONCE(ret == T_OK); + ret = tfw_h2_parse_req_hdr(c->data, c->len, req, last, false); + if (unlikely(ret < T_POSTPONE)) + return ret; + } + return ret ? T_DROP : T_OK; +} + +static inline int +__hpack_process_hdr_value(TfwHttpReq *req) +{ + const TfwStr *chunk, *end; + TfwMsgParseIter *it = &req->pit; + const TfwStr *hdr = &it->hdr, *next = it->next; + int ret = T_BAD; + + BUG_ON(TFW_STR_DUP(hdr)); + if (TFW_STR_PLAIN(hdr)) { + WARN_ON_ONCE(hdr != next); + chunk = hdr; + end = hdr + 1; + } else { + /* + * In case of compound @hdr the @next can point either to the + * @hdr itself (if only header's value has been Huffman-decoded, + * i.e. in case of indexed or raw header's name), or to some + * chunk inside the @hdr (if both, the name and the value, has + * been Huffman-decoded). + */ + chunk = (hdr != next) ? next : next->chunks; + end = hdr->chunks + hdr->nchunks; + } + + while (chunk < end) { + bool last = chunk + 1 == end; + + WARN_ON_ONCE(ret == T_OK); + ret = tfw_h2_parse_req_hdr(chunk->data, chunk->len, + req, last, true); + if (unlikely(ret < T_POSTPONE)) + return ret; + ++chunk; + } + return ret ? T_DROP : T_OK; +} + #define HPACK_DECODE_PROCESS_STRING(field, len) \ do { \ T_DBG3("%s: decoding, len=%lu, n=%lu, tail=%lu\n", \ @@ -299,7 +357,7 @@ do { \ WARN_ON_ONCE(hp->length); \ hp->hctx = 0; \ tfw_huffman_init(hp); \ - if ((r = tfw_hpack_process_hdr_##field(req))) \ + if ((r = __hpack_process_hdr_##field(req))) \ goto out; \ T_DBG3("%s: processed decoded, tail=%lu\n", __func__, \ last - src); \ @@ -593,7 +651,6 @@ tfw_huffman_decode(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, return T_DROP; } - static int tfw_hpack_set_entry(TfwPool *__restrict h_pool, TfwMsgParseIter *__restrict it, TfwHPackEntry *__restrict entry, bool *__restrict np) @@ -990,64 +1047,6 @@ tfw_hpack_reinit(TfwHPack *__restrict hp, TfwMsgParseIter *__restrict it) sizeof(*hp) - offsetof(TfwHPack, __off)); } -static inline int -tfw_hpack_process_hdr_name(TfwHttpReq *req) -{ - const TfwStr *c, *end; - TfwMsgParseIter *it = &req->pit; - const TfwStr *hdr = &it->hdr, *next = it->next; - int ret = T_BAD; - - WARN_ON_ONCE(next != hdr); - TFW_STR_FOR_EACH_CHUNK(c, next, end) { - bool last = c + 1 == end; - - WARN_ON_ONCE(ret == T_OK); - ret = tfw_h2_parse_req_hdr(c->data, c->len, req, last, false); - if (unlikely(ret < T_POSTPONE)) - return ret; - } - return ret ? T_DROP : T_OK; -} - -static inline int -tfw_hpack_process_hdr_value(TfwHttpReq *req) -{ - const TfwStr *chunk, *end; - TfwMsgParseIter *it = &req->pit; - const TfwStr *hdr = &it->hdr, *next = it->next; - int ret = T_BAD; - - BUG_ON(TFW_STR_DUP(hdr)); - if (TFW_STR_PLAIN(hdr)) { - WARN_ON_ONCE(hdr != next); - chunk = hdr; - end = hdr + 1; - } else { - /* - * In case of compound @hdr the @next can point either to the - * @hdr itself (if only header's value has been Huffman-decoded, - * i.e. in case of indexed or raw header's name), or to some - * chunk inside the @hdr (if both, the name and the value, has - * been Huffman-decoded). - */ - chunk = (hdr != next) ? next : next->chunks; - end = hdr->chunks + hdr->nchunks; - } - - while (chunk < end) { - bool last = chunk + 1 == end; - - WARN_ON_ONCE(ret == T_OK); - ret = tfw_h2_parse_req_hdr(chunk->data, chunk->len, - req, last, true); - if (unlikely(ret < T_POSTPONE)) - return ret; - ++chunk; - } - return ret ? T_DROP : T_OK; -} - static int tfw_hpack_hdr_name_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, const TfwHPackEntry *__restrict entry) diff --git a/tempesta_fw/hpack.h b/tempesta_fw/hpack.h index 7b7b296426..01209831e5 100644 --- a/tempesta_fw/hpack.h +++ b/tempesta_fw/hpack.h @@ -144,7 +144,8 @@ typedef enum { * @name_len - length of the header's name part; * @name_num - chunks count of the header's name part; * @tag - tag of the indexed header; - * @last - flag bit indicating that corresponding header is the last on the page. + * @last - flag bit indicating that corresponding header is the last on + * the page. */ typedef struct { TfwStr *hdr; diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index cac3816428..bb01e2cd2b 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3090,10 +3090,9 @@ do { \ * Create special buffer to write headers block into the target HTTP/1.1 * representation. */ - if (!(pg = alloc_pages(GFP_ATOMIC, get_order(it->hdrs_len)))) + if (!(dst = pg_skb_alloc(it->hdrs_len, GFP_ATOMIC, NUMA_NO_NODE))) return -ENOMEM; - - dst = (char *)page_address(pg); + pg = virt_to_page(dst); /* Add request-line into the HTTP/1.1 request. */ BUG_ON(TFW_STR_EMPTY(&req->uri_path)); @@ -3153,7 +3152,7 @@ do { \ r = ss_skb_replace_page(&req->msg.skb_head, pg, it->hdrs_len, it->hb_len); if (r) { - __free_pages(pg, get_order(it->hdrs_len)); + put_page(pg); return r; } @@ -3641,6 +3640,7 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) TfwStr *tgt = &ht->tbl[hid]; TfwStr *first = TFW_STR_CHUNK(tgt, 0); TfwHdrModsDesc *f_desc = NULL; + const TfwStr *val; if (TFW_STR_DUP(tgt)) tgt = TFW_STR_CHUNK(tgt, d_num); @@ -3668,69 +3668,68 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) break; } } + if (!f_desc) + goto def; - if (f_desc) { - const TfwStr *val = TFW_STR_CHUNK(f_desc->hdr, 2); - /* - * If this is a duplicate of already processed header, - * leave this duplicate as is (for transformation - * in-place) in case of appending operation, and remove - * it (by skipping) in case of substitution or deletion - * operations. - */ - if (test_bit(k, mit->found)) { - if (!val || !f_desc->append) - continue; - - mit->bnd = first->data; - next->s_hdr = *tgt; - next->op = TFW_H2_TRANS_INPLACE; - - break; - } - - __set_bit(k, mit->found); - - /* - * If header configured with empty value, it should be - * removed from the response; so, just skip such header. - */ - if (!val) + *val = TFW_STR_CHUNK(f_desc->hdr, 2); + /* + * If this is a duplicate of already processed header, + * leave this duplicate as is (for transformation + * in-place) in case of appending operation, and remove + * it (by skipping) in case of substitution or deletion + * operations. + */ + if (test_bit(k, mit->found)) { + if (!val || !f_desc->append) continue; mit->bnd = first->data; + next->s_hdr = *tgt; + next->op = TFW_H2_TRANS_INPLACE; - /* - * If the header configured for value appending, - * concatenate it with the target header in skb for - * subsequent in-place rewriting. - */ - if (f_desc->append) { - TfwStr h_app = { - .chunks = (TfwStr []){ - { .data = ", ", .len = 2 }, - { .data = val->data, - .len = val->len } - }, - .len = val->len + 2, - .nchunks = 2 - }; - - r = tfw_strcat(resp->pool, tgt, &h_app); - if (unlikely(r)) - return r; - - next->s_hdr = *tgt; - next->op = TFW_H2_TRANS_INPLACE; + break; + } - break; - } + __set_bit(k, mit->found); - next->s_hdr = *f_desc->hdr; - next->op = TFW_H2_TRANS_SUB; + /* + * If header configured with empty value, it should be + * removed from the response; so, just skip such header. + */ + if (!val) + continue; + mit->bnd = first->data; + + /* + * If the header configured for value appending, + * concatenate it with the target header in skb for + * subsequent in-place rewriting. + */ + if (f_desc->append) { + TfwStr h_app = { + .chunks = (TfwStr []){ + { .data = ", ", .len = 2 }, + { .data = val->data, + .len = val->len } + }, + .len = val->len + 2, + .nchunks = 2 + }; + + r = tfw_strcat(resp->pool, tgt, &h_app); + if (unlikely(r)) + return r; + + next->s_hdr = *tgt; + next->op = TFW_H2_TRANS_INPLACE; break; } + + next->s_hdr = *f_desc->hdr; + next->op = TFW_H2_TRANS_SUB; + break; + def: /* * Remove 'Connection', 'Keep-Alive' headers and all hop-by-hop From a5519f891395edd39e4442582b5728f2d8aa5a1c Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 24/64] Hpack state machine table in no more generated, product of generated file is used in the source tree --- Makefile | 5 +- tempesta_fw/.gitignore | 1 - tempesta_fw/fill_hpack_hdr.pl | 963 --------------------------------- tempesta_fw/hpack/.gitignore | 1 + tempesta_fw/hpack/Makefile | 37 ++ tempesta_fw/hpack/Readme.md | 12 + tempesta_fw/{ => hpack}/hgen.c | 0 tempesta_fw/hpack_tbl.h | 6 +- 8 files changed, 54 insertions(+), 971 deletions(-) delete mode 100644 tempesta_fw/.gitignore delete mode 100755 tempesta_fw/fill_hpack_hdr.pl create mode 100644 tempesta_fw/hpack/.gitignore create mode 100644 tempesta_fw/hpack/Makefile create mode 100644 tempesta_fw/hpack/Readme.md rename tempesta_fw/{ => hpack}/hgen.c (100%) diff --git a/Makefile b/Makefile index 754183a8c6..bfb770f2b4 100644 --- a/Makefile +++ b/Makefile @@ -74,10 +74,7 @@ obj-m += lib/ tempesta_db/core/ tempesta_fw/ tls/ all: build -generate_headers: - tempesta_fw/fill_hpack_hdr.pl tempesta_fw/hpack_tbl.h.tpl - -build: generate_headers +build: ifdef ERROR $(error $(ERROR)) endif diff --git a/tempesta_fw/.gitignore b/tempesta_fw/.gitignore deleted file mode 100644 index edd04ab3be..0000000000 --- a/tempesta_fw/.gitignore +++ /dev/null @@ -1 +0,0 @@ -hpack_tbl.h diff --git a/tempesta_fw/fill_hpack_hdr.pl b/tempesta_fw/fill_hpack_hdr.pl deleted file mode 100755 index 4309441d57..0000000000 --- a/tempesta_fw/fill_hpack_hdr.pl +++ /dev/null @@ -1,963 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (C) 2019 Tempesta Technologies, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, -# or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. -use 5.16.0; -use strict; -use warnings; -use File::Basename; -use POSIX; -use Template; -use Cwd 'abs_path'; - -#my $table_size; -my ($template) = @ARGV; -if (!$template || $template !~ '.h.tpl$' ) -{ - die "Bad args!\n" -} - -sub generate_table -{ - say "Generate HPACK table"; - - my $table_size = 3392; - my $table = << 'ARRAY_END'; - /* --- [TABLE-128: offset = 0] --- */ - {5, 48}, /* 5: '0' (48) */ - {5, 48}, /* 5: '0' (48) */ - {5, 48}, /* 5: '0' (48) */ - {5, 48}, /* 5: '0' (48) */ - {5, 49}, /* 5: '1' (49) */ - {5, 49}, /* 5: '1' (49) */ - {5, 49}, /* 5: '1' (49) */ - {5, 49}, /* 5: '1' (49) */ - {5, 50}, /* 5: '2' (50) */ - {5, 50}, /* 5: '2' (50) */ - {5, 50}, /* 5: '2' (50) */ - {5, 50}, /* 5: '2' (50) */ - {5, 97}, /* 5: 'a' (97) */ - {5, 97}, /* 5: 'a' (97) */ - {5, 97}, /* 5: 'a' (97) */ - {5, 97}, /* 5: 'a' (97) */ - {5, 99}, /* 5: 'c' (99) */ - {5, 99}, /* 5: 'c' (99) */ - {5, 99}, /* 5: 'c' (99) */ - {5, 99}, /* 5: 'c' (99) */ - {5, 101}, /* 5: 'e' (101) */ - {5, 101}, /* 5: 'e' (101) */ - {5, 101}, /* 5: 'e' (101) */ - {5, 101}, /* 5: 'e' (101) */ - {5, 105}, /* 5: 'i' (105) */ - {5, 105}, /* 5: 'i' (105) */ - {5, 105}, /* 5: 'i' (105) */ - {5, 105}, /* 5: 'i' (105) */ - {5, 111}, /* 5: 'o' (111) */ - {5, 111}, /* 5: 'o' (111) */ - {5, 111}, /* 5: 'o' (111) */ - {5, 111}, /* 5: 'o' (111) */ - {5, 115}, /* 5: 's' (115) */ - {5, 115}, /* 5: 's' (115) */ - {5, 115}, /* 5: 's' (115) */ - {5, 115}, /* 5: 's' (115) */ - {5, 116}, /* 5: 't' (116) */ - {5, 116}, /* 5: 't' (116) */ - {5, 116}, /* 5: 't' (116) */ - {5, 116}, /* 5: 't' (116) */ - {6, 32}, /* 6: ' ' (32) */ - {6, 32}, /* 6: ' ' (32) */ - {6, 37}, /* 6: '%' (37) */ - {6, 37}, /* 6: '%' (37) */ - {6, 45}, /* 6: '-' (45) */ - {6, 45}, /* 6: '-' (45) */ - {6, 46}, /* 6: '.' (46) */ - {6, 46}, /* 6: '.' (46) */ - {6, 47}, /* 6: '/' (47) */ - {6, 47}, /* 6: '/' (47) */ - {6, 51}, /* 6: '3' (51) */ - {6, 51}, /* 6: '3' (51) */ - {6, 52}, /* 6: '4' (52) */ - {6, 52}, /* 6: '4' (52) */ - {6, 53}, /* 6: '5' (53) */ - {6, 53}, /* 6: '5' (53) */ - {6, 54}, /* 6: '6' (54) */ - {6, 54}, /* 6: '6' (54) */ - {6, 55}, /* 6: '7' (55) */ - {6, 55}, /* 6: '7' (55) */ - {6, 56}, /* 6: '8' (56) */ - {6, 56}, /* 6: '8' (56) */ - {6, 57}, /* 6: '9' (57) */ - {6, 57}, /* 6: '9' (57) */ - {6, 61}, /* 6: '=' (61) */ - {6, 61}, /* 6: '=' (61) */ - {6, 65}, /* 6: 'A' (65) */ - {6, 65}, /* 6: 'A' (65) */ - {6, 95}, /* 6: '_' (95) */ - {6, 95}, /* 6: '_' (95) */ - {6, 98}, /* 6: 'b' (98) */ - {6, 98}, /* 6: 'b' (98) */ - {6, 100}, /* 6: 'd' (100) */ - {6, 100}, /* 6: 'd' (100) */ - {6, 102}, /* 6: 'f' (102) */ - {6, 102}, /* 6: 'f' (102) */ - {6, 103}, /* 6: 'g' (103) */ - {6, 103}, /* 6: 'g' (103) */ - {6, 104}, /* 6: 'h' (104) */ - {6, 104}, /* 6: 'h' (104) */ - {6, 108}, /* 6: 'l' (108) */ - {6, 108}, /* 6: 'l' (108) */ - {6, 109}, /* 6: 'm' (109) */ - {6, 109}, /* 6: 'm' (109) */ - {6, 110}, /* 6: 'n' (110) */ - {6, 110}, /* 6: 'n' (110) */ - {6, 112}, /* 6: 'p' (112) */ - {6, 112}, /* 6: 'p' (112) */ - {6, 114}, /* 6: 'r' (114) */ - {6, 114}, /* 6: 'r' (114) */ - {6, 117}, /* 6: 'u' (117) */ - {6, 117}, /* 6: 'u' (117) */ - {7, 58}, /* 7: ':' (58) */ - {7, 66}, /* 7: 'B' (66) */ - {7, 67}, /* 7: 'C' (67) */ - {7, 68}, /* 7: 'D' (68) */ - {7, 69}, /* 7: 'E' (69) */ - {7, 70}, /* 7: 'F' (70) */ - {7, 71}, /* 7: 'G' (71) */ - {7, 72}, /* 7: 'H' (72) */ - {7, 73}, /* 7: 'I' (73) */ - {7, 74}, /* 7: 'J' (74) */ - {7, 75}, /* 7: 'K' (75) */ - {7, 76}, /* 7: 'L' (76) */ - {7, 77}, /* 7: 'M' (77) */ - {7, 78}, /* 7: 'N' (78) */ - {7, 79}, /* 7: 'O' (79) */ - {7, 80}, /* 7: 'P' (80) */ - {7, 81}, /* 7: 'Q' (81) */ - {7, 82}, /* 7: 'R' (82) */ - {7, 83}, /* 7: 'S' (83) */ - {7, 84}, /* 7: 'T' (84) */ - {7, 85}, /* 7: 'U' (85) */ - {7, 86}, /* 7: 'V' (86) */ - {7, 87}, /* 7: 'W' (87) */ - {7, 89}, /* 7: 'Y' (89) */ - {7, 106}, /* 7: 'j' (106) */ - {7, 107}, /* 7: 'k' (107) */ - {7, 113}, /* 7: 'q' (113) */ - {7, 118}, /* 7: 'v' (118) */ - {7, 119}, /* 7: 'w' (119) */ - {7, 120}, /* 7: 'x' (120) */ - {7, 121}, /* 7: 'y' (121) */ - {7, 122}, /* 7: 'z' (122) */ - {-7, 640}, /* 7: ---> TABLE 640 */ - {-7, 648}, /* 7: ---> TABLE 648 */ - {-7, 656}, /* 7: ---> TABLE 656 */ - {-7, 128}, /* 7: ---> TABLE 128 */ -/* --- [TABLE-128: offset = 128] --- */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 33}, /* 3: '!' (33) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 34}, /* 3: '"' (34) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 40}, /* 3: '(' (40) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 41}, /* 3: ')' (41) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {3, 63}, /* 3: '?' (63) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 39}, /* 4: '\'' (39) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 43}, /* 4: '+' (43) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {4, 124}, /* 4: '|' (124) */ - {5, 35}, /* 5: '#' (35) */ - {5, 35}, /* 5: '#' (35) */ - {5, 35}, /* 5: '#' (35) */ - {5, 35}, /* 5: '#' (35) */ - {5, 62}, /* 5: '>' (62) */ - {5, 62}, /* 5: '>' (62) */ - {5, 62}, /* 5: '>' (62) */ - {5, 62}, /* 5: '>' (62) */ - {6, 0}, /* 6: '\x00' (0) */ - {6, 0}, /* 6: '\x00' (0) */ - {6, 36}, /* 6: '$' (36) */ - {6, 36}, /* 6: '$' (36) */ - {6, 64}, /* 6: '@' (64) */ - {6, 64}, /* 6: '@' (64) */ - {6, 91}, /* 6: '[' (91) */ - {6, 91}, /* 6: '[' (91) */ - {6, 93}, /* 6: ']' (93) */ - {6, 93}, /* 6: ']' (93) */ - {6, 126}, /* 6: '~' (126) */ - {6, 126}, /* 6: '~' (126) */ - {7, 94}, /* 7: '^' (94) */ - {7, 125}, /* 7: '}' (125) */ - {-7, 664}, /* 7: ---> TABLE 664 */ - {-7, 256}, /* 7: ---> TABLE 256 */ -/* --- [TABLE-128: offset = 256] --- */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {1, 123}, /* 1: '{' (123) */ - {5, 92}, /* 5: '\\' (92) */ - {5, 92}, /* 5: '\\' (92) */ - {5, 92}, /* 5: '\\' (92) */ - {5, 92}, /* 5: '\\' (92) */ - {5, -61}, /* 5: '\xC3' (195) */ - {5, -61}, /* 5: '\xC3' (195) */ - {5, -61}, /* 5: '\xC3' (195) */ - {5, -61}, /* 5: '\xC3' (195) */ - {5, -48}, /* 5: '\xD0' (208) */ - {5, -48}, /* 5: '\xD0' (208) */ - {5, -48}, /* 5: '\xD0' (208) */ - {5, -48}, /* 5: '\xD0' (208) */ - {6, -128}, /* 6: '\x80' (128) */ - {6, -128}, /* 6: '\x80' (128) */ - {6, -126}, /* 6: '\x82' (130) */ - {6, -126}, /* 6: '\x82' (130) */ - {6, -125}, /* 6: '\x83' (131) */ - {6, -125}, /* 6: '\x83' (131) */ - {6, -94}, /* 6: '\xA2' (162) */ - {6, -94}, /* 6: '\xA2' (162) */ - {6, -72}, /* 6: '\xB8' (184) */ - {6, -72}, /* 6: '\xB8' (184) */ - {6, -62}, /* 6: '\xC2' (194) */ - {6, -62}, /* 6: '\xC2' (194) */ - {6, -32}, /* 6: '\xE0' (224) */ - {6, -32}, /* 6: '\xE0' (224) */ - {6, -30}, /* 6: '\xE2' (226) */ - {6, -30}, /* 6: '\xE2' (226) */ - {7, -103}, /* 7: '\x99' (153) */ - {7, -95}, /* 7: '\xA1' (161) */ - {7, -89}, /* 7: '\xA7' (167) */ - {7, -84}, /* 7: '\xAC' (172) */ - {7, -80}, /* 7: '\xB0' (176) */ - {7, -79}, /* 7: '\xB1' (177) */ - {7, -77}, /* 7: '\xB3' (179) */ - {7, -47}, /* 7: '\xD1' (209) */ - {7, -40}, /* 7: '\xD8' (216) */ - {7, -39}, /* 7: '\xD9' (217) */ - {7, -29}, /* 7: '\xE3' (227) */ - {7, -27}, /* 7: '\xE5' (229) */ - {7, -26}, /* 7: '\xE6' (230) */ - {-7, 672}, /* 7: ---> TABLE 672 */ - {-7, 680}, /* 7: ---> TABLE 680 */ - {-7, 688}, /* 7: ---> TABLE 688 */ - {-7, 696}, /* 7: ---> TABLE 696 */ - {-7, 704}, /* 7: ---> TABLE 704 */ - {-7, 712}, /* 7: ---> TABLE 712 */ - {-7, 720}, /* 7: ---> TABLE 720 */ - {-7, 728}, /* 7: ---> TABLE 728 */ - {-7, 736}, /* 7: ---> TABLE 736 */ - {-7, 744}, /* 7: ---> TABLE 744 */ - {-7, 752}, /* 7: ---> TABLE 752 */ - {-7, 760}, /* 7: ---> TABLE 760 */ - {-7, 768}, /* 7: ---> TABLE 768 */ - {-7, 776}, /* 7: ---> TABLE 776 */ - {-7, 784}, /* 7: ---> TABLE 784 */ - {-7, 792}, /* 7: ---> TABLE 792 */ - {-7, 800}, /* 7: ---> TABLE 800 */ - {-7, 808}, /* 7: ---> TABLE 808 */ - {-7, 816}, /* 7: ---> TABLE 816 */ - {-7, 824}, /* 7: ---> TABLE 824 */ - {-7, 832}, /* 7: ---> TABLE 832 */ - {-7, 384}, /* 7: ---> TABLE 384 */ - {-7, 512}, /* 7: ---> TABLE 512 */ -/* --- [TABLE-128: offset = 384] --- */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -85}, /* 3: '\xAB' (171) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -50}, /* 3: '\xCE' (206) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -41}, /* 3: '\xD7' (215) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -31}, /* 3: '\xE1' (225) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -20}, /* 3: '\xEC' (236) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {3, -19}, /* 3: '\xED' (237) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -57}, /* 4: '\xC7' (199) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -49}, /* 4: '\xCF' (207) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -22}, /* 4: '\xEA' (234) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ - {4, -21}, /* 4: '\xEB' (235) */ -/* --- [TABLE-128: offset = 512] --- */ - {5, -64}, /* 5: '\xC0' (192) */ - {5, -64}, /* 5: '\xC0' (192) */ - {5, -64}, /* 5: '\xC0' (192) */ - {5, -64}, /* 5: '\xC0' (192) */ - {5, -63}, /* 5: '\xC1' (193) */ - {5, -63}, /* 5: '\xC1' (193) */ - {5, -63}, /* 5: '\xC1' (193) */ - {5, -63}, /* 5: '\xC1' (193) */ - {5, -56}, /* 5: '\xC8' (200) */ - {5, -56}, /* 5: '\xC8' (200) */ - {5, -56}, /* 5: '\xC8' (200) */ - {5, -56}, /* 5: '\xC8' (200) */ - {5, -55}, /* 5: '\xC9' (201) */ - {5, -55}, /* 5: '\xC9' (201) */ - {5, -55}, /* 5: '\xC9' (201) */ - {5, -55}, /* 5: '\xC9' (201) */ - {5, -54}, /* 5: '\xCA' (202) */ - {5, -54}, /* 5: '\xCA' (202) */ - {5, -54}, /* 5: '\xCA' (202) */ - {5, -54}, /* 5: '\xCA' (202) */ - {5, -51}, /* 5: '\xCD' (205) */ - {5, -51}, /* 5: '\xCD' (205) */ - {5, -51}, /* 5: '\xCD' (205) */ - {5, -51}, /* 5: '\xCD' (205) */ - {5, -46}, /* 5: '\xD2' (210) */ - {5, -46}, /* 5: '\xD2' (210) */ - {5, -46}, /* 5: '\xD2' (210) */ - {5, -46}, /* 5: '\xD2' (210) */ - {5, -43}, /* 5: '\xD5' (213) */ - {5, -43}, /* 5: '\xD5' (213) */ - {5, -43}, /* 5: '\xD5' (213) */ - {5, -43}, /* 5: '\xD5' (213) */ - {5, -38}, /* 5: '\xDA' (218) */ - {5, -38}, /* 5: '\xDA' (218) */ - {5, -38}, /* 5: '\xDA' (218) */ - {5, -38}, /* 5: '\xDA' (218) */ - {5, -37}, /* 5: '\xDB' (219) */ - {5, -37}, /* 5: '\xDB' (219) */ - {5, -37}, /* 5: '\xDB' (219) */ - {5, -37}, /* 5: '\xDB' (219) */ - {5, -18}, /* 5: '\xEE' (238) */ - {5, -18}, /* 5: '\xEE' (238) */ - {5, -18}, /* 5: '\xEE' (238) */ - {5, -18}, /* 5: '\xEE' (238) */ - {5, -16}, /* 5: '\xF0' (240) */ - {5, -16}, /* 5: '\xF0' (240) */ - {5, -16}, /* 5: '\xF0' (240) */ - {5, -16}, /* 5: '\xF0' (240) */ - {5, -14}, /* 5: '\xF2' (242) */ - {5, -14}, /* 5: '\xF2' (242) */ - {5, -14}, /* 5: '\xF2' (242) */ - {5, -14}, /* 5: '\xF2' (242) */ - {5, -13}, /* 5: '\xF3' (243) */ - {5, -13}, /* 5: '\xF3' (243) */ - {5, -13}, /* 5: '\xF3' (243) */ - {5, -13}, /* 5: '\xF3' (243) */ - {5, -1}, /* 5: '\xFF' (255) */ - {5, -1}, /* 5: '\xFF' (255) */ - {5, -1}, /* 5: '\xFF' (255) */ - {5, -1}, /* 5: '\xFF' (255) */ - {6, -53}, /* 6: '\xCB' (203) */ - {6, -53}, /* 6: '\xCB' (203) */ - {6, -52}, /* 6: '\xCC' (204) */ - {6, -52}, /* 6: '\xCC' (204) */ - {6, -45}, /* 6: '\xD3' (211) */ - {6, -45}, /* 6: '\xD3' (211) */ - {6, -44}, /* 6: '\xD4' (212) */ - {6, -44}, /* 6: '\xD4' (212) */ - {6, -42}, /* 6: '\xD6' (214) */ - {6, -42}, /* 6: '\xD6' (214) */ - {6, -35}, /* 6: '\xDD' (221) */ - {6, -35}, /* 6: '\xDD' (221) */ - {6, -34}, /* 6: '\xDE' (222) */ - {6, -34}, /* 6: '\xDE' (222) */ - {6, -33}, /* 6: '\xDF' (223) */ - {6, -33}, /* 6: '\xDF' (223) */ - {6, -15}, /* 6: '\xF1' (241) */ - {6, -15}, /* 6: '\xF1' (241) */ - {6, -12}, /* 6: '\xF4' (244) */ - {6, -12}, /* 6: '\xF4' (244) */ - {6, -11}, /* 6: '\xF5' (245) */ - {6, -11}, /* 6: '\xF5' (245) */ - {6, -10}, /* 6: '\xF6' (246) */ - {6, -10}, /* 6: '\xF6' (246) */ - {6, -9}, /* 6: '\xF7' (247) */ - {6, -9}, /* 6: '\xF7' (247) */ - {6, -8}, /* 6: '\xF8' (248) */ - {6, -8}, /* 6: '\xF8' (248) */ - {6, -6}, /* 6: '\xFA' (250) */ - {6, -6}, /* 6: '\xFA' (250) */ - {6, -5}, /* 6: '\xFB' (251) */ - {6, -5}, /* 6: '\xFB' (251) */ - {6, -4}, /* 6: '\xFC' (252) */ - {6, -4}, /* 6: '\xFC' (252) */ - {6, -3}, /* 6: '\xFD' (253) */ - {6, -3}, /* 6: '\xFD' (253) */ - {6, -2}, /* 6: '\xFE' (254) */ - {6, -2}, /* 6: '\xFE' (254) */ - {7, 2}, /* 7: '\x02' (2) */ - {7, 3}, /* 7: '\x03' (3) */ - {7, 4}, /* 7: '\x04' (4) */ - {7, 5}, /* 7: '\x05' (5) */ - {7, 6}, /* 7: '\x06' (6) */ - {7, 7}, /* 7: '\x07' (7) */ - {7, 8}, /* 7: '\x08' (8) */ - {7, 11}, /* 7: '\x0B' (11) */ - {7, 12}, /* 7: '\x0C' (12) */ - {7, 14}, /* 7: '\x0E' (14) */ - {7, 15}, /* 7: '\x0F' (15) */ - {7, 16}, /* 7: '\x10' (16) */ - {7, 17}, /* 7: '\x11' (17) */ - {7, 18}, /* 7: '\x12' (18) */ - {7, 19}, /* 7: '\x13' (19) */ - {7, 20}, /* 7: '\x14' (20) */ - {7, 21}, /* 7: '\x15' (21) */ - {7, 23}, /* 7: '\x17' (23) */ - {7, 24}, /* 7: '\x18' (24) */ - {7, 25}, /* 7: '\x19' (25) */ - {7, 26}, /* 7: '\x1A' (26) */ - {7, 27}, /* 7: '\x1B' (27) */ - {7, 28}, /* 7: '\x1C' (28) */ - {7, 29}, /* 7: '\x1D' (29) */ - {7, 30}, /* 7: '\x1E' (30) */ - {7, 31}, /* 7: '\x1F' (31) */ - {7, 127}, /* 7: '\x7F' (127) */ - {7, -36}, /* 7: '\xDC' (220) */ - {7, -7}, /* 7: '\xF9' (249) */ - {-7, 840}, /* 7: ---> TABLE 840 */ -/* --- [TABLE-8: offset = 640] --- */ - {5, 38}, /* 1: '&' (38) */ - {5, 38}, /* 1: '&' (38) */ - {5, 38}, /* 1: '&' (38) */ - {5, 38}, /* 1: '&' (38) */ - {5, 42}, /* 1: '*' (42) */ - {5, 42}, /* 1: '*' (42) */ - {5, 42}, /* 1: '*' (42) */ - {5, 42}, /* 1: '*' (42) */ -/* --- [TABLE-8: offset = 648] --- */ - {5, 44}, /* 1: ',' (44) */ - {5, 44}, /* 1: ',' (44) */ - {5, 44}, /* 1: ',' (44) */ - {5, 44}, /* 1: ',' (44) */ - {5, 59}, /* 1: ';' (59) */ - {5, 59}, /* 1: ';' (59) */ - {5, 59}, /* 1: ';' (59) */ - {5, 59}, /* 1: ';' (59) */ -/* --- [TABLE-8: offset = 656] --- */ - {5, 88}, /* 1: 'X' (88) */ - {5, 88}, /* 1: 'X' (88) */ - {5, 88}, /* 1: 'X' (88) */ - {5, 88}, /* 1: 'X' (88) */ - {5, 90}, /* 1: 'Z' (90) */ - {5, 90}, /* 1: 'Z' (90) */ - {5, 90}, /* 1: 'Z' (90) */ - {5, 90}, /* 1: 'Z' (90) */ -/* --- [TABLE-8: offset = 664] --- */ - {5, 60}, /* 1: '<' (60) */ - {5, 60}, /* 1: '<' (60) */ - {5, 60}, /* 1: '<' (60) */ - {5, 60}, /* 1: '<' (60) */ - {5, 96}, /* 1: '`' (96) */ - {5, 96}, /* 1: '`' (96) */ - {5, 96}, /* 1: '`' (96) */ - {5, 96}, /* 1: '`' (96) */ -/* --- [TABLE-8: offset = 672] --- */ - {5, -127}, /* 1: '\x81' (129) */ - {5, -127}, /* 1: '\x81' (129) */ - {5, -127}, /* 1: '\x81' (129) */ - {5, -127}, /* 1: '\x81' (129) */ - {5, -124}, /* 1: '\x84' (132) */ - {5, -124}, /* 1: '\x84' (132) */ - {5, -124}, /* 1: '\x84' (132) */ - {5, -124}, /* 1: '\x84' (132) */ -/* --- [TABLE-8: offset = 680] --- */ - {5, -123}, /* 1: '\x85' (133) */ - {5, -123}, /* 1: '\x85' (133) */ - {5, -123}, /* 1: '\x85' (133) */ - {5, -123}, /* 1: '\x85' (133) */ - {5, -122}, /* 1: '\x86' (134) */ - {5, -122}, /* 1: '\x86' (134) */ - {5, -122}, /* 1: '\x86' (134) */ - {5, -122}, /* 1: '\x86' (134) */ -/* --- [TABLE-8: offset = 688] --- */ - {5, -120}, /* 1: '\x88' (136) */ - {5, -120}, /* 1: '\x88' (136) */ - {5, -120}, /* 1: '\x88' (136) */ - {5, -120}, /* 1: '\x88' (136) */ - {5, -110}, /* 1: '\x92' (146) */ - {5, -110}, /* 1: '\x92' (146) */ - {5, -110}, /* 1: '\x92' (146) */ - {5, -110}, /* 1: '\x92' (146) */ -/* --- [TABLE-8: offset = 696] --- */ - {5, -102}, /* 1: '\x9A' (154) */ - {5, -102}, /* 1: '\x9A' (154) */ - {5, -102}, /* 1: '\x9A' (154) */ - {5, -102}, /* 1: '\x9A' (154) */ - {5, -100}, /* 1: '\x9C' (156) */ - {5, -100}, /* 1: '\x9C' (156) */ - {5, -100}, /* 1: '\x9C' (156) */ - {5, -100}, /* 1: '\x9C' (156) */ -/* --- [TABLE-8: offset = 704] --- */ - {5, -96}, /* 1: '\xA0' (160) */ - {5, -96}, /* 1: '\xA0' (160) */ - {5, -96}, /* 1: '\xA0' (160) */ - {5, -96}, /* 1: '\xA0' (160) */ - {5, -93}, /* 1: '\xA3' (163) */ - {5, -93}, /* 1: '\xA3' (163) */ - {5, -93}, /* 1: '\xA3' (163) */ - {5, -93}, /* 1: '\xA3' (163) */ -/* --- [TABLE-8: offset = 712] --- */ - {5, -92}, /* 1: '\xA4' (164) */ - {5, -92}, /* 1: '\xA4' (164) */ - {5, -92}, /* 1: '\xA4' (164) */ - {5, -92}, /* 1: '\xA4' (164) */ - {5, -87}, /* 1: '\xA9' (169) */ - {5, -87}, /* 1: '\xA9' (169) */ - {5, -87}, /* 1: '\xA9' (169) */ - {5, -87}, /* 1: '\xA9' (169) */ -/* --- [TABLE-8: offset = 720] --- */ - {5, -86}, /* 1: '\xAA' (170) */ - {5, -86}, /* 1: '\xAA' (170) */ - {5, -86}, /* 1: '\xAA' (170) */ - {5, -86}, /* 1: '\xAA' (170) */ - {5, -83}, /* 1: '\xAD' (173) */ - {5, -83}, /* 1: '\xAD' (173) */ - {5, -83}, /* 1: '\xAD' (173) */ - {5, -83}, /* 1: '\xAD' (173) */ -/* --- [TABLE-8: offset = 728] --- */ - {5, -78}, /* 1: '\xB2' (178) */ - {5, -78}, /* 1: '\xB2' (178) */ - {5, -78}, /* 1: '\xB2' (178) */ - {5, -78}, /* 1: '\xB2' (178) */ - {5, -75}, /* 1: '\xB5' (181) */ - {5, -75}, /* 1: '\xB5' (181) */ - {5, -75}, /* 1: '\xB5' (181) */ - {5, -75}, /* 1: '\xB5' (181) */ -/* --- [TABLE-8: offset = 736] --- */ - {5, -71}, /* 1: '\xB9' (185) */ - {5, -71}, /* 1: '\xB9' (185) */ - {5, -71}, /* 1: '\xB9' (185) */ - {5, -71}, /* 1: '\xB9' (185) */ - {5, -70}, /* 1: '\xBA' (186) */ - {5, -70}, /* 1: '\xBA' (186) */ - {5, -70}, /* 1: '\xBA' (186) */ - {5, -70}, /* 1: '\xBA' (186) */ -/* --- [TABLE-8: offset = 744] --- */ - {5, -69}, /* 1: '\xBB' (187) */ - {5, -69}, /* 1: '\xBB' (187) */ - {5, -69}, /* 1: '\xBB' (187) */ - {5, -69}, /* 1: '\xBB' (187) */ - {5, -67}, /* 1: '\xBD' (189) */ - {5, -67}, /* 1: '\xBD' (189) */ - {5, -67}, /* 1: '\xBD' (189) */ - {5, -67}, /* 1: '\xBD' (189) */ -/* --- [TABLE-8: offset = 752] --- */ - {5, -66}, /* 1: '\xBE' (190) */ - {5, -66}, /* 1: '\xBE' (190) */ - {5, -66}, /* 1: '\xBE' (190) */ - {5, -66}, /* 1: '\xBE' (190) */ - {5, -60}, /* 1: '\xC4' (196) */ - {5, -60}, /* 1: '\xC4' (196) */ - {5, -60}, /* 1: '\xC4' (196) */ - {5, -60}, /* 1: '\xC4' (196) */ -/* --- [TABLE-8: offset = 760] --- */ - {5, -58}, /* 1: '\xC6' (198) */ - {5, -58}, /* 1: '\xC6' (198) */ - {5, -58}, /* 1: '\xC6' (198) */ - {5, -58}, /* 1: '\xC6' (198) */ - {5, -28}, /* 1: '\xE4' (228) */ - {5, -28}, /* 1: '\xE4' (228) */ - {5, -28}, /* 1: '\xE4' (228) */ - {5, -28}, /* 1: '\xE4' (228) */ -/* --- [TABLE-8: offset = 768] --- */ - {5, -24}, /* 1: '\xE8' (232) */ - {5, -24}, /* 1: '\xE8' (232) */ - {5, -24}, /* 1: '\xE8' (232) */ - {5, -24}, /* 1: '\xE8' (232) */ - {5, -23}, /* 1: '\xE9' (233) */ - {5, -23}, /* 1: '\xE9' (233) */ - {5, -23}, /* 1: '\xE9' (233) */ - {5, -23}, /* 1: '\xE9' (233) */ -/* --- [TABLE-8: offset = 776] --- */ - {6, 1}, /* 2: '\x01' (1) */ - {6, 1}, /* 2: '\x01' (1) */ - {6, -121}, /* 2: '\x87' (135) */ - {6, -121}, /* 2: '\x87' (135) */ - {6, -119}, /* 2: '\x89' (137) */ - {6, -119}, /* 2: '\x89' (137) */ - {6, -118}, /* 2: '\x8A' (138) */ - {6, -118}, /* 2: '\x8A' (138) */ -/* --- [TABLE-8: offset = 784] --- */ - {6, -117}, /* 2: '\x8B' (139) */ - {6, -117}, /* 2: '\x8B' (139) */ - {6, -116}, /* 2: '\x8C' (140) */ - {6, -116}, /* 2: '\x8C' (140) */ - {6, -115}, /* 2: '\x8D' (141) */ - {6, -115}, /* 2: '\x8D' (141) */ - {6, -113}, /* 2: '\x8F' (143) */ - {6, -113}, /* 2: '\x8F' (143) */ -/* --- [TABLE-8: offset = 792] --- */ - {6, -109}, /* 2: '\x93' (147) */ - {6, -109}, /* 2: '\x93' (147) */ - {6, -107}, /* 2: '\x95' (149) */ - {6, -107}, /* 2: '\x95' (149) */ - {6, -106}, /* 2: '\x96' (150) */ - {6, -106}, /* 2: '\x96' (150) */ - {6, -105}, /* 2: '\x97' (151) */ - {6, -105}, /* 2: '\x97' (151) */ -/* --- [TABLE-8: offset = 800] --- */ - {6, -104}, /* 2: '\x98' (152) */ - {6, -104}, /* 2: '\x98' (152) */ - {6, -101}, /* 2: '\x9B' (155) */ - {6, -101}, /* 2: '\x9B' (155) */ - {6, -99}, /* 2: '\x9D' (157) */ - {6, -99}, /* 2: '\x9D' (157) */ - {6, -98}, /* 2: '\x9E' (158) */ - {6, -98}, /* 2: '\x9E' (158) */ -/* --- [TABLE-8: offset = 808] --- */ - {6, -91}, /* 2: '\xA5' (165) */ - {6, -91}, /* 2: '\xA5' (165) */ - {6, -90}, /* 2: '\xA6' (166) */ - {6, -90}, /* 2: '\xA6' (166) */ - {6, -88}, /* 2: '\xA8' (168) */ - {6, -88}, /* 2: '\xA8' (168) */ - {6, -82}, /* 2: '\xAE' (174) */ - {6, -82}, /* 2: '\xAE' (174) */ -/* --- [TABLE-8: offset = 816] --- */ - {6, -81}, /* 2: '\xAF' (175) */ - {6, -81}, /* 2: '\xAF' (175) */ - {6, -76}, /* 2: '\xB4' (180) */ - {6, -76}, /* 2: '\xB4' (180) */ - {6, -74}, /* 2: '\xB6' (182) */ - {6, -74}, /* 2: '\xB6' (182) */ - {6, -73}, /* 2: '\xB7' (183) */ - {6, -73}, /* 2: '\xB7' (183) */ -/* --- [TABLE-8: offset = 824] --- */ - {6, -68}, /* 2: '\xBC' (188) */ - {6, -68}, /* 2: '\xBC' (188) */ - {6, -65}, /* 2: '\xBF' (191) */ - {6, -65}, /* 2: '\xBF' (191) */ - {6, -59}, /* 2: '\xC5' (197) */ - {6, -59}, /* 2: '\xC5' (197) */ - {6, -25}, /* 2: '\xE7' (231) */ - {6, -25}, /* 2: '\xE7' (231) */ -/* --- [TABLE-8: offset = 832] --- */ - {6, -17}, /* 2: '\xEF' (239) */ - {6, -17}, /* 2: '\xEF' (239) */ - {7, 9}, /* 3: '\x09' (9) */ - {7, -114}, /* 3: '\x8E' (142) */ - {7, -112}, /* 3: '\x90' (144) */ - {7, -111}, /* 3: '\x91' (145) */ - {7, -108}, /* 3: '\x94' (148) */ - {7, -97}, /* 3: '\x9F' (159) */ -/* --- [TABLE-8: offset = 840] --- */ - {6, 10}, /* 2: '\x0A' (10) */ - {6, 10}, /* 2: '\x0A' (10) */ - {6, 13}, /* 2: '\x0D' (13) */ - {6, 13}, /* 2: '\x0D' (13) */ - {6, 22}, /* 2: '\x16' (22) */ - {6, 22}, /* 2: '\x16' (22) */ - {6, 0}, /* 2: EOS */ - {6, 0}, /* 2: EOS */ -ARRAY_END - - return($table, $table_size); -} - -# Assemble HTML templates and minify resulting files. -sub assemble -{ - my ($src) = @_; - my $dir = dirname($src); - $src = basename($src); - (my $dest = $src) =~ s/.tpl$//; - my $t = Template->new({ - INCLUDE_PATH => "$dir", - OUTPUT_PATH => "$dir", - }); - - say "Assemble template $src -> $dest"; - - my ($table, $table_size) = generate_table(); - my $t_vals = { - TABLE_SIZE => $table_size, - TABLE => $table - }; - $t->process($src, $t_vals, $dest) - || die $t->error(), "\n"; -} - - -assemble(abs_path($template)); - -__END__ - -=head1 NAME - -HPACK decoder state machine generator for TempestaFW. - -=head1 SYNOPSIS - -./fill_hpack_hdr.pl FILE - -FILE - Path to template file for the header. File extension must - be '.h.tpl'. - - -=cut diff --git a/tempesta_fw/hpack/.gitignore b/tempesta_fw/hpack/.gitignore new file mode 100644 index 0000000000..8ffde9978d --- /dev/null +++ b/tempesta_fw/hpack/.gitignore @@ -0,0 +1 @@ +hgen diff --git a/tempesta_fw/hpack/Makefile b/tempesta_fw/hpack/Makefile new file mode 100644 index 0000000000..72516b4a5c --- /dev/null +++ b/tempesta_fw/hpack/Makefile @@ -0,0 +1,37 @@ +# Tempesta FW +# +# User-space unit tests. +# +# Copyright (C) 2019 Tempesta Technologies, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, +# or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +ifndef CC + CC = gcc +endif + +CFLAGS = -O0 -ggdb -Wall -Werror +TARGETS = hgen + +.PHONY = all clean + +all : $(TARGETS) + +hgen : hgen.c + $(CC) $(CFLAGS) -o $@ $^ + +clean : + rm -f *.o *~ *.orig $(TARGETS) + diff --git a/tempesta_fw/hpack/Readme.md b/tempesta_fw/hpack/Readme.md new file mode 100644 index 0000000000..5adce37723 --- /dev/null +++ b/tempesta_fw/hpack/Readme.md @@ -0,0 +1,12 @@ +Huffman decoder state machine generator +--------------------------------------- + +The `hpack_tbl.h` file contains objects generated bu `hgen.c`. + +To update the header, remove everything from `hpack_tbl.h` after +`DO NOT EDIT IT BY HANDS!` statement and put output of the `hgen.c` there. Don't +forget to about include guards in the end of the file. + + make && hgen >> ../hpack_tbl.h + + diff --git a/tempesta_fw/hgen.c b/tempesta_fw/hpack/hgen.c similarity index 100% rename from tempesta_fw/hgen.c rename to tempesta_fw/hpack/hgen.c diff --git a/tempesta_fw/hpack_tbl.h b/tempesta_fw/hpack_tbl.h index d44a806dc2..b278eb8348 100644 --- a/tempesta_fw/hpack_tbl.h +++ b/tempesta_fw/hpack_tbl.h @@ -48,11 +48,11 @@ typedef struct { } __attribute__((packed)) HTState; /* - * All the below is generated by hgen.c. DO NOT EDIT IT BY HANDS! - * If you need to change the constants, then update the genertion program, + * All the below is generated by hpack/hgen.c. DO NOT EDIT IT BY HANDS! + * If you need to change the constants, then update the generation program, * remove all the below the comment and run: * - * $ gcc ./hgen.c && ./a.out >> hpack_tbl.h + * $ cd hpack && make && hgen >> ../hpack_tbl.h */ #define HT_NBITS 7 From 5c77db5255db19a08d16ffc20c61286d62ba1cb5 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 25/64] port method override detection feature to HTTP2 --- tempesta_fw/http_parser.c | 319 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 302 insertions(+), 17 deletions(-) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index d5ea2b14b3..47b46e9b13 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -3778,7 +3778,6 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, parser->_i_st = &&Req_HdrX_Method_OverrideV; __FSM_MOVE_n(RGen_LWS, 23); } - if (likely(__data_available(p, 18) && *(p + 1) == '-' && *(p + 9) == '-' @@ -6405,6 +6404,170 @@ __h2_req_parse_x_forwarded_for(TfwHttpMsg *hm, unsigned char *data, size_t len, } STACK_FRAME_NON_STANDARD(__h2_req_parse_x_forwarded_for); +/* Parse method override request headers. */ +static int +__h2_req_parse_m_override(TfwHttpReq *req, unsigned char *data, size_t len, + bool fin) +{ + int r = CSTR_NEQ; + __FSM_DECLARE_VARS(req); + + __FSM_START(parser->_i_st); + + __FSM_STATE(I_Meth_Start) { + switch (TFW_LC(c)) { + case 'c': + __FSM_I_JMP(I_Meth_C); + case 'd': + __FSM_I_JMP(I_Meth_D); + case 'g': + __FSM_I_JMP(I_Meth_G); + case 'h': + __FSM_I_JMP(I_Meth_H); + case 'l': + __FSM_I_JMP(I_Meth_L); + case 'm': + __FSM_I_JMP(I_Meth_M); + case 'o': + __FSM_I_JMP(I_Meth_O); + case 'p': + __FSM_I_JMP(I_Meth_P); + case 't': + __FSM_I_JMP(I_Meth_T); + case 'u': + __FSM_I_JMP(I_Meth_U); + } + __FSM_I_MOVE(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_C) { + H2_TRY_STR_LAMBDA("copy", { + req->method_override = TFW_HTTP_METH_COPY; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_C, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_D) { + H2_TRY_STR_LAMBDA("delete", { + req->method_override = TFW_HTTP_METH_DELETE; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_D, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_G) { + H2_TRY_STR_LAMBDA("get", { + req->method_override = TFW_HTTP_METH_GET; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_G, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_H) { + H2_TRY_STR_LAMBDA("head", { + req->method_override = TFW_HTTP_METH_HEAD; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_H, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_L) { + H2_TRY_STR_LAMBDA("lock", { + req->method_override = TFW_HTTP_METH_LOCK; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_L, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_M) { + H2_TRY_STR_LAMBDA("mkcol", { + req->method_override = TFW_HTTP_METH_MKCOL; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_M, I_EoT); + H2_TRY_STR_LAMBDA("move", { + req->method_override = TFW_HTTP_METH_MOVE; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_M, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_O) { + H2_TRY_STR_LAMBDA("options", { + req->method_override = TFW_HTTP_METH_OPTIONS; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_O, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_P) { + H2_TRY_STR_LAMBDA("patch", { + req->method_override = TFW_HTTP_METH_PATCH; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_P, I_EoT); + H2_TRY_STR_LAMBDA("post", { + req->method_override = TFW_HTTP_METH_POST; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_P, I_EoT); + H2_TRY_STR_LAMBDA("propfind", { + req->method_override = TFW_HTTP_METH_PROPFIND; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_P, I_EoT); + H2_TRY_STR_LAMBDA("proppatch", { + req->method_override = TFW_HTTP_METH_PROPPATCH; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_P, I_EoT); + H2_TRY_STR_LAMBDA("put", { + req->method_override = TFW_HTTP_METH_PUT; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_P, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_T) { + H2_TRY_STR_LAMBDA("trace", { + req->method_override = TFW_HTTP_METH_TRACE; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_T, I_EoT); + TRY_STR_INIT(); + } + + __FSM_STATE(I_Meth_U) { + H2_TRY_STR_LAMBDA("unlock", { + req->method_override = TFW_HTTP_METH_UNLOCK; + __FSM_EXIT(CSTR_EQ); + } , I_Meth_U, I_EoT); + TRY_STR_INIT(); + __FSM_I_JMP(I_Meth_Unknown); + } + + __FSM_STATE(I_Meth_Unknown) { + __FSM_I_MATCH_MOVE(token, I_Meth_Unknown); + req->method_override = _TFW_HTTP_METH_UNKNOWN; + __FSM_H2_I_MOVE_n(I_EoT, __fsm_sz); + } + + __FSM_STATE(I_EoT) { + if (IS_TOKEN(c)) + __FSM_H2_I_MOVE(I_Meth_Unknown); + if (IS_WS(c)) + __FSM_H2_I_MOVE(I_EoT); + return CSTR_NEQ; + } + +done: + return r; +} +STACK_FRAME_NON_STANDARD(__h2_req_parse_m_override); + static int __h2_req_parse_mark(TfwHttpReq *req, unsigned char *data, size_t len, bool fin) { @@ -6672,6 +6835,17 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, __FSM_H2_DROP(Req_HdrTransfer_Encoding); } __FSM_H2_NEXT(Req_HdrT); + case 'u': + if (likely(__data_available(p, 10) + && C4_INT(p, 'u', 's', 'e', 'r') + && *(p + 4) == '-' + && C4_INT(p + 5, 'a', 'g', 'e', 'n') + && *(p + 9) == 't')) + { + __FSM_H2_FIN(Req_HdrUser_AgentV, 10, + TFW_TAG_HDR_USER_AGENT); + } + __FSM_H2_NEXT(Req_HdrU); case 'x': if (likely(__data_available(p, 15) && C8_INT(p + 1, '-', 'f', 'o', 'r', 'w', @@ -6683,18 +6857,48 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, __FSM_H2_FIN(Req_HdrX_Forwarded_ForV, 15, TFW_TAG_HDR_X_FORWARDED_FOR); } - __FSM_H2_NEXT(Req_HdrX); - case 'u': - if (likely(__data_available(p, 10) - && C4_INT(p, 'u', 's', 'e', 'r') - && *(p + 4) == '-' - && C4_INT(p + 5, 'a', 'g', 'e', 'n') - && *(p + 9) == 't')) + if (likely(__data_available(p, 14) + && *(p + 1) == '-' + && *(p + 7) == '-' + /* Safe match: '-' is checked above. */ + && C8_INT_LCM(p, 'x', '-', 'h', 't', + 't', 'p', '-', 'm') + && C4_INT_LCM(p + 8, 'e', 't', 'h', 'o') + && TFW_LC(*(p + 12) == 'd') + && *(p + 10) == ':')) { - __FSM_H2_FIN(Req_HdrUser_AgentV, 10, - TFW_TAG_HDR_USER_AGENT); + __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 14, + TFW_TAG_HDR_RAW); } - __FSM_H2_NEXT(Req_HdrU); + if (likely(__data_available(p, 23) + && *(p + 1) == '-' + && *(p + 7) == '-' + && *(p + 14) == '-' + /* Safe match: '-' is checked above. */ + && C8_INT_LCM(p, 'x', '-', 'h', 't', + 't', 'p', '-', 'm') + && C8_INT_LCM(p + 8, 'e', 't', 'h', 'o', + 'd', '-', 'o', 'v') + && C8_INT7_LCM(p + 16, 'v', 'e', 'r', 'r', + 'i', 'd', 'e', ':'))) + { + __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 23, + TFW_TAG_HDR_RAW); + } + if (likely(__data_available(p, 18) + && *(p + 1) == '-' + && *(p + 9) == '-' + /* Safe match: '-' is checked above. */ + && C8_INT_LCM(p + 2, 'm', 'e', 't', 'h', + 'o', 'd', '-', 'o') + && C8_INT7_LCM(p + 10, 'v', 'e', 'r', 'r', + 'i', 'd', 'e', ':'))) + { + + __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 18, + TFW_TAG_HDR_RAW); + } + __FSM_H2_NEXT(Req_HdrX); default: __FSM_JMP(RGen_HdrOtherN); } @@ -6946,11 +7150,6 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, TFW_H2_PARSE_HDR_VAL(Req_HdrRefererV, msg, __h2_req_parse_referer, TFW_HTTP_HDR_REFERER, 1); - /* 'x-forwarded-for' is read, process field-value. */ - TFW_H2_PARSE_HDR_VAL(Req_HdrX_Forwarded_ForV, msg, - __h2_req_parse_x_forwarded_for, - TFW_HTTP_HDR_X_FORWARDED_FOR, 0); - /* 'user-agent' is read, process field-value. */ TFW_H2_PARSE_HDR_VAL(Req_HdrUser_AgentV, msg, __h2_req_parse_user_agent, TFW_HTTP_HDR_USER_AGENT, 1); @@ -6959,6 +7158,18 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, TFW_H2_PARSE_HDR_VAL(Req_HdrCookieV, msg, __h2_req_parse_cookie, TFW_HTTP_HDR_COOKIE, 0); + /* 'x-forwarded-for' is read, process field-value. */ + TFW_H2_PARSE_HDR_VAL(Req_HdrX_Forwarded_ForV, msg, + __h2_req_parse_x_forwarded_for, + TFW_HTTP_HDR_X_FORWARDED_FOR, 0); + + /* + * 'X-HTTP-Method:*OWS' OR 'X-HTTP-Method-Override:*OWS' OR + * 'X-Method-Override:*OWS' is read, process field-value. + */ + TFW_H2_PARSE_HDR_VAL(Req_HdrX_Method_OverrideV, req, + __h2_req_parse_m_override, TFW_HTTP_HDR_RAW, 1); + __FSM_STATE(RGen_HdrOtherV) { if (!H2_MSG_VERIFY(TFW_HTTP_HDR_RAW)) __FSM_H2_DROP(RGen_HdrOtherV); @@ -7214,7 +7425,20 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, __FSM_H2_TX_AF_DROP(Req_HdrTransfer_Encodin, 'g'); __FSM_H2_TX_AF(Req_HdrX, '-', Req_HdrX_); - __FSM_H2_TX_AF(Req_HdrX_, 'f', Req_HdrX_F); + __FSM_STATE(Req_HdrX_, cold) { + switch (c) { + case 'f': + __FSM_H2_NEXT(Req_HdrX_F); + case 'h': + __FSM_H2_NEXT(Req_HdrX_H); + case 'm': + __FSM_H2_NEXT(Req_HdrX_M); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + + /* X-Forwarded-For header processing. */ __FSM_H2_TX_AF(Req_HdrX_F, 'o', Req_HdrX_Fo); __FSM_H2_TX_AF(Req_HdrX_Fo, 'r', Req_HdrX_For); __FSM_H2_TX_AF(Req_HdrX_For, 'w', Req_HdrX_Forw); @@ -7229,6 +7453,65 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, __FSM_H2_TX_AF_FIN(Req_HdrX_Forwarded_Fo, 'r', Req_HdrX_Forwarded_ForV, TFW_TAG_HDR_X_FORWARDED_FOR); + /* X-Method-Override header processing. */ + __FSM_H2_TX_AF(Req_HdrX_M, 'e', Req_HdrX_Me); + __FSM_H2_TX_AF(Req_HdrX_Me, 't', Req_HdrX_Met); + __FSM_H2_TX_AF(Req_HdrX_Met, 'h', Req_HdrX_Meth); + __FSM_H2_TX_AF(Req_HdrX_Meth, 'o', Req_HdrX_Metho); + __FSM_H2_TX_AF(Req_HdrX_Metho, 'd', Req_HdrX_Method); + __FSM_H2_TX_AF(Req_HdrX_Method, '-', Req_HdrX_Method_); + __FSM_H2_TX_AF(Req_HdrX_Method_, 'o', Req_HdrX_Method_O); + __FSM_H2_TX_AF(Req_HdrX_Method_O, 'v', Req_HdrX_Method_Ov); + __FSM_H2_TX_AF(Req_HdrX_Method_Ov, 'e', Req_HdrX_Method_Ove); + __FSM_H2_TX_AF(Req_HdrX_Method_Ove, 'r', Req_HdrX_Method_Over); + __FSM_H2_TX_AF(Req_HdrX_Method_Over, 'r', Req_HdrX_Method_Overr); + __FSM_H2_TX_AF(Req_HdrX_Method_Overr, 'i', Req_HdrX_Method_Overri); + __FSM_H2_TX_AF(Req_HdrX_Method_Overri, 'd', Req_HdrX_Method_Overrid); + __FSM_H2_TX_AF_FIN(Req_HdrX_Method_Overrid, 'e', Req_HdrX_Method_OverrideV, + TFW_TAG_HDR_RAW); + + /* X-HTTP-Method header processing */ + __FSM_H2_TX_AF(Req_HdrX_H, 't', Req_HdrX_Ht); + __FSM_H2_TX_AF(Req_HdrX_Ht, 't', Req_HdrX_Htt); + __FSM_H2_TX_AF(Req_HdrX_Htt, 'p', Req_HdrX_Http); + __FSM_H2_TX_AF(Req_HdrX_Http, '-', Req_HdrX_Http_); + __FSM_H2_TX_AF(Req_HdrX_Http_, 'm', Req_HdrX_Http_M); + __FSM_H2_TX_AF(Req_HdrX_Http_M, 'e', Req_HdrX_Http_Me); + __FSM_H2_TX_AF(Req_HdrX_Http_Me, 't', Req_HdrX_Http_Met); + __FSM_H2_TX_AF(Req_HdrX_Http_Met, 'h', Req_HdrX_Http_Meth); + __FSM_H2_TX_AF(Req_HdrX_Http_Meth, 'o', Req_HdrX_Http_Metho); + /* + * Same as __FSM_H2_TX_AF_FIN, but jump to X-HTTP-Method-Override + * header if more data is found afer 'd' + */ + __FSM_STATE(Req_HdrX_Http_Metho, cold) { + if (c != 'd') + __FSM_JMP(RGen_HdrOtherN); + p += 1; + T_DBG3("%s: name fin, h_tag=%d, to=Req_HdrX_Http_Metho len=%lu, " + "off=%lu\n", + __func__, TFW_TAG_HDR_RAW, len, __data_off(p)); + if (unlikely(__data_off(p) < len)) + goto RGen_HdrOtherN; + __msg_hdr_chunk_fixup(data, len); + if (unlikely(!fin)) + __FSM_H2_POSTPONE(Req_HdrX_Http_Method); + it->tag = TFW_TAG_HDR_RAW; + __FSM_H2_OK(Req_HdrX_Method_OverrideV); + } + + /* X-HTTP-Method-Override processing */ + __FSM_H2_TX_AF(Req_HdrX_Http_Method, '-', Req_HdrX_Http_Method_); + __FSM_H2_TX_AF(Req_HdrX_Http_Method_, 'o', Req_HdrX_Http_Method_O); + __FSM_H2_TX_AF(Req_HdrX_Http_Method_O, 'v', Req_HdrX_Http_Method_Ov); + __FSM_H2_TX_AF(Req_HdrX_Http_Method_Ov, 'e', Req_HdrX_Http_Method_Ove); + __FSM_H2_TX_AF(Req_HdrX_Http_Method_Ove, 'r', Req_HdrX_Http_Method_Over); + __FSM_H2_TX_AF(Req_HdrX_Http_Method_Over, 'r', Req_HdrX_Http_Method_Overr); + __FSM_H2_TX_AF(Req_HdrX_Http_Method_Overr, 'i', Req_HdrX_Http_Method_Overri); + __FSM_H2_TX_AF(Req_HdrX_Http_Method_Overri, 'd', Req_HdrX_Http_Method_Overrid); + __FSM_H2_TX_AF_FIN(Req_HdrX_Http_Method_Overrid, 'e', Req_HdrX_Method_OverrideV, + TFW_TAG_HDR_RAW); + __FSM_H2_TX_AF(Req_HdrU, 's', Req_HdrUs); __FSM_H2_TX_AF(Req_HdrUs, 'e', Req_HdrUse); __FSM_H2_TX_AF(Req_HdrUse, 'r', Req_HdrUser); @@ -7245,6 +7528,8 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, __FSM_H2_TX_AF_FIN(Req_HdrCooki, 'e', Req_HdrCookieV, TFW_TAG_HDR_COOKIE); + + /* Improbable states of method values processing. */ __FSM_STATE(Req_RareMethods_3, cold) { From a7de0eb7535d039c1663dc4c2c448ad7a7a0f02c Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 17 Dec 2019 18:46:49 +0500 Subject: [PATCH 26/64] Fix pointer assignment --- tempesta_fw/http.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index bb01e2cd2b..c2371f273a 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3640,7 +3640,7 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) TfwStr *tgt = &ht->tbl[hid]; TfwStr *first = TFW_STR_CHUNK(tgt, 0); TfwHdrModsDesc *f_desc = NULL; - const TfwStr *val; + const TfwStr *val = NULL; if (TFW_STR_DUP(tgt)) tgt = TFW_STR_CHUNK(tgt, d_num); @@ -3671,7 +3671,7 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) if (!f_desc) goto def; - *val = TFW_STR_CHUNK(f_desc->hdr, 2); + val = TFW_STR_CHUNK(f_desc->hdr, 2); /* * If this is a duplicate of already processed header, * leave this duplicate as is (for transformation From db602d20a1fdb2c0cdb49b95832880a3f3b68b0d Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sun, 29 Dec 2019 20:21:36 +0500 Subject: [PATCH 27/64] replace bug_on with warn_on --- tempesta_fw/http.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index c2371f273a..83e574280b 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -5330,7 +5330,8 @@ tfw_http_resp_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data) int tfw_http_msg_process_generic(TfwConn *conn, TfwStream *stream, TfwFsmData *data) { - BUG_ON(!stream); + if (WARN_ON_ONCE(!stream)) + return -EINVAL; if (unlikely(!stream->msg)) { stream->msg = tfw_http_conn_msg_alloc(conn, stream); if (!stream->msg) { From 54fab94f2f71f18f6610c30ee5ef68bcea3b7e0c Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sun, 29 Dec 2019 20:21:43 +0500 Subject: [PATCH 28/64] Port parsing accept header according to rfc to http2 --- tempesta_fw/http_parser.c | 134 +++++++++++++++++++++++++++++++++----- 1 file changed, 119 insertions(+), 15 deletions(-) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 47b46e9b13..f7fd13d499 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -5164,17 +5164,63 @@ __h2_req_parse_accept(TfwHttpReq *req, unsigned char *data, size_t len, __FSM_START(parser->_i_st); + __FSM_STATE(Req_I_WSAccept) { + if (IS_WS(c)) + __FSM_H2_I_MOVE(Req_I_WSAccept); + /* Fall through. */ + } + __FSM_STATE(Req_I_Accept) { - H2_TRY_STR_LAMBDA("text/html", { - __set_bit(TFW_HTTP_B_ACCEPT_HTML, req->flags); - __FSM_EXIT(CSTR_EQ); - }, Req_I_Accept, Req_I_AcceptHtml); - H2_TRY_STR_LAMBDA("*/*", { + H2_TRY_STR_LAMBDA("text", { + __FSM_EXIT(CSTR_NEQ); + }, Req_I_Accept, Req_I_AfterText); + /* + * TRY_STR() compares the string with the substring at the + * beginning of the chunk sequence, but @c is the first + * non-matching character with the string of the previous + * TRY_STR(). If we will use @c to compare with "*", then we will + * catch matches not only with "*", but also with "t*", "te*", + * "tex*". + */ + H2_TRY_STR_LAMBDA("*", { + __FSM_EXIT(CSTR_NEQ); + }, Req_I_Accept, Req_I_AfterStar); + TRY_STR_INIT(); + if (IS_TOKEN(c)) + __FSM_I_JMP(Req_I_Type); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_AfterText) { + if (c == '/') + __FSM_H2_I_MOVE(Req_I_AfterTextSlash); + + __FSM_H2_I_MOVE(Req_I_Type); + } + + __FSM_STATE(Req_I_AfterTextSlash) { + H2_TRY_STR_LAMBDA("html", { __set_bit(TFW_HTTP_B_ACCEPT_HTML, req->flags); __FSM_EXIT(CSTR_EQ); - }, Req_I_Accept, Req_I_AcceptHtml); + }, Req_I_AfterTextSlash, Req_I_AcceptHtml); TRY_STR_INIT(); - __FSM_I_JMP(Req_I_AcceptOther); + __FSM_I_JMP(Req_I_Subtype); + } + + __FSM_STATE(Req_I_AfterStar) { + if (c == '/') + __FSM_H2_I_MOVE(Req_I_StarSlashStar); + if (IS_TOKEN(c)) + __FSM_I_JMP(Req_I_Type); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_StarSlashStar) { + if (c == '*') + __FSM_H2_I_MOVE_LAMBDA_n(Req_I_AcceptHtml, 1, { + __set_bit(TFW_HTTP_B_ACCEPT_HTML, req->flags); + }); + return CSTR_NEQ; } __FSM_STATE(Req_I_AcceptHtml) { @@ -5182,26 +5228,84 @@ __h2_req_parse_accept(TfwHttpReq *req, unsigned char *data, size_t len, __set_bit(TFW_HTTP_B_ACCEPT_HTML, req->flags); __FSM_I_JMP(I_EoT); } - /* Fall through. */ + __FSM_I_JMP(Req_I_Subtype); + } + + __FSM_STATE(Req_I_Type) { + __FSM_H2_I_MATCH_MOVE_LAMBDA(token, Req_I_Type, { + __FSM_EXIT(CSTR_NEQ); + }); + c = *(p + __fsm_sz); + if (c == '/') + __FSM_H2_I_MOVE_n(Req_I_Slash, __fsm_sz + 1); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_Slash) { + if (IS_TOKEN(c)) + __FSM_I_JMP(Req_I_Subtype); + if (c == '*') + __FSM_H2_I_MOVE(I_EoT); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_Subtype) { + __FSM_H2_I_MATCH_MOVE(token, Req_I_Subtype); + __FSM_H2_I_MOVE_n(I_EoT, __fsm_sz); + } + + __FSM_STATE(Req_I_QValue) { + if (isdigit(c) || c == '.') + __FSM_H2_I_MOVE(Req_I_QValue); + __FSM_I_JMP(I_EoT); + return CSTR_NEQ; + } + + __FSM_STATE(Req_I_WSAcceptOther) { + if (IS_WS(c)) + __FSM_H2_I_MOVE(Req_I_WSAcceptOther); + if (IS_TOKEN(c)) + __FSM_I_JMP(Req_I_AcceptOther); + return CSTR_NEQ; } __FSM_STATE(Req_I_AcceptOther) { - __FSM_H2_I_MATCH_MOVE(uri, Req_I_AcceptOther); + H2_TRY_STR_LAMBDA("q=", { + __FSM_EXIT(CSTR_NEQ); + }, Req_I_AcceptOther, Req_I_QValue); + TRY_STR_INIT(); + __FSM_H2_I_MATCH_MOVE(token, Req_I_AcceptOther); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_H2_I_MOVE_n(I_EoT, __fsm_sz + 1); + if (c == '=') + __FSM_H2_I_MOVE_n(Req_I_ParamValue, __fsm_sz + 1); return CSTR_NEQ; } + __FSM_STATE(Req_I_ParamValue) { + if (c == '\"') + __FSM_H2_I_MOVE_NEQ(Req_I_QuotedString, 1); + __FSM_H2_I_MATCH_MOVE(token, Req_I_ParamValue); + __FSM_H2_I_MOVE_n(I_EoT, __fsm_sz); + } + + __FSM_STATE(Req_I_QuotedString) { + __FSM_H2_I_MATCH_MOVE_LAMBDA(token, Req_I_QuotedString, { + __FSM_EXIT(CSTR_NEQ); + }); + if (c != '"') + __FSM_H2_I_MOVE_NEQ(Req_I_QuotedString, 1); + __FSM_H2_I_MOVE(I_EoT); + } + /* End of term. */ __FSM_STATE(I_EoT) { - if (IS_WS(c) || c == ',') + if (IS_WS(c)) __FSM_H2_I_MOVE(I_EoT); + if (c == ',') + __FSM_H2_I_MOVE(Req_I_WSAccept); if (c == ';') /* Skip weight parameter. */ - __FSM_H2_I_MOVE(Req_I_AcceptOther); - if (IS_TOKEN(c)) - __FSM_I_JMP(Req_I_Accept); + __FSM_H2_I_MOVE(Req_I_WSAcceptOther); return CSTR_NEQ; } From 82b7376bda02f46b8ba76b2cc6f2f896c10a6cfb Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sun, 29 Dec 2019 20:21:57 +0500 Subject: [PATCH 29/64] Port encoding headers to Huffman encoding --- tempesta_fw/hpack.c | 333 +++++++++++++++++++++++++++++++++----------- 1 file changed, 255 insertions(+), 78 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index 839b465f6d..18a1de7b50 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -2742,6 +2742,253 @@ write_int(unsigned long index, unsigned short max, unsigned short mask, res_idx->sz = size; } +static unsigned long +tfw_huffman_encode_string_len(TfwStr *str) +{ + TfwStr *c, *end; + unsigned long n = 0; + + BUG_ON(TFW_STR_DUP(str)); + + TFW_STR_FOR_EACH_CHUNK(c, str, end) { + unsigned long i; + + for (i = 0; i < c->len; i++) { + n += ht_length[(unsigned char)c->data[i]]; + } + } + + return (n + 7) >> 3; +} + +static int +tfw_huffman_encode_copy(TfwStr *__restrict src, TfwStr *__restrict dst_str) +{ + TfwStr *c, *end; + int off = 0; + u64 last_word = 0; + u64 *dst = (u64 *)dst_str->data; + const char *dst_end = dst_str->data + dst_str->len; + +#define JOIN_BITS(a, len, b) (((a) << (len)) | (b)) +#define write_bytes(n) \ +do { \ + unsigned int __n = n; \ + do { \ + if (unlikely(dst_end - (char *)dst == 0)) \ + return -EINVAL; \ + *(char *)dst++ = (char)last_word; \ + last_word >>= 8; \ + } while (--__n); \ +} while (0) + + BUG_ON(!TFW_STR_PLAIN(dst_str)); + + TFW_STR_FOR_EACH_CHUNK(c, src, end) { + unsigned long i; + + for (i = 0; i < c->len; i++) { + const unsigned int s = c->data[i]; + const unsigned int d = 64 - off; + const unsigned int e = ht_encode[s]; + const unsigned int l = ht_length[s]; + + off += l; + if (l <= d) { + last_word = JOIN_BITS(last_word, l, e); + } else { + off -= 64; + last_word = JOIN_BITS(last_word, d, e >> off); + if (dst_end - (char *)dst >= sizeof(u64)) { + last_word = cpu_to_be64(last_word); + *dst = last_word; + dst++; + } else { + write_bytes(sizeof(u64)); + } + last_word = e; + } + } + } + + if (off) { + unsigned int tail = off & 7; + + if (tail) { + unsigned int d = 8 - tail; + + last_word = JOIN_BITS(last_word, d, HT_EOS_HIGH >> tail); + off += d; + } + last_word <<= 64 - off; + last_word = cpu_to_be64(last_word); + if (off == 64) { + if (dst_end - (char *)dst >= sizeof(u64)) { + *dst = last_word; + dst++; + } else { + write_bytes(sizeof(u64)); + } + goto done; + } + if (off > 31) { + if (dst_end - (char *)dst >= sizeof(u32)) { + *(u32 *)dst = (u32)last_word; + dst = (u64 *)((u32 *)dst + 1); + } else { + write_bytes(sizeof(u32)); + } + last_word >>= 32; + off -= 32; + } + if (off > 15) { + if (dst_end - (char *)dst >= sizeof(u16)) { + *(u16 *)dst = (u16)last_word; + dst = (u64 *)((u16 *)dst + 1); + } else { + write_bytes(sizeof(u16)); + } + last_word >>= 16; + off -= 16; + } + if (off) { + if (unlikely(dst_end - (char *)dst == 0)) + return -EINVAL; + *(u8 *)dst = (u8)last_word; + } + } + +#undef JOIN_BITS +#undef write_bytes + +done: + return 0; +} + +static TfwStr * +tfw_huffman_encode_string(TfwStr *str, TfwPool *pool) +{ + unsigned long enc_len = tfw_huffman_encode_string_len(str); + TfwStr *encoded; + int r; + + if (!enc_len) + return ERR_PTR(-EINVAL); + encoded = tfw_pool_alloc(pool, sizeof(TfwStr) + enc_len); + if (!encoded) + return ERR_PTR(-ENOMEM); + + TFW_STR_INIT(encoded); + encoded->data = (char *)(encoded + 1); + encoded->len = enc_len; + + r = tfw_huffman_encode_copy(str, encoded); + + return r ? ERR_PTR(r) : encoded; +} + +static int +tfw_hpack_str_add_raw(TfwHttpTransIter *mit, TfwStr *str, bool in_huffman) +{ + int r = 0; + TfwHPackInt len; + TfwStr len_str = { 0 }; + unsigned short mask = in_huffman ? 0x80 : 0x0; + + write_int(str->len, 0x7F, mask, &len); + len_str.data = len.buf; + len_str.len = len.sz; + + r = tfw_h2_msg_rewrite_data(mit, &len_str, mit->bnd); + if (unlikely(r)) + return r; + + return tfw_h2_msg_rewrite_data(mit, str, mit->bnd); +} + +static int +tfw_hpack_str_expand_raw(TfwHttpTransIter *mit, TfwMsgIter *it, + struct sk_buff **skb_head, TfwStr *str, + bool in_huffman) +{ + int r; + TfwHPackInt len; + TfwStr len_str = { 0 }; + unsigned short mask = in_huffman ? 0x80 : 0x0; + + write_int(str->len, 0x7F, mask, &len); + len_str.data = len.buf; + len_str.len = len.sz; + + r = tfw_http_msg_expand_data(it, skb_head, &len_str); + if (unlikely(r)) + return r; + mit->acc_len += len_str.len; + + r = tfw_http_msg_expand_data(it, skb_head, str); + if (unlikely(r)) + return r; + mit->acc_len += str->len; + + return 0; +} + +/* + * Family of functions to add new or append of h2 response. Huffman encoding + * is never used due to performance considerations: + * - We must write first the size of the string, it has variable size, and + * string encoded with Huffman codes may be shorter or longer than original + * string. The size will be known only after the string is completely encoded. + * - We can't encode @str in-place in @str: due to variable-sized coding, we + * can't overwrite symbol-by symbol in place. But this is also not possible + * due to @str origin, some strings are predefined strings from const memory + * region, others are shared for all messages to the same vhost. + * - For skb management we need to have exact data size before using API. + * + * Since these reasons we can't encode @str on the fly or encode it while + * writing to skb, so allocate a new string and copy @str encoded value there. + * It heavily affect performance, since we have two allocations and two copy + * operations here. We decided to keep the code, but our core use cases shows + * no performance improvement opportunities. + * + * TODO: We are free to choose encoding when we're adding a new header, but + * we have to (?) preserve original encoding when modifying already existent + * headers. + */ +static inline int +tfw_hpack_str_add(TfwHttpTransIter *mit, TfwStr *str, TfwPool *pool) +{ + bool in_huffman = false; + + if (0) { + str = tfw_huffman_encode_string(str, pool); + + if (IS_ERR(str)) + return PTR_ERR(str); + in_huffman = true; + } + + return tfw_hpack_str_add_raw(mit, str, in_huffman); +} + +static inline int +tfw_hpack_str_expand(TfwHttpTransIter *mit, TfwMsgIter *it, + struct sk_buff **skb_head, TfwStr *str, + TfwPool *pool) +{ + bool in_huffman = false; + + if (0) { + str = tfw_huffman_encode_string(str, pool); + + if (IS_ERR(str)) + return PTR_ERR(str); + in_huffman = true; + } + + return tfw_hpack_str_expand_raw(mit, it, skb_head, str, in_huffman); +} + /* * Add header @hdr in HTTP/2 HPACK format with metadata @idx into the * response @resp. @@ -2751,10 +2998,9 @@ tfw_hpack_hdr_add(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, bool name_indexed) { int r; - TfwHPackInt vlen; TfwStr *c, *end; TfwHttpTransIter *mit = &resp->mit; - TfwStr s_val, s_vlen = {}; + TfwStr s_val; const TfwStr s_idx = { .data = idx->buf, .len = idx->sz, @@ -2774,21 +3020,7 @@ tfw_hpack_hdr_add(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return -EINVAL; if (unlikely(!name_indexed)) { - TfwHPackInt nlen; - TfwStr *s_n = TFW_STR_CHUNK(hdr, 0); - TfwStr s_name = { - .chunks = (TfwStr []){ {}, {} }, - .nchunks = 2 - }; - - write_int(s_n->len, 0x7F, 0, &nlen); - __TFW_STR_CH(&s_name, 0)->data = nlen.buf; - __TFW_STR_CH(&s_name, 0)->len = nlen.sz; - __TFW_STR_CH(&s_name, 1)->data = s_n->data; - __TFW_STR_CH(&s_name, 1)->len = s_n->len; - s_name.len = nlen.sz + s_n->len; - - r = tfw_h2_msg_rewrite_data(mit, &s_name, mit->bnd); + r = tfw_hpack_str_add(mit, TFW_STR_CHUNK(hdr, 0), resp->pool); if (unlikely(r)) return r; } @@ -2813,19 +3045,7 @@ tfw_hpack_hdr_add(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, end = hdr->chunks + hdr->nchunks; tfw_str_collect_cmp(c, end, &s_val, NULL); - write_int(s_val.len, 0x7F, 0, &vlen); - s_vlen.data = vlen.buf; - s_vlen.len = vlen.sz; - - r = tfw_h2_msg_rewrite_data(mit, &s_vlen, mit->bnd); - if (unlikely(r)) - return r; - - r = tfw_h2_msg_rewrite_data(mit, &s_val, mit->bnd); - if (unlikely(r)) - return r; - - return 0; + return tfw_hpack_str_add(mit,&s_val, resp->pool); } /* @@ -2837,12 +3057,11 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, TfwHPackInt *__restrict idx, bool name_indexed) { int ret; - TfwHPackInt nlen, vlen; - TfwStr *c, *end, *s_name; + TfwStr *c, *end; TfwHttpTransIter *mit = &resp->mit; TfwMsgIter *iter = &mit->iter; struct sk_buff **skb_head = &resp->msg.skb_head; - TfwStr s_val, s_nlen = {}, s_vlen = {}; + TfwStr s_val; TfwStr idx_str = { .data = idx->buf, .len = idx->sz, @@ -2865,30 +3084,10 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, return -EINVAL; if (unlikely(!name_indexed)) { - s_name = TFW_STR_CHUNK(hdr, 0); - write_int(s_name->len, 0x7F, 0, &nlen); - s_nlen.data = nlen.buf; - s_nlen.len = nlen.sz; - - ret = tfw_http_msg_expand_data(iter, skb_head, &s_nlen); - if (unlikely(ret)) - return ret; - - mit->acc_len += s_nlen.len; - - T_DBG3("%s: name len, acc_len=%lu, s_nlen.len=%lu, s_nlen.data" - "='%.*s'\n", __func__, mit->acc_len, s_nlen.len, - (int)s_nlen.len, s_nlen.data); - - ret = tfw_http_msg_expand_data(iter, skb_head, s_name); + ret = tfw_hpack_str_expand(mit, iter, skb_head, + TFW_STR_CHUNK(hdr, 0), NULL); if (unlikely(ret)) return ret; - - mit->acc_len += s_name->len; - - T_DBG3("%s: name str, acc_len=%lu, s_name.len=%lu, s_name.data" - "='%.*s'\n", __func__, mit->acc_len, s_name->len, - (int)s_name->len, s_name->data); } /* @@ -2911,29 +3110,7 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, end = hdr->chunks + hdr->nchunks; tfw_str_collect_cmp(c, end, &s_val, NULL); - write_int(s_val.len, 0x7F, 0, &vlen); - s_vlen.data = vlen.buf; - s_vlen.len = vlen.sz; - - ret = tfw_http_msg_expand_data(iter, skb_head, &s_vlen); - if (unlikely(ret)) - return ret; - - mit->acc_len += s_vlen.len; - - T_DBG3("%s: val len, acc_len=%lu, s_vlen.len=%lu, s_vlen.data='%.*s'\n", - __func__, mit->acc_len, s_vlen.len, (int)s_vlen.len, s_vlen.data); - - ret = tfw_http_msg_expand_data(iter, skb_head, &s_val); - if (unlikely(ret)) - return ret; - - mit->acc_len += s_val.len; - - T_DBG3("%s: val str, acc_len=%lu, s_val.len=%lu, s_val.data='%.*s'\n", - __func__, mit->acc_len, s_val.len, (int)s_val.len, s_val.data); - - return 0; + return tfw_hpack_str_expand(mit, iter, skb_head, &s_val, NULL); } /* From 66752e4b84fec84d45ba2085f7bed19118df6224 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sun, 29 Dec 2019 22:15:19 +0500 Subject: [PATCH 30/64] Remove unneeded template file --- tempesta_fw/hpack_tbl.h.tpl | 60 ------------------------------------- 1 file changed, 60 deletions(-) delete mode 100644 tempesta_fw/hpack_tbl.h.tpl diff --git a/tempesta_fw/hpack_tbl.h.tpl b/tempesta_fw/hpack_tbl.h.tpl deleted file mode 100644 index 4ac4079150..0000000000 --- a/tempesta_fw/hpack_tbl.h.tpl +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Tempesta FW - * - * Copyright (C) 2019 Tempesta Technologies, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, - * or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ -#ifndef __TFW_HTTP_HPACK_TBL_H__ -#define __TFW_HTTP_HPACK_TBL_H__ - -/* This file is auto generated, please do not edit it... */ - -/** - * Huffman decoder state machine: - * |shift| = modulus of the "shift" field always contains - * number of bits in the Huffman-encoded representation, - * which must be taken by decoder on this step. - * NB! for short tables (f.e. 8-entries tables) difference - * in the prefix length (f.e. 7 bits for the big tables - * minus 3 bits for short tables = 4 bits) was pre-added - * to the value of "shift" field (just to speedup the decoder), - * thus true value of the Huffman-encoded prefix, which must - * be taken by decoder on this step is equal to the "shift" - * minus three. - * shift > 0 ---> normal symbol: - * offset = signed char representation of the decoded symbol. - * shift < 0 && offset == 0 ---> EOS. - * |shift| = number of bits in the truncated path. - * shift < 0 && offset > 0 ---> jump to next table: - * offset = offset of the next table. - */ -typedef struct { - short shift; - short offset; -} HTState; - -#define HT_NBITS 7 -#define HT_MBITS 3 -#define HT_NMASK 127 -#define HT_MMASK 7 -#define HT_SMALL 640 -#define HT_EOS_HIGH 0xFF - -static const HTState ht_decode[ [% TABLE_SIZE %] ] ____cacheline_aligned = { - [% TABLE %] -}; - -#endif /* __TFW_HTTP_HPACK_TBL_H__ */ From 2d2480bc10027025d606ca06bee41ee6479fe7c4 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Mon, 30 Dec 2019 10:00:43 +0500 Subject: [PATCH 31/64] Add tests for huffman encoding --- tempesta_fw/t/unit/test_hpack.c | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index 0845da09ab..5e6d946a56 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -1163,6 +1163,41 @@ TEST(hpack, dec_huffman) #undef HDR_VALUE_3 } +TEST(hpack, enc_huffman) +{ + /* + * The test dec_huffman checks that the following values was + * correctly parsed from the Huffman encoded strings, check that we + * can create exactly the same encoded strings. + */ + DEFINE_TFW_STR(custom_key_raw, "custom-key"); + DEFINE_TFW_STR(custom_key_exp, "\x25\xA8\x49\xE9\x5B\xA9\x7D\x7F"); + DEFINE_TFW_STR(custom_val_raw, "custom-value"); + DEFINE_TFW_STR(custom_val_exp, "\x25\xA8\x49\xE9\x5B\xB8\xE8\xB4\xBF"); + DEFINE_TFW_STR(no_cache_raw, "no-cache"); + DEFINE_TFW_STR(no_cache_exp, "\xA8\xEB\x10\x64\x9C\xBF"); + DEFINE_TFW_STR(www_raw, "www.example.com"); + DEFINE_TFW_STR(www_exp, + "\xF1\xE3\xC2\xE5\xF2\x3A\x6B\xA0\xAB\x90\xF4\xFF"); + TfwStr *custom_key_enc, *custom_val_enc, *no_cache_enc, *www_enc; + + custom_key_enc = tfw_huffman_encode_string(&custom_key_raw, str_pool); + EXPECT_TRUE(!IS_ERR_OR_NULL(custom_key_enc)); + EXPECT_OK(tfw_strcmp(custom_key_enc, &custom_key_exp)); + + custom_val_enc = tfw_huffman_encode_string(&custom_val_raw, str_pool); + EXPECT_TRUE(!IS_ERR_OR_NULL(custom_val_enc)); + EXPECT_OK(tfw_strcmp(custom_val_enc, &custom_val_exp)); + + no_cache_enc = tfw_huffman_encode_string(&no_cache_raw, str_pool); + EXPECT_TRUE(!IS_ERR_OR_NULL(no_cache_enc)); + EXPECT_OK(tfw_strcmp(no_cache_enc, &no_cache_exp)); + + www_enc = tfw_huffman_encode_string(&www_raw, str_pool); + EXPECT_TRUE(!IS_ERR_OR_NULL(www_enc)); + EXPECT_OK(tfw_strcmp(www_enc, &www_exp)); +} + TEST(hpack, enc_table_hdr_write) { char *buf; @@ -1813,6 +1848,7 @@ TEST_SUITE(hpack) TEST_RUN(hpack, dec_raw); TEST_RUN(hpack, dec_indexed); TEST_RUN(hpack, dec_huffman); + TEST_RUN(hpack, enc_huffman); TEST_RUN(hpack, enc_table_hdr_write); TEST_RUN(hpack, enc_table_index); TEST_RUN(hpack, enc_table_rbtree); From 4d7eb6623ac45f777109beabe5505672a68206dc Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Mon, 30 Dec 2019 10:04:59 +0500 Subject: [PATCH 32/64] Add unit tests for hufman encoding --- tempesta_fw/t/unit/test_hpack.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index 5e6d946a56..b0c72ea7ef 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -1176,7 +1176,7 @@ TEST(hpack, enc_huffman) DEFINE_TFW_STR(custom_val_exp, "\x25\xA8\x49\xE9\x5B\xB8\xE8\xB4\xBF"); DEFINE_TFW_STR(no_cache_raw, "no-cache"); DEFINE_TFW_STR(no_cache_exp, "\xA8\xEB\x10\x64\x9C\xBF"); - DEFINE_TFW_STR(www_raw, "www.example.com"); + TfwStr *www_raw = make_compound_str("www.example.com"); DEFINE_TFW_STR(www_exp, "\xF1\xE3\xC2\xE5\xF2\x3A\x6B\xA0\xAB\x90\xF4\xFF"); TfwStr *custom_key_enc, *custom_val_enc, *no_cache_enc, *www_enc; @@ -1193,9 +1193,14 @@ TEST(hpack, enc_huffman) EXPECT_TRUE(!IS_ERR_OR_NULL(no_cache_enc)); EXPECT_OK(tfw_strcmp(no_cache_enc, &no_cache_exp)); - www_enc = tfw_huffman_encode_string(&www_raw, str_pool); + www_enc = tfw_huffman_encode_string(www_raw, str_pool); EXPECT_TRUE(!IS_ERR_OR_NULL(www_enc)); EXPECT_OK(tfw_strcmp(www_enc, &www_exp)); + + /* + * TODO: add more test cases: encode<->decode the same text using + * Tempesta functions. + */ } TEST(hpack, enc_table_hdr_write) From d647d63a1276749d1f2176211a1db41003da58ca Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Mon, 30 Dec 2019 18:17:53 +0500 Subject: [PATCH 33/64] Fix code review comments --- tempesta_fw/hpack.c | 9 ++++----- tempesta_fw/http.c | 4 ++-- tempesta_fw/http_parser.c | 6 ++---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index 18a1de7b50..fde33a77ec 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -2949,11 +2949,10 @@ tfw_hpack_str_expand_raw(TfwHttpTransIter *mit, TfwMsgIter *it, * writing to skb, so allocate a new string and copy @str encoded value there. * It heavily affect performance, since we have two allocations and two copy * operations here. We decided to keep the code, but our core use cases shows - * no performance improvement opportunities. - * - * TODO: We are free to choose encoding when we're adding a new header, but - * we have to (?) preserve original encoding when modifying already existent - * headers. + * no performance improvement opportunities. According to RFC we are free to + * choose encoding (static/dynamic/Huffman) when modifying already existent + * headers (e.g. in cases of HTTP/1.1=>HTTP/2 or HTTP/2=>HTTP/2 response proxy), + * thus avoiding Huffman encodings is completely RFC-compliant behaviour. */ static inline int tfw_hpack_str_add(TfwHttpTransIter *mit, TfwStr *str, TfwPool *pool) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 83e574280b..76c8a26ff2 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -740,7 +740,7 @@ tfw_h2_pseudo_index(unsigned short status) } /** - * Write HTTP/2 Pseudo-header fields. Only ':status' is defined as response + * Write HTTP/2 pseudo-header fields. Only ':status' is defined as response * pseudo-header and all HTTP/2 responses must contain that. * https://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.4 */ @@ -3640,7 +3640,7 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) TfwStr *tgt = &ht->tbl[hid]; TfwStr *first = TFW_STR_CHUNK(tgt, 0); TfwHdrModsDesc *f_desc = NULL; - const TfwStr *val = NULL; + const TfwStr *val; if (TFW_STR_DUP(tgt)) tgt = TFW_STR_CHUNK(tgt, d_num); diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index f7fd13d499..3a270d6fc1 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -4457,7 +4457,7 @@ Req_Method_1CharStep: __attribute__((cold)) __FSM_TX_AF(Req_HdrX_Method_Overrid, 'e', Req_HdrX_Method_Override); __FSM_TX_AF_OWS(Req_HdrX_Method_Override, Req_HdrX_Method_OverrideV); - /* X-HTTP-Method header processing */ + /* X-HTTP-Method header processing. */ __FSM_TX_AF(Req_HdrX_H, 't', Req_HdrX_Ht); __FSM_TX_AF(Req_HdrX_Ht, 't', Req_HdrX_Htt); __FSM_TX_AF(Req_HdrX_Htt, 'p', Req_HdrX_Http); @@ -4480,7 +4480,7 @@ Req_Method_1CharStep: __attribute__((cold)) } } - /* X-HTTP-Method-Override processing */ + /* X-HTTP-Method-Override processing. */ __FSM_TX_AF(Req_HdrX_Http_Method_, 'o', Req_HdrX_Http_Method_O); __FSM_TX_AF(Req_HdrX_Http_Method_O, 'v', Req_HdrX_Http_Method_Ov); __FSM_TX_AF(Req_HdrX_Http_Method_Ov, 'e', Req_HdrX_Http_Method_Ove); @@ -7632,8 +7632,6 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, __FSM_H2_TX_AF_FIN(Req_HdrCooki, 'e', Req_HdrCookieV, TFW_TAG_HDR_COOKIE); - - /* Improbable states of method values processing. */ __FSM_STATE(Req_RareMethods_3, cold) { From d306c1b9c1084409c40fae9649ab0e548da1cdb6 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Wed, 18 Dec 2019 15:45:55 +0500 Subject: [PATCH 34/64] Fix request h2 -> http1.1 conversion Main changes: 1. Keep h_tbl always valid since we need to read the table during response caching. 2. Optimize header alteration for h2, use compatibility mode for http1.1 3. Use tfw_msg_write functions to take care of all possible caveats at message creation stage. (E.g. headers may be longer than one page) Known issues: 1. Body and trailer conversion is not tested due to parsing issues: body is not appended to the request and WARN_ON is triggered. --- tempesta_fw/http.c | 475 ++++++++++++++++++++++++----------------- tempesta_fw/http_msg.c | 8 +- tempesta_fw/ss_skb.c | 2 - tempesta_fw/ss_skb.h | 16 ++ 4 files changed, 301 insertions(+), 200 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 76c8a26ff2..489f144591 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -150,12 +150,12 @@ static struct { #define S_503 "HTTP/1.1 503 Service Unavailable" #define S_504 "HTTP/1.1 504 Gateway Timeout" -#define S_CONT_TYPE "content-type" #define S_XFF "x-forwarded-for" #define S_F_HOST "host: " #define S_F_DATE "date: " #define S_F_CONTENT_LENGTH "content-length: " +#define S_F_CONTENT_TYPE "content-type: " #define S_F_LOCATION "location: " #define S_F_CONNECTION "connection: " #define S_F_ETAG "etag: " @@ -168,6 +168,7 @@ static struct { #define S_V_CONN_CLOSE "close" #define S_V_CONN_KA "keep-alive" #define S_V_RETRY_AFTER "10" +#define S_V_MULTIPART "multipart/form-data; boundary=" #define S_H_CONN_KA S_F_CONNECTION S_V_CONN_KA S_CRLFCRLF #define S_H_CONN_CLOSE S_F_CONNECTION S_V_CONN_CLOSE S_CRLFCRLF @@ -2664,7 +2665,6 @@ tfw_http_add_x_forwarded_for(TfwHttpMsg *hm) static int tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req) { - int r; size_t i; int mod_type = (hm == (TfwHttpMsg *)req) ? TFW_VHOST_HDRMOD_REQ : TFW_VHOST_HDRMOD_RESP; @@ -2675,7 +2675,26 @@ tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req) for (i = 0; i < h_mods->sz; ++i) { TfwHdrModsDesc *d = &h_mods->hdrs[i]; - r = tfw_http_msg_hdr_xfrm_str(hm, d->hdr, d->hid, d->append); + /* + * Header is stored optimized for HTTP2: without delimiter + * between header and value. Add it as separate chunk as + * required for tfw_http_msg_hdr_xfrm_str. + */ + TfwStr h_mdf = { + .chunks = (TfwStr []){ + {}, + { .data = S_DLM, .len = SLEN(S_DLM) }, + {}, + }, + .len = SLEN(S_DLM), + }; + int r; + + h_mdf.chunks[0] = d->hdr->chunks[0]; + h_mdf.chunks[2] = d->hdr->chunks[1]; + h_mdf.len += d->hdr->len; + h_mdf.nchunks = d->hdr->nchunks + 1; + r = tfw_http_msg_hdr_xfrm_str(hm, &h_mdf, d->hid, d->append); if (r) { T_ERR("can't update location-specific header in msg %p\n", hm); @@ -2792,6 +2811,21 @@ __h2_hdrs_dup_decrease(TfwHttpReq *req, const TfwStr *hdr) } } +/** + * Apply header modification. @hdr contains exactly two chunks: header name and + * value. + * + * The @hdr descriptor (top level TfwStr is copied directly into header table. + * It looks dangerous but it's safe and we avoid extra copy operations. + * @req holds a reference to vhost, and @hdr value is stored inside that + * vhost/location description. On reconfiguration a new vhost instance is + * created instead of alternating its settings in place. So @hdr will valid + * until request itself is alive. + * + * Since h2 requests are always converted to http1 and all headers are + * recreated from cratch, there is no need to fragment underlying skbs and copy + * data there, it's enough to put safe pointers inside header table. + */ static int __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) { @@ -2799,9 +2833,8 @@ __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) TfwHttpMsg *hm = (TfwHttpMsg *)req; TfwHttpHdrTbl *ht = hm->h_tbl; TfwMsgParseIter *it = &req->pit; - const TfwStr *s_val = TFW_STR_CHUNK(hdr, 2); + const TfwStr *s_val = TFW_STR_CHUNK(hdr, 1); - BUG_ON(TFW_STR_CHUNK(hdr, 1)->len != SLEN(S_DLM)); if (WARN_ON_ONCE(!ht)) return -EINVAL; @@ -2813,14 +2846,14 @@ __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) if (hid < TFW_HTTP_HDR_RAW) { orig_hdr = &ht->tbl[hid]; /* - * Insert special header if empty and exit we have nothing + * Insert special header if empty and exit if we have nothing * to insert. */ if (TFW_STR_EMPTY(orig_hdr)) { if (unlikely(!s_val)) return 0; ++it->hdrs_cnt; - it->hdrs_len += hdr->len - SLEN(S_DLM); + it->hdrs_len += hdr->len; *orig_hdr = *hdr; return 0; } @@ -2843,7 +2876,7 @@ __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) */ ++ht->off; ++it->hdrs_cnt; - it->hdrs_len += hdr->len - SLEN(S_DLM); + it->hdrs_len += hdr->len; ht->tbl[hid] = *hdr; return 0; } @@ -2886,7 +2919,7 @@ __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) */ __h2_hdrs_dup_decrease(req, orig_hdr); ++it->hdrs_cnt; - it->hdrs_len += hdr->len - SLEN(S_DLM); + it->hdrs_len += hdr->len; *orig_hdr = *hdr; return 0; @@ -2916,125 +2949,145 @@ tfw_h2_req_set_loc_hdrs(TfwHttpReq *req) } /** - * Generation and adjusting HTTP/1.1 request (applicable in - * HTTP/2=>HTTP/1.1 transformation). + * Transform h2 request to http1.1 request before forward it to backend server. + * Usually we prefer in-place header modifications avoid copying, but here + * we have to insert a lot of information into header, like delimiters between + * header name and value, and between headers. To avoid creating extreme number + * of skb fragments we cut off skbs with h2 headers from the beginning of the + * request and replace them with http1.1 headers. + * + * Note, that we keep original headers in h_tbl untouched, since the response + * may want to access the request headers: the cache subsystem reads `Host` + * header and `uri` part, also if 'Vary' header controls response + * representation, any header listed inside 'Vary' one may be also read on + * response processing (not implemented yet). */ static int tfw_h2_adjust_req(TfwHttpReq *req) { int r; - char *dst; - struct page *pg; - unsigned int hid; - TfwStr method; - const TfwStr *fld, *fld_end, *hdr, *dup_end, *chunk, *end; - TfwHttpMsg *hm = (TfwHttpMsg *)req; + TfwMsgParseIter *pit = &req->pit; + ssize_t h1_hdrs_sz = 0; TfwHttpHdrTbl *ht = req->h_tbl; - TfwMsgParseIter *it = &req->pit; bool auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); bool host = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_HOST]); - TfwGlobal *g_vhost = tfw_vhost_get_global(); + size_t pseudo_num = auth ? 4 : 3; + TfwStr meth = {}, *field, *end; + struct sk_buff *new_head = NULL, *old_head = NULL; + TfwMsgIter it; + const DEFINE_TFW_STR(sp, " "); + const DEFINE_TFW_STR(dlm, S_DLM); + const DEFINE_TFW_STR(crlf, S_CRLF); + const DEFINE_TFW_STR(fl_end, " " S_VERSION11 S_CRLF S_F_HOST); char *buf = *this_cpu_ptr(&g_buf); - char *xff_ptr = ss_skb_fmt_src_addr(req->msg.skb_head, buf); - TfwStr h_xff = { + char *xff_end = ss_skb_fmt_src_addr(req->msg.skb_head, buf); + const TfwStr h_xff = { .chunks = (TfwStr []){ { .data = S_XFF, .len = SLEN(S_XFF) }, - { .data = buf, .len = xff_ptr - buf, - .flags = TFW_STR_HDR_VALUE } - }, - .len = SLEN(S_XFF) + xff_ptr - buf, - .nchunks = 2 - }; - TfwStr h_ct_new = { - .chunks = (TfwStr []) { - { .data = S_CONT_TYPE, .len = SLEN(S_CONT_TYPE) }, - TFW_STR_F_STRING("multipart/form-data; boundary=", - TFW_STR_HDR_VALUE), - req->multipart_boundary_raw + { .data = S_DLM, .len = SLEN(S_DLM) }, + { .data = buf, .len = xff_end - buf }, + { .data = S_CRLF, .len = SLEN(S_CRLF) }, }, - .nchunks = 3 + .len = SLEN(S_XFF) + SLEN(S_DLM) + xff_end - buf + SLEN(S_CRLF), + .nchunks = 4 }; + TfwGlobal *g_vhost = tfw_vhost_get_global(); const TfwStr h_via = { .chunks = (TfwStr []) { { .data = S_F_VIA, .len = SLEN(S_F_VIA) }, { .data = "1.1 ", .len = 4 }, - { .data = buf, .len = g_vhost->hdr_via_len }, + { .data = (char *)g_vhost->hdr_via, + .len = g_vhost->hdr_via_len }, { .data = S_CRLF, .len = SLEN(S_CRLF) } }, .len = SLEN(S_F_VIA) + 4 + g_vhost->hdr_via_len + SLEN(S_CRLF), .nchunks = 4 }; - TfwStr h_conn = { - .chunks = (TfwStr []){ - { .data = S_F_CONNECTION, .len = SLEN(S_F_CONNECTION) }, - { .data = S_V_CONN_KA, .len = SLEN(S_V_CONN_KA) }, + const TfwStr h_ct = { + .chunks = (TfwStr []) { + { .data = S_F_CONTENT_TYPE S_V_MULTIPART, + .len = SLEN(S_F_CONTENT_TYPE S_V_MULTIPART) }, + req->multipart_boundary_raw, { .data = S_CRLF, .len = SLEN(S_CRLF) } }, - .len = SLEN(S_F_CONNECTION) + SLEN(S_V_CONN_KA) + SLEN(S_CRLF), - .nchunks = 3 + .nchunks = 3, + .len = SLEN(S_F_CONTENT_TYPE S_V_MULTIPART) + + req->multipart_boundary_raw.len + SLEN(S_CRLF) }; + unsigned int xff_hid; + int h_ct_replace = 0; -#define WRITE_LIT(buf, lit) \ -do { \ - memcpy_fast(buf, lit, SLEN(lit)); \ - buf += SLEN(lit); \ -} while (0) - -#define WRITE_STR(buf, str) \ -do { \ - TFW_STR_FOR_EACH_CHUNK(chunk, str, end) { \ - memcpy_fast(buf, chunk->data, chunk->len); \ - buf += chunk->len; \ - } \ -} while (0) - -#define WRITE_HDR(buf, hdr) \ -({ \ - const TfwStr *c, *end; \ - bool val_found = false; \ - TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { \ - if (!val_found) { \ - if (c->flags & TFW_STR_HDR_VALUE) { \ - WRITE_LIT(buf, S_DLM); \ - val_found = true; \ - } else if (*c->data == ':') { \ - /* For statically configured adjustments. */\ - WARN_ON_ONCE(c->len != SLEN(S_DLM)); \ - WRITE_LIT(buf, S_DLM); \ - val_found = true; \ - continue; \ - } \ - } \ - memcpy_fast(buf, c->data, c->len); \ - buf += c->len; \ - } \ - WRITE_LIT(buf, S_CRLF); \ -}) - - T_DBG3("%s: it->hdrs_len=%lu, it->hdrs_cnt=%u\n", __func__, - it->hdrs_len, it->hdrs_cnt); - - /* Substitution/addition of 'x-forwarded-for' header. */ - hid = __h2_hdr_lookup(hm, TFW_STR_CHUNK(&h_xff, 0)); - if (unlikely(hid == ht->size)) - if (tfw_http_msg_grow_hdr_tbl(hm)) - return -ENOMEM; - if (hid < ht->off) - __h2_hdrs_dup_decrease(req, &ht->tbl[hid]); - else - ++ht->off; + T_DBG3("%s: req [%p] to be converted to http1.1\n", __func__, req); - ++it->hdrs_cnt; - it->hdrs_len += h_xff.len; - ht->tbl[hid] = h_xff; + /* H2 client may use either authority or host header but at least one + * is sufficient for correct conversion. + */ + if (WARN_ON_ONCE(!auth && !host)) + return -EINVAL; /* - * Conditional substitution/addition/deletion or appending of statically - * configured headers. + * First apply message modifications defined by admin in configuration + * file. Ideally we should do it al last stage, when h2 headers are + * copied into h1 buffer and apply modifications during copying. But + * this doesn't allow us to predict h1 headers size before memory + * allocation. Header modifications manual on wiki is already has a + * warning about performance impact, so just live it as is, a more + * robust algorithm will be used here if really required. */ if ((r = tfw_h2_req_set_loc_hdrs(req))) return r; + /* + * Calculate http1.1 headers size. H2 request contains pseudo headers + * that are represented in different way in the http1.1 requests. + * pit->hdrs_cnt is aware of header duplicates. Redirection mark is + * ignored and not copied. + */ + h1_hdrs_sz = pit->hdrs_len + + (pit->hdrs_cnt - pseudo_num) * (SLEN(S_DLM) + SLEN(S_CRLF)) + - req->mark.len; + /* First request line: remove pseudo headers names, all values are on + * the same line. + */ + h1_hdrs_sz += (long int)2 + SLEN(S_VERSION11) + SLEN(S_CRLF) + - ht->tbl[TFW_HTTP_HDR_H2_SCHEME].len + - SLEN(S_H2_METHOD) + - SLEN(S_H2_PATH) + + SLEN(S_CRLF) /* After headers */; + /* :authority pseudo header */ + if (auth) { + /* RFC 7540: + * An intermediary that converts an HTTP/2 request to HTTP/1.1 + * MUST create a Host header field if one is not present in a + * request by copying the value of the :authority pseudo-header + * field. + * AND + * Clients that generate HTTP/2 requests directly SHOULD use + * the :authority pseudo-header field instead of the Host + * header field. + */ + if (host) { + h1_hdrs_sz -= ht->tbl[TFW_HTTP_HDR_HOST].len; + h1_hdrs_sz -= SLEN(S_H2_AUTH); + h1_hdrs_sz += SLEN(S_F_HOST); + } + else { + h1_hdrs_sz -= SLEN(S_H2_AUTH); + h1_hdrs_sz += SLEN(S_F_HOST) + SLEN(S_CRLF); + } + } + + /* 'x-forwarded-for' header must be updated. */ + xff_hid = __h2_hdr_lookup((TfwHttpMsg *)req, TFW_STR_CHUNK(&h_xff, 0)); + if (xff_hid < ht->off) { + TfwStr *dup, *dup_end; + TFW_STR_FOR_EACH_DUP(dup, &ht->tbl[xff_hid], dup_end) { + h1_hdrs_sz -= dup->len + SLEN(S_DLM) + SLEN(S_CRLF); + } + } + h1_hdrs_sz += h_xff.len; + h1_hdrs_sz += h_via.len; + /* * Conditional substitution/additions of 'content-type' header. This is * singular header, so we can avoid duplicates processing. @@ -3043,124 +3096,138 @@ do { \ test_bit(TFW_HTTP_B_CT_MULTIPART, req->flags) && tfw_http_should_validate_post_req(req)) { - TfwStr *h_ct = &ht->tbl[TFW_HTTP_HDR_CONTENT_TYPE]; - unsigned long ct_len = TFW_STR_EMPTY(h_ct) ? 0 : h_ct->len; - TfwStr *c = h_ct_new.chunks; - - BUG_ON(!TFW_STR_PLAIN(&req->multipart_boundary_raw)); - h_ct_new.len = c[0].len + c[1].len + c[2].len; + TfwStr *h_ct_old = &ht->tbl[TFW_HTTP_HDR_CONTENT_TYPE]; - if (TFW_STR_EMPTY(h_ct)) - ++it->hdrs_cnt; + if (WARN_ON_ONCE(!TFW_STR_PLAIN(&req->multipart_boundary_raw) + || TFW_STR_EMPTY(h_ct_old))) + return -EINVAL; - it->hdrs_len += h_ct_new.len - ct_len; - *h_ct = h_ct_new; + h1_hdrs_sz -= h_ct_old->len + SLEN(S_DLM) + SLEN(S_CRLF); + h1_hdrs_sz += h_ct.len; + h_ct_replace = 1; } - /* - * Calculation of the entire buffer size needed for headers. Request-line - * should be written at first and separately, thus, the total headers - * count and length is correcting by three (or four) pseudo-headers - * count and length. - */ - it->hdrs_cnt -= 4; - it->hdrs_len += 2 + SLEN(S_VERSION11) + SLEN(S_CRLF) - - ht->tbl[TFW_HTTP_HDR_H2_SCHEME].len - - SLEN(S_H2_METHOD) - - SLEN(S_H2_PATH); + if (WARN_ON_ONCE(h1_hdrs_sz < 0)) + return -EINVAL; + if ((r = tfw_msg_iter_setup(&it, &new_head, h1_hdrs_sz, 0))) + return r; + /* First line. */ + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_METHOD], &meth); + r = tfw_msg_write(&it, &meth); + r |= tfw_msg_write(&it, &sp); + r |= tfw_msg_write(&it, &req->uri_path); + r |= tfw_msg_write(&it, &fl_end); /* start of Host: header */ if (auth) { - BUG_ON(TFW_STR_EMPTY(&req->host)); - it->hdrs_len += SLEN(S_HTTP) - - SLEN(S_H2_AUTH) - + SLEN(S_F_HOST) - + req->host.len - + SLEN(S_CRLF); - if (host) { - --it->hdrs_cnt; - it->hdrs_len -= ht->tbl[TFW_HTTP_HDR_HOST].len; - } + r |= tfw_msg_write(&it, &req->host); + } else if (host) { + TfwStr hostval = {}; + + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_HOST], &hostval); + r |= tfw_msg_write(&it, &hostval); } + r |= tfw_msg_write(&it, &crlf); - it->hdrs_len += it->hdrs_cnt * (SLEN(S_DLM) + SLEN(S_CRLF)); - it->hdrs_len += h_conn.len + h_via.len; - it->hdrs_len += SLEN(S_CRLF); + /* Skip host header: it's already written. */ + FOR_EACH_HDR_FIELD_FROM(field, end, req, TFW_HTTP_HDR_REGULAR) { + TfwStr *dup, *dup_end; + unsigned int hid = field - ht->tbl; - /* - * Create special buffer to write headers block into the target HTTP/1.1 - * representation. - */ - if (!(dst = pg_skb_alloc(it->hdrs_len, GFP_ATOMIC, NUMA_NO_NODE))) - return -ENOMEM; - pg = virt_to_page(dst); + /* Skip already written headers and headers to be modified. */ + if (hid == TFW_HTTP_HDR_HOST || hid == xff_hid) + continue; + if ((hid == TFW_HTTP_HDR_CONTENT_TYPE) && h_ct_replace) { + r |= tfw_msg_write(&it, &h_ct); + continue; + } - /* Add request-line into the HTTP/1.1 request. */ - BUG_ON(TFW_STR_EMPTY(&req->uri_path)); - __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_METHOD], &method); + if (TFW_STR_EMPTY(field)) + continue; + TFW_STR_FOR_EACH_DUP(dup, field, dup_end) { + TfwStr *chunk, *chunk_end, hval = {}; - WRITE_STR(dst, &method); - WRITE_LIT(dst, " "); - if (auth) { - WRITE_LIT(dst, S_HTTP); - WRITE_STR(dst, &req->host); - } - WRITE_STR(dst, &req->uri_path); - WRITE_LIT(dst, " "); - WRITE_LIT(dst, S_VERSION11); - WRITE_LIT(dst, S_CRLF); + if (unlikely(TFW_STR_PLAIN(dup))) { + r = -EINVAL; + break; + } - /* Add 'host' as the first header in request (RFC 7230 section 5.4). */ - WRITE_LIT(dst, S_F_HOST); - if (auth) { - WRITE_STR(dst, &req->host); - } else if (host) { - BUG_ON(!TFW_STR_EMPTY(&req->host)); - __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_HOST], &req->host); - WRITE_HDR(dst, &req->host); + hval.chunks = dup->chunks; + TFW_STR_FOR_EACH_CHUNK(chunk, dup, chunk_end) { + if (chunk->flags & TFW_STR_HDR_VALUE) + break; + hval.nchunks++; + hval.len += chunk->len; + } + r |= tfw_msg_write(&it, &hval); + r |= tfw_msg_write(&it, &dlm); + hval.chunks += hval.nchunks; + hval.nchunks = dup->nchunks - hval.nchunks; + hval.len = dup->len - hval.len; + r |= tfw_msg_write(&it, &hval); + + r |= tfw_msg_write(&it, &crlf); + } + if (unlikely(r)) + goto err; } - WRITE_LIT(dst, S_CRLF); - /* - * We have already added 'host' header, thus, it won't be needed in - * the writing cycle for all headers below. - */ - TFW_STR_INIT(&ht->tbl[TFW_HTTP_HDR_HOST]); + r |= tfw_msg_write(&it, &h_xff); + r |= tfw_msg_write(&it, &h_via); + /* Finally close headers. */ + r |= tfw_msg_write(&it, &crlf); - FOR_EACH_HDR_FIELD_FROM(fld, fld_end, req, TFW_HTTP_HDR_REGULAR) { - if (TFW_STR_EMPTY(fld)) - continue; - TFW_STR_FOR_EACH_DUP(hdr, fld, dup_end) - WRITE_HDR(dst, hdr); - } + if (unlikely(r)) + goto err; - /* - * Headers, which should be added unconditionally, inserted in - * the end. - */ - memcpy_fast(__TFW_STR_CH(&h_via, 2)->data, g_vhost->hdr_via, - g_vhost->hdr_via_len); - WRITE_STR(dst, &h_via); - WRITE_STR(dst, &h_conn); - WRITE_LIT(dst, S_CRLF); + T_DBG3("%s: req [%p] converted to http1.1\n", __func__, req); - T_DBG3("%s: req adjusted, it->hb_len=%lu, it->hdrs_len=%lu," - " it->hdrs_cnt=%u, dst=[%p], pg=[%p]\n", __func__, it->hb_len, - it->hdrs_len, it->hdrs_cnt, dst, (char *)page_address(pg)); + old_head = req->msg.skb_head; + req->msg.skb_head = new_head; - WARN_ON_ONCE(dst - (char *)page_address(pg) != it->hdrs_len); + /* Http chains might add a mark for the message, keep it. */ + new_head->mark = old_head->mark; - r = ss_skb_replace_page(&req->msg.skb_head, pg, it->hdrs_len, - it->hb_len); - if (r) { - put_page(pg); - return r; + if (!TFW_STR_EMPTY(&req->body)) { + /* + * Request has a body. we have to detach it from the old + * skb_head and append to a new one. There might be trailing + * headers after the body, but we're already copied them before + * body. This is not a problem, but we have to drop the trailer + * part after the body to avoid sending the same headers twice. + * + * Body travels in a separate DATA frame thus it's always in + * it's own skb. + */ + struct sk_buff *b_skbs = old_head, *trailer; + size_t len = 0; + + do { + b_skbs = b_skbs->next; + if (WARN_ON_ONCE((b_skbs == old_head))) + goto err; + } while (b_skbs != req->body.skb); + + ss_skb_queue_split(old_head, b_skbs); + trailer = b_skbs; + do { + len += trailer->len; + trailer = trailer->next; + + } while ((trailer != b_skbs) || (len != req->body.len)); + ss_skb_queue_tail(&req->msg.skb_head, b_skbs); + if (trailer != b_skbs) { + ss_skb_queue_split(req->msg.skb_head, trailer); + ss_skb_queue_tail(&old_head, trailer); + } } + ss_skb_queue_purge(&old_head); return 0; - -#undef WRITE_HDR -#undef WRITE_STR -#undef WRITE_LIT +err: + ss_skb_queue_purge(&new_head); + T_DBG3("%s: req [%p] convertation to http1.1 has failed\n", + __func__, req); + return r; } /** @@ -3595,17 +3662,37 @@ tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) static int tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods) { - int r; unsigned int i; TfwHttpTransIter *mit = &resp->mit; + if (!h_mods) + return 0; + for (i = 0; i < h_mods->sz; ++i) { const TfwHdrModsDesc *desc = &h_mods->hdrs[i]; + /* + * Header is stored optimized for HTTP2 request processing: + * without delimiter between header and value. + */ + TfwStr h_mdf = { + .chunks = (TfwStr []){ + {}, + { .data = S_DLM, .len = SLEN(S_DLM) }, + {}, + }, + .len = SLEN(S_DLM), + }; + int r; + + h_mdf.chunks[0] = desc->hdr->chunks[0]; + h_mdf.chunks[2] = desc->hdr->chunks[1]; + h_mdf.len += desc->hdr->len; + h_mdf.nchunks = desc->hdr->nchunks + 1; - if (test_bit(i, mit->found) || !TFW_STR_CHUNK(desc->hdr, 2)) + if (test_bit(i, mit->found) || !TFW_STR_CHUNK(&h_mdf, 2)) continue; - r = __hdr_h2_add(resp, desc->hdr); + r = __hdr_h2_add(resp, &h_mdf); if (unlikely(r)) return r; } diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index eb9a3c2a3d..748ecf7cef 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -45,12 +45,12 @@ tfw_http_msg_make_hdr(TfwPool *pool, const char *name, const char *val) const TfwStr tmp_hdr = { .chunks = (TfwStr []){ { .data = (void *)name, .len = n_len }, - { .data = S_DLM, .len = SLEN(S_DLM) }, - { .data = (void *)val, .len = v_len }, + { .data = (void *)val, .len = v_len, + .flags = TFW_STR_HDR_VALUE}, }, - .len = n_len + SLEN(S_DLM) + v_len, + .len = n_len + v_len, .eolen = 2, - .nchunks = (val ? 3 : 2) + .nchunks = (val ? 2 : 1) }; return tfw_strdup(pool, &tmp_hdr); diff --git a/tempesta_fw/ss_skb.c b/tempesta_fw/ss_skb.c index a2c98cf7d5..da375dff16 100644 --- a/tempesta_fw/ss_skb.c +++ b/tempesta_fw/ss_skb.c @@ -1720,5 +1720,3 @@ ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, return 0; } - - diff --git a/tempesta_fw/ss_skb.h b/tempesta_fw/ss_skb.h index d284aff89c..15b8c1e8ff 100644 --- a/tempesta_fw/ss_skb.h +++ b/tempesta_fw/ss_skb.h @@ -90,6 +90,22 @@ ss_skb_peek_tail(struct sk_buff **skb_head) return *skb_head ? (*skb_head)->prev : NULL; } +/** + * Split single queue into two, where the @skb will be a head of a new queue. + */ +static inline void +ss_skb_queue_split(struct sk_buff *skb_head, struct sk_buff *skb) +{ + struct sk_buff *prev = skb->prev; + WARN_ON_ONCE(skb_head == skb); + + skb->prev = skb_head->prev; + prev->next = skb_head; + + skb->prev->next = skb; + skb_head->prev = prev; +} + /** * Almost a copy of standard skb_dequeue() except it works with skb list * instead of sk_buff_head. Several crucial data include skb list and we don't From 0d6e7c665dd2b0b4c08669bbd63e145f0605dfd5 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Mon, 30 Dec 2019 22:49:17 +0500 Subject: [PATCH 35/64] Fix code review comments --- tempesta_fw/http.c | 64 +++++++++++--------- tempesta_fw/http_parser.c | 6 +- tempesta_fw/msg.h | 2 - tempesta_fw/ss_skb.c | 124 -------------------------------------- tempesta_fw/ss_skb.h | 2 - 5 files changed, 39 insertions(+), 159 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 489f144591..980e621d12 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -2967,12 +2967,12 @@ tfw_h2_adjust_req(TfwHttpReq *req) { int r; TfwMsgParseIter *pit = &req->pit; - ssize_t h1_hdrs_sz = 0; + ssize_t h1_hdrs_sz; TfwHttpHdrTbl *ht = req->h_tbl; bool auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); bool host = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_HOST]); size_t pseudo_num = auth ? 4 : 3; - TfwStr meth = {}, *field, *end; + TfwStr meth = {}, host_val = {}, *field, *end; struct sk_buff *new_head = NULL, *old_head = NULL; TfwMsgIter it; const DEFINE_TFW_STR(sp, " "); @@ -3014,20 +3014,22 @@ tfw_h2_adjust_req(TfwHttpReq *req) .len = SLEN(S_F_CONTENT_TYPE S_V_MULTIPART) + req->multipart_boundary_raw.len + SLEN(S_CRLF) }; - unsigned int xff_hid; int h_ct_replace = 0; T_DBG3("%s: req [%p] to be converted to http1.1\n", __func__, req); /* H2 client may use either authority or host header but at least one - * is sufficient for correct conversion. + * is required for correct conversion. */ - if (WARN_ON_ONCE(!auth && !host)) + if (!auth && !host) { + T_WARN("Cant convert h2 request to http/1.1: no authority " + "found\n"); return -EINVAL; + } /* * First apply message modifications defined by admin in configuration - * file. Ideally we should do it al last stage, when h2 headers are + * file. Ideally we should do it at last stage, when h2 headers are * copied into h1 buffer and apply modifications during copying. But * this doesn't allow us to predict h1 headers size before memory * allocation. Header modifications manual on wiki is already has a @@ -3067,21 +3069,25 @@ tfw_h2_adjust_req(TfwHttpReq *req) * header field. */ if (host) { - h1_hdrs_sz -= ht->tbl[TFW_HTTP_HDR_HOST].len; + h1_hdrs_sz -= ht->tbl[TFW_HTTP_HDR_HOST].len + + SLEN(S_DLM) + SLEN(S_CRLF); h1_hdrs_sz -= SLEN(S_H2_AUTH); - h1_hdrs_sz += SLEN(S_F_HOST); + /* S_F_HOST already contains S_DLM */ + h1_hdrs_sz += SLEN(S_F_HOST) - SLEN(S_DLM); } else { h1_hdrs_sz -= SLEN(S_H2_AUTH); - h1_hdrs_sz += SLEN(S_F_HOST) + SLEN(S_CRLF); + /* S_F_HOST already contains S_DLM */ + h1_hdrs_sz += SLEN(S_F_HOST) - SLEN(S_DLM); } } /* 'x-forwarded-for' header must be updated. */ - xff_hid = __h2_hdr_lookup((TfwHttpMsg *)req, TFW_STR_CHUNK(&h_xff, 0)); - if (xff_hid < ht->off) { + if (!TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR])) { + TfwStr *xff_hdr = &ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR]; TfwStr *dup, *dup_end; - TFW_STR_FOR_EACH_DUP(dup, &ht->tbl[xff_hid], dup_end) { + + TFW_STR_FOR_EACH_DUP(dup, xff_hdr, dup_end) { h1_hdrs_sz -= dup->len + SLEN(S_DLM) + SLEN(S_CRLF); } } @@ -3118,27 +3124,32 @@ tfw_h2_adjust_req(TfwHttpReq *req) r |= tfw_msg_write(&it, &sp); r |= tfw_msg_write(&it, &req->uri_path); r |= tfw_msg_write(&it, &fl_end); /* start of Host: header */ - if (auth) { - r |= tfw_msg_write(&it, &req->host); - } else if (host) { - TfwStr hostval = {}; - - __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_HOST], &hostval); - r |= tfw_msg_write(&it, &hostval); - } + if (auth) + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY], &host_val); + else if (host) + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_HOST], &host_val); + r |= tfw_msg_write(&it, &host_val); r |= tfw_msg_write(&it, &crlf); /* Skip host header: it's already written. */ FOR_EACH_HDR_FIELD_FROM(field, end, req, TFW_HTTP_HDR_REGULAR) { TfwStr *dup, *dup_end; - unsigned int hid = field - ht->tbl; - /* Skip already written headers and headers to be modified. */ - if (hid == TFW_HTTP_HDR_HOST || hid == xff_hid) - continue; - if ((hid == TFW_HTTP_HDR_CONTENT_TYPE) && h_ct_replace) { - r |= tfw_msg_write(&it, &h_ct); + switch (field - ht->tbl) + { + case TFW_HTTP_HDR_HOST: + continue; /* Already written. */ + case TFW_HTTP_HDR_X_FORWARDED_FOR: + r |= tfw_msg_write(&it, &h_xff); continue; + case TFW_HTTP_HDR_CONTENT_TYPE: + if (h_ct_replace) { + r |= tfw_msg_write(&it, &h_ct); + continue; + } + break; + default: + break; } if (TFW_STR_EMPTY(field)) @@ -3171,7 +3182,6 @@ tfw_h2_adjust_req(TfwHttpReq *req) goto err; } - r |= tfw_msg_write(&it, &h_xff); r |= tfw_msg_write(&it, &h_via); /* Finally close headers. */ r |= tfw_msg_write(&it, &crlf); diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 3a270d6fc1..94dd382a37 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -7980,12 +7980,10 @@ tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, *parsed = 0; - if (likely(type != HTTP2_DATA)) { + if (likely(type != HTTP2_DATA)) r = tfw_hpack_decode(&ctx->hpack, data, len, req, parsed); - req->pit.hb_len += *parsed; - } else { + else r = tfw_h2_parse_body(data, len, req, parsed); - } if (r != T_OK && r != T_POSTPONE) return r; diff --git a/tempesta_fw/msg.h b/tempesta_fw/msg.h index 936c255e3e..f25c5f1381 100644 --- a/tempesta_fw/msg.h +++ b/tempesta_fw/msg.h @@ -66,7 +66,6 @@ typedef struct { * * @pool - allocation pool for target buffer of decoded headers; * @parsed_hdr - pointer to the message header which is currently processed; - * @hb_len - length of the message HTTP/2 headers block; * @hdrs_len - accumulated length of message's decoded and parsed headers; * @hdrs_cnt - count of all headers from message headers block; * @__off - offset for iterator reinitializing before next processing @@ -84,7 +83,6 @@ typedef struct { typedef struct { TfwPool *pool; TfwStr *parsed_hdr; - unsigned long hb_len; unsigned long hdrs_len; unsigned int hdrs_cnt; char __off[0]; diff --git a/tempesta_fw/ss_skb.c b/tempesta_fw/ss_skb.c index da375dff16..83bef2c68b 100644 --- a/tempesta_fw/ss_skb.c +++ b/tempesta_fw/ss_skb.c @@ -1503,130 +1503,6 @@ ss_skb_to_sgvec_with_new_pages(struct sk_buff *skb, struct scatterlist *sgl, return out_frags; } -/* - * Remove @offset amount of data from head side of the skb list, and insert - * new page @pg instead. - */ -int -ss_skb_replace_page(struct sk_buff **skb_head, struct page *pg, - unsigned long pg_len, unsigned long offset) -{ - struct sk_buff *skb; - unsigned int head_len; - struct skb_shared_info *si; - int m_len, i = 0; - - /* Remove skbs which are fully included in the @offset quantity. */ - while ((*skb_head)->len < offset) { - skb = ss_skb_dequeue(skb_head); - offset -= skb->len; - __kfree_skb(skb); - if (WARN_ON_ONCE(!*skb_head)) - return -EIO; - } - skb = *skb_head; - T_DBG3("%s: offset=%#lx, skb=[%pK], (head=[%pK], data=[%pK], tail=[%pK]" - " end=[%pK], len=%u, data_len=%u, nr_frags=%u)\n", __func__, - offset, skb, skb->head, skb->data, skb_tail_pointer(skb), - skb_end_pointer(skb), skb->len, skb->data_len, - skb_shinfo(skb)->nr_frags); - - /* - * Erase @offset amount of data from linear part and fragments of - * the first remaining skb. - */ - si = skb_shinfo(skb); - m_len = min_t(int, skb_headlen(skb), offset); - if (m_len) { - __skb_pull(skb, m_len); - offset -= m_len; - } - while (offset) { - skb_frag_t *frag; - - if (WARN_ON_ONCE(i >= si->nr_frags)) - return -EIO; - - frag = &si->frags[i]; - m_len = min_t(int, skb_frag_size(frag), offset); - - T_DBG3("%s: skb=[%p], m_len=%d, fragsize=%d\n", __func__, skb, - m_len, skb_frag_size(frag)); - - if (unlikely(m_len == skb_frag_size(frag))) { - ss_skb_adjust_data_len(skb, m_len); - __skb_frag_unref(frag); - ++i; - } - else { - frag->page_offset += m_len; - skb_frag_size_sub(frag, m_len); - skb->len -= m_len; - skb->data_len -= m_len; - } - offset -= m_len; - } - - /* - * If one or more fragments have been fully removed, we can just - * insert fragment with new data into the freed slot, without - * extending fragments/skbs. - */ - if (i > 0) { - si->nr_frags -= i; - - /* - * If all data have been removed from the skb or only first - * fragment (from zero slot) have been removed, we don't - * need to shift anything. - */ - if (skb->len && i > 1) { - WARN_ON_ONCE(!si->nr_frags); - memmove(&si->frags[1], &si->frags[i], - si->nr_frags * sizeof(skb_frag_t)); - } - ++si->nr_frags; - - goto done; - } - - head_len = skb_headlen(skb); - - /* - * Extend fragments to make room for additional fragments (maximum 2) - * to hold new data page and skb linear part (if it still exists after - * @offset removal). - */ - if (__extend_pgfrags(*skb_head, skb, 0, 1 + !!head_len)) - return -ENOMEM; - - if (head_len) { - struct page *head_pg = virt_to_head_page(skb->head); - char *head_ptr = (char *)page_address(head_pg); - unsigned int head_off = (char *)skb->data - head_ptr; - - __skb_fill_page_desc(skb, 1, head_pg, head_off, head_len); - get_page(head_pg); - - skb->tail -= head_len; - skb->data_len += head_len; - - /* - * Prevent @skb->tail from moving forward, since we have linked - * all the data from the linear part (which is after @skb->tail - * now) with the second fragment. - */ - skb->tail_lock = 1; - } - -done: - /* Set up new fragment with data page into the zero slot. */ - __skb_fill_page_desc(skb, 0, pg, 0, pg_len); - ss_skb_adjust_data_len(skb, pg_len); - - return 0; -} - /** * Evict extra data between @curr and @stop pointers, beginning from the @skb * and the @frag_num fragment. diff --git a/tempesta_fw/ss_skb.h b/tempesta_fw/ss_skb.h index 15b8c1e8ff..8e1679c450 100644 --- a/tempesta_fw/ss_skb.h +++ b/tempesta_fw/ss_skb.h @@ -208,8 +208,6 @@ void ss_skb_init_for_xmit(struct sk_buff *skb); void ss_skb_dump(struct sk_buff *skb); int ss_skb_to_sgvec_with_new_pages(struct sk_buff *skb, struct scatterlist *sgl, struct page ***old_pages); -int ss_skb_replace_page(struct sk_buff **skb_head, struct page *pg, - unsigned long pg_len, unsigned long offset); int ss_skb_cut_extra_data(struct sk_buff *skb_head, struct sk_buff *skb, int frag_num, char *curr, const char *stop); From 1b15a89274716fbdc007460f7690c9ea39f8ddbe Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 31 Dec 2019 09:33:40 +0500 Subject: [PATCH 36/64] Update saver pointer ht after tfw_http_msg_grow_hdr_tbl() calls When tfw_http_msg_grow_hdr_tbl() is called it relocates the hm->ht, but all code around the call still uses the previous pointer. Thus changes are written to previous table, not current. --- tempesta_fw/http.c | 15 ++++++++++++--- tempesta_fw/http_msg.c | 8 ++++++-- tempesta_fw/t/unit/test_http_match.c | 9 +++------ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 980e621d12..6e8a8c95a0 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -2866,9 +2866,11 @@ __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) * to delete. */ return 0; - if (unlikely(hid == ht->size)) + if (unlikely(hid == ht->size)) { if (tfw_http_msg_grow_hdr_tbl(hm)) return -ENOMEM; + ht = hm->h_tbl; + } if (hid == ht->off) { /* * The raw header not found, but we have the new @@ -2971,7 +2973,7 @@ tfw_h2_adjust_req(TfwHttpReq *req) TfwHttpHdrTbl *ht = req->h_tbl; bool auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); bool host = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_HOST]); - size_t pseudo_num = auth ? 4 : 3; + size_t pseudo_num; TfwStr meth = {}, host_val = {}, *field, *end; struct sk_buff *new_head = NULL, *old_head = NULL; TfwMsgIter it; @@ -3038,7 +3040,14 @@ tfw_h2_adjust_req(TfwHttpReq *req) */ if ((r = tfw_h2_req_set_loc_hdrs(req))) return r; - + /* + * tfw_h2_req_set_loc_hdrs() may realloc header table and user may + * defined headers modifications, even headers we rely on, recheck them. + */ + ht = req->h_tbl; + auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); + host = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_HOST]); + pseudo_num = auth ? 4 : 3; /* * Calculate http1.1 headers size. H2 request contains pseudo headers * that are represented in different way in the http1.1 requests. diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 748ecf7cef..44f8a4431b 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -862,9 +862,11 @@ tfw_http_msg_hdr_xfrm_str(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid, if (hid == ht->off && !s_val) /* Not found, nothing to delete. */ return 0; - if (hid == ht->size) + if (hid == ht->size) { if (tfw_http_msg_grow_hdr_tbl(hm)) return -ENOMEM; + ht = hm->h_tbl; + } if (hid == ht->off) ++ht->off; else @@ -1097,9 +1099,11 @@ tfw_http_msg_hdr_add(TfwHttpMsg *hm, const TfwStr *hdr) ht = hm->h_tbl; hid = ht->off; - if (hid == ht->size) + if (hid == ht->size) { if (tfw_http_msg_grow_hdr_tbl(hm)) return -ENOMEM; + ht = hm->h_tbl; + } ++ht->off; return __hdr_add(hm, hdr, hid); diff --git a/tempesta_fw/t/unit/test_http_match.c b/tempesta_fw/t/unit/test_http_match.c index d038207431..cf5bd68631 100644 --- a/tempesta_fw/t/unit/test_http_match.c +++ b/tempesta_fw/t/unit/test_http_match.c @@ -165,16 +165,13 @@ set_tfw_str(TfwStr *str, const char *cstr) static void set_raw_hdr(const char *cstr) { - unsigned int hid; - TfwHttpHdrTbl *h_tbl = test_req->h_tbl; + unsigned int hid = test_req->h_tbl->off; - hid = h_tbl->off; - - if (hid == h_tbl->size && + if (hid == test_req->h_tbl->size && tfw_http_msg_grow_hdr_tbl((TfwHttpMsg *)test_req)) return; - ++h_tbl->off; + ++test_req->h_tbl->off; set_tfw_str(&test_req->h_tbl->tbl[hid], cstr); } From 9a4c20cd61f32a1d11fdd2a851ae0bd5680bfab3 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 31 Dec 2019 09:49:01 +0500 Subject: [PATCH 37/64] Fix setting local headers --- tempesta_fw/http.c | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 6e8a8c95a0..b0c293cd4e 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -2685,15 +2685,17 @@ tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req) {}, { .data = S_DLM, .len = SLEN(S_DLM) }, {}, + { .data = S_CRLF, .len = SLEN(S_CRLF) }, }, - .len = SLEN(S_DLM), + .len = SLEN(S_DLM) + SLEN(S_CRLF), + .nchunks = 2 }; int r; h_mdf.chunks[0] = d->hdr->chunks[0]; h_mdf.chunks[2] = d->hdr->chunks[1]; h_mdf.len += d->hdr->len; - h_mdf.nchunks = d->hdr->nchunks + 1; + h_mdf.nchunks += d->hdr->nchunks; r = tfw_http_msg_hdr_xfrm_str(hm, &h_mdf, d->hid, d->append); if (r) { T_ERR("can't update location-specific header in msg %p\n", @@ -3168,7 +3170,7 @@ tfw_h2_adjust_req(TfwHttpReq *req) if (unlikely(TFW_STR_PLAIN(dup))) { r = -EINVAL; - break; + goto err; } hval.chunks = dup->chunks; @@ -3689,29 +3691,12 @@ tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods) for (i = 0; i < h_mods->sz; ++i) { const TfwHdrModsDesc *desc = &h_mods->hdrs[i]; - /* - * Header is stored optimized for HTTP2 request processing: - * without delimiter between header and value. - */ - TfwStr h_mdf = { - .chunks = (TfwStr []){ - {}, - { .data = S_DLM, .len = SLEN(S_DLM) }, - {}, - }, - .len = SLEN(S_DLM), - }; int r; - h_mdf.chunks[0] = desc->hdr->chunks[0]; - h_mdf.chunks[2] = desc->hdr->chunks[1]; - h_mdf.len += desc->hdr->len; - h_mdf.nchunks = desc->hdr->nchunks + 1; - - if (test_bit(i, mit->found) || !TFW_STR_CHUNK(&h_mdf, 2)) + if (test_bit(i, mit->found) || !TFW_STR_CHUNK(desc->hdr, 1)) continue; - r = __hdr_h2_add(resp, &h_mdf); + r = __hdr_h2_add(resp, desc->hdr); if (unlikely(r)) return r; } From b4e2719509d7afd010fba99ae7556781d1c0b056 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Fri, 3 Jan 2020 13:39:46 +0500 Subject: [PATCH 38/64] fix header calculations --- tempesta_fw/http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index b0c293cd4e..74882a0f15 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3049,7 +3049,7 @@ tfw_h2_adjust_req(TfwHttpReq *req) ht = req->h_tbl; auth = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]); host = !TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_HOST]); - pseudo_num = auth ? 4 : 3; + pseudo_num = 3; /* Count authority as usual header for now. */ /* * Calculate http1.1 headers size. H2 request contains pseudo headers * that are represented in different way in the http1.1 requests. From ebe2b83b4626a6379e8945297341c464097cbc0b Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sun, 5 Jan 2020 17:55:46 +0500 Subject: [PATCH 39/64] Fix h2 fsm transitions - Don't switch fsm state on processing DATA frame header, update the fsm state only after the full DATA frame is processed. Otherwise the fsm is triggered twice and fsm closes the connection when a new portion of DATA frame is expected. - The HTTP2_STREAM_REM_HALF_CLOSED flag is set when the h2 frames are processed, this flag may be evalueted by http parser only when all the data from current h2 frame is parsed --- tempesta_fw/http_frame.c | 2 -- tempesta_fw/http_parser.c | 38 ++++++++++++++++++++++++-------------- tempesta_fw/http_parser.h | 1 + tempesta_fw/http_stream.c | 20 +++++++++++++++----- 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index 01643412fd..0d1d714ec4 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -1207,8 +1207,6 @@ tfw_h2_frame_type_process(TfwH2Ctx *ctx) if (tfw_h2_flow_control(ctx)) return T_DROP; - STREAM_RECV_PROCESS(ctx, hdr); - ctx->data_off = FRAME_HEADER_SIZE; ctx->plen = ctx->hdr.length; diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 94dd382a37..17b5943471 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -7985,8 +7985,6 @@ tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, else r = tfw_h2_parse_body(data, len, req, parsed); - if (r != T_OK && r != T_POSTPONE) - return r; /* * Parsing of HTTP/2 frames' payload never gives T_OK result since * request can be assembled from different number of frames; only @@ -7997,23 +7995,35 @@ tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, * procedures yet, it must be deleted during corresponding stream * removal. */ - if (tfw_h2_stream_req_complete(req->stream)) { - TfwHttpHdrTbl *ht = req->h_tbl; + return (r == T_OK) ? T_POSTPONE : r; +} - if (req->content_length - && req->content_length != req->body.len) - return T_DROP; +/** + * Finish parsing h2 request. The request may consist of multiple skbs and + * multiple h2 frames. Last frame is marked with End Stream flag, it's the + * only way to indicate message end. + */ +int +tfw_h2_parse_req_finish(TfwHttpReq *req) +{ + TfwHttpHdrTbl *ht = req->h_tbl; - __set_bit(TFW_HTTP_B_FULLY_PARSED, req->flags); + if (WARN_ON_ONCE(!tfw_h2_stream_req_complete(req->stream))) + return T_DROP; - __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY], - &req->host); - __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_PATH], - &req->uri_path); - return T_OK; + if (req->content_length + && req->content_length != req->body.len) + { + return T_DROP; } + __set_bit(TFW_HTTP_B_FULLY_PARSED, req->flags); + + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY], + &req->host); + __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_PATH], + &req->uri_path); - return T_POSTPONE; + return T_OK; } /* diff --git a/tempesta_fw/http_parser.h b/tempesta_fw/http_parser.h index 9ff392331a..c15e298a04 100644 --- a/tempesta_fw/http_parser.h +++ b/tempesta_fw/http_parser.h @@ -132,6 +132,7 @@ int tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req bool fin, bool value_stage); int tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, unsigned int *parsed); +int tfw_h2_parse_req_finish(TfwHttpReq *req); int tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, unsigned int *parsed); int tfw_http_parse_terminate(TfwHttpMsg *hm); diff --git a/tempesta_fw/http_stream.c b/tempesta_fw/http_stream.c index 54456fe3e9..6a0f8357c6 100644 --- a/tempesta_fw/http_stream.c +++ b/tempesta_fw/http_stream.c @@ -105,18 +105,28 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, * (remote)' state. */ if (type == HTTP2_HEADERS) { - if (flags & (HTTP2_F_END_STREAM | HTTP2_F_END_HEADERS)) { + switch (flags + & (HTTP2_F_END_HEADERS | HTTP2_F_END_STREAM)) + { + case HTTP2_F_END_HEADERS | HTTP2_F_END_STREAM: stream->state = HTTP2_STREAM_REM_HALF_CLOSED; - } + break; + case HTTP2_F_END_HEADERS: + /* + * Headers is ended, next frame in the stream + * should be DATA frame. + */ + break; /* * If END_HEADERS flag is not received, move stream into * the states of waiting CONTINUATION frame. */ - else if (flags & HTTP2_F_END_STREAM) { + case HTTP2_F_END_STREAM: stream->state = HTTP2_STREAM_CONT_CLOSED; - } - else if (!(flags & HTTP2_F_END_HEADERS)) { + break; + default: stream->state = HTTP2_STREAM_CONT; + break; } } else if (type == HTTP2_DATA && (flags & HTTP2_F_END_STREAM)) { From 06d75c25a278e32c0f0d5030820b03e8acee7160 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Thu, 9 Jan 2020 19:01:45 +0500 Subject: [PATCH 40/64] Don't close h2 request body until it's fully received --- tempesta_fw/http_msg.c | 3 ++- tempesta_fw/http_parser.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 44f8a4431b..2fd4b548ac 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -621,7 +621,8 @@ int __tfw_http_msg_add_str_data(TfwHttpMsg *hm, TfwStr *str, void *data, size_t len, struct sk_buff *skb) { - BUG_ON(str->flags & (TFW_STR_DUPLICATE | TFW_STR_COMPLETE)); + if (WARN_ON_ONCE(str->flags & (TFW_STR_DUPLICATE | TFW_STR_COMPLETE))) + return -EINVAL; T_DBG3("store field chunk len=%lu data=%p(%c) field=<%#x,%lu,%p>\n", len, data, isprint(*(char *)data) ? *(char *)data : '.', diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 17b5943471..20a7812936 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -7956,7 +7956,6 @@ tfw_h2_parse_body(char *data, unsigned long len, TfwHttpReq *req, return T_DROP; parser->to_read = -1; - req->body.flags |= TFW_STR_COMPLETE; ret = T_OK; out: @@ -8016,6 +8015,7 @@ tfw_h2_parse_req_finish(TfwHttpReq *req) { return T_DROP; } + req->body.flags |= TFW_STR_COMPLETE; __set_bit(TFW_HTTP_B_FULLY_PARSED, req->flags); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY], From bee32462b71271281d4d1e48782c33951a87b4a2 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Thu, 9 Jan 2020 19:03:20 +0500 Subject: [PATCH 41/64] Fix moving request body on h2->h1 transformation Body it skb list by itself, so a new function is required. --- tempesta_fw/http.c | 6 +++--- tempesta_fw/ss_skb.h | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 74882a0f15..a833cf314d 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3234,11 +3234,11 @@ tfw_h2_adjust_req(TfwHttpReq *req) len += trailer->len; trailer = trailer->next; - } while ((trailer != b_skbs) || (len != req->body.len)); - ss_skb_queue_tail(&req->msg.skb_head, b_skbs); + } while ((trailer != b_skbs) && (len != req->body.len)); + ss_skb_queue_append(&req->msg.skb_head, b_skbs); if (trailer != b_skbs) { ss_skb_queue_split(req->msg.skb_head, trailer); - ss_skb_queue_tail(&old_head, trailer); + ss_skb_queue_append(&old_head, trailer); } } ss_skb_queue_purge(&old_head); diff --git a/tempesta_fw/ss_skb.h b/tempesta_fw/ss_skb.h index 8e1679c450..2d56370646 100644 --- a/tempesta_fw/ss_skb.h +++ b/tempesta_fw/ss_skb.h @@ -50,7 +50,7 @@ typedef int ss_skb_actor_t(void *conn, unsigned char *data, size_t len, unsigned int *read); /** - * Add new @skb to the queue in FIFO order. + * Add new _single_ @skb to the queue in FIFO order. */ static inline void ss_skb_queue_tail(struct sk_buff **skb_head, struct sk_buff *skb) @@ -67,6 +67,26 @@ ss_skb_queue_tail(struct sk_buff **skb_head, struct sk_buff *skb) skb->next->prev = skb->prev->next = skb; } +/** + * Append list of @skb to the queue in FIFO order. + */ +static inline void +ss_skb_queue_append(struct sk_buff **skb_head, struct sk_buff *skb) +{ + struct sk_buff *tail; + + if (WARN_ON_ONCE(!*skb_head)) { + *skb_head = skb; + return; + } + + skb->prev->next = *skb_head; + (*skb_head)->prev = skb->prev; + tail = (*skb_head)->prev; + skb->prev = tail; + tail->next = skb; +} + static inline void ss_skb_unlink(struct sk_buff **skb_head, struct sk_buff *skb) { From a31df74f6c9f3095e8c35347683ee73a77d5197f Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Thu, 9 Jan 2020 19:03:43 +0500 Subject: [PATCH 42/64] Fix crash on debug builds --- tempesta_fw/http.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index a833cf314d..25568e7753 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3742,7 +3742,8 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) return -EINVAL; T_DBG3("%s: hid=%hu, d_num=%hu, nchunks=%u, h_mods->sz=%lu\n", - __func__, hid, d_num, ht->tbl[hid].nchunks, h_mods->sz); + __func__, hid, d_num, ht->tbl[hid].nchunks, + h_mods ? h_mods->sz : 0); if (!h_mods) goto def; From 04c0fc9444c57b07da0fa312f4dff008b15e414e Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Thu, 9 Jan 2020 19:05:50 +0500 Subject: [PATCH 43/64] Add todo: split message into multiple frames h2 recipient will reject frame if it's too big for it. Honor remote peer settings when send messages, or fail fast otherwise, since the connection will be closed anyway. --- tempesta_fw/http.c | 90 +++++++++++++++++++++++++++------------- tempesta_fw/http_frame.c | 2 +- tempesta_fw/http_frame.h | 1 + 3 files changed, 64 insertions(+), 29 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 25568e7753..f4235b8c2b 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -849,6 +849,15 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) mit = &resp->mit; skb_head = &resp->msg.skb_head; body = TFW_STR_BODY_CH(msg); + if (WARN_ON_ONCE(body->len > ctx->rsettings.max_frame_sz)) { + /* + * TODO: split body to frames it it's too long. + * Since full-featured response is not constructed construct + * here, it's possible to avoid framing body for each h2 client + * individually. Every client must support 16384-byte frames. + */ + goto err_setup; + } /* Create frame header for HEADERS. Note, that we leave the length of * HEADERS frame unset, to be filled later (below) after all headers @@ -919,6 +928,17 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) * head part of skb (also, see description of frame header format in * RFC 7540 section 4.1). */ + if (WARN_ON_ONCE(mit->acc_len > ctx->rsettings.max_frame_sz)) { + /* + * TODO: multiple frames might be required here. + * + * Too long frames will be rejected by remote peer and + * the connection will be closed by remote side. There is no + * point to send such response, let's fail fast and close the + * connection on our own. + */ + goto err_setup; + } len_be = htonl((unsigned int)mit->acc_len); src_len_p = (unsigned char *)&len_be; dst_len_p = (*skb_head)->data; @@ -3868,18 +3888,29 @@ tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, unsigned char buf[FRAME_HEADER_SIZE]; TfwHttpTransIter *mit = &resp->mit; unsigned long b_len = resp->body.len; + TfwH2Ctx *ctx = tfw_h2_context(resp->req->conn); frame_hdr.stream_id = stream_id; + /* + * TODO: create multiple frames. Remote peer will reject the frame if + * it's bigger than was allowed by remote. + */ + if (h_len > ctx->rsettings.max_frame_sz) { + /* TODO: split headers by frames. */ + T_WARN("Unable to make HTTP/2 HEADERS frame: too big header" + " block fragment (%lu)\n", h_len); + return -E2BIG; + } + if (b_len > ctx->rsettings.max_frame_sz) { + T_WARN("Unable to make HTTP/2 DATA frame: too big" + " message body (%lu)\n", b_len); + return -E2BIG; + } + if (b_len) { TfwStr s_hdr = {}; - if (b_len > FRAME_MAX_LENGTH) { - T_WARN("Unable to make HTTP/2 DATA frame: too big" - " message body (%lu)\n", b_len); - return -E2BIG; - } - /* * Set frame header for DATA, if body part of HTTP/1.1 * response exists. @@ -3898,12 +3929,6 @@ tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, return r; } - if (h_len > FRAME_MAX_LENGTH) { - T_WARN("Unable to make HTTP/2 HEADERS frame: too big header" - " block fragment (%lu)\n", h_len); - return -E2BIG; - } - /* Set frame header for HEADERS. */ frame_hdr.length = h_len; frame_hdr.type = HTTP2_HEADERS; @@ -4724,23 +4749,32 @@ tfw_http_req_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data) "Request parsing inconsistency"); return TFW_BLOCK; } - r = tfw_gfsm_move(&conn->state, TFW_HTTP_FSM_REQ_CHUNK, - &data_up); - T_DBG3("TFW_HTTP_FSM_REQ_CHUNK return code %d\n", r); - if (r == TFW_BLOCK) { - TFW_INC_STAT_BH(clnt.msgs_filtout); - tfw_http_req_parse_block(req, 403, - "postponed request has been" - " filtered out"); - return TFW_BLOCK; + if (TFW_MSG_H2(req) && tfw_h2_stream_req_complete(req->stream)) { + if (tfw_h2_parse_req_finish(req)) { + TFW_INC_STAT_BH(clnt.msgs_otherr); + tfw_http_req_parse_block(req, 500, + "Request parsing inconsistency"); + return TFW_BLOCK; + } + } + else { + r = tfw_gfsm_move(&conn->state, TFW_HTTP_FSM_REQ_CHUNK, + &data_up); + T_DBG3("TFW_HTTP_FSM_REQ_CHUNK return code %d\n", r); + if (r == TFW_BLOCK) { + TFW_INC_STAT_BH(clnt.msgs_filtout); + tfw_http_req_parse_block(req, 403, + "postponed request has been filtered out"); + return TFW_BLOCK; + } + /* + * TFW_POSTPONE status means that parsing succeeded + * but more data is needed to complete it. Lower layers + * just supply data for parsing. They only want to know + * if processing of a message should continue or not. + */ + return TFW_PASS; } - /* - * TFW_POSTPONE status means that parsing succeeded - * but more data is needed to complete it. Lower layers - * just supply data for parsing. They only want to know - * if processing of a message should continue or not. - */ - return TFW_PASS; case TFW_PASS: /* * The request is fully parsed, diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index 0d1d714ec4..b1868064a7 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -214,7 +214,7 @@ tfw_h2_context_init(TfwH2Ctx *ctx) lset->hdr_tbl_sz = rset->hdr_tbl_sz = HPACK_TABLE_DEF_SIZE; lset->push = rset->push = 1; lset->max_streams = rset->max_streams = 0xffffffff; - lset->max_frame_sz = rset->max_frame_sz = 1 << 14; + lset->max_frame_sz = rset->max_frame_sz = FRAME_DEF_LENGTH; lset->max_lhdr_sz = rset->max_lhdr_sz = UINT_MAX; /* * We ignore client's window size until #498, so currently diff --git a/tempesta_fw/http_frame.h b/tempesta_fw/http_frame.h index 029ed36dbb..6a96dabb52 100644 --- a/tempesta_fw/http_frame.h +++ b/tempesta_fw/http_frame.h @@ -28,6 +28,7 @@ #define FRAME_STREAM_ID_MASK ((1U << 31) - 1) #define FRAME_RESERVED_BIT_MASK (~FRAME_STREAM_ID_MASK) #define FRAME_MAX_LENGTH ((1U << 24) - 1) +#define FRAME_DEF_LENGTH (16384) /** * HTTP/2 frame types (RFC 7540 section 6). From f9c16449699bc8e1ad5c81ba4fff7a5ea29c9f64 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Thu, 9 Jan 2020 19:10:37 +0500 Subject: [PATCH 44/64] Add todo: h2 protocol requires lower case headers, not mixed case --- tempesta_fw/http.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index f4235b8c2b..60ab0843ac 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -4219,7 +4219,13 @@ tfw_http_req_block(TfwHttpReq *req, int status, const char *msg) tfw_http_cli_error_resp_and_log(req, status, msg, true, false); } - +/* + * TODO: http1 headers usually use mixed case, while h2 uses only lower case + * header names. It seems that usual browsers are OK with mixed case in h2 + * responses, but this behaviour is not defined by RFC and implementation + * dependent. Curl is not happy with the mixed case and return parser errors. + * The headers should be converted to lower case before transmission. + */ static void tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) { From 2ad4c3f032bd733032cdc47d80fbb6a0ec902407 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Fri, 10 Jan 2020 12:30:08 +0500 Subject: [PATCH 45/64] h2: content-length is extra header, don't block messages if it's absent --- tempesta_fw/http.c | 8 ++++++-- tempesta_fw/http_parser.c | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 60ab0843ac..f090bc2d2c 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -4786,8 +4786,12 @@ tfw_http_req_process(TfwConn *conn, TfwStream *stream, const TfwFsmData *data) * The request is fully parsed, * fall through and process it. */ - BUG_ON(!test_bit(TFW_HTTP_B_CHUNKED, req->flags) - && (req->content_length != req->body.len)); + + if (WARN_ON_ONCE(!test_bit(TFW_HTTP_B_CHUNKED, req->flags) + && (req->content_length != req->body.len))) + { + return TFW_BLOCK; + } } /* diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 20a7812936..7372881599 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -8015,6 +8015,15 @@ tfw_h2_parse_req_finish(TfwHttpReq *req) { return T_DROP; } + /* + * RFC 7540 8.1.2.6: A request or response that includes a payload + * body can include a content-length header field. + * + * Since the h2 provides explicit message framing, the content-length + * header is not required at all. But our code may rely on + * req->content_length field value, fill it. + */ + req->content_length = req->body.len; req->body.flags |= TFW_STR_COMPLETE; __set_bit(TFW_HTTP_B_FULLY_PARSED, req->flags); From 2b69ff42e5460560fb5a532430eea3735098520a Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sat, 11 Jan 2020 14:06:03 +0500 Subject: [PATCH 46/64] h2: close connection if remote sends too huge frames --- tempesta_fw/http_frame.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index b1868064a7..b9d78cc795 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -1177,6 +1177,9 @@ tfw_h2_frame_type_process(TfwH2Ctx *ctx) T_DBG3("%s: hdr->type=%hhu, ctx->state=%d\n", __func__, hdr->type, ctx->state); + if (unlikely(ctx->hdr.length > ctx->lsettings.max_frame_sz)) + goto conn_term; + switch (hdr->type) { case HTTP2_DATA: if (!hdr->stream_id) { From e6b400db2103214808546603d7e2c2703dc5b605 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sat, 11 Jan 2020 14:22:36 +0500 Subject: [PATCH 47/64] Fix null pointer dereference on modifications on h1 headers and incorrenct additions of CRLF --- tempesta_fw/http.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index f090bc2d2c..8a9cab311a 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -2704,18 +2704,21 @@ tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req) .chunks = (TfwStr []){ {}, { .data = S_DLM, .len = SLEN(S_DLM) }, - {}, - { .data = S_CRLF, .len = SLEN(S_CRLF) }, + {} }, - .len = SLEN(S_DLM) + SLEN(S_CRLF), - .nchunks = 2 + .len = SLEN(S_DLM), + .nchunks = 2 /* header name + delimeter. */ }; int r; h_mdf.chunks[0] = d->hdr->chunks[0]; - h_mdf.chunks[2] = d->hdr->chunks[1]; + if (d->hdr->nchunks == 2) { + h_mdf.chunks[2] = d->hdr->chunks[1]; + h_mdf.nchunks += 1; + } h_mdf.len += d->hdr->len; - h_mdf.nchunks += d->hdr->nchunks; + h_mdf.flags = d->hdr->flags; + h_mdf.eolen += d->hdr->eolen; r = tfw_http_msg_hdr_xfrm_str(hm, &h_mdf, d->hid, d->append); if (r) { T_ERR("can't update location-specific header in msg %p\n", From e257d63ee15bd86dd75bf02a614900d956305a61 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sat, 11 Jan 2020 17:14:53 +0500 Subject: [PATCH 48/64] h2: check settings before apply them. Reject if limits defined in RFC is broken --- tempesta_fw/http_frame.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index b9d78cc795..4dea4b2529 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -915,6 +915,8 @@ tfw_h2_apply_settings_entry(TfwH2Ctx *ctx, unsigned short id, break; case HTTP2_SETTINGS_ENABLE_PUSH: + if (val > 1) + return T_BAD; dest->push = val; break; @@ -923,10 +925,14 @@ tfw_h2_apply_settings_entry(TfwH2Ctx *ctx, unsigned short id, break; case HTTP2_SETTINGS_INIT_WND_SIZE: + if (val > MAX_WND_SIZE) + return T_BAD; dest->wnd_sz = val; break; case HTTP2_SETTINGS_MAX_FRAME_SIZE: + if (val < FRAME_DEF_LENGTH || val > FRAME_MAX_LENGTH) + return T_BAD; dest->max_frame_sz = val; break; From ee0dced70f2ec206f1de7aad162c3af8d1517b6a Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sat, 11 Jan 2020 17:33:44 +0500 Subject: [PATCH 49/64] Add issue number for TODOs --- tempesta_fw/http.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 8a9cab311a..d26ab93023 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -851,7 +851,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) body = TFW_STR_BODY_CH(msg); if (WARN_ON_ONCE(body->len > ctx->rsettings.max_frame_sz)) { /* - * TODO: split body to frames it it's too long. + * TODO #1378: split body to frames it it's too long. * Since full-featured response is not constructed construct * here, it's possible to avoid framing body for each h2 client * individually. Every client must support 16384-byte frames. @@ -930,7 +930,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) */ if (WARN_ON_ONCE(mit->acc_len > ctx->rsettings.max_frame_sz)) { /* - * TODO: multiple frames might be required here. + * TODO #1378: multiple frames might be required here. * * Too long frames will be rejected by remote peer and * the connection will be closed by remote side. There is no @@ -3896,8 +3896,8 @@ tfw_h2_make_frames(TfwHttpResp *resp, unsigned int stream_id, frame_hdr.stream_id = stream_id; /* - * TODO: create multiple frames. Remote peer will reject the frame if - * it's bigger than was allowed by remote. + * TODO #1378: create multiple frames. Remote peer will reject the frame + * if it's bigger than was allowed by remote. */ if (h_len > ctx->rsettings.max_frame_sz) { /* TODO: split headers by frames. */ From 6849d753719e87a5c90e6a29fd123465351231d7 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sat, 11 Jan 2020 19:14:25 +0500 Subject: [PATCH 50/64] Update macro documentations --- tempesta_fw/http_frame.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tempesta_fw/http_frame.h b/tempesta_fw/http_frame.h index 6a96dabb52..b44b1ecb65 100644 --- a/tempesta_fw/http_frame.h +++ b/tempesta_fw/http_frame.h @@ -24,6 +24,7 @@ #include "http_stream.h" #include "hpack.h" +/* RFC 7540 Section 4.1 frame header constants. */ #define FRAME_HEADER_SIZE 9 #define FRAME_STREAM_ID_MASK ((1U << 31) - 1) #define FRAME_RESERVED_BIT_MASK (~FRAME_STREAM_ID_MASK) From 3cb5ec09bc8edb36113750a4e402316bda29089a Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sat, 11 Jan 2020 19:30:22 +0500 Subject: [PATCH 51/64] h2: Add several TODO comments --- tempesta_fw/http.c | 9 ++++++++- tempesta_fw/http_frame.c | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index d26ab93023..d23c8d480c 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3054,6 +3054,12 @@ tfw_h2_adjust_req(TfwHttpReq *req) return -EINVAL; } + /* + * TODO: remove from header table all hop-by-hop headers, including + * 'Connection:' header (hdrs with flag TFW_STR_HBH_HDR). If do this + * before tfw_h2_req_set_loc_hdrs(), we may free some space in the table + * and avoid the table reallocation. + */ /* * First apply message modifications defined by admin in configuration * file. Ideally we should do it at last stage, when h2 headers are @@ -3734,6 +3740,8 @@ tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods) * each iteration function produces the boundary pointer @mit->bnd for current * iteration and the operation instance @mit->next - for the next iteration * (including source header @mit->next.s_hdr). + * + * TODO #1103: This function should be treated as a foundation for #1103 issue. */ static int tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) @@ -3878,7 +3886,6 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) mit->curr = i + 1; return 0; - } static int diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index 4dea4b2529..306ca5eebe 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -1186,6 +1186,14 @@ tfw_h2_frame_type_process(TfwH2Ctx *ctx) if (unlikely(ctx->hdr.length > ctx->lsettings.max_frame_sz)) goto conn_term; + /* + * TODO: RFC 7540 Section 6.2: + * A HEADERS frame without the END_HEADERS flag set MUST be followed + * by a CONTINUATION frame for the same stream. A receiver MUST treat + * the receipt of any other type of frame or a frame on a different + * stream as a connection error (Section 5.4.1) of type PROTOCOL_ERROR. + */ + switch (hdr->type) { case HTTP2_DATA: if (!hdr->stream_id) { From f91139feb7d4f1b8aea8c167c19af0948f7fe3c9 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sat, 11 Jan 2020 22:21:24 +0500 Subject: [PATCH 52/64] h2: fix too early fsm update for CONTINUATION frames Don't switch fsm state on processing frame header, update the fsm state only after the full frame is processed. Otherwise the fsm is triggered twice and fsm closes the connection when a new fragment of frame is expected. --- tempesta_fw/http_frame.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index 306ca5eebe..32dcbb2f67 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -1432,8 +1432,6 @@ tfw_h2_frame_type_process(TfwH2Ctx *ctx) goto conn_term; } - STREAM_RECV_PROCESS(ctx, hdr); - ctx->data_off = FRAME_HEADER_SIZE; ctx->plen = ctx->hdr.length; From e7bd235b735ed28410b538c58491f9dd8d8af801 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sat, 11 Jan 2020 23:17:28 +0500 Subject: [PATCH 53/64] h2: Improve check for mandatory pseudo headers presense --- tempesta_fw/http_parser.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 7372881599..6cae9c32f1 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -8010,6 +8010,17 @@ tfw_h2_parse_req_finish(TfwHttpReq *req) if (WARN_ON_ONCE(!tfw_h2_stream_req_complete(req->stream))) return T_DROP; + /* + * TFW_HTTP_B_H2_HDRS_FULL flag is set on first TFW_HTTP_HDR_REGULAR + * header, if no present, need to check mandatory pseudo headers. + */ + if (unlikely(!test_bit(TFW_HTTP_B_H2_HDRS_FULL, req->flags) + && (TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_METHOD]) + || TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_SCHEME]) + || TFW_STR_EMPTY(&ht->tbl[TFW_HTTP_HDR_H2_PATH])))) + { + return T_DROP; + } if (req->content_length && req->content_length != req->body.len) { From c114bede4a56a07b1989e7a5d4be390a12ddf754 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 21 Jan 2020 17:32:46 +0500 Subject: [PATCH 54/64] Fix code review comments --- tempesta_fw/http.c | 24 ++++++++++-------------- tempesta_fw/http_parser.c | 19 +++++++++---------- tempesta_fw/ss_skb.h | 2 +- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index d23c8d480c..7222054bbe 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -851,9 +851,9 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) body = TFW_STR_BODY_CH(msg); if (WARN_ON_ONCE(body->len > ctx->rsettings.max_frame_sz)) { /* - * TODO #1378: split body to frames it it's too long. - * Since full-featured response is not constructed construct - * here, it's possible to avoid framing body for each h2 client + * TODO #1378: split body to frames if it's too long. + * Since full-featured response is not constructed here, it's + * possible to avoid framing body for each h2 client * individually. Every client must support 16384-byte frames. */ goto err_setup; @@ -3054,12 +3054,6 @@ tfw_h2_adjust_req(TfwHttpReq *req) return -EINVAL; } - /* - * TODO: remove from header table all hop-by-hop headers, including - * 'Connection:' header (hdrs with flag TFW_STR_HBH_HDR). If do this - * before tfw_h2_req_set_loc_hdrs(), we may free some space in the table - * and avoid the table reallocation. - */ /* * First apply message modifications defined by admin in configuration * file. Ideally we should do it at last stage, when h2 headers are @@ -4230,11 +4224,13 @@ tfw_http_req_block(TfwHttpReq *req, int status, const char *msg) } /* - * TODO: http1 headers usually use mixed case, while h2 uses only lower case - * header names. It seems that usual browsers are OK with mixed case in h2 - * responses, but this behaviour is not defined by RFC and implementation - * dependent. Curl is not happy with the mixed case and return parser errors. - * The headers should be converted to lower case before transmission. + * TODO: RFC 7540 8.1.2 + * However, header field names MUST be converted to lowercase prior to + * their encoding in HTTP/2. A request or response containing uppercase + * header field names MUST be treated as malformed (Section 8.1.2.6). + * + * Major browsers and curl ignore that RFC requirement an work well. But + * that is definitely an RFC violation and implementation specific behaviour. */ static void tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 6cae9c32f1..e0a3e8e3d1 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -7963,6 +7963,15 @@ tfw_h2_parse_body(char *data, unsigned long len, TfwHttpReq *req, return ret; } +/** + * Parse h2 request. + * + * Parsing of HTTP/2 frames' payload never gives T_OK result since request + * can be assembled from different number of frames; only stream's state can + * indicate the moment when request is completed. After all parts of request are + * fully received and parsed, call the @tfw_h2_parse_req_finish() to check the + * parser state. + */ int tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, unsigned int *parsed) @@ -7984,16 +7993,6 @@ tfw_h2_parse_req(void *req_data, unsigned char *data, size_t len, else r = tfw_h2_parse_body(data, len, req, parsed); - /* - * Parsing of HTTP/2 frames' payload never gives T_OK result since - * request can be assembled from different number of frames; only - * stream's state can indicate the moment when request is completed. - * Note, due to persistent linkage Stream<->Request in HTTP/2 mode - * special flag is necessary for tracking the request completeness; if - * the request is not fully assembled and is not passed to forwarding - * procedures yet, it must be deleted during corresponding stream - * removal. - */ return (r == T_OK) ? T_POSTPONE : r; } diff --git a/tempesta_fw/ss_skb.h b/tempesta_fw/ss_skb.h index 2d56370646..0935733698 100644 --- a/tempesta_fw/ss_skb.h +++ b/tempesta_fw/ss_skb.h @@ -80,9 +80,9 @@ ss_skb_queue_append(struct sk_buff **skb_head, struct sk_buff *skb) return; } + tail = (*skb_head)->prev; skb->prev->next = *skb_head; (*skb_head)->prev = skb->prev; - tail = (*skb_head)->prev; skb->prev = tail; tail->next = skb; } From c721aaa577660cf8f18d0fb1be6253a57748831e Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Tue, 18 Feb 2020 17:10:51 +0300 Subject: [PATCH 55/64] HTTP/2 Parser implementation: HTTP/2-cache (#309). --- tempesta_fw/cache.c | 1191 ++++++++++++++++++++++++++----------- tempesta_fw/hpack.c | 500 ++++++++++++++-- tempesta_fw/hpack.h | 57 +- tempesta_fw/http.c | 734 +++++++++++++++-------- tempesta_fw/http.h | 64 +- tempesta_fw/http_frame.c | 30 +- tempesta_fw/http_frame.h | 1 + tempesta_fw/http_msg.c | 33 +- tempesta_fw/http_msg.h | 18 +- tempesta_fw/http_parser.c | 49 +- tempesta_fw/http_sess.c | 39 +- tempesta_fw/http_sess.h | 2 +- tempesta_fw/http_types.h | 2 + tempesta_fw/str.c | 26 + tempesta_fw/str.h | 1 + tempesta_fw/tempesta_fw.h | 1 + tempesta_fw/vhost.c | 2 +- tempesta_fw/vhost.h | 8 +- 18 files changed, 2075 insertions(+), 683 deletions(-) diff --git a/tempesta_fw/cache.c b/tempesta_fw/cache.c index 2183245ee9..3cb38944c1 100644 --- a/tempesta_fw/cache.c +++ b/tempesta_fw/cache.c @@ -34,6 +34,7 @@ #include "vhost.h" #include "cache.h" #include "http_msg.h" +#include "http_sess.h" #include "procfs.h" #include "sync_socket.h" #include "work_queue.h" @@ -67,10 +68,11 @@ static const TfwStr tfw_cache_raw_headers_304[] = { /* * @trec - Database record descriptor; * @key_len - length of key (URI + Host header); - * @status_len - length of response status line; + * @status_len - length of response status code; + * @rph_len - length of response reason phrase; * @hdr_num - number of headers; * @hdr_len - length of whole headers data; - * @hdr_len_304 - length of headers data used to build 304 response; + * @body_len - length of the response body; * @method - request method, part of the key; * @flags - various cache entry flags; * @age - the value of response Age: header field; @@ -80,9 +82,9 @@ static const TfwStr tfw_cache_raw_headers_304[] = { * @lifetime - the cache entry's current lifetime; * @last_modified - the value of response Last-Modified: header field; * @key - the cache entry key (URI + Host header); - * @status - pointer to status line (with trailing CRLFs); - * @hdrs - pointer to list of HTTP headers (with trailing CRLFs); - * @body - pointer to response body (with a prepending CRLF); + * @status - pointer to status line; + * @hdrs - pointer to list of HTTP headers; + * @body - pointer to response body; * @hdrs_304 - pointers to headers used to build 304 response; * @version - HTTP version of the response; * @resp_status - Http status of the cached response. @@ -94,9 +96,10 @@ typedef struct { #define ce_body key_len unsigned int key_len; unsigned int status_len; + unsigned int rph_len; unsigned int hdr_num; unsigned int hdr_len; - unsigned int hdr_len_304; + unsigned int body_len; unsigned int method: 4; unsigned int flags: 28; time_t age; @@ -169,6 +172,8 @@ typedef struct { static CaNode c_nodes[MAX_NUMNODES]; +typedef int tfw_cache_write_actor_t(TDB *, TdbVRec **, TfwHttpResp *, char **, + size_t, TfwDecodeCacheIter *); /* * TODO the thread doesn't do anything for now, however, kthread_stop() crashes * on restarts, so comment to logic out. @@ -178,25 +183,25 @@ static struct task_struct *cache_mgr_thr; #endif static DEFINE_PER_CPU(TfwWorkTasklet, cache_wq); +static DEFINE_PER_CPU(char[RESP_BUF_LEN], g_c_buf); + static TfwStr g_crlf = { .data = S_CRLF, .len = SLEN(S_CRLF) }; /* Iterate over request URI and Host header to process request key. */ -#define TFW_CACHE_REQ_KEYITER(c, req, u_end, h_start, h_end) \ - if (TFW_STR_PLAIN(&req->uri_path)) { \ - c = &req->uri_path; \ - u_end = &req->uri_path + 1; \ +#define TFW_CACHE_REQ_KEYITER(c, uri_path, host, u_end, h_start, h_end) \ + if (TFW_STR_PLAIN(uri_path)) { \ + c = uri_path; \ + u_end = (uri_path) + 1; \ } else { \ - c = req->uri_path.chunks; \ - u_end = req->uri_path.chunks \ - + req->uri_path.nchunks; \ + c = (uri_path)->chunks; \ + u_end = (uri_path)->chunks + (uri_path)->nchunks; \ } \ - if (TFW_STR_PLAIN(&req->h_tbl->tbl[TFW_HTTP_HDR_HOST])) { \ - h_start = req->h_tbl->tbl + TFW_HTTP_HDR_HOST; \ - h_end = req->h_tbl->tbl + TFW_HTTP_HDR_HOST + 1; \ + if (!TFW_STR_EMPTY(host)) { \ + BUG_ON(TFW_STR_PLAIN(host)); \ + h_start = (host)->chunks; \ + h_end = (host)->chunks + (host)->nchunks; \ } else { \ - h_start = req->h_tbl->tbl[TFW_HTTP_HDR_HOST].chunks; \ - h_end = req->h_tbl->tbl[TFW_HTTP_HDR_HOST].chunks \ - + req->h_tbl->tbl[TFW_HTTP_HDR_HOST].nchunks; \ + h_start = h_end = u_end; \ } \ for ( ; c != h_end; ++c, c = (c == u_end) ? h_start : c) @@ -531,29 +536,38 @@ tfw_cache_cond_none_match(TfwHttpReq *req, TfwCacheEntry *ce) } /** - * Add a new data chunk size of @len starting from @data to HTTP response @resp. - * Properly set up @hdr if not NULL. + * Add a new header with size of @len starting from @data to HTTP response @resp, + * expanding the @resp with new skb/frags if needed. */ static int -tfw_cache_write_field(TDB *db, TdbVRec **trec, TfwHttpResp *resp, - TfwMsgIter *it, char **data, size_t len, TfwStr *hdr) +tfw_cache_h2_write(TDB *db, TdbVRec **trec, TfwHttpResp *resp, char **data, + size_t len, TfwDecodeCacheIter *dc_iter) { - int r, copied = 0; - TdbVRec *tr = *trec; TfwStr c = { 0 }; + TdbVRec *tr = *trec; + TfwHttpTransIter *mit = &resp->mit; + TfwMsgIter *it = &mit->iter; + int r = 0, copied = 0; while (1) { c.data = *data; - c.len = min(tr->data + tr->len - *data, - (long)(len - copied)); - r = hdr - ? tfw_http_msg_add_data(it, (TfwHttpMsg *)resp, hdr, &c) - : tfw_msg_write(it, &c); - if (r) - return r; + c.len = min(tr->data + tr->len - *data, (long)(len - copied)); + if (!dc_iter->skip) { + r = tfw_http_msg_expand_data(it, &resp->msg.skb_head, + &c, &mit->start_off); + if (unlikely(r)) + break; + + dc_iter->acc_len += c.len; + } copied += c.len; *data += c.len; + + T_DBG3("%s: len='%zu', copied='%d', dc_iter->acc_len='%lu'," + " dc_iter->skip='%d'\n", __func__, len, copied, + dc_iter->acc_len, dc_iter->skip); + if (copied == len) break; @@ -562,56 +576,154 @@ tfw_cache_write_field(TDB *db, TdbVRec **trec, TfwHttpResp *resp, *data = tr->data; } - /* Every non-empty header contains CRLF at the end. We need to translate - * it to { str, eolen } presentation. */ - if (hdr && hdr->len) - tfw_str_fixup_eol(hdr, SLEN(S_CRLF)); + return r; +} + +/** + * Same as @tfw_cache_h2_write(), but also decode the header from HTTP/2 format + * before writing it into the response (used e.g. for HTTP/1.1-response creation + * from cache). + */ +static int +tfw_cache_h2_decode_write(TDB *db, TdbVRec **trec, TfwHttpResp *resp, + char **data, size_t len, TfwDecodeCacheIter *dc_iter) +{ + unsigned long m_len; + TdbVRec *tr = *trec; + int r = 0, acc = 0; + TfwHPack hp = {}; + + while (1) { + m_len = min(tr->data + tr->len - *data, (long)(len - acc)); + if (!dc_iter->skip) { + r = tfw_hpack_cache_decode_expand(&hp, resp, *data, + m_len, dc_iter); + if (unlikely(r)) + break; + } + + acc += m_len; + *data += m_len; + if (acc == len) { + bzero_fast(dc_iter->__off, + sizeof(*dc_iter) + - offsetof(TfwDecodeCacheIter, __off)); + break; + } + + tr = *trec = tdb_next_rec_chunk(db, tr); + BUG_ON(!tr); + *data = tr->data; + } + + return r; +} + +static inline int +tfw_cache_h2_set_status(TDB *db, TfwCacheEntry *ce, TfwHttpResp *resp, + TdbVRec **trec, char **p, unsigned long *acc_len) +{ + int r; + TfwMsgIter *it = &resp->mit.iter; + struct sk_buff **skb_head = &resp->msg.skb_head; + bool h2_mode = TFW_MSG_H2(resp->req); + TfwDecodeCacheIter dc_iter = {}; + + if (h2_mode) + resp->mit.start_off = FRAME_HEADER_SIZE; + else + dc_iter.skip = true; + + + r = tfw_cache_h2_write(db, trec, resp, p, ce->status_len, &dc_iter); + if (unlikely(r)) + return r; + + if (!h2_mode) { + char buf[H2_STAT_VAL_LEN]; + TfwStr s_line = { + .chunks = (TfwStr []){ + { .data = S_0, .len = SLEN(S_0) }, + { .data = buf, .len = H2_STAT_VAL_LEN}, + { .data = " ", .len = 1 } + }, + .len = SLEN(S_0) + H2_STAT_VAL_LEN + 1, + .nchunks = 3 + }; + + if (!tfw_ultoa(ce->resp_status, __TFW_STR_CH(&s_line, 1)->data, + H2_STAT_VAL_LEN)) + return -E2BIG; + + r = tfw_http_msg_expand_data(it, skb_head, &s_line, NULL); + if (unlikely(r)) + return r; + + *acc_len += s_line.len; + } + + dc_iter.skip = h2_mode ? true : false; + + r = tfw_cache_h2_write(db, trec, resp, p, ce->rph_len, &dc_iter); + if (unlikely(r)) + return r; + + *acc_len += dc_iter.acc_len; + + if (!h2_mode) { + r = tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL); + if (unlikely(r)) + return r; + + *acc_len += g_crlf.len; + } return 0; } /** * Write HTTP header to skb data. - * The headers are likely to be adjusted, so copy them. */ static int -tfw_cache_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwStr *hdr, - TdbVRec **trec, TfwMsgIter *it, char **p) +tfw_cache_h2_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwHdrMods *hmods, + TdbVRec **trec, char **p, unsigned long *acc_len) { - int r, d, dn; - TfwStr *dups; + tfw_cache_write_actor_t *write_actor; TfwCStr *s = (TfwCStr *)*p; + TfwHttpReq *req = resp->req; + TfwDecodeCacheIter dc_iter = { .h_mods = hmods }; + int d, dn, r = 0; + + BUG_ON(!req); + + write_actor = !TFW_MSG_H2(req) || dc_iter.h_mods + ? tfw_cache_h2_decode_write + : tfw_cache_h2_write; *p += TFW_CSTR_HDRLEN; BUG_ON(*p > (*trec)->data + (*trec)->len); - if (likely(!(s->flags & TFW_STR_DUPLICATE))) - return tfw_cache_write_field(db, trec, resp, it, p, s->len, - hdr); + if (likely(!(s->flags & TFW_STR_DUPLICATE))) { + r = write_actor(db, trec, resp, p, s->len, &dc_iter); + if (likely(!r)) + *acc_len += dc_iter.acc_len; + goto out; + } /* Process duplicated headers. */ dn = s->len; - dups = tfw_pool_alloc(resp->pool, dn * sizeof(TfwStr)); - if (!dups) - return -ENOMEM; - for (d = 0; d < dn; ++d) { s = (TfwCStr *)*p; BUG_ON(s->flags); - TFW_STR_INIT(&dups[d]); *p += TFW_CSTR_HDRLEN; - if ((r = tfw_cache_write_field(db, trec, resp, it, p, s->len, - &dups[d]))) - return r; - } - - if (hdr) { - hdr->chunks = dups; - hdr->nchunks = dn; - hdr->flags |= TFW_STR_DUPLICATE; + r = write_actor(db, trec, resp, p, s->len, &dc_iter); + if (unlikely(r)) + break; + *acc_len += dc_iter.acc_len;; } - return 0; +out: + return r; } /** @@ -620,37 +732,49 @@ tfw_cache_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwStr *hdr, * Last-Modified might be used if the response does not have an ETag field. * * The 304 response should be as short as possible, we don't need to add - * extra headers with tfw_http_adjust_resp(). Use quicker tfw_msg_write() - * instead of tfw_http_msg_add_data() used to build full response. + * extra headers. */ static void tfw_cache_send_304(TfwHttpReq *req, TfwCacheEntry *ce) { - TfwHttpResp *resp; - TfwMsgIter it; - int i; char *p; + int r, i; + TfwMsgIter *it; + TfwHttpResp *resp; + TfwFrameHdr frame_hdr; + struct sk_buff **skb_head; + unsigned char buf[FRAME_HEADER_SIZE]; + unsigned int stream_id = 0; + unsigned long h_len = 0; TdbVRec *trec = &ce->trec; TDB *db = node_db(); WARN_ON_ONCE(!list_empty(&req->fwd_list)); WARN_ON_ONCE(!list_empty(&req->nip_list)); - if (TFW_MSG_H2(req)) { - /* - * TODO #309: add separate flow for HTTP/2 response preparing - * and sending (HPACK index, encode in HTTP/2 format, add frame - * headers and send via @tfw_h2_resp_fwd()). - */ - return; - } - if (!(resp = tfw_http_msg_alloc_resp_light(req))) goto err_create; - if (tfw_http_prep_304((TfwHttpMsg *)resp, req, &it, - ce->hdr_len_304)) - goto err_setup; + it = &resp->mit.iter; + skb_head = &resp->msg.skb_head; + + if (!TFW_MSG_H2(req)) { + r = tfw_http_prep_304(req, skb_head, it); + if (unlikely(r)) + goto err_setup; + } else { + stream_id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!stream_id)) + goto err_setup; + + resp->mit.start_off = FRAME_HEADER_SIZE; + + r = tfw_h2_resp_status_write(resp, 304, TFW_H2_TRANS_EXPAND, + true); + if (unlikely(r)) + goto err_setup; + } /* Put 304 headers */ for (i = 0; i < ARRAY_SIZE(ce->hdrs_304); ++i) { @@ -662,14 +786,32 @@ tfw_cache_send_304(TfwHttpReq *req, TfwCacheEntry *ce) trec = tdb_next_rec_chunk(db, trec); BUG_ON(!trec); - if (tfw_cache_build_resp_hdr(db, resp, NULL, &trec, &it, &p)) + if (tfw_cache_h2_build_resp_hdr(db, resp, NULL, &trec, &p, + &h_len)) goto err_setup; } - if (tfw_msg_write(&it, &g_crlf)) + if (!TFW_MSG_H2(req)) { + if (tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL)) + goto err_setup; + + tfw_http_resp_fwd(resp); + + return; + } + + if (h_len > FRAME_MAX_LENGTH) goto err_setup; - tfw_http_resp_fwd(resp); + frame_hdr.stream_id = stream_id; + frame_hdr.length = h_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS | HTTP2_F_END_STREAM; + + tfw_h2_pack_frame_header(buf, &frame_hdr); + memcpy_fast((*skb_head)->data, buf, sizeof(buf)); + + tfw_h2_resp_fwd(resp); return; err_setup: @@ -749,15 +891,24 @@ tfw_cache_entry_key_eq(TDB *db, TfwHttpReq *req, TfwCacheEntry *ce) int n, c_off = 0, t_off; TdbVRec *trec = &ce->trec; TfwStr *c, *h_start, *u_end, *h_end; + TfwStr host_val, *host = &req->h_tbl->tbl[TFW_HTTP_HDR_HOST]; if ((req->method != TFW_HTTP_METH_PURGE) && (ce->method != req->method)) return false; - if (req->uri_path.len - + req->h_tbl->tbl[TFW_HTTP_HDR_HOST].len != ce->key_len) + + /* + * Get 'host' header value (from HTTP/2 or HTTP/1.1 request) for + * strict comparison. + */ + tfw_http_msg_clnthdr_val(req, host, TFW_HTTP_HDR_HOST, &host_val); + + if (req->uri_path.len + host_val.len != ce->key_len) return false; t_off = CE_BODY_SIZE; - TFW_CACHE_REQ_KEYITER(c, req, u_end, h_start, h_end) { + TFW_CACHE_REQ_KEYITER(c, &req->uri_path, &host_val, u_end, h_start, + h_end) + { if (!trec) return false; this_chunk: @@ -836,20 +987,6 @@ tfw_cache_dbce_put(TfwCacheEntry *ce) tdb_rec_put(ce); } -static void -tfw_cache_str_write_hdr(const TfwStr *str, char *p) -{ - TfwCStr *s = (TfwCStr *)p; - - if (TFW_STR_DUP(str)) { - s->flags = TFW_STR_DUPLICATE; - s->len = str->nchunks; - } else { - s->flags = 0; - s->len = str->len ? str->len + SLEN(S_CRLF) : 0; - } -} - /** * Copies plain TfwStr @src to TdbRec @trec. * @return number of copied bytes (@src length). @@ -915,18 +1052,31 @@ tfw_cache_strcpy_lc(char **p, TdbVRec **trec, TfwStr *src, size_t tot_len) return __tfw_cache_strcpy(p, trec, src, tot_len, tfw_cstrtolower); } +#define CSTR_MOVE_HDR() \ +do { \ + cs = (TfwCStr *)*p; \ + *p += TFW_CSTR_HDRLEN; \ + *tot_len -= TFW_CSTR_HDRLEN; \ + ce->hdr_len += TFW_CSTR_HDRLEN; \ +} while (0) + +#define CSTR_WRITE_HDR(f, l) \ +do { \ + cs->flags = f; \ + cs->len = l; \ +} while (0) + /** - * Copies plain or compound (chunked) TfwStr @src to TdbRec @trec may be - * appending EOL marker at the end. + * Copies plain or compound (chunked) TfwStr @src to TdbRec @trec. * - * @src is copied (possibly with EOL appended) + * @src is copied * @return number of copied bytes on success and negative value otherwise. */ -static long -tfw_cache_strcpy_eol(char **p, TdbVRec **trec, - TfwStr *src, size_t *tot_len, bool eol) +static int +tfw_cache_h2_copy_str(unsigned int *acc_len, char **p, TdbVRec **trec, + TfwStr *src, size_t *tot_len) { - long n, copied = 0; + long n; TfwStr *c, *end; BUG_ON(TFW_STR_DUP(src)); @@ -936,73 +1086,181 @@ tfw_cache_strcpy_eol(char **p, TdbVRec **trec, TFW_STR_FOR_EACH_CHUNK(c, src, end) { if ((n = tfw_cache_strcpy(p, trec, c, *tot_len)) < 0) { - T_ERR("Cache: cannot copy chunk of string\n"); + T_ERR("Cache: cannot copy chunk of HTTP/2 string\n"); return -ENOMEM; } - *tot_len -= n; - copied += n; - } - if (eol) { - if ((n = tfw_cache_strcpy(p, trec, &g_crlf, *tot_len)) < 0) - return -ENOMEM; - BUG_ON(n != SLEN(S_CRLF)); *tot_len -= n; - copied += n; + *acc_len += n; } - return copied; + return 0; +} + +static inline int +tfw_cache_h2_copy_int(unsigned int *acc_len, unsigned long src, + unsigned short max, char **p, TdbVRec **trec, + size_t *tot_len) +{ + int r; + TfwHPackInt hp_int; + TfwStr str = {}; + + write_int(src, max, 0, &hp_int); + + str.data = hp_int.buf; + str.len = hp_int.sz; + + if ((r = tfw_cache_h2_copy_str(acc_len, p, trec, &str, tot_len))) + return r; + + return 0; } /** * Deep HTTP header copy to TdbRec. - * @src is copied in depth first fashion to speed up upcoming scans. + * @hdr is copied in depth first fashion to speed up upcoming scans. * @return number of copied bytes on success and negative value otherwise. */ static long -tfw_cache_copy_hdr(char **p, TdbVRec **trec, TfwStr *src, size_t *tot_len) +tfw_cache_h2_copy_hdr(TfwPool *pool, TfwCacheEntry *ce, char **p, + TdbVRec **trec, TfwStr *hdr, size_t *tot_len) { - long n = sizeof(TfwCStr), copied; - TfwStr *dup, *dup_end; + TfwCStr *cs; + long n = sizeof(TfwCStr); + unsigned short st_index = 0; + bool dupl = TFW_STR_DUP(hdr); + unsigned int init_len = ce->hdr_len; + TfwStr s_nm, s_val, *dup, *dup_end; - if (unlikely(src->len >= TFW_CSTR_MAXLEN)) { - T_WARN("Cache: trying to store too big string %lx\n", - src->len); - return -E2BIG; - } - /* Don't split short strings. */ - if (likely(!TFW_STR_DUP(src)) - && sizeof(TfwCStr) + src->len <= L1_CACHE_BYTES) - n += src->len; + T_DBG3("%s: ce=[%p] p=[%p], trec=[%p], tot_len='%zu'\n", __func__, ce, + *p, *trec, *tot_len); -#define CSTR_WRITE_HDR(str) \ - tfw_cache_str_write_hdr(str, *p); \ - *p += TFW_CSTR_HDRLEN; \ - *tot_len -= TFW_CSTR_HDRLEN; \ - copied = TFW_CSTR_HDRLEN; + if (likely(!dupl)) { + unsigned long h_len; + + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + + /* + * We cannot use just @tfw_http_hdr_split() here, since the + * response must be forwarded to the client and its headers + * will be processed further, so we must not invalidate headers + * descriptors (see also description of @tfw_http_hdr_split_cp() + * and @tfw_http_hdr_split() for additional details). + */ + if (tfw_http_hdr_split_cp(pool, hdr, &s_nm, &s_val)) + return -ENOMEM; + + st_index = hdr->hpack_idx; + h_len = tfw_h2_hdr_size(s_nm.len, s_val.len, st_index); + + /* Don't split short stprings. */ + if (sizeof(TfwCStr) + h_len <= L1_CACHE_BYTES) + n += h_len; + } *p = tdb_entry_get_room(node_db(), trec, *p, n, *tot_len); - if (!*p) { + if (unlikely(!*p)) { T_WARN("Cache: cannot allocate TDB space\n"); return -ENOMEM; } - CSTR_WRITE_HDR(src); + CSTR_MOVE_HDR(); - if (!TFW_STR_DUP(src)) { - if ((n = tfw_cache_strcpy_eol(p, trec, src, tot_len, 1)) < 0) - return n; - return copied + n; + if (TFW_STR_DUP(hdr)) { + CSTR_WRITE_HDR(TFW_STR_DUPLICATE, hdr->nchunks); + CSTR_MOVE_HDR(); } - TFW_STR_FOR_EACH_DUP(dup, src, dup_end) { - CSTR_WRITE_HDR(dup); - if ((n = tfw_cache_strcpy_eol(p, trec, dup, tot_len, 1)) < 0) - return n; - copied += n; + TFW_STR_FOR_EACH_DUP(dup, hdr, dup_end) { + unsigned int prev_len = ce->hdr_len; + + if (dupl) { + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + if (tfw_http_hdr_split_cp(pool, dup, &s_nm, &s_val)) + return -ENOMEM; + + st_index = dup->hpack_idx; + CSTR_MOVE_HDR(); + } + + if (st_index) { + if (tfw_cache_h2_copy_int(&ce->hdr_len, st_index, 0xF, + p, trec, tot_len)) + return -ENOMEM; + } + else { + if (tfw_cache_h2_copy_int(&ce->hdr_len, 0, 0xF, p, trec, + tot_len) + || tfw_cache_h2_copy_int(&ce->hdr_len, s_nm.len, + 0x7f, p, trec, tot_len) + || tfw_cache_h2_copy_str(&ce->hdr_len, p, trec, + &s_nm, tot_len)) + return -ENOMEM; + } + + if (tfw_cache_h2_copy_int(&ce->hdr_len, s_val.len, 0x7f, p, + trec, tot_len) + || tfw_cache_h2_copy_str(&ce->hdr_len, p, trec, &s_val, + tot_len)) + return -ENOMEM; + + CSTR_WRITE_HDR(0, ce->hdr_len - prev_len); } - return copied; + T_DBG3("%s: p=[%p], trec=[%p], ce->hdr_len='%u', tot_len='%zu'\n", + __func__, *p, *trec, ce->hdr_len, *tot_len); + + return ce->hdr_len - init_len; +} + +static long +tfw_cache_h2_add_hdr(TfwCacheEntry *ce, char **p, TdbVRec **trec, + unsigned short st_idx, TfwStr *val, size_t *tot_len) +{ + TfwCStr *cs; + unsigned long len; + unsigned int prev_len = ce->hdr_len; + long n = TFW_CSTR_HDRLEN; + + BUG_ON(!st_idx || TFW_STR_EMPTY(val)); + + len = tfw_hpack_int_size(st_idx, 0xF) + + tfw_hpack_int_size(val->len, 0x7F) + + val->len; + + if (unlikely(len >= TFW_CSTR_MAXLEN)) { + T_WARN("Cache: trying to store too big string %lx\n", len); + return -E2BIG; + } + + /* Don't split short strings. */ + if (TFW_CSTR_HDRLEN + len <= L1_CACHE_BYTES) + n += len; + + *p = tdb_entry_get_room(node_db(), trec, *p, n, *tot_len); + if (unlikely(!*p)) { + T_WARN("jCache: cannot allocate TDB space\n"); + return -ENOMEM; + } + + CSTR_MOVE_HDR(); + + if (tfw_cache_h2_copy_int(&ce->hdr_len, st_idx, 0xF, p, trec, tot_len)) + return -ENOMEM; + + if (tfw_cache_h2_copy_int(&ce->hdr_len, val->len, 0x7f, p, trec, + tot_len)) + return -ENOMEM; + + if (tfw_cache_h2_copy_str(&ce->hdr_len, p, trec, val, tot_len)) + return -ENOMEM; + + CSTR_WRITE_HDR(0, ce->hdr_len - prev_len - TFW_CSTR_HDRLEN); + + return ce->hdr_len - prev_len; } /** @@ -1020,13 +1278,30 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec, char *curr_p, TdbVRec **curr_trec) { char *e_p; - size_t len = 0; - TfwStr etag_val, *c, *end, *h = &resp->h_tbl->tbl[TFW_HTTP_HDR_ETAG]; + size_t c_size; + unsigned long n_len, v_off, v_len; TDB *db = node_db(); + size_t len = 0; + unsigned short flags = 0; + TfwStr h_val, *c, *end, *h = &resp->h_tbl->tbl[TFW_HTTP_HDR_ETAG]; + +#define CHECK_REC_SPACE() \ + while (c_size) { \ + size_t tail = h_trec->len - (e_p - h_trec->data); \ + if (c_size > tail) { \ + c_size -= tail; \ + h_trec = tdb_next_rec_chunk(db, h_trec); \ + e_p = h_trec->data; \ + } else { \ + e_p += c_size; \ + c_size = 0; \ + } \ + } if (TFW_STR_EMPTY(h)) return 0; - etag_val = tfw_str_next_str_val(h); /* not empty after http parser. */ + + tfw_http_msg_srvhdr_val(h, TFW_HTTP_HDR_ETAG, &h_val); /* Update supposed Etag offset to real value. */ /* FIXME: #803 */ @@ -1035,32 +1310,32 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec, h_trec = tdb_next_rec_chunk(db, h_trec); e_p = h_trec->data; } - /* Skip anything that is not a etag value. */ + /* + * Skip anything that is not etag value. Note, since headers are + * stored in cache in HTTP/2 representation (and the name of 'etag' + * header is always statically indexed), we should also skip index + * and value length fields; the 'etag' header has the 34 index in + * HPACK static table, thus we definitely now here, that it will + * occupy 2 bytes (RFC 7541 section 6.2.2). + */ e_p += TFW_CSTR_HDRLEN; - TFW_STR_FOR_EACH_CHUNK(c, h, end) { - size_t c_size = c->len; - - if (c->flags & TFW_STR_VALUE) + tfw_h2_msg_hdr_length(h, &n_len, &v_off, &v_len, TFW_H2_TRANS_INPLACE); + c_size = 2 + tfw_hpack_int_size(v_len, 0x7F); + CHECK_REC_SPACE(); + TFW_STR_FOR_EACH_CHUNK(c, &h_val, end) { + if (c->flags & TFW_STR_VALUE) { + flags = c->flags; break; - while (c_size) { - size_t tail = h_trec->len - (e_p - h_trec->data); - if (c_size > tail) { - c_size -= tail; - h_trec = tdb_next_rec_chunk(db, h_trec); - e_p = h_trec->data; - } - else { - e_p += c_size; - c_size = 0; - } } + c_size = c->len; + CHECK_REC_SPACE(); } for ( ; (c < end) && (c->flags & TFW_STR_VALUE); ++c) len += c->len; /* Create TfWStr that contains only entity-tag value. */ ce->etag.data = e_p; - ce->etag.flags = TFW_STR_CHUNK(&etag_val, 0)->flags; + ce->etag.flags = flags; ce->etag.len = min(len, (size_t)(h_trec->len - (e_p - h_trec->data))); len -= ce->etag.len; @@ -1090,6 +1365,8 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec, } return 0; + +#undef CHECK_REC_SPACE } /** @@ -1137,26 +1414,36 @@ __save_hdr_304_off(TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr *hdr, long off) /** * Copy response skbs to database mapped area. - * @tot_len - total length of actual data to write w/o TfwCStr's etc. + * @tot_len - total length of actual data to write w/o TfwCStr's etc; + * @rph - response reason-phrase to be saved in the cache. * * It's nasty to copy data on CPU, but we can't use DMA for mmaped file * as well as for unaligned memory areas. - * - * TODO Store the cache entries as a templates with placeholders for - * changeable headers. That we can faster build the final answers instead - * of adjusting skb's. */ static int -tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) +tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr *rph, + size_t tot_len) { - TfwHttpReq *req = resp->req; - long n, etag_off = 0; + int r, i; char *p; - TdbVRec *trec = &ce->trec, *etag_trec = NULL; - TfwStr *s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; + unsigned short status_idx; + TfwStr *field, *h, *end1, *end2; TDB *db = node_db(); - TfwStr *field, *h, *end1, *end2, empty = {}; - int r, i; + TdbVRec *trec = &ce->trec, *etag_trec = NULL; + long n, etag_off = 0; + TfwHttpReq *req = resp->req; + TfwGlobal *g_vhost = tfw_vhost_get_global(); + TfwStr h_val, *host = &req->h_tbl->tbl[TFW_HTTP_HDR_HOST]; + TfwStr val_srv = TFW_STR_STRING(TFW_SERVER); + TfwStr val_via = { + .chunks = (TfwStr []) { + { .data = S_VIA_H2_PROTO, .len = SLEN(S_VIA_H2_PROTO) }, + { .data = *this_cpu_ptr(&g_c_buf), + .len = g_vhost->hdr_via_len }, + }, + .len = SLEN(S_VIA_H2_PROTO) + g_vhost->hdr_via_len, + .nchunks = 2 + }; p = (char *)(ce + 1); tot_len -= CE_BODY_SIZE; @@ -1164,7 +1451,13 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) /* Write record key (URI + Host header). */ ce->key = TDB_OFF(db->hdr, p); ce->key_len = 0; - TFW_CACHE_REQ_KEYITER(field, req, end1, h, end2) { + + /* + * Get 'host' header value (from HTTP/2 or HTTP/1.1 request) for + * strict comparison. + */ + tfw_http_msg_clnthdr_val(req, host, TFW_HTTP_HDR_HOST, &h_val); + TFW_CACHE_REQ_KEYITER(field, &req->uri_path, &h_val, end1, h, end2) { if ((n = tfw_cache_strcpy_lc(&p, &trec, field, tot_len)) < 0) { T_ERR("Cache: cannot copy request key\n"); return -ENOMEM; @@ -1173,58 +1466,109 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) tot_len -= n; ce->key_len += n; } + /* Request method is a part of the cache record key. */ ce->method = req->method; + /* Write ':status' pseudo-header. */ ce->status = TDB_OFF(db->hdr, p); - if ((n = tfw_cache_strcpy_eol(&p, &trec, s_line, &tot_len, 1)) < 0) { - T_ERR("Cache: cannot copy HTTP status line\n"); - return -ENOMEM; + status_idx = tfw_h2_pseudo_index(resp->status); + if (status_idx) { + TfwHPackInt hp_idx; + TfwStr str = {}; + + write_int(status_idx, 0x7F, 0x80, &hp_idx); + str.data = hp_idx.buf; + str.len = hp_idx.sz; + + if (tfw_cache_h2_copy_str(&ce->status_len, &p, &trec, &str, + &tot_len)) + return -ENOMEM; + } else { + char buf[H2_STAT_VAL_LEN]; + TfwStr str = { + .data = buf, + .len = H2_STAT_VAL_LEN + }; + + /* + * If the ':status' pseudo-header is not fully indexed, set + * the default static index (8) just for the name. + */ + if (tfw_cache_h2_copy_int(&ce->status_len, 8, 0xF, &p, &trec, + &tot_len) + || tfw_cache_h2_copy_int(&ce->status_len, H2_STAT_VAL_LEN, + 0x7f, &p, &trec, &tot_len)) + return -ENOMEM; + + if (!tfw_ultoa(resp->status, str.data, H2_STAT_VAL_LEN)) + return -E2BIG; + + if (tfw_cache_h2_copy_str(&ce->status_len, &p, &trec, &str, + &tot_len)) + return -ENOMEM; } - ce->status_len += n; + + if (tfw_cache_h2_copy_str(&ce->rph_len, &p, &trec, rph, &tot_len)) + return -ENOMEM; ce->hdrs = TDB_OFF(db->hdr, p); ce->hdr_len = 0; ce->hdr_num = resp->h_tbl->off; FOR_EACH_HDR_FIELD_FROM(field, end1, resp, TFW_HTTP_HDR_REGULAR) { - bool hdr_304 = false; - - /* Skip hop-by-hop headers. */ - if (!(field->flags & TFW_STR_HBH_HDR)) { - h = field; - } else if (field - resp->h_tbl->tbl < TFW_HTTP_HDR_RAW) { - h = ∅ - } else { + int hid = field - resp->h_tbl->tbl; + /* + * Skip hop-by-hop headers. Also skip 'Server' header (with + * possible duplicates), since we will substitute it with our + * version of this header. + */ + if ((field->flags & TFW_STR_HBH_HDR) + || hid == TFW_HTTP_HDR_SERVER + || TFW_STR_EMPTY(field)) + { --ce->hdr_num; continue; } - if (field - resp->h_tbl->tbl == TFW_HTTP_HDR_ETAG) { - /* Must be updated after tfw_cache_copy_hdr(). */ + + if (hid == TFW_HTTP_HDR_ETAG) { + /* Must be updated after tfw_cache_h2_copy_hdr(). */ etag_off = TDB_OFF(db->hdr, p); etag_trec = trec; } - hdr_304 = __save_hdr_304_off(ce, resp, field, - TDB_OFF(db->hdr, p)); - n = tfw_cache_copy_hdr(&p, &trec, h, &tot_len); - if (n < 0) { - T_ERR("Cache: cannot copy HTTP header\n"); - return -ENOMEM; - } else if (hdr_304) { - ce->hdr_len_304 += n; - } - ce->hdr_len += n; + __save_hdr_304_off(ce, resp, field, TDB_OFF(db->hdr, p)); + + n = tfw_cache_h2_copy_hdr(resp->pool, ce, &p, &trec, field, + &tot_len); + if (unlikely(n < 0)) + return n; } + /* Add 'server' header. */ + n = tfw_cache_h2_add_hdr(ce, &p, &trec, 54, &val_srv, &tot_len); + if (unlikely(n < 0)) + return n; + + /* Add 'via' header. */ + memcpy_fast(__TFW_STR_CH(&val_via, 1)->data, g_vhost->hdr_via, + g_vhost->hdr_via_len); + n = tfw_cache_h2_add_hdr(ce, &p, &trec, 60, &val_via, &tot_len); + if (unlikely(n < 0)) + return n; + + ce->hdr_num += 2; + /* Write HTTP response body. */ ce->body = TDB_OFF(db->hdr, p); - n = tfw_cache_strcpy_eol(&p, &trec, &resp->body, &tot_len, - test_bit(TFW_HTTP_B_CHUNKED, resp->flags)); - if (n < 0) { + r = tfw_cache_h2_copy_str(&ce->body_len, &p, &trec, &resp->body, + &tot_len); + if (unlikely(r)) { T_ERR("Cache: cannot copy HTTP body\n"); return -ENOMEM; } - BUG_ON(tot_len != 0); + + if (WARN_ON_ONCE(tot_len != 0)) + return -EINVAL; ce->version = resp->version; tfw_http_copy_flags(ce->hmflags, resp->flags); @@ -1271,56 +1615,119 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) return 0; } -static size_t +static long __cache_entry_size(TfwHttpResp *resp) { + TfwStr host_val, *hdr, *hdr_end; + unsigned long n_len, v_off, v_len; TfwHttpReq *req = resp->req; - size_t size = CE_BODY_SIZE; - TfwStr *h, *hdr, *hdr_end, *dup, *dup_end, empty = {}; + long size, res_size = CE_BODY_SIZE; + TfwStr *host = &req->h_tbl->tbl[TFW_HTTP_HDR_HOST]; + unsigned long via_sz = SLEN(S_VIA_H2_PROTO) + + tfw_vhost_get_global()->hdr_via_len; /* Add compound key size */ - size += req->uri_path.len; - size += req->h_tbl->tbl[TFW_HTTP_HDR_HOST].len; + res_size += req->uri_path.len; + tfw_http_msg_clnthdr_val(req, host, TFW_HTTP_HDR_HOST, &host_val); + res_size += host_val.len; + + /* + * Add the length of ':status' pseudo-header: one byte if fully indexed, + * or five bytes (one byte for name index, one for status code length + * and three for status code itself) if only name is indexed. + */ + ++res_size; + res_size += (1 + H2_STAT_VAL_LEN) * !tfw_h2_pseudo_index(resp->status); /* Add all the headers size */ FOR_EACH_HDR_FIELD_FROM(hdr, hdr_end, resp, TFW_HTTP_HDR_REGULAR) { - /* Skip hop-by-hop headers. */ - if (!(hdr->flags & TFW_STR_HBH_HDR)) - h = hdr; - else if (hdr - resp->h_tbl->tbl < TFW_HTTP_HDR_RAW) - h = ∅ - else + TfwStr *d, *d_end; + int hid = hdr - resp->h_tbl->tbl; + /* + * Skip hop-by-hop headers. Also skip 'Server' header (with + * possible duplicates), since we will substitute it with our + * version of this header. + */ + if ((hdr->flags & TFW_STR_HBH_HDR) + || hid == TFW_HTTP_HDR_SERVER + || TFW_STR_EMPTY(hdr)) continue; - if (!TFW_STR_DUP(h)) { - size += sizeof(TfwCStr); - size += h->len ? (h->len + SLEN(S_CRLF)) : 0; + size = sizeof(TfwCStr); + + if (!TFW_STR_DUP(hdr)) { + tfw_h2_msg_hdr_length(hdr, &n_len, &v_off, &v_len, + TFW_H2_TRANS_INPLACE); + size += tfw_h2_hdr_size(n_len, v_len, hdr->hpack_idx); } else { - size += sizeof(TfwCStr); - TFW_STR_FOR_EACH_DUP(dup, h, dup_end) { + TFW_STR_FOR_EACH_DUP(d, hdr, d_end) { size += sizeof(TfwCStr); - size += dup->len + SLEN(S_CRLF); + tfw_h2_msg_hdr_length(d, &n_len, &v_off, &v_len, + TFW_H2_TRANS_INPLACE); + size += tfw_h2_hdr_size(n_len, v_len, + d->hpack_idx); } } + + if (unlikely(size >= TFW_CSTR_MAXLEN)) + goto err; + res_size += size; } - /* Add status line length + CRLF */ - size += resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE].len + SLEN(S_CRLF); + /* + * Add the length of our version of 'Server' header and 'Via' header. + * Note, that we need two bytes for static index, since in the first + * byte we have only four available bits for the index (we do not use + * dynamic indexing during headers storing into the cache, thus `without + * indexing` code must be set in the first index byte, see RFC 7541 + * section 6.2.2 for details), and the 'Server' (as well as 'Via') + * static index doesn't fit to that space. + */ + res_size += sizeof(TfwCStr); + res_size += 2; + res_size += tfw_hpack_int_size(SLEN(TFW_SERVER), 0x7F); + res_size += SLEN(TFW_SERVER); + + res_size += sizeof(TfwCStr); + res_size += 2; + res_size += tfw_hpack_int_size(via_sz, 0x7F); + res_size += via_sz; - /* Add body size accounting CRLF after the last chunk */ - size += resp->body.len; - if (test_bit(TFW_HTTP_B_CHUNKED, resp->flags)) - size += SLEN(S_CRLF); + /* Add body size. */ + res_size += resp->body.len; - return size; + return res_size; +err: + T_WARN("Cache: trying to store too big string %ld\n", size); + return -E2BIG; } static void __cache_add_node(TDB *db, TfwHttpResp *resp, unsigned long key) { + size_t len; TfwCacheEntry *ce; - size_t data_len = __cache_entry_size(resp); - size_t len = data_len; + TfwStr rph, *s_line; + long data_len = __cache_entry_size(resp); + + if (unlikely(data_len < 0)) + return; + + /* + * We need to save the reason-phrase for the case of HTTP/1.1-response + * creation from cache. Note, that reason-phrase is always the last part + * of status-line with the TFW_STR_VALUE flag set. + */ + s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; + rph = tfw_str_next_str_val(s_line); + if (WARN_ON_ONCE(TFW_STR_EMPTY(&rph))) + return; + + data_len += rph.len; + len = data_len; + + T_DBG3("%s: db=[%p] resp=[%p], req=[%p], key='%lu', data_len='%ld'\n", + __func__, db, resp, resp->req, key, data_len); /* TODO #788: revalidate existing entries before inserting a new one. */ @@ -1334,13 +1741,11 @@ __cache_add_node(TDB *db, TfwHttpResp *resp, unsigned long key) if (!ce) return; - T_DBG3("cache db=%p resp=%p/req=%p/ce=%p: alloc_len=%lu\n", - db, resp, resp->req, ce, len); + T_DBG3("%s: ce=[%p], alloc_len='%lu'\n", __func__, ce, len); - if (tfw_cache_copy_resp(ce, resp, data_len)) { + if (tfw_cache_copy_resp(ce, resp, &rph, data_len)) { /* TODO delete the probably partially built TDB entry. */ } - } static void @@ -1450,38 +1855,30 @@ tfw_cache_purge_method(TfwHttpReq *req) * See do_tcp_sendpages() as reference. */ static int -tfw_cache_build_resp_body(TDB *db, TfwHttpResp *resp, TdbVRec *trec, - TfwMsgIter *it, char *p) +tfw_cache_h2_build_resp_body(TDB *db, TdbVRec *trec, TfwMsgIter *it, char *p) { - int off, f_size, r; + int r; - if (WARN_ON_ONCE(!it->skb)) + if (WARN_ON_ONCE(!it->skb_head)) return -EINVAL; /* - * If headers perfectly fit allocated skbs, then - * it->skb == it->skb_head, see tfw_msg_iter_next_data_frag(). - * Normally all the headers fit single skb, but these two situations - * can't be distinguished. Start after last fragment of last skb in list. + * If all skbs/frags are used up (see @tfw_http_msg_expand_data()), + * create new skb with empty frags to reference the cached body; + * otherwise, use next empty frag in current skb. */ - if ((it->skb == it->skb_head) || (it->frag == -1)) { - it->skb = ss_skb_peek_tail(&it->skb_head); - it->frag = skb_shinfo(it->skb)->nr_frags; - } - else { - skb_frag_t *frag = &skb_shinfo(it->skb)->frags[it->frag]; - if (skb_frag_size(frag)) - ++it->frag; + if (!it->skb) { + if ((r = tfw_msg_iter_append_skb(it))) + return r; + } else { + ++it->frag; + if (it->frag >= MAX_SKB_FRAGS + && (r = tfw_msg_iter_append_skb(it))) + return r; } BUG_ON(it->frag < 0); - if (it->frag >= MAX_SKB_FRAGS - 1 - && (r = tfw_msg_iter_append_skb(it))) - return r; - while (1) { - if (it->frag == MAX_SKB_FRAGS - && (r = tfw_msg_iter_append_skb(it))) - return r; + int off, f_size; /* TDB keeps data by pages and we can reuse the pages. */ off = (unsigned long)p & ~PAGE_MASK; @@ -1491,22 +1888,74 @@ tfw_cache_build_resp_body(TDB *db, TfwHttpResp *resp, TdbVRec *trec, off, f_size); skb_frag_ref(it->skb, it->frag); ss_skb_adjust_data_len(it->skb, f_size); - - if (__tfw_http_msg_add_str_data((TfwHttpMsg *)resp, - &resp->body, p, f_size, - it->skb)) - return - ENOMEM; ++it->frag; } if (!(trec = tdb_next_rec_chunk(db, trec))) break; BUG_ON(trec && !f_size); p = trec->data; + + if (it->frag == MAX_SKB_FRAGS + && (r = tfw_msg_iter_append_skb(it))) + return r; } return 0; } +static int +tfw_cache_set_hdr_age(TfwHttpResp *resp, TfwCacheEntry *ce) +{ + int r; + size_t digs; + bool to_h2 = TFW_MSG_H2(resp->req); + TfwHttpTransIter *mit = &resp->mit; + struct sk_buff **skb_head = &resp->msg.skb_head; + time_t age = tfw_cache_entry_age(ce); + char cstr_age[TFW_ULTOA_BUF_SIZ] = {0}; + char *name = to_h2 ? "age" : "age" S_DLM; + unsigned int nlen = to_h2 ? SLEN("age") : SLEN("age" S_DLM); + TfwStr h_age = { + .chunks = (TfwStr []){ + { .data = name, .len = nlen }, + {} + }, + .len = nlen, + .nchunks = 2 + }; + + if (!(digs = tfw_ultoa(age, cstr_age, TFW_ULTOA_BUF_SIZ))) { + r = -E2BIG; + goto err; + } + + __TFW_STR_CH(&h_age, 1)->data = cstr_age; + __TFW_STR_CH(&h_age, 1)->len = digs; + h_age.len += digs; + + if (to_h2) { + h_age.hpack_idx = 21; + if ((r = tfw_hpack_encode(resp, &h_age, TFW_H2_TRANS_EXPAND, + false))) + goto err; + } else { + if ((r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &h_age, NULL))) + goto err; + + if ((r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &g_crlf, NULL))) + goto err; + } + + return 0; + +err: + T_WARN("Unable to add Age: header, cached response [%p] dropped" + " (err: %d)\n", resp, r); + return r; +} + /** * Build response that can be sent via TCP socket. * @@ -1529,44 +1978,33 @@ tfw_cache_build_resp_body(TDB *db, TfwHttpResp *resp, TdbVRec *trec, * TODO use iterator and passed skbs to be called from net_tx_action. */ static TfwHttpResp * -tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce) +tfw_cache_h2_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, time_t lifetime, + unsigned int stream_id) { int h; - char *p; - TfwStr *s_line; + TfwMsgIter *it; TfwHttpResp *resp; - TdbVRec *trec = &ce->trec; + char *p, *head_ptr; + TfwHttpTransIter *mit; + TfwFrameHdr frame_hdr; TDB *db = node_db(); - TfwMsgIter it; - + unsigned long h_len = 0; + struct sk_buff **skb_head; + TdbVRec *trec = &ce->trec; + unsigned char buf[FRAME_HEADER_SIZE]; + TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, + TFW_VHOST_HDRMOD_RESP); /* * The allocated response won't be checked by any filters and * is used for sending response data only, so don't initialize * connection and GFSM fields. */ if (!(resp = tfw_http_msg_alloc_resp(req))) - return NULL; - - /* - * Cached responses need SKBTX_SHARED_FRAG flag if they are going to be - * encrypted, since by default encryption happens in-place. As data - * pages are reused, overwriting will damage the date. - */ - if (tfw_http_msg_setup((TfwHttpMsg *)resp, &it, ce->hdr_len + 2, - TFW_CONN_TLS(req->conn) ? SKBTX_SHARED_FRAG : 0)) - { - goto free; - } + goto out; - /* - * Allocate HTTP headers table of proper size. - * There were no other allocations since the table is allocated, - * so realloc() just grows the table and returns the same pointer. - */ - h = (ce->hdr_num + 2 * TFW_HTTP_HDR_NUM - 1) & ~(TFW_HTTP_HDR_NUM - 1); - p = tfw_pool_realloc(resp->pool, resp->h_tbl, TFW_HHTBL_SZ(1), - TFW_HHTBL_EXACTSZ(h)); - BUG_ON(p != (char *)resp->h_tbl); + /* Copy version information and flags */ + resp->version = ce->version; + tfw_http_copy_flags(resp->flags, ce->hmflags); /* Skip record key until status line. */ for (p = TDB_PTR(db->hdr, ce->status); @@ -1576,53 +2014,119 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce) if (unlikely(!trec)) { T_WARN("Huh, partially stored cache entry (key=%lx)?\n", ce->key); - goto err; + goto free; } - s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; - if (tfw_cache_write_field(db, &trec, resp, &it, &p, - ce->status_len, s_line)) - goto err; + if (tfw_cache_h2_set_status(db, ce, resp, &trec, &p, &h_len)) + goto free; - resp->h_tbl->off = ce->hdr_num; for (h = TFW_HTTP_HDR_REGULAR; h < ce->hdr_num; ++h) { - TFW_STR_INIT(resp->h_tbl->tbl + h); - if (tfw_cache_build_resp_hdr(db, resp, resp->h_tbl->tbl + h, - &trec, &it, &p)) - goto err; + if (tfw_cache_h2_build_resp_hdr(db, resp, h_mods, &trec, &p, + &h_len)) + goto free; } - if (tfw_http_msg_add_data(&it, (TfwHttpMsg *)resp, &resp->crlf, - &g_crlf)) - goto err; + mit = &resp->mit; + skb_head = &resp->msg.skb_head; + WARN_ON_ONCE(mit->acc_len); + it = &mit->iter; - BUG_ON(p != TDB_PTR(db->hdr, ce->body)); - if (tfw_cache_build_resp_body(db, resp, trec, &it, p)) - goto err; + /* + * Set 'set-cookie' header if needed, for HTTP/2 or HTTP/1.1 + * response. + */ + if (tfw_http_sess_resp_process(resp, true)) + goto free; + /* + * RFC 7234 p.4 Constructing Responses from Caches: + * When a stored response is used to satisfy a request without + * validation, a cache MUST generate an Age header field. + */ + if (tfw_cache_set_hdr_age(resp, ce)) + goto free; - resp->version = ce->version; - tfw_http_copy_flags(resp->flags, ce->hmflags); + if (!TFW_MSG_H2(req)) { + /* + * Set additional headers and final CRLF for HTTP/1.1 + * response. + */ + if (tfw_http_expand_hbh(resp, ce->resp_status) + || tfw_http_set_loc_hdrs((TfwHttpMsg *)resp, req, true) + || (lifetime > ce->lifetime + && tfw_http_expand_stale_warn(resp)) + || (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags) + && tfw_http_expand_hdr_date(resp)) + || tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL)) + goto free; + + goto write_body; + } + + /* Set additional headers for HTTP/2 response. */ + if (tfw_h2_resp_add_loc_hdrs(resp, h_mods, true) + || (lifetime > ce->lifetime + && tfw_h2_set_stale_warn(resp)) + || (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags) + && tfw_h2_add_hdr_date(resp, TFW_H2_TRANS_EXPAND, true))) + goto free; + + h_len += mit->acc_len; + + if (h_len > FRAME_MAX_LENGTH || ce->body_len > FRAME_MAX_LENGTH) + goto free; + + frame_hdr.stream_id = stream_id; + /* + * Set header for HTTP/2 DATA frame, if body part of response + * exists. + */ + if (ce->body_len) { + TfwStr h_body = { + .data = buf, + .len = sizeof(buf) + }; + + frame_hdr.length = ce->body_len; + frame_hdr.type = HTTP2_DATA; + frame_hdr.flags = HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, &frame_hdr); + + if (tfw_http_msg_expand_data(it, skb_head, &h_body, NULL)) + goto free; + } + + /* Set header for HTTP/2 HEADERS frame. */ + frame_hdr.length = h_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + + if (!ce->body_len) + frame_hdr.flags |= HTTP2_F_END_STREAM; + + head_ptr = (*skb_head)->data; + tfw_h2_pack_frame_header(buf, &frame_hdr); + memcpy_fast(head_ptr, buf, sizeof(buf)); + +write_body: + /* Fill skb with body from cache for HTTP/2 or HTTP/1.1 response. */ + BUG_ON(p != TDB_PTR(db->hdr, ce->body)); + if (ce->body_len) { + if (tfw_cache_h2_build_resp_body(db, trec, it, p)) + goto free; + if (!TFW_MSG_H2(req) + && test_bit(TFW_HTTP_B_CHUNKED, resp->flags) + && tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL)) + goto free; + } return resp; -err: - T_WARN("Cannot use cached response, key=%lx\n", ce->key); free: tfw_http_msg_free((TfwHttpMsg *)resp); - return NULL; -} - -static inline int -tfw_cache_set_hdr_age(TfwHttpMsg *hmresp, TfwCacheEntry *ce) -{ - size_t digs; - char cstr_age[TFW_ULTOA_BUF_SIZ] = {0}; - time_t age = tfw_cache_entry_age(ce); - - if (!(digs = tfw_ultoa(age, cstr_age, TFW_ULTOA_BUF_SIZ))) - return -E2BIG; +out: + T_WARN("Cannot use cached response, key=%lx\n", ce->key); + TFW_INC_STAT_BH(clnt.msgs_otherr); - return tfw_http_msg_hdr_xfrm(hmresp, "Age", sizeof("Age") - 1, - cstr_age, digs, TFW_HTTP_HDR_RAW, 0); + return NULL; } static void @@ -1630,6 +2134,7 @@ cache_req_process_node(TfwHttpReq *req, tfw_http_cache_cb_t action) { TfwCacheEntry *ce = NULL; TfwHttpResp *resp = NULL; + unsigned int id = 0; TDB *db = node_db(); TdbIter iter; time_t lifetime; @@ -1651,23 +2156,35 @@ cache_req_process_node(TfwHttpReq *req, tfw_http_cache_cb_t action) if (!tfw_handle_validation_req(req, ce)) goto put; - if (!(resp = tfw_cache_build_resp(req, ce))) - goto out; /* - * RFC 7234 p.4 Constructing Responses from Caches: - * When a stored response is used to satisfy a request without - * validation, a cache MUST generate an Age header field. + * If the stream for HTTP/2-request is already closed (due to some + * error or just reset from the client side), there is no sense to + * forward request to the server. */ - if (tfw_cache_set_hdr_age((TfwHttpMsg *)resp, ce)) { - T_WARN("Unable to add Age: header, cached response [%p] " - "dropped\n", resp); - TFW_INC_STAT_BH(clnt.msgs_otherr); - tfw_http_msg_free((TfwHttpMsg *)resp); - resp = NULL; - goto out; + if (TFW_MSG_H2(req)) { + id = tfw_h2_stream_id(req); + if (unlikely(!id)) { + tfw_http_conn_msg_free((TfwHttpMsg *)req); + goto put; + } + } + + resp = tfw_cache_h2_build_resp(req, ce, lifetime, id); + /* + * The stream of HTTP/2-request should be closed here since we have + * successfully created the resulting response from cache and will + * send this response to the client (without forwarding request to + * the backend), thus the stream will be finished. + */ + if (resp && TFW_MSG_H2(req)) { + id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!id)) { + tfw_http_msg_free((TfwHttpMsg *)resp); + tfw_http_conn_msg_free((TfwHttpMsg *)req); + goto put; + } } - if (lifetime > ce->lifetime) - __set_bit(TFW_HTTP_B_RESP_STALE, resp->flags); out: if (!resp && (req->cache_ctl.flags & TFW_HTTP_CC_OIFCACHED)) tfw_http_send_resp(req, 504, "resource not cached"); diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index fde33a77ec..ebc85edc70 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -186,7 +186,7 @@ do { \ * variable-length integer greater than defined limit, this is the malformed * request and we should drop the parsing process. */ -#define GET_FLEXIBLE(x, new_state) \ +#define GET_FLEXIBLE_lambda(x, new_state, lambda) \ do { \ unsigned int __m = 0; \ unsigned int __c; \ @@ -194,6 +194,7 @@ do { \ if (src >= last) { \ hp->shift = __m; \ NEXT_STATE(new_state); \ + lambda; \ goto out; \ } \ __c = *src++; \ @@ -206,11 +207,14 @@ do { \ } while (__c > 127); \ } while (0) +#define GET_FLEXIBLE(x, new_state) \ + GET_FLEXIBLE_lambda(x, new_state, {}) + /* Continue decoding after interruption due to absence of the next fragment. * If the variable-length integer greater than defined limit, this is the * malformed request and we should drop the parsing process. */ -#define GET_CONTINUE(x) \ +#define GET_CONTINUE_lambda(x, lambda) \ do { \ unsigned int __m = hp->shift; \ unsigned int __c = *src++; \ @@ -224,6 +228,7 @@ do { \ while (__c > 127) { \ if (src >= last) { \ hp->shift = __m; \ + lambda; \ goto out; \ } \ __c = *src++; \ @@ -234,8 +239,12 @@ do { \ goto out; \ } \ } \ + lambda; \ } while (0) +#define GET_CONTINUE(x) \ + GET_CONTINUE_lambda(x, {}) + #define SET_NEXT() \ do { \ hp->curr += 8; \ @@ -379,6 +388,30 @@ do { \ static unsigned long act_hp_str_n; +void +write_int(unsigned long index, unsigned short max, unsigned short mask, + TfwHPackInt *__restrict res_idx) +{ + unsigned int size = 1; + unsigned char *dst = res_idx->buf; + + if (likely(index < max)) { + index |= mask; + } + else { + ++size; + *dst++ = max | mask; + index -= max; + while (index > 0x7F) { + ++size; + *dst++ = (index & 0x7F) | 0x80; + index >>= 7; + } + } + *dst = index; + res_idx->sz = size; +} + static inline TfwStr * tfw_hpack_exp_hdr(TfwPool *__restrict pool, unsigned long len, TfwMsgParseIter *__restrict it) @@ -1158,6 +1191,14 @@ tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, done: switch (entry->tag) { case TFW_TAG_HDR_H2_METHOD: + if (hp->index == 2) { + req->method = TFW_HTTP_METH_GET; + } else if (hp->index == 3) { + req->method = TFW_HTTP_METH_POST; + } else { + WARN_ON_ONCE(1); + return T_DROP; + } parser->_hdr_tag = TFW_HTTP_HDR_H2_METHOD; break; case TFW_TAG_HDR_H2_SCHEME: @@ -1585,6 +1626,406 @@ tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, return r; } +/* + * Modified version of HPACK decoder FSM - for cache entries processing, + * HTTP/2-headers decoding (either into HTTP/2 or HTTP/1.1 format) and skb + * expanding at once; only static indexing is allowed, no service HPACK codes, + * no Huffman decoding and no parsing; only a limited subset of HPACK decoder + * FSM states is used. + */ +int +tfw_hpack_cache_decode_expand(TfwHPack *__restrict hp, + TfwHttpResp *__restrict resp, + unsigned char *__restrict src, unsigned long n, + TfwDecodeCacheIter *__restrict dc_iter) +{ + unsigned char c; + unsigned int state; + int r = T_OK; + TfwStr exp_str = {}; + TfwHttpTransIter *mit = &resp->mit; + TfwMsgIter *it = &mit->iter; + bool h2_mode = TFW_MSG_H2(resp->req); + const unsigned char *prev, *last = src + n; + struct sk_buff **skb_head = &resp->msg.skb_head; + +#define GET_NEXT_DATA(cond) \ +do { \ + if (unlikely(cond)) \ + goto out; \ +} while (0) + +#define FIXUP_DATA(str, data, len) \ + if (__tfw_http_msg_add_str_data((TfwHttpMsg *)resp, str, data, \ + len, NULL)) \ + { \ + r = T_DROP; \ + goto out; \ + } + +#define FIXUP_H2_DATA(str, data, len) \ +do { \ + if (h2_mode) \ + FIXUP_DATA(str, data, len); \ +} while (0) + +#define EXPAND_STR_DATA(str) \ +do { \ + if (tfw_http_msg_expand_data(it, skb_head, str, NULL)) { \ + r = T_DROP; \ + goto out; \ + } \ + dc_iter->acc_len += (str)->len; \ +} while (0) + +#define EXPAND_DATA(ptr, length) \ +do { \ + exp_str.data = ptr; \ + exp_str.len = length; \ + EXPAND_STR_DATA(&exp_str); \ +} while (0) + +#define EXPAND_H2_DATA(data, len) \ +do { \ + if (h2_mode) \ + EXPAND_DATA(data, len); \ +} while (0) + + WARN_ON_ONCE(!n); + + state = hp->state; + + T_DBG3("%s: header processing, n=%lu, to_parse=%lu, state=%d\n", + __func__, n, last - src, state); + + switch (state & HPACK_STATE_MASK) { + case HPACK_STATE_READY: + prev = src; + c = *src++; + + /* + * We use only static indexing during headers storing + * into the cache, thus `without indexing` code must be + * always set in the first index byte (RFC 7541 section + * 6.2.2) of cached response; besides, since response + * regular headers have no full indexes in HPACK static + * table, only header's name is allowed to be indexed. + */ + if (WARN_ON_ONCE(c & 0xF0)) { + r = T_DROP; + goto out; + } + + T_DBG3("%s: reference with value...\n", __func__); + + hp->index = c & 0x0F; + if (hp->index == 0x0F) { + GET_FLEXIBLE_lambda(hp->index, + HPACK_STATE_INDEX, { + FIXUP_H2_DATA(&dc_iter->h2_data, src, + src - prev); + }); + } + + T_DBG3("%s: name index: %lu\n", __func__, hp->index); + + FIXUP_H2_DATA(&dc_iter->h2_data, src, src - prev); + + NEXT_STATE(hp->index + ? HPACK_STATE_INDEXED_NAME_TEXT + : HPACK_STATE_NAME); + + GET_NEXT_DATA(src >= last); + + if (hp->index) + goto get_indexed_name; + + /* Fall through. */ + + case HPACK_STATE_NAME: + prev = src; + c = *src++; + + T_DBG3("%s: decode header name length...\n", __func__); + WARN_ON_ONCE(hp->length); + WARN_ON_ONCE(c & 0x80); + + hp->length = c & 0x7F; + if (unlikely(hp->length == 0x7F)) { + GET_FLEXIBLE_lambda(hp->length, + HPACK_STATE_NAME_LENGTH, { + FIXUP_H2_DATA(&dc_iter->h2_data, src, + src - prev); + }); + } + else if (unlikely(hp->length == 0)) { + r = T_DROP; + goto out; + } + + T_DBG3("%s: name length: %lu\n", __func__, hp->length); + + FIXUP_H2_DATA(&dc_iter->h2_data, src, src - prev); + + NEXT_STATE(HPACK_STATE_NAME_TEXT); + + GET_NEXT_DATA(src >= last); + + goto get_name_text; + + case HPACK_STATE_INDEXED_NAME_TEXT: + { + const TfwHPackEntry *entry; +get_indexed_name: + T_DBG3("%s: decode indexed (%lu) header name...\n", + __func__, hp->index); + if (WARN_ON_ONCE(!hp->index + || hp->index > HPACK_STATIC_ENTRIES)) + { + r = T_DROP; + goto out; + } + + entry = static_table + hp->index - 1; + if (WARN_ON_ONCE(entry->name_num != 1)) { + r = T_DROP; + goto out; + } + + dc_iter->hdr_data.len = entry->name_len; + dc_iter->hdr_data.data = __TFW_STR_CH(entry->hdr, 0)->data; + + goto check_name_text; + + } + case HPACK_STATE_NAME_TEXT: + { + int i; + TfwHdrMods *h_mods; + unsigned long m_len; +get_name_text: + m_len = min((unsigned long)(last - src), hp->length); + + T_DBG3("%s: decoding header name, m_len=%lu\n", __func__, m_len); + + FIXUP_DATA(&dc_iter->hdr_data, src, m_len); + + hp->length -= m_len; + src += m_len; + + GET_NEXT_DATA(hp->length); +check_name_text: + i = 0; + h_mods = dc_iter->h_mods; + WARN_ON_ONCE(dc_iter->desc); + if (h_mods) { + for (; i < h_mods->sz; ++i) { + TfwHdrModsDesc *d = &h_mods->hdrs[i]; + + if (!__hdr_name_cmp(&dc_iter->hdr_data, + TFW_STR_CHUNK(d->hdr, 0))) + { + dc_iter->desc = d; + break; + } + } + } + + if (dc_iter->desc) { + /* All duplicate headers must be skipped by caller. */ + WARN_ON_ONCE(test_bit(i, mit->found)); + __set_bit(i, mit->found); + if (!TFW_STR_CHUNK(dc_iter->desc->hdr, 2)) { + dc_iter->skip = true; + goto out; + } + + } + + if (h2_mode) + EXPAND_STR_DATA(&dc_iter->h2_data); + + EXPAND_STR_DATA(&dc_iter->hdr_data); + TFW_STR_INIT(&dc_iter->hdr_data); + + if (!h2_mode) + EXPAND_DATA(S_DLM, SLEN(S_DLM)); + + T_DBG3("%s: name copied, n=%lu, tail=%lu, hp->length=%lu\n", + __func__, n, last - src, hp->length); + + NEXT_STATE(HPACK_STATE_VALUE); + + GET_NEXT_DATA(src >= last); + + /* Fall through. */ + } + case HPACK_STATE_VALUE: + T_DBG3("%s: decode header value length...\n", __func__); + + prev = src; + c = *src++; + WARN_ON_ONCE(hp->length); + WARN_ON_ONCE(c & 0x80); + + hp->length = c & 0x7F; + if (unlikely(hp->length == 0x7F)) + GET_FLEXIBLE_lambda(hp->length, + HPACK_STATE_VALUE_LENGTH, { + if (!dc_iter->desc) + EXPAND_H2_DATA(src, src - prev); + }); + + T_DBG3("%s: value length: %lu\n", __func__, hp->length); + + if (!dc_iter->desc) + EXPAND_H2_DATA(src, src - prev); + + NEXT_STATE(HPACK_STATE_VALUE_TEXT); + + GET_NEXT_DATA(src >= last); + + /* Fall through. */ + + case HPACK_STATE_VALUE_TEXT: + { + unsigned long m_len; +get_value_text: + T_DBG3("%s: decode header value...\n", __func__); + m_len = min((unsigned long)(last - src), hp->length); + + if (dc_iter->desc && dc_iter->desc->append && h2_mode) { + /* + * If the header value must be appended, we need to + * collect the value for HTTP/2-header, since it should + * be re-encoded in this case. + */ + FIXUP_DATA(&dc_iter->hdr_data, src, m_len); + } + else if (!dc_iter->desc || dc_iter->desc->append) { + EXPAND_DATA(src, m_len); + } + + hp->length -= m_len; + src += m_len; + + GET_NEXT_DATA(hp->length); + + if (dc_iter->desc) { + TfwStr *val, *h = dc_iter->desc->hdr; + TfwStr n_val = { + .chunks = (TfwStr []){ + { .data = ", ", .len = 2 }, + { .data = __TFW_STR_CH(h, 2)->data, + .len = __TFW_STR_CH(h, 2)->len } + }, + .len = __TFW_STR_CH(h, 2)->len + 2, + .nchunks = 2 + }; + + dc_iter->skip = true; + + if (h2_mode) { + TfwHPackInt vlen; + + if (dc_iter->desc->append) { + val = &dc_iter->hdr_data; + if (tfw_strcat(resp->pool, val, &n_val)) + { + r = T_DROP; + goto out; + } + } + else { + val = __TFW_STR_CH(&n_val, 1); + } + + write_int(val->len, 0x7F, 0, &vlen); + + EXPAND_DATA(vlen.buf, vlen.sz); + EXPAND_STR_DATA(val); + + break; + } + + val = dc_iter->desc->append + ? &n_val + : __TFW_STR_CH(&n_val, 1); + + EXPAND_STR_DATA(val); + } + + if (!h2_mode) + EXPAND_DATA(S_CRLF, SLEN(S_CRLF)); + + break; + } + case HPACK_STATE_INDEX: + prev = src; + GET_CONTINUE_lambda(hp->index, { + FIXUP_H2_DATA(&dc_iter->h2_data, src, src - prev); + }); + T_DBG3("%s: index finally decoded: %lu\n", __func__, hp->index); + + NEXT_STATE(HPACK_STATE_INDEXED_NAME_TEXT); + + GET_NEXT_DATA(src >= last); + + goto get_indexed_name; + + case HPACK_STATE_NAME_LENGTH: + prev = src; + GET_CONTINUE_lambda(hp->length, { + FIXUP_H2_DATA(&dc_iter->h2_data, src, src - prev); + }); + T_DBG3("%s: name length finally decoded: %lu\n", __func__, + hp->length); + + NEXT_STATE(HPACK_STATE_NAME_TEXT); + + GET_NEXT_DATA(src >= last); + + goto get_name_text; + + case HPACK_STATE_VALUE_LENGTH: + prev = src; + GET_CONTINUE_lambda(hp->length, { + if (!dc_iter->desc) + EXPAND_H2_DATA(src, src - prev); + }); + T_DBG3("%s: value length finally decoded: %lu\n", __func__, + hp->length); + + NEXT_STATE(HPACK_STATE_VALUE_TEXT); + + GET_NEXT_DATA(src >= last); + + goto get_value_text; + + default: + WARN_ON_ONCE(1); + r = T_DROP; + goto out; + } + + T_DBG3("%s: new header added\n", __func__); + + WARN_ON_ONCE(src != last); + + return T_OK; +out: + WARN_ON_ONCE(src > last); + hp->state = state; + return r; + +#undef GET_NEXT_DATA +#undef FIXUP_DATA +#undef FIXUP_H2_DATA +#undef EXPAND_STR_DATA +#undef EXPAND_DATA +#undef EXPAND_H2_DATA +} + /** * ------------------------------------------------------------------------ * HPACK Encoder functionality @@ -2716,30 +3157,8 @@ tfw_hpack_enc_release(TfwHPack *__restrict hp, unsigned long *flags) WARN_ON_ONCE(!atomic64_read(&tbl->guard)); atomic64_dec(&tbl->guard); } -} - -static void -write_int(unsigned long index, unsigned short max, unsigned short mask, - TfwHPackInt *__restrict res_idx) -{ - unsigned int size = 1; - unsigned char *dst = res_idx->buf; - if (likely(index < max)) { - index |= mask; - } - else { - ++size; - *dst++ = max | mask; - index -= max; - while (index > 0x7F) { - ++size; - *dst++ = (index & 0x7F) | 0x80; - index >>= 7; - } - } - *dst = index; - res_idx->sz = size; + __clear_bit(TFW_HTTP_B_H2_TRANS_ENTERED, flags); } static unsigned long @@ -2920,12 +3339,12 @@ tfw_hpack_str_expand_raw(TfwHttpTransIter *mit, TfwMsgIter *it, len_str.data = len.buf; len_str.len = len.sz; - r = tfw_http_msg_expand_data(it, skb_head, &len_str); + r = tfw_http_msg_expand_data(it, skb_head, &len_str, NULL); if (unlikely(r)) return r; mit->acc_len += len_str.len; - r = tfw_http_msg_expand_data(it, skb_head, str); + r = tfw_http_msg_expand_data(it, skb_head, str, NULL); if (unlikely(r)) return r; mit->acc_len += str->len; @@ -3066,7 +3485,8 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, .len = idx->sz, }; - ret = tfw_http_msg_expand_data(iter, skb_head, &idx_str); + ret = tfw_http_msg_expand_data(iter, skb_head, &idx_str, + &mit->start_off); if (unlikely(ret)) return ret; @@ -3079,9 +3499,6 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, if (!hdr) return 0; - if (WARN_ON_ONCE(TFW_STR_PLAIN(hdr))) - return -EINVAL; - if (unlikely(!name_indexed)) { ret = tfw_hpack_str_expand(mit, iter, skb_head, TFW_STR_CHUNK(hdr, 0), NULL); @@ -3095,10 +3512,16 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, * * { name [S_DLM] value1 [value2 [value3 ...]] }. * + * Besides, we can get here the source header which contains only the + * name (e.g. due to creation of headers separately by parts on the + * upper HTTP level, during internal responses generation) - this is the + * valid case for expanding procedure and we should return control + * upstairs in this case - in order the header creation to be continued. + * */ c = TFW_STR_CHUNK(hdr, 1); - if (WARN_ON_ONCE(!c)) - return -EINVAL; + if (!(c = TFW_STR_CHUNK(hdr, 1))) + return 0; if (c->len == SLEN(S_DLM) && *(short *)c->data == *(short *)S_DLM) { c = TFW_STR_CHUNK(hdr, 2); @@ -3137,10 +3560,7 @@ tfw_hpack_hdr_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, if (!hdr || WARN_ON_ONCE(TFW_STR_PLAIN(hdr) || TFW_STR_DUP(hdr))) return -EINVAL; - r = tfw_http_hdr_split(hdr, &s_name, &s_val); - - if (unlikely(r)) - return r; + tfw_http_hdr_split(hdr, &s_name, &s_val); if (unlikely(!name_indexed)) { TfwHPackInt nlen; @@ -3203,14 +3623,14 @@ tfw_hpack_hdr_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, */ int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, - TfwH2TransOp op) + TfwH2TransOp op, bool dyn_indexing) { TfwHPackInt idx; bool st_full_index; unsigned short st_index, index = 0; TfwH2Ctx *ctx = tfw_h2_context(resp->req->conn); TfwHPackETbl *tbl = &ctx->hpack.enc_tbl; - int r = 0; + int r = HPACK_IDX_ST_NOT_FOUND; if (WARN_ON_ONCE(!hdr || TFW_STR_EMPTY(hdr))) return -EINVAL; @@ -3221,7 +3641,7 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, T_DBG3("%s: op=%d, st_index=%hu, st_full_index=%d\n", __func__, op, st_index, st_full_index); - if (!st_full_index) { + if (!st_full_index && dyn_indexing) { r = tfw_hpack_encoder_index(tbl, hdr, &index, resp->flags, op); if (r < 0) return r; diff --git a/tempesta_fw/hpack.h b/tempesta_fw/hpack.h index 01209831e5..7e05394444 100644 --- a/tempesta_fw/hpack.h +++ b/tempesta_fw/hpack.h @@ -20,6 +20,8 @@ #ifndef __TFW_HPACK_H__ #define __TFW_HPACK_H__ +#include "http_types.h" + /** * Default allowed size for dynamic tables (in bytes). Note, that for encoder's * dynamic table - this is also the maximum allowed size and the maximum storage @@ -219,10 +221,35 @@ typedef struct { (DIV_ROUND_UP(sizeof(unsigned long), 7) + 1) typedef struct { - unsigned int sz; - unsigned char buf[HPACK_MAX_INT]; + unsigned int sz; + unsigned char buf[HPACK_MAX_INT]; } TfwHPackInt; + +/** + * Iterator for the message headers decoding from HTTP/2-cache. + * + * @h_mods - pointer to the headers configured to be changed; + * @skip - flag to skip particular cached data in order to switch + * between HTTP/2 and HTTP/1.1 resulting representation during + * decoding from HTTP/2-cache; + * @__off - offset to reinitialize the iterator non-persistent part; + * @desc - pointer to the found header configured to be changed; + * @acc_len - accumulated length of the resulting headers part of the + * response; + * @hdr_data - header's data currently received from cache; + * @h2_data - HTTP/2-specific data currently received from cache. + */ +typedef struct { + TfwHdrMods *h_mods; + bool skip; + char __off[0]; + TfwHdrModsDesc *desc; + unsigned long acc_len; + TfwStr hdr_data; + TfwStr h2_data; +} TfwDecodeCacheIter; + #define BUFFER_GET(len, it) \ do { \ BUG_ON(!(len)); \ @@ -234,15 +261,39 @@ do { \ (it)->pos, (unsigned long)(it)->pos); \ } while (0) +void write_int(unsigned long index, unsigned short max, unsigned short mask, + TfwHPackInt *__restrict res_idx); int tfw_hpack_init(TfwHPack *__restrict hp, unsigned int htbl_sz); void tfw_hpack_clean(TfwHPack *__restrict hp); int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, - TfwH2TransOp op); + TfwH2TransOp op, bool dyn_indexing); void tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl, unsigned short new_size); int tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, unsigned long n, TfwHttpReq *__restrict req, unsigned int *__restrict parsed); +int tfw_hpack_cache_decode_expand(TfwHPack *__restrict hp, + TfwHttpResp *__restrict resp, + unsigned char *__restrict src, unsigned long n, + TfwDecodeCacheIter *__restrict cd_iter); void tfw_hpack_enc_release(TfwHPack *__restrict hp, unsigned long *flags); +static inline unsigned int +tfw_hpack_int_size(unsigned long index, unsigned short max) +{ + unsigned int size = 1; + + if (likely(index < max)) + return size; + + ++size; + index -= max; + while (index > 0x7F) { + ++size; + index >>= 7; + } + + return size; +} + #endif /* __TFW_HPACK_H__ */ diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 7222054bbe..0d6f088509 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -112,14 +112,12 @@ #define S_H2_AUTH ":authority" #define S_H2_PATH ":path" #define S_H2_STAT ":status" -#define H2_STAT_VAL_LEN 3 #define T_WARN_ADDR_STATUS(msg, addr_ptr, print_port, status) \ TFW_WITH_ADDR_FMT(addr_ptr, print_port, addr_str, \ T_WARN("%s, status %d: %s\n", \ msg, status, addr_str)) -#define RESP_BUF_LEN 128 static DEFINE_PER_CPU(char[RESP_BUF_LEN], g_buf); int ghprio; /* GFSM hook priority. */ @@ -135,9 +133,7 @@ static struct { #define S_CRLFCRLF "\r\n\r\n" #define S_HTTP "http://" #define S_HTTPS "https://" -#define S_VERSION11 "HTTP/1.1" -#define S_0 S_VERSION11 " " #define S_200 "HTTP/1.1 200 OK" #define S_302 "HTTP/1.1 302 Found" #define S_304 "HTTP/1.1 304 Not Modified" @@ -151,17 +147,17 @@ static struct { #define S_504 "HTTP/1.1 504 Gateway Timeout" #define S_XFF "x-forwarded-for" +#define S_WARN "warning" #define S_F_HOST "host: " #define S_F_DATE "date: " #define S_F_CONTENT_LENGTH "content-length: " #define S_F_CONTENT_TYPE "content-type: " -#define S_F_LOCATION "location: " #define S_F_CONNECTION "connection: " #define S_F_ETAG "etag: " #define S_F_RETRY_AFTER "retry-after: " #define S_F_SERVER "server: " -#define S_F_VIA "via: " + #define S_V_DATE "Sun, 06 Nov 1994 08:49:37 GMT" #define S_V_CONTENT_LENGTH "9999" @@ -169,6 +165,7 @@ static struct { #define S_V_CONN_KA "keep-alive" #define S_V_RETRY_AFTER "10" #define S_V_MULTIPART "multipart/form-data; boundary=" +#define S_V_WARN "110 - Response is stale" #define S_H_CONN_KA S_F_CONNECTION S_V_CONN_KA S_CRLFCRLF #define S_H_CONN_CLOSE S_F_CONNECTION S_V_CONN_CLOSE S_CRLFCRLF @@ -414,6 +411,155 @@ tfw_http_prep_date(char *buf) tfw_http_prep_date_from(buf, tfw_current_timestamp()); } +int +tfw_h2_prep_redirect(TfwHttpResp *resp, unsigned short status, TfwStr *rmark, + TfwStr *cookie, TfwStr *body) +{ + int r; + TfwHPackInt vlen; + TfwFrameHdr frame_hdr; + unsigned int stream_id; + unsigned long hdrs_len, loc_val_len; + unsigned char buf[FRAME_HEADER_SIZE]; + TfwHttpReq *req = resp->req; + TfwHttpTransIter *mit = &resp->mit; + TfwMsgIter *iter = &mit->iter; + struct sk_buff **skb_head = &resp->msg.skb_head; + static TfwStr h_loc = TFW_STR_STRING(S_LOCATION); + static TfwStr proto = TFW_STR_STRING(S_HTTPS); + static TfwStr h_sc = TFW_STR_STRING(S_SET_COOKIE); + TfwStr host, *host_ptr = &host, s_vlen = {}; + + stream_id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!stream_id)) + return -ENOENT; + + frame_hdr.stream_id = stream_id; + + /* Set HTTP/2 ':status' pseudo-header. */ + mit->start_off = FRAME_HEADER_SIZE; + r = tfw_h2_resp_status_write(resp, status, TFW_H2_TRANS_EXPAND, false); + if (unlikely(r)) + return r; + + /* Add 'date' header. */ + r = tfw_h2_add_hdr_date(resp, TFW_H2_TRANS_EXPAND, false); + if (unlikely(r)) + return r; + + /* Add 'location' header (possibly, with redirection mark). */ + h_loc.hpack_idx = 46; + r = tfw_hpack_encode(resp, &h_loc, TFW_H2_TRANS_EXPAND, false); + if (unlikely(r)) + return r; + + if (req->host.len) + host_ptr = &req->host; + else + __h2_msg_hdr_val(&req->h_tbl->tbl[TFW_HTTP_HDR_HOST], &host); + + loc_val_len = req->uri_path.len; + loc_val_len += host_ptr->len ? host_ptr->len + proto.len : 0; + loc_val_len += rmark->len; + + write_int(loc_val_len, 0x7F, 0, &vlen); + s_vlen.data = vlen.buf; + s_vlen.len = vlen.sz; + + r = tfw_http_msg_expand_data(iter, skb_head, &s_vlen, NULL); + if (unlikely(r)) + return r; + + if (host_ptr->len) { + r = tfw_http_msg_expand_data(iter, skb_head, &proto, NULL); + if (unlikely(r)) + return r; + + r = tfw_http_msg_expand_data(iter, skb_head, host_ptr, NULL); + if (unlikely(r)) + return r; + } + + if (rmark->len) { + r = tfw_http_msg_expand_data(iter, skb_head, rmark, NULL); + if (unlikely(r)) + return r; + } + + r = tfw_http_msg_expand_data(iter, skb_head, &req->uri_path, NULL); + if (unlikely(r)) + return r; + + hdrs_len = s_vlen.len + loc_val_len; + + /* Add 'set-cookie' header. */ + h_sc.hpack_idx = 55; + r = tfw_hpack_encode(resp, &h_sc, TFW_H2_TRANS_EXPAND, false); + if (unlikely(r)) + return r; + + write_int(cookie->len, 0x7F, 0, &vlen); + s_vlen.data = vlen.buf; + s_vlen.len = vlen.sz; + + r = tfw_http_msg_expand_data(iter, skb_head, &s_vlen, NULL); + if (unlikely(r)) + return r; + + r = tfw_http_msg_expand_data(iter, skb_head, cookie, NULL); + if (unlikely(r)) + return r; + + hdrs_len += s_vlen.len + cookie->len; + + /* + * Set message body (if exists) into DATA frame header. In this case + * 'content-length' header also must be added into HEADERS frame. + */ + if (body) { + TfwStr f_hdr = { + .data = buf, + .len = sizeof(buf) + }; + + if (WARN_ON_ONCE(!body->len)) + return -EINVAL; + + if (body->len > FRAME_MAX_LENGTH) + return -E2BIG; + + frame_hdr.length = body->len; + frame_hdr.type = HTTP2_DATA; + frame_hdr.flags = HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, &frame_hdr); + + r = tfw_http_msg_expand_data(iter, skb_head, &f_hdr, NULL); + if (unlikely(r)) + return r; + + r = tfw_http_msg_expand_data(iter, skb_head, body, NULL); + if (unlikely(r)) + return r; + } + + hdrs_len += mit->acc_len; + if (hdrs_len > FRAME_MAX_LENGTH) + return -E2BIG; + + /* Set frame header for HEADERS. */ + frame_hdr.length = hdrs_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + if (!body) + frame_hdr.flags |= HTTP2_F_END_STREAM; + + tfw_h2_pack_frame_header(buf, &frame_hdr); + memcpy_fast((*skb_head)->data, buf, sizeof(buf)); + + return 0; +} + #define S_REDIR_302 S_302 S_CRLF #define S_REDIR_503 S_503 S_CRLF #define S_REDIR_GEN " Redirection" S_CRLF @@ -428,11 +574,11 @@ tfw_http_prep_date(char *buf) * The response redirects the client to the same URI as the original request, * but it includes 'Set-Cookie:' header field that sets Tempesta sticky cookie. * If JS challenge is enabled, then body contained JS challenge is provided. - * Body string contains the 'Content-Legth' header, CRLF and body itself. + * Body string contains the 'Content-Length' header, CRLF and body itself. */ int -tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, TfwStr *rmark, - TfwStr *cookie, TfwStr *body) +tfw_h1_prep_redirect(TfwHttpResp *resp, unsigned short status, TfwStr *rmark, + TfwStr *cookie, TfwStr *body) { TfwHttpReq *req = resp->req; size_t data_len; @@ -511,7 +657,7 @@ tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, TfwStr *rmark, data_len += req->uri_path.len + h_common_2.len + cookie->len; data_len += cookie_crlf->len + r_end->len; - if (tfw_http_msg_setup(resp, &it, data_len, 0)) + if (tfw_http_msg_setup((TfwHttpMsg *)resp, &it, data_len, 0)) return TFW_BLOCK; tfw_http_prep_date(__TFW_STR_CH(&h_common_1, 1)->data); @@ -542,14 +688,13 @@ tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, TfwStr *rmark, #define S_304_PART_01 S_304 S_CRLF #define S_304_KEEP S_F_CONNECTION S_V_CONN_KA S_CRLF #define S_304_CLOSE S_F_CONNECTION S_V_CONN_CLOSE S_CRLF + /* - * HTTP 304 response: Not Modified. + * Preparing 304 response (Not Modified) for HTTP/1.1-client. */ int -tfw_http_prep_304(TfwHttpMsg *resp, TfwHttpReq *req, TfwMsgIter *it, - size_t hdrs_size) +tfw_http_prep_304(TfwHttpReq *req, struct sk_buff **skb_head, TfwMsgIter *it) { - size_t data_len = SLEN(S_304_PART_01); int ret = 0; static TfwStr rh = { .data = S_304_PART_01, .len = SLEN(S_304_PART_01) }; @@ -565,21 +710,19 @@ tfw_http_prep_304(TfwHttpMsg *resp, TfwHttpReq *req, TfwMsgIter *it, else if (test_bit(TFW_HTTP_B_CONN_KA, req->flags)) end = &crlf_keep; - /* Add variable part of data length to get the total */ - data_len += hdrs_size; - if (end) - data_len += end->len; - - if (tfw_http_msg_setup(resp, it, data_len, 0)) - return TFW_BLOCK; + ret = tfw_http_msg_expand_data(it, skb_head, &rh, NULL); + if (unlikely(ret)) + return ret; - ret = tfw_msg_write(it, &rh); - if (end) - ret |= tfw_msg_write(it, end); + if (end) { + ret = tfw_http_msg_expand_data(it, skb_head, end, NULL); + if (unlikely(ret)) + return ret; + } T_DBG("Send HTTP 304 response\n"); - return ret ? TFW_BLOCK : TFW_PASS; + return 0; } /* @@ -713,43 +856,17 @@ tfw_http_enum_resp_code(int status) } } -/* - * Static index determination for response ':status' pseudo-header (see RFC - * 7541 Appendix A for details). - */ -static inline unsigned short -tfw_h2_pseudo_index(unsigned short status) -{ - switch (status) { - case 200: - return 8; - case 204: - return 9; - case 206: - return 10; - case 304: - return 11; - case 400: - return 12; - case 404: - return 13; - case 500: - return 14; - default: - return 0; - } -} - /** - * Write HTTP/2 pseudo-header fields. Only ':status' is defined as response - * pseudo-header and all HTTP/2 responses must contain that. - * https://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.4 + * Write HTTP/2 ':status' pseudo-header. The ':status' is only defined + * pseudo-header for the response and all HTTP/2 responses must contain it. + * https://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.4. */ -static int -tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) +int +tfw_h2_resp_status_write(TfwHttpResp *resp, unsigned short status, + TfwH2TransOp op, bool cache) { int ret; - unsigned short index = tfw_h2_pseudo_index(resp->status); + unsigned short index = tfw_h2_pseudo_index(status); char buf[H2_STAT_VAL_LEN]; TfwStr s_hdr = { .chunks = (TfwStr []){ @@ -771,17 +888,16 @@ tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) s_hdr.flags |= TFW_STR_FULL_INDEX; } - if (!tfw_ultoa(resp->status, __TFW_STR_CH(&s_hdr, 1)->data, - H2_STAT_VAL_LEN)) + if (!tfw_ultoa(status, __TFW_STR_CH(&s_hdr, 1)->data, H2_STAT_VAL_LEN)) return -E2BIG; - if ((ret = tfw_hpack_encode(resp, &s_hdr, op))) + if ((ret = tfw_hpack_encode(resp, &s_hdr, op, !cache))) return ret; return 0; } -static inline void +void tfw_h2_resp_fwd(TfwHttpResp *resp) { TfwHttpReq *req = resp->req; @@ -811,9 +927,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) TfwFrameHdr frame_hdr; TfwHttpTransIter *mit; char *date_val, *data_ptr; - unsigned int len_be; unsigned long nlen, vlen; - unsigned char *dst_len_p, *src_len_p; unsigned char buf[FRAME_HEADER_SIZE]; TfwStr *start, *date, *clen, *srv, *body; TfwH2Ctx *ctx = tfw_h2_context(req->conn); @@ -835,6 +949,8 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) } } + frame_hdr.stream_id = stream_id; + code = tfw_http_enum_resp_code(status); if (code == RESP_NUM) { T_WARN("Unexpected response error code: [%d]\n", status); @@ -859,24 +975,9 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) goto err_setup; } - /* Create frame header for HEADERS. Note, that we leave the length of - * HEADERS frame unset, to be filled later (below) after all headers - * will be processed, indexed (if needed) and written into the target - * skbs. */ - frame_hdr.stream_id = stream_id; - frame_hdr.length = 0; - frame_hdr.type = HTTP2_HEADERS; - frame_hdr.flags = HTTP2_F_END_HEADERS; - if (!body->data) - frame_hdr.flags |= HTTP2_F_END_STREAM; - tfw_h2_pack_frame_header(buf, &frame_hdr); - - /* Set frame header and HTTP/2 ':status' pseudo-header. */ - if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr)) - goto err_setup; - - resp->status = (unsigned short)status; - if (tfw_h2_pseudo_write(resp, TFW_H2_TRANS_EXPAND)) + /* Set HTTP/2 ':status' pseudo-header. */ + mit->start_off = FRAME_HEADER_SIZE; + if (tfw_h2_resp_status_write(resp, status, TFW_H2_TRANS_EXPAND, false)) goto err_setup; /* @@ -893,7 +994,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) __TFW_STR_CH(&hdr, 1)->len = vlen = date->len; hdr.len = nlen + vlen; hdr.hpack_idx = 33; - if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND, true)) goto err_setup; clen = TFW_STR_CLEN_CH(msg); @@ -904,7 +1005,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) - data_ptr - SLEN(S_CRLF); hdr.len = nlen + vlen; hdr.hpack_idx = 28; - if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND, true)) goto err_setup; srv = TFW_STR_SRV_CH(msg); @@ -915,19 +1016,12 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) - data_ptr - SLEN(S_CRLF); hdr.len = nlen + vlen; hdr.hpack_idx = 54; - if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND, true)) goto err_setup; if (WARN_ON_ONCE(!mit->acc_len)) goto err_setup; - /* - * Get the pointer to head skb data to write the frame's accumulated - * length. We can do this here, since we always allocate new skb in - * @tfw_http_msg_expand_data() with SKB_MAX_HEADER length for the - * head part of skb (also, see description of frame header format in - * RFC 7540 section 4.1). - */ if (WARN_ON_ONCE(mit->acc_len > ctx->rsettings.max_frame_sz)) { /* * TODO #1378: multiple frames might be required here. @@ -939,14 +1033,6 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) */ goto err_setup; } - len_be = htonl((unsigned int)mit->acc_len); - src_len_p = (unsigned char *)&len_be; - dst_len_p = (*skb_head)->data; - - *(unsigned short *)dst_len_p = *(unsigned short *)++src_len_p; - src_len_p += 2; - dst_len_p += 2; - *dst_len_p = *src_len_p; /* Create and set frame header and set payload for DATA. */ if (body->data) { @@ -955,11 +1041,26 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) frame_hdr.flags = HTTP2_F_END_STREAM; tfw_h2_pack_frame_header(buf, &frame_hdr); - if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr) - || tfw_http_msg_expand_data(&resp->mit.iter, skb_head, body)) + if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr, + NULL) + || tfw_http_msg_expand_data(&resp->mit.iter, skb_head, body, + NULL)) goto err_setup; } + /* + * Set the frame header for HEADERS. Note, that we leave the place for + * it at the beginning of the response - due to @mit->start_off setting + * above (before the first data is written into the target skb). + */ + frame_hdr.length = mit->acc_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + if (!body->data) + frame_hdr.flags |= HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, &frame_hdr); + memcpy_fast((*skb_head)->data, buf, sizeof(buf)); + /* Send resulting HTTP/2 response and release HPACK encoder index. */ tfw_h2_resp_fwd(resp); @@ -2540,6 +2641,36 @@ tfw_http_set_hdr_date(TfwHttpMsg *hm) return r; } +/* + * Expand HTTP response with 'Date:' header field. + */ +int +tfw_http_expand_hdr_date(TfwHttpResp *resp) +{ + int r; + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwHttpTransIter *mit = &resp->mit; + char *date = *this_cpu_ptr(&g_buf); + TfwStr h_date = { + .chunks = (TfwStr []){ + { .data = S_F_DATE, .len = SLEN(S_F_DATE) }, + { .data = date, .len = SLEN(S_V_DATE) }, + { .data = S_CRLF, .len = SLEN(S_CRLF) } + }, + .len = SLEN(S_F_DATE) + SLEN(S_V_DATE) + SLEN(S_CRLF), + .nchunks = 3 + }; + + tfw_http_prep_date_from(date, resp->date); + r = tfw_http_msg_expand_data(&mit->iter, skb_head, &h_date, NULL); + if (r) + T_ERR("Unable to expand resp [%p] with 'Date:' header\n", resp); + else + T_DBG2("Epanded resp [%p] with 'Date:' header\n", resp); + + return r; +} + /** * Connection is to be closed after response for the request @req is forwarded * to the client. Don't process new requests from the client and update @@ -2552,6 +2683,51 @@ tfw_http_req_set_conn_close(TfwHttpReq *req) set_bit(TFW_HTTP_B_CONN_CLOSE, req->flags); } +/** + * Expand HTTP/1.1 response with hop-by-hop headers. It is implied that this + * procedure should be used only for cases when original hop-by-hop headers + * is already removed from the response: e.g. creation HTTP/1.1-response from + * the cache (see also comments for tfw_http_set_hdr_connection(), + * tfw_http_set_hdr_keep_alive() and tfw_http_adjust_resp()). + */ +int +tfw_http_expand_hbh(TfwHttpResp *resp, unsigned short status) +{ + TfwHttpReq *req = resp->req; + TfwHttpTransIter *mit = &resp->mit; + struct sk_buff **skb_head = &resp->msg.skb_head; + bool proxy_close = test_bit(TFW_HTTP_B_CONN_CLOSE, resp->flags) + && (status / 100 == 4); + TfwStr h_conn = { + .chunks = (TfwStr []){ + { .data = S_F_CONNECTION, .len = SLEN(S_F_CONNECTION) }, + {}, + { .data = S_CRLF, .len = SLEN(S_CRLF) } + }, + .len = SLEN(S_F_CONNECTION) + SLEN(S_CRLF), + .nchunks = 3 + }; + + if (unlikely(test_bit(TFW_HTTP_B_CONN_CLOSE, req->flags) + || proxy_close)) + { + __TFW_STR_CH(&h_conn, 1)->data = S_V_CONN_CLOSE; + __TFW_STR_CH(&h_conn, 1)->len = SLEN(S_V_CONN_CLOSE); + h_conn.len += SLEN(S_V_CONN_CLOSE); + } + else if (test_bit(TFW_HTTP_B_CONN_KA, req->flags)) + { + __TFW_STR_CH(&h_conn, 1)->data = S_V_CONN_KA; + __TFW_STR_CH(&h_conn, 1)->len = SLEN(S_V_CONN_KA); + h_conn.len += SLEN(S_V_CONN_KA); + } + + if (unlikely(proxy_close)) + tfw_http_req_set_conn_close(req); + + return tfw_http_msg_expand_data(&mit->iter, skb_head, &h_conn, NULL); +} + /** * Remove Connection header from HTTP message @msg if @conn_flg is zero, * and replace or set a new header value otherwise. @@ -2626,6 +2802,29 @@ tfw_http_set_hdr_keep_alive(TfwHttpMsg *hm, unsigned long conn_flg) } } +/* + * In case if response is stale, we should pass it with a warning. + */ +int +tfw_http_expand_stale_warn(TfwHttpResp *resp) +{ + /* TODO: adjust for #865 */ + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwHttpTransIter *mit = &resp->mit; + TfwStr wh = { + .chunks = (TfwStr []){ + { .data = S_WARN, .len = SLEN(S_WARN) }, + { .data = S_DLM, .len = SLEN(S_DLM) }, + { .data = S_V_WARN, .len = SLEN(S_V_WARN) }, + { .data = S_CRLF, .len = SLEN(S_CRLF) } + }, + .len = SLEN(S_WARN) + SLEN(S_DLM) + SLEN(S_V_WARN) + SLEN(S_CRLF), + .nchunks = 4, + }; + + return tfw_http_msg_expand_data(&mit->iter, skb_head, &wh, NULL); +} + static int tfw_http_add_hdr_via(TfwHttpMsg *hm) { @@ -2682,55 +2881,6 @@ tfw_http_add_x_forwarded_for(TfwHttpMsg *hm) return r; } -static int -tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req) -{ - size_t i; - int mod_type = (hm == (TfwHttpMsg *)req) ? TFW_VHOST_HDRMOD_REQ - : TFW_VHOST_HDRMOD_RESP; - TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, - mod_type); - if (!h_mods) - return 0; - - for (i = 0; i < h_mods->sz; ++i) { - TfwHdrModsDesc *d = &h_mods->hdrs[i]; - /* - * Header is stored optimized for HTTP2: without delimiter - * between header and value. Add it as separate chunk as - * required for tfw_http_msg_hdr_xfrm_str. - */ - TfwStr h_mdf = { - .chunks = (TfwStr []){ - {}, - { .data = S_DLM, .len = SLEN(S_DLM) }, - {} - }, - .len = SLEN(S_DLM), - .nchunks = 2 /* header name + delimeter. */ - }; - int r; - - h_mdf.chunks[0] = d->hdr->chunks[0]; - if (d->hdr->nchunks == 2) { - h_mdf.chunks[2] = d->hdr->chunks[1]; - h_mdf.nchunks += 1; - } - h_mdf.len += d->hdr->len; - h_mdf.flags = d->hdr->flags; - h_mdf.eolen += d->hdr->eolen; - r = tfw_http_msg_hdr_xfrm_str(hm, &h_mdf, d->hid, d->append); - if (r) { - T_ERR("can't update location-specific header in msg %p\n", - hm); - return r; - } - T_DBG2("updated location-specific header in msg %p\n", hm); - } - - return 0; -} - /** * Compose Content-Type header field from scratch. * @@ -2783,6 +2933,76 @@ tfw_http_should_validate_post_req(TfwHttpReq *req) return false; } +int +tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req, bool cache) +{ + size_t i; + bool hm_req = (hm == (TfwHttpMsg *)req); + int mod_type = hm_req ? TFW_VHOST_HDRMOD_REQ : TFW_VHOST_HDRMOD_RESP; + TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, + mod_type); + BUG_ON(hm_req && cache); + if (!h_mods) + return 0; + + for (i = 0; i < h_mods->sz; ++i) { + int r; + TfwHdrModsDesc *d = &h_mods->hdrs[i]; + /* + * Header is stored optimized for HTTP2: without delimiter + * between header and value. Add it as separate chunk as + * required for tfw_http_msg_hdr_xfrm_str. + */ + TfwStr h_mdf = { + .chunks = (TfwStr []){ + {}, + { .data = S_DLM, .len = SLEN(S_DLM) }, + {} + }, + .len = SLEN(S_DLM), + .nchunks = 2 /* header name + delimeter. */ + }; + + h_mdf.chunks[0] = d->hdr->chunks[0]; + if (d->hdr->nchunks == 2) { + h_mdf.chunks[2] = d->hdr->chunks[1]; + h_mdf.nchunks += 1; + } + h_mdf.len += d->hdr->len; + h_mdf.flags = d->hdr->flags; + h_mdf.eolen += d->hdr->eolen; + + if (!hm_req && cache) { + TfwHttpResp *resp = (TfwHttpResp *)hm; + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwHttpTransIter *mit = &resp->mit; + /* + * Skip the configured header if we have already + * processed it during cache reading, or if the header + * is configured for deletion (without value chunk). + */ + if (test_bit(i, mit->found) || h_mdf.nchunks < 3) + continue; + + r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &h_mdf, NULL); + } else { + r = tfw_http_msg_hdr_xfrm_str(hm, &h_mdf, d->hid, + d->append); + } + + if (r) { + T_ERR("can't update location-specific header in msg %p\n", + hm); + return r; + } + + T_DBG2("updated location-specific header in msg %p\n", hm); + } + + return 0; +} + /** * Adjust the request before proxying it to real server. */ @@ -2808,7 +3028,7 @@ tfw_h1_adjust_req(TfwHttpReq *req) if (r < 0) return r; - r = tfw_http_set_loc_hdrs(hm, req); + r = tfw_http_set_loc_hdrs(hm, req, false); if (r < 0) return r; @@ -3305,7 +3525,7 @@ tfw_http_adjust_resp(TfwHttpResp *resp) conn_flg = BIT(TFW_HTTP_B_CONN_KA); } - r = tfw_http_sess_resp_process(resp); + r = tfw_http_sess_resp_process(resp, false); if (r < 0) return r; @@ -3325,24 +3545,10 @@ tfw_http_adjust_resp(TfwHttpResp *resp) if (r < 0) return r; - r = tfw_http_set_loc_hdrs(hm, req); + r = tfw_http_set_loc_hdrs(hm, req, false); if (r < 0) return r; - if (test_bit(TFW_HTTP_B_RESP_STALE, resp->flags)) { -#define S_WARN_110 "Warning: 110 - Response is stale" - /* TODO: adjust for #865 */ - TfwStr wh = { - .data = S_WARN_110, - .len = SLEN(S_WARN_110), - .eolen = 2 - }; - r = tfw_http_msg_hdr_add(hm, &wh); - if (r) - return r; -#undef S_WARN_110 - } - if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { r = tfw_http_set_hdr_date(hm); if (r < 0) @@ -3536,18 +3742,16 @@ tfw_h2_hdr_map(TfwHttpResp *resp, const TfwStr *hdr, unsigned int id) static int tfw_h2_add_hdr_via(TfwHttpResp *resp) { -#define NM_VIA "via" -#define V_PROTO "2.0 " int r; TfwGlobal *g_vhost = tfw_vhost_get_global(); TfwStr via = { .chunks = (TfwStr []) { - { .data = NM_VIA, .len = SLEN(NM_VIA) }, - { .data = V_PROTO, .len = SLEN(V_PROTO) }, + { .data = S_VIA, .len = SLEN(S_VIA) }, + { .data = S_VIA_H2_PROTO, .len = SLEN(S_VIA_H2_PROTO) }, { .data = *this_cpu_ptr(&g_buf), .len = g_vhost->hdr_via_len }, }, - .len = SLEN(NM_VIA) + SLEN(V_PROTO) + g_vhost->hdr_via_len, + .len = SLEN(S_VIA) + SLEN(S_VIA_H2_PROTO) + g_vhost->hdr_via_len, .nchunks = 3 }; @@ -3556,29 +3760,37 @@ tfw_h2_add_hdr_via(TfwHttpResp *resp) via.hpack_idx = 60; - r = __hdr_h2_add(resp, &via); + r = tfw_hpack_encode(resp, &via, TFW_H2_TRANS_ADD, true); if (unlikely(r)) T_ERR("HTTP/2: unable to add 'via' header (resp=[%p])\n", resp); else T_DBG3("%s: added 'via' header, resp=[%p]\n", __func__, resp); return r; -#undef NM_VIA -#undef V_PROTO } /* * Same as @tfw_http_set_hdr_date(), but intended for usage in HTTP/1.1=>HTTP/2 * transformation. */ -static int -tfw_h2_add_hdr_date(TfwHttpResp *resp) +int +tfw_h2_add_hdr_date(TfwHttpResp *resp, TfwH2TransOp op, bool cache) { int r; char *s_date = *this_cpu_ptr(&g_buf); + TfwStr hdr = { + .chunks = (TfwStr []){ + { .data = "date", .len = SLEN("date") }, + { .data = s_date, .len = SLEN(S_V_DATE) }, + }, + .len = SLEN("date") + SLEN(S_V_DATE), + .nchunks = 2 + }; tfw_http_prep_date_from(s_date, resp->date); - r = tfw_h2_msg_hdr_add(resp, "date", SLEN("date"), s_date, - SLEN(S_V_DATE), TFW_HTTP_HDR_RAW, 33); + + hdr.hpack_idx = 33; + + r = tfw_hpack_encode(resp, &hdr, op, !cache); if (unlikely(r)) T_ERR("HTTP/2: unable to add 'date' header to response" " [%p]\n", resp); @@ -3588,27 +3800,22 @@ tfw_h2_add_hdr_date(TfwHttpResp *resp) return r; } -static inline int +/* + * In case if response is stale, we should pass it with a warning. + */ +int tfw_h2_set_stale_warn(TfwHttpResp *resp) { -#define WARN_NM "warning" -#define WARN_VAL "110 - Response is stale" - if (test_bit(TFW_HTTP_B_RESP_STALE, resp->flags)) { - TfwStr wh = { - .chunks = (TfwStr []){ - { .data = WARN_NM, .len = SLEN(WARN_NM) }, - { .data = WARN_VAL, .len = SLEN(WARN_VAL) } - }, - .len = SLEN(WARN_NM) + SLEN(WARN_VAL), - .nchunks = 2 - }; - - return __hdr_h2_add(resp, &wh); - } + TfwStr wh = { + .chunks = (TfwStr []){ + { .data = S_WARN, .len = SLEN(S_WARN) }, + { .data = S_V_WARN, .len = SLEN(S_V_WARN) } + }, + .len = SLEN(S_WARN) + SLEN(S_V_WARN), + .nchunks = 2 + }; - return 0; -#undef WARN_NM -#undef WARN_VAL + return tfw_hpack_encode(resp, &wh, TFW_H2_TRANS_EXPAND, false); } /* @@ -3621,7 +3828,7 @@ tfw_h2_set_stale_warn(TfwHttpResp *resp) * forwarding), or when the descriptors has been copied previously (e.g. * via @tfw_strcpy_desc() function). */ -int +void tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) { bool name_found = false, val_found = false; @@ -3688,26 +3895,78 @@ tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) } } + /* The header value is empty. */ + if (unlikely(!val_found)) + return; + + if (WARN_ON_ONCE(!last_chunk)) + return; + T_DBG3("%s: hdr_tail=%lu, val_out->len=%lu, last_tail=%lu," " last_chunk->len=%lu, last_chunk->data='%.*s'\n", __func__, hdr_tail, val_out->len, last_tail, last_chunk->len, (int)last_chunk->len, last_chunk->data); - if (WARN_ON_ONCE(!last_chunk)) - return -EINVAL; - val_out->nchunks = chunk - val_out->chunks; val_out->len -= hdr_tail; last_chunk->len -= last_tail; +} + +/* + * Split header in two parts: name and value, with descriptors copying (see + * description of @tfw_http_hdr_split() for details). + * + * TODO: this procedure should be evicted after HTTP/1.1-parser extending + * (and @tfw_http_hdr_split()/@tfw_h2_msg_hdr_length procedures upgrading) in + * order to split headers into separate chunks for name, ':', LWS, value, and + * RWS - during the HTTP-message parsing stage. + */ +int +tfw_http_hdr_split_cp(TfwPool *pool, TfwStr *hdr, TfwStr *name_out, + TfwStr *val_out) +{ + TfwStr *h; + + BUG_ON(TFW_STR_DUP(hdr)); + + h = tfw_strdup_desc(pool, hdr); + if (unlikely(!h)) + return -ENOMEM; + + tfw_http_hdr_split(h, name_out, val_out); return 0; } -static int -tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods) +unsigned long +tfw_h2_hdr_size(unsigned long n_len, unsigned long v_len, + unsigned short st_index) +{ + unsigned long size; + + if (st_index) { + size = tfw_hpack_int_size(st_index, 0xF); + } else { + size = 1; + size += tfw_hpack_int_size(n_len, 0x7F); + size += n_len; + } + size += tfw_hpack_int_size(v_len, 0x7F); + size += v_len; + + return size; +} + +int +tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods, + bool cache) { unsigned int i; TfwHttpTransIter *mit = &resp->mit; + TfwH2TransOp op = cache ? TFW_H2_TRANS_EXPAND : TFW_H2_TRANS_ADD; + + if (!h_mods) + return 0; if (!h_mods) return 0; @@ -3719,7 +3978,7 @@ tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods) if (test_bit(i, mit->found) || !TFW_STR_CHUNK(desc->hdr, 1)) continue; - r = __hdr_h2_add(resp, desc->hdr); + r = tfw_hpack_encode(resp, desc->hdr, op, !cache); if (unlikely(r)) return r; } @@ -4044,13 +4303,16 @@ tfw_h2_error_resp(TfwHttpReq *req, int status, bool reply, bool attack, /* * If reply should be sent and this is not the attack case - we * can just send error response, leave the connection alive and - * drop request's corresponding stream; in this case stream will be - * switched into closed state locally (in @tfw_h2_resp_adjust_fwd() - * during error response sending, or below in @tfw_h2_stream_id_close(), - * if no error response is needed), and remotely - due to END_STREAM - * flag set in the last frame of error response; in case of attack - * we must close entire connection, and GOAWAY frame should be sent - * (RFC 7540 section 6.8) after error response. + * drop request's corresponding stream; in this case stream either + * is already in locally closed state (switched in + * @tfw_h2_stream_id_close() during failed proxy/internal response + * creation) or will be switched into locally closed state in + * @tfw_h2_send_resp() (or in @tfw_h2_stream_id_close() if no error + * response is needed) below; remotely (i.e. on client side) stream + * will be closed - due to END_STREAM flag set in the last frame of + * error response; in case of attack we must close entire connection, + * and GOAWAY frame should be sent (RFC 7540 section 6.8) after + * error response. */ if (reply) { tfw_h2_send_resp(req, status, 0); @@ -4267,7 +4529,8 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) if (unlikely(r)) goto clean; - r = tfw_h2_pseudo_write(resp, TFW_H2_TRANS_SUB); + r = tfw_h2_resp_status_write(resp, resp->status, TFW_H2_TRANS_SUB, + false); if (unlikely(r)) goto clean; @@ -4289,7 +4552,7 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) hdrs_end = true; } - r = tfw_hpack_encode(resp, &hdr, op); + r = tfw_hpack_encode(resp, &hdr, op, true); if (unlikely(r)) goto clean; @@ -4301,7 +4564,7 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) * processed above and which have non-empty value (i.e. configured * not for deletion). */ - r = tfw_http_sess_resp_process(resp); + r = tfw_http_sess_resp_process(resp, false); if (unlikely(r)) goto clean; @@ -4309,22 +4572,17 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) if (unlikely(r)) goto clean; - r = tfw_h2_set_stale_warn(resp); - if (unlikely(r)) - goto clean; - if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { - r = tfw_h2_add_hdr_date(resp); + r = tfw_h2_add_hdr_date(resp, TFW_H2_TRANS_ADD, false); if (unlikely(r)) goto clean; } - r = TFW_H2_MSG_HDR_ADD(resp, "server", TFW_NAME "/" TFW_VERSION, - TFW_HTTP_HDR_SERVER, 54); + r = TFW_H2_MSG_HDR_ADD(resp, "server", TFW_SERVER, 54); if (unlikely(r)) goto clean; - r = tfw_h2_resp_add_loc_hdrs(resp, h_mods); + r = tfw_h2_resp_add_loc_hdrs(resp, h_mods, false); if (unlikely(r)) goto clean; @@ -4367,18 +4625,11 @@ tfw_http_req_cache_service(TfwHttpResp *resp) WARN_ON_ONCE(!list_empty(&req->fwd_list)); WARN_ON_ONCE(!list_empty(&req->nip_list)); - if (TFW_MSG_H2(req)) { - tfw_h2_resp_adjust_fwd(resp); - } else { - if (tfw_http_adjust_resp(resp)) { - tfw_http_conn_msg_free((TfwHttpMsg *)resp); - tfw_http_send_resp(req, 500, "response dropped:" - " processing error"); - TFW_INC_STAT_BH(clnt.msgs_otherr); - return; - } + if (TFW_MSG_H2(req)) + tfw_h2_resp_fwd(resp); + else tfw_http_resp_fwd(resp); - } + TFW_INC_STAT_BH(clnt.msgs_fromcache); } @@ -5621,6 +5872,21 @@ tfw_http_req_key_calc(TfwHttpReq *req) } EXPORT_SYMBOL(tfw_http_req_key_calc); +TfwHdrModsDesc * +tfw_http_find_desc(const TfwStr *hdr, const TfwHdrMods *h_mods) +{ + int i; + + for (i = 0; i < h_mods->sz; ++i) { + TfwHdrModsDesc *desc = &h_mods->hdrs[i]; + + if (!__hdr_name_cmp(hdr, TFW_STR_CHUNK(desc->hdr, 0))) + return desc; + } + + return NULL; +} + static TfwConnHooks http_conn_hooks = { .conn_init = tfw_http_conn_init, .conn_repair = tfw_http_conn_repair, diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index 632333e762..944ddfb4e0 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -284,8 +284,6 @@ enum { TFW_HTTP_B_HDR_DATE, /* Response has header 'Last-Modified:'. */ TFW_HTTP_B_HDR_LMODIFIED, - /* Response is stale, but pass with a warning. */ - TFW_HTTP_B_RESP_STALE, /* Response is fully processed and ready to be forwarded to the client. */ TFW_HTTP_B_RESP_READY, @@ -302,6 +300,9 @@ enum { ((!hmmsg->conn || TFW_CONN_TYPE(hmmsg->conn) & Conn_Srv) && \ hmmsg->pair && TFW_MSG_H2(hmmsg->pair)) +#define H2_STAT_VAL_LEN 3 +#define RESP_BUF_LEN 128 + /** * The structure to hold data for an HTTP error response. * An error response is sent later in an unlocked queue context. @@ -508,6 +509,8 @@ typedef struct { * Iterator for message HTTP/2 transformation process. * * @map - indirection map for tracking headers order in skb; + * @start_off - initial offset during copying response data into + * skb (for subsequent insertion of HTTP/2 frame header); * @curr - current header index in the @map; * @next - operation (with necessary attributes) which should be executed * with next header; @@ -520,6 +523,7 @@ typedef struct { */ typedef struct { TfwHttpHdrMap *map; + unsigned int start_off; unsigned int curr; TfwNextHdrOp next; DECLARE_BITMAP (found, TFW_USRHDRS_ARRAY_SZ); @@ -614,6 +618,33 @@ tfw_http_resp_code_range(const int n) return n <= HTTP_CODE_MAX && n >= HTTP_CODE_MIN; } +/* + * Static index determination for response ':status' pseudo-header (see RFC + * 7541 Appendix A for details). + */ +static inline unsigned short +tfw_h2_pseudo_index(unsigned short status) +{ + switch (status) { + case 200: + return 8; + case 204: + return 9; + case 206: + return 10; + case 304: + return 11; + case 400: + return 12; + case 404: + return 13; + case 500: + return 14; + default: + return 0; + } +} + typedef void (*tfw_http_cache_cb_t)(TfwHttpMsg *); /* External HTTP functions. */ @@ -626,20 +657,37 @@ void tfw_http_resp_fwd(TfwHttpResp *resp); void tfw_http_resp_build_error(TfwHttpReq *req); int tfw_cfgop_parse_http_status(const char *status, int *out); void tfw_http_hm_srv_send(TfwServer *srv, char *data, unsigned long len); +int tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req, bool cache); +int tfw_http_expand_stale_warn(TfwHttpResp *resp); +int tfw_http_expand_hdr_date(TfwHttpResp *resp); +int tfw_http_expand_hbh(TfwHttpResp *resp, unsigned short status); +void tfw_h2_resp_fwd(TfwHttpResp *resp); int tfw_h2_hdr_map(TfwHttpResp *resp, const TfwStr *hdr, unsigned int id); - +int tfw_h2_add_hdr_date(TfwHttpResp *resp, TfwH2TransOp op, bool cache); +int tfw_h2_set_stale_warn(TfwHttpResp *resp); +int tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods, + bool cache); +int tfw_h2_resp_status_write(TfwHttpResp *resp, unsigned short status, + TfwH2TransOp op, bool cache); /* * Functions to send an HTTP error response to a client. */ -int tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, - TfwStr *rmark, TfwStr *cookie, TfwStr *body); -int tfw_http_prep_304(TfwHttpMsg *resp, TfwHttpReq *req, TfwMsgIter *msg_it, - size_t hdrs_size); +int tfw_h2_prep_redirect(TfwHttpResp *resp, unsigned short status, + TfwStr *rmark, TfwStr *cookie, TfwStr *body); +int tfw_h1_prep_redirect(TfwHttpResp *resp, unsigned short status, + TfwStr *rmark, TfwStr *cookie, TfwStr *body); +int tfw_http_prep_304(TfwHttpReq *req, struct sk_buff **skb_head, + TfwMsgIter *it); void tfw_http_conn_msg_free(TfwHttpMsg *hm); void tfw_http_send_resp(TfwHttpReq *req, int status, const char *reason); /* Helper functions */ char *tfw_http_msg_body_dup(const char *filename, size_t *len); -int tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out); +void tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out); +int tfw_http_hdr_split_cp(TfwPool *pool, TfwStr *hdr, TfwStr *name_out, + TfwStr *val_out); +unsigned long tfw_h2_hdr_size(unsigned long n_len, unsigned long v_len, + unsigned short st_index); +TfwHdrModsDesc *tfw_http_find_desc(const TfwStr *hdr, const TfwHdrMods *h_mods); #endif /* __TFW_HTTP_H__ */ diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index 32dcbb2f67..ec344de8d4 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -694,11 +694,30 @@ tfw_h2_stream_close(TfwH2Ctx *ctx, unsigned int id, TfwStream **stream, } /* - * Get stream ID for upper layer to prepare and send frame (of type specified - * in @type and with flags set in @flags) with response to client. This - * procedure also unlinks request from corresponding stream (if linked) and - * moves the stream to the queue of closed streams (if it is not contained - * there yet). + * Get stream ID for upper layer to create frames info. + */ +unsigned int +tfw_h2_stream_id(TfwHttpReq *req) +{ + unsigned int id = 0; + TfwH2Ctx *ctx = tfw_h2_context(req->conn); + + spin_lock(&ctx->lock); + + if (req->stream) + id = req->stream->id; + + spin_unlock(&ctx->lock); + + return id; +} + +/* + * Get stream ID for upper layer to prepare and send frame with response to + * client, and process stream FSM for the frame (of type specified in @type + * and with flags set in @flags). This procedure also unlinks request from + * corresponding stream (if linked) and moves the stream to the queue of + * closed streams (if it is not contained there yet). */ unsigned int tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type, @@ -722,6 +741,7 @@ tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type, id = stream->id; } + req->stream = NULL; stream->msg = NULL; __tfw_h2_stream_add_closed(&ctx->hclosed_streams, stream); diff --git a/tempesta_fw/http_frame.h b/tempesta_fw/http_frame.h index b44b1ecb65..ea83dde2b5 100644 --- a/tempesta_fw/http_frame.h +++ b/tempesta_fw/http_frame.h @@ -206,6 +206,7 @@ int tfw_h2_context_init(TfwH2Ctx *ctx); void tfw_h2_context_clear(TfwH2Ctx *ctx); int tfw_h2_frame_process(void *c, TfwFsmData *data); void tfw_h2_conn_streams_cleanup(TfwH2Ctx *ctx); +unsigned int tfw_h2_stream_id(TfwHttpReq *req); unsigned int tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type, unsigned char flags); void tfw_h2_conn_terminate_close(TfwH2Ctx *ctx, TfwH2Err err_code, bool close); diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 2fd4b548ac..004fce0623 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -168,6 +168,13 @@ tfw_http_msg_req_spec_hid(const TfwStr *hdr) /** * Fills @val with second part of special HTTP header containing the header * value. + * + * TODO: this function should be brought to a uniform look with other similar + * procedures( @tfw_http_hdr_split() and @tfw_h2_msg_hdr_length()), more + * likely - aggregated into one general-purpose procedure, after HTTP/1.1-parser + * extending (see also TODO-comment for @tfw_http_hdr_split_cp() procedure); for + * now this procedure leaves RWS in the resulting @val and this is not correct + * in context of HTTP/2<=>HTTP/1.1 comparisons, transformations etc. */ void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client) @@ -698,12 +705,6 @@ __hdr_add(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid) return 0; } -int -__hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr) -{ - return tfw_hpack_encode(resp, hdr, TFW_H2_TRANS_ADD); -} - /** * Expand @orig_hdr by appending or replacing with the @hdr. * (CRLF is not accounted in TfwStr representation of HTTP headers). @@ -1316,9 +1317,12 @@ tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, *name_len = *val_off = *val_len = 0; + if (TFW_STR_EMPTY(hdr)) + return 0; + if (op != TFW_H2_TRANS_INPLACE) { /* - * During headers addition (or message expansion) the the source + * During headers addition (or message expansion) the source * @hdr must have the following chunk structure (without the * OWS): * @@ -1466,7 +1470,7 @@ tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, - const TfwStr *src) + const TfwStr *src, unsigned int *start_off) { const TfwStr *c, *end; @@ -1477,10 +1481,21 @@ tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, if (!it->skb) { if (!(it->skb = ss_skb_alloc(SKB_MAX_HEADER))) return -ENOMEM; + /* + * Expanding skb is always used for TLS client + * connections. + */ + skb_shinfo(it->skb)->tx_flags |= SKBTX_SHARED_FRAG; ss_skb_queue_tail(skb_head, it->skb); it->frag = -1; - if (!it->skb_head) + if (!it->skb_head) { it->skb_head = *skb_head; + + if (start_off && *start_off) { + skb_put(it->skb_head, *start_off); + *start_off = 0; + } + } T_DBG3("message expanded by new skb [%p]\n", it->skb); } diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 92b00d7c81..b4e96b4ff3 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -27,6 +27,13 @@ #define S_DLM ": " #define S_SET_COOKIE "set-cookie" #define S_F_SET_COOKIE S_SET_COOKIE S_DLM +#define S_LOCATION "location" +#define S_F_LOCATION S_LOCATION S_DLM +#define S_VIA "via" +#define S_F_VIA S_VIA S_DLM +#define S_VIA_H2_PROTO "2.0 " +#define S_VERSION11 "HTTP/1.1" +#define S_0 S_VERSION11 " " #define SLEN(s) (sizeof(s) - 1) @@ -50,7 +57,6 @@ __tfw_http_msg_set_str_data(TfwStr *str, void *data, struct sk_buff *skb) __tfw_http_msg_set_str_data(str, data, \ ss_skb_peek_tail(&hm->msg.skb_head)) -int __hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr); void __h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name); void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val); void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client); @@ -125,7 +131,7 @@ tfw_h2_msg_transform_setup(TfwHttpTransIter *mit, struct sk_buff *skb, static inline int tfw_h2_msg_hdr_add(TfwHttpResp *resp, char *name, size_t nlen, char *val, - size_t vlen, unsigned int hid, unsigned short idx) + size_t vlen, unsigned short idx) { TfwStr hdr = { .chunks = (TfwStr []){ @@ -137,7 +143,7 @@ tfw_h2_msg_hdr_add(TfwHttpResp *resp, char *name, size_t nlen, char *val, .hpack_idx = idx }; - return __hdr_h2_add(resp, &hdr); + return tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_ADD, true); } int __tfw_http_msg_add_str_data(TfwHttpMsg *hm, TfwStr *str, void *data, @@ -173,7 +179,7 @@ int tfw_http_msg_hdr_close(TfwHttpMsg *hm); int tfw_http_msg_grow_hdr_tbl(TfwHttpMsg *hm); void tfw_http_msg_free(TfwHttpMsg *m); int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, - const TfwStr *src); + const TfwStr *src, unsigned int *start_off); int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name); int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name); unsigned long tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, @@ -186,8 +192,8 @@ void tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, int tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, const char *stop); -#define TFW_H2_MSG_HDR_ADD(hm, name, val, hid, idx) \ +#define TFW_H2_MSG_HDR_ADD(hm, name, val, idx) \ tfw_h2_msg_hdr_add(hm, name, sizeof(name) - 1, val, \ - sizeof(val) - 1, hid, idx) + sizeof(val) - 1, idx) #endif /* __TFW_HTTP_MSG_H__ */ diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index e0a3e8e3d1..6fef26c549 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -83,6 +83,15 @@ do { \ (field)->flags |= TFW_STR_COMPLETE; \ } while (0) +#define __msg_field_chunk_flags(field, flag) \ +do { \ + T_DBG3("parser: add chunk flags: %u\n", flag); \ + TFW_STR_CURR(field)->flags |= flag; \ +} while (0) + +#define __msg_chunk_flags(flag) \ + __msg_field_chunk_flags(&msg->stream->parser.hdr, flag) + #define __msg_hdr_chunk_fixup(data, len) \ tfw_http_msg_add_str_data(msg, &msg->stream->parser.hdr, data, len) @@ -183,7 +192,7 @@ do { \ /* The same as __FSM_MOVE_n(), but exactly for jumps w/o data moving. */ #define __FSM_JMP(to) do { goto to; } while (0) -#define __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, fixup_pos) \ +#define __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, flag, fixup_pos) \ do { \ __fsm_n = __data_remain(p); \ __fsm_sz = tfw_match_##alphabet(p, __fsm_n); \ @@ -194,20 +203,21 @@ do { \ __msg_field_fixup_pos(field, p, __fsm_sz); \ else \ __msg_field_fixup(field, data + len); \ + __msg_field_chunk_flags(field, flag); \ parser->state = &&to; \ p += __fsm_sz; \ __FSM_EXIT(TFW_POSTPONE); \ } \ } while (0) -#define __FSM_MATCH_MOVE_f(alphabet, to, field) \ - __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, false) +#define __FSM_MATCH_MOVE_f(alphabet, to, field, flag) \ + __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, flag, false) -#define __FSM_MATCH_MOVE_pos_f(alphabet, to, field) \ - __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, true) +#define __FSM_MATCH_MOVE_pos_f(alphabet, to, field, flag) \ + __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, flag, true) -#define __FSM_MATCH_MOVE(alphabet, to) \ - __FSM_MATCH_MOVE_f(alphabet, to, &msg->stream->parser.hdr) +#define __FSM_MATCH_MOVE(alphabet, to, flag) \ + __FSM_MATCH_MOVE_f(alphabet, to, &msg->stream->parser.hdr, flag) #define __FSM_MATCH_MOVE_nofixup(alphabet, to) \ do { \ @@ -226,13 +236,10 @@ do { \ * FSMs, and so _I_ in the name means "interior" FSM. */ #define __FSM_I_field_chunk_flags(field, flag) \ -do { \ - T_DBG3("parser: add chunk flags: %u\n", flag); \ - TFW_STR_CURR(field)->flags |= flag; \ -} while (0) + __msg_field_chunk_flags(field, flag) #define __FSM_I_chunk_flags(flag) \ - __FSM_I_field_chunk_flags(&msg->stream->parser.hdr, flag) + __msg_chunk_flags(flag) #define __FSM_I_MOVE_BY_REF_n(to, n) \ do { \ @@ -1004,7 +1011,7 @@ __FSM_STATE(st_curr) { \ */ #define RGEN_HDR_OTHER() \ __FSM_STATE(RGen_HdrOtherN) { \ - __FSM_MATCH_MOVE(token, RGen_HdrOtherN); \ + __FSM_MATCH_MOVE(token, RGen_HdrOtherN, 0); \ if (likely(*(p + __fsm_sz) == ':')) { \ parser->_i_st = &&RGen_HdrOtherV; \ __FSM_MOVE_n(RGen_LWS, __fsm_sz + 1); \ @@ -1017,7 +1024,7 @@ __FSM_STATE(RGen_HdrOtherV) { \ * so pass ctext and VCHAR. \ */ \ __FSM_MATCH_MOVE_pos_f(ctext_vchar, RGen_HdrOtherV, \ - &msg->stream->parser.hdr); \ + &msg->stream->parser.hdr, 0); \ if (!IS_CRLF(*(p + __fsm_sz))) \ TFW_PARSER_BLOCK(RGen_HdrOtherV); \ __msg_hdr_chunk_fixup(p, __fsm_sz); \ @@ -3551,7 +3558,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, __msg_field_finish_pos(&req->uri_path, p, 0); __FSM_MOVE_nofixup(Req_HttpVer); } - __FSM_MATCH_MOVE_pos_f(uri, Req_UriAbsPath, &req->uri_path); + __FSM_MATCH_MOVE_pos_f(uri, Req_UriAbsPath, &req->uri_path, 0); if (unlikely(*(p + __fsm_sz) != ' ')) TFW_PARSER_BLOCK(Req_UriAbsPath); __msg_field_finish_pos(&req->uri_path, p, __fsm_sz); @@ -8620,11 +8627,15 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, * reason-phrase = *( HTAB / SP / VCHAR / obs-text ) */ __FSM_STATE(Resp_ReasonPhrase) { - __FSM_MATCH_MOVE(ctext_vchar, Resp_ReasonPhrase); - p += __fsm_sz; - if (IS_CRLF(*p)) { + /* Store reason-phrase in separate chunk(s). */ + __msg_hdr_chunk_fixup(data, p - data); + __FSM_MATCH_MOVE_pos_f(ctext_vchar, Resp_ReasonPhrase, + &parser->hdr, TFW_STR_VALUE); + if (IS_CRLF(*(p + __fsm_sz))) { parser->_hdr_tag = TFW_HTTP_STATUS_LINE; - __msg_hdr_chunk_fixup(data, __data_off(p)); + __msg_hdr_chunk_fixup(p, __fsm_sz); + __msg_chunk_flags(TFW_STR_VALUE); + p += __fsm_sz; __FSM_JMP(RGen_EoL); } TFW_PARSER_BLOCK(Resp_ReasonPhrase); diff --git a/tempesta_fw/http_sess.c b/tempesta_fw/http_sess.c index b2b1828902..2702386a14 100644 --- a/tempesta_fw/http_sess.c +++ b/tempesta_fw/http_sess.c @@ -206,15 +206,6 @@ tfw_http_sticky_build_redirect(TfwHttpReq *req, StickyVal *sv, RedirMarkVal *mv) if (!tfw_http_sticky_redirect_applied(req)) return TFW_HTTP_SESS_JS_NOT_SUPPORTED; - if (TFW_MSG_H2(req)) { - /* - * TODO #309: add separate flow for HTTP/2 response preparing - * and sending (HPACK index, encode in HTTP/2 format, add frame - * headers and send via @tfw_h2_resp_fwd()). - */ - return TFW_HTTP_SESS_REDIRECT_NEED; - } - if (!(resp = tfw_http_msg_alloc_resp_light(req))) return TFW_HTTP_SESS_FAILURE; @@ -250,8 +241,11 @@ tfw_http_sticky_build_redirect(TfwHttpReq *req, StickyVal *sv, RedirMarkVal *mv) cookie.nchunks++; } - r = tfw_http_prep_redirect((TfwHttpMsg *)resp, sticky->redirect_code, - &rmark, &cookie, body); + r = TFW_MSG_H2(req) + ? tfw_h2_prep_redirect(resp, sticky->redirect_code, &rmark, + &cookie, body) + : tfw_h1_prep_redirect(resp, sticky->redirect_code, &rmark, + &cookie, body); if (r) { tfw_http_msg_free((TfwHttpMsg *)resp); return TFW_HTTP_SESS_FAILURE; @@ -424,11 +418,11 @@ tfw_http_sticky_calc(TfwHttpReq *req, StickyVal *sv) /* * Add Tempesta sticky cookie to an HTTP response. * - * Create a complete 'Set-Cookie:' header field, and add it + * Create a complete 'set-cookie' header field, and add it * to the HTTP response' header block. */ static int -tfw_http_sticky_add(TfwHttpResp *resp) +tfw_http_sticky_add(TfwHttpResp *resp, bool cache) { int r; static const unsigned int len = sizeof(StickyVal) * 2; @@ -460,8 +454,21 @@ tfw_http_sticky_add(TfwHttpResp *resp) PR_TFW_STR(&sticky->name), len, buf); if (to_h2) { + TfwH2TransOp op = cache ? TFW_H2_TRANS_EXPAND : TFW_H2_TRANS_ADD; + set_cookie.hpack_idx = 55; - r = __hdr_h2_add(resp, &set_cookie); + r = tfw_hpack_encode(resp, &set_cookie, op, !cache); + } + else if (cache) { + TfwHttpTransIter *mit = &resp->mit; + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwStr crlf = { .data = S_CRLF, .len = SLEN(S_CRLF) }; + + r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &set_cookie, NULL); + if (!r) + r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &crlf, NULL); } else { r = tfw_http_msg_hdr_add((TfwHttpMsg *)resp, &set_cookie); @@ -825,7 +832,7 @@ tfw_http_sess_req_process(TfwHttpReq *req) * Add Tempesta sticky cookie to an HTTP response if needed. */ int -tfw_http_sess_resp_process(TfwHttpResp *resp) +tfw_http_sess_resp_process(TfwHttpResp *resp, bool cache) { TfwHttpReq *req = resp->req; TfwStickyCookie *sticky = req->vhost->cookie; @@ -846,7 +853,7 @@ tfw_http_sess_resp_process(TfwHttpResp *resp) */ if (test_bit(TFW_HTTP_B_HAS_STICKY, req->flags)) return 0; - return tfw_http_sticky_add(resp); + return tfw_http_sticky_add(resp, cache); } /** diff --git a/tempesta_fw/http_sess.h b/tempesta_fw/http_sess.h index 513bf8ae9e..fce10917aa 100644 --- a/tempesta_fw/http_sess.h +++ b/tempesta_fw/http_sess.h @@ -168,7 +168,7 @@ enum { int tfw_http_sess_obtain(TfwHttpReq *req); void tfw_http_sess_learn(TfwHttpResp *resp); int tfw_http_sess_req_process(TfwHttpReq *req); -int tfw_http_sess_resp_process(TfwHttpResp *resp); +int tfw_http_sess_resp_process(TfwHttpResp *resp, bool cache); void tfw_http_sess_put(TfwHttpSess *sess); void tfw_http_sess_pin_vhost(TfwHttpSess *sess, TfwVhost *vhost); diff --git a/tempesta_fw/http_types.h b/tempesta_fw/http_types.h index 18a8d60967..4ef7a380a7 100644 --- a/tempesta_fw/http_types.h +++ b/tempesta_fw/http_types.h @@ -26,6 +26,8 @@ typedef struct tfw_http_msg_t TfwHttpMsg; typedef struct tfw_http_req_t TfwHttpReq; typedef struct tfw_http_resp_t TfwHttpResp; typedef struct tfw_vhost_t TfwVhost; +typedef struct tfw_hdr_mods_desc_t TfwHdrModsDesc; +typedef struct tfw_hdr_mods_t TfwHdrMods; typedef struct frang_global_cfg_t FrangGlobCfg; typedef struct frang_vhost_cfg_t FrangVhostCfg; typedef struct tfw_http_cookie_t TfwStickyCookie; diff --git a/tempesta_fw/str.c b/tempesta_fw/str.c index 7c3f691121..4a4c53837b 100644 --- a/tempesta_fw/str.c +++ b/tempesta_fw/str.c @@ -771,6 +771,32 @@ tfw_strcpy_desc(TfwStr *dst, TfwStr *src) } EXPORT_SYMBOL(tfw_strcpy_desc); +/** + * Same as @tfw_strcpy_desc(), but allocate chunk descriptors for result string. + */ +TfwStr * +tfw_strdup_desc(TfwPool *pool, const TfwStr *src) +{ + size_t sz; + TfwStr *dst, *d; + const TfwStr *s, *end; + + sz = (src->nchunks + 1) * sizeof(TfwStr); + if (!(dst = (TfwStr *)tfw_pool_alloc(pool, sz))) + return NULL; + + *dst = *src; + dst->chunks = dst + 1; + + d = TFW_STR_CHUNK(dst, 0); + TFW_STR_FOR_EACH_CHUNK(s, src, end) { + *d = *s; + ++d; + } + + return dst; +} + static inline int __tfw_str_insert(TfwPool *pool, TfwStr *dst, TfwStr *src, unsigned int chunk) { diff --git a/tempesta_fw/str.h b/tempesta_fw/str.h index 405bc9c847..7273a5ccce 100644 --- a/tempesta_fw/str.h +++ b/tempesta_fw/str.h @@ -390,6 +390,7 @@ typedef enum { int tfw_strcpy(TfwStr *dst, const TfwStr *src); TfwStr *tfw_strdup(TfwPool *pool, const TfwStr *src); int tfw_strcpy_desc(TfwStr *dst, TfwStr *src); +TfwStr *tfw_strdup_desc(TfwPool *pool, const TfwStr *src); int tfw_strcat(TfwPool *pool, TfwStr *dst, TfwStr *src); int tfw_str_insert(TfwPool *pool, TfwStr *dst, TfwStr *src, unsigned int chunk); diff --git a/tempesta_fw/tempesta_fw.h b/tempesta_fw/tempesta_fw.h index 2d30d8d3c2..81cab61cc4 100644 --- a/tempesta_fw/tempesta_fw.h +++ b/tempesta_fw/tempesta_fw.h @@ -34,6 +34,7 @@ #define TFW_AUTHOR "Tempesta Technologies, Inc" #define TFW_NAME "Tempesta FW" #define TFW_VERSION "0.7.0" +#define TFW_SERVER TFW_NAME "/" TFW_VERSION #define DEF_MAX_PORTS 8 diff --git a/tempesta_fw/vhost.c b/tempesta_fw/vhost.c index db7295ea8d..e9178e6d8d 100644 --- a/tempesta_fw/vhost.c +++ b/tempesta_fw/vhost.c @@ -333,7 +333,7 @@ tfw_vhost_get_hdr_mods(TfwLocation *loc, TfwVhost *vhost, int mod_type) loc = vhost->loc_dflt; if (!loc || !loc->mod_hdrs[mod_type].sz) loc = vh_dflt ? vh_dflt->loc_dflt : NULL; - if (!loc) + if (!loc || !loc->mod_hdrs[mod_type].sz) return NULL; return &loc->mod_hdrs[mod_type]; diff --git a/tempesta_fw/vhost.h b/tempesta_fw/vhost.h index 7dab3334c9..3ec2b3911a 100644 --- a/tempesta_fw/vhost.h +++ b/tempesta_fw/vhost.h @@ -68,11 +68,11 @@ typedef struct { * @hdr - Header string, see @tfw_http_msg_hdr_xfrm_str(); * @add_hdrs - Headers to modify; */ -typedef struct { +struct tfw_hdr_mods_desc_t { TfwStr *hdr; unsigned int hid; bool append; -} TfwHdrModsDesc; +}; /** * Headers modification before forwarding HTTP message. @@ -80,10 +80,10 @@ typedef struct { * @sz - Number of headers to modify; * @hdrs - Headers to modify; */ -typedef struct { +struct tfw_hdr_mods_t { size_t sz; TfwHdrModsDesc *hdrs; -} TfwHdrMods; +}; enum { TFW_VHOST_HDRMOD_REQ, From 4dec0e819720ff10e5288390f477fb87502f9a6a Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Thu, 27 Feb 2020 14:40:47 +0300 Subject: [PATCH 56/64] HTTP/2: changes in processing of headers strings (#309). Parse name, colon, LWS, value and RWS of HTTP/1.1-response headers into separate chunks to facilitate the name/value splitting and colon/OWS eviction during HTTP/1.1=>HTTP/2 response transformation. --- tempesta_fw/cache.c | 44 ++- tempesta_fw/hpack.c | 46 ++- tempesta_fw/http.c | 138 +++++---- tempesta_fw/http.h | 5 +- tempesta_fw/http_msg.c | 174 ----------- tempesta_fw/http_msg.h | 7 - tempesta_fw/http_parser.c | 408 ++++++++++++++++++-------- tempesta_fw/str.h | 3 + tempesta_fw/t/unit/test_hpack.c | 155 ++++++---- tempesta_fw/t/unit/test_http_parser.c | 27 +- 10 files changed, 527 insertions(+), 480 deletions(-) diff --git a/tempesta_fw/cache.c b/tempesta_fw/cache.c index 3cb38944c1..069b68effe 100644 --- a/tempesta_fw/cache.c +++ b/tempesta_fw/cache.c @@ -1123,8 +1123,8 @@ tfw_cache_h2_copy_int(unsigned int *acc_len, unsigned long src, * @return number of copied bytes on success and negative value otherwise. */ static long -tfw_cache_h2_copy_hdr(TfwPool *pool, TfwCacheEntry *ce, char **p, - TdbVRec **trec, TfwStr *hdr, size_t *tot_len) +tfw_cache_h2_copy_hdr(TfwCacheEntry *ce, char **p, TdbVRec **trec, TfwStr *hdr, + size_t *tot_len) { TfwCStr *cs; long n = sizeof(TfwCStr); @@ -1141,16 +1141,7 @@ tfw_cache_h2_copy_hdr(TfwPool *pool, TfwCacheEntry *ce, char **p, TFW_STR_INIT(&s_nm); TFW_STR_INIT(&s_val); - - /* - * We cannot use just @tfw_http_hdr_split() here, since the - * response must be forwarded to the client and its headers - * will be processed further, so we must not invalidate headers - * descriptors (see also description of @tfw_http_hdr_split_cp() - * and @tfw_http_hdr_split() for additional details). - */ - if (tfw_http_hdr_split_cp(pool, hdr, &s_nm, &s_val)) - return -ENOMEM; + tfw_http_hdr_split(hdr, &s_nm, &s_val, true); st_index = hdr->hpack_idx; h_len = tfw_h2_hdr_size(s_nm.len, s_val.len, st_index); @@ -1179,9 +1170,7 @@ tfw_cache_h2_copy_hdr(TfwPool *pool, TfwCacheEntry *ce, char **p, if (dupl) { TFW_STR_INIT(&s_nm); TFW_STR_INIT(&s_val); - if (tfw_http_hdr_split_cp(pool, dup, &s_nm, &s_val)) - return -ENOMEM; - + tfw_http_hdr_split(dup, &s_nm, &s_val, true); st_index = dup->hpack_idx; CSTR_MOVE_HDR(); } @@ -1279,11 +1268,11 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec, { char *e_p; size_t c_size; - unsigned long n_len, v_off, v_len; TDB *db = node_db(); size_t len = 0; unsigned short flags = 0; TfwStr h_val, *c, *end, *h = &resp->h_tbl->tbl[TFW_HTTP_HDR_ETAG]; + TfwStr s_dummy = {}, s_val = {}; #define CHECK_REC_SPACE() \ while (c_size) { \ @@ -1319,8 +1308,8 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec, * occupy 2 bytes (RFC 7541 section 6.2.2). */ e_p += TFW_CSTR_HDRLEN; - tfw_h2_msg_hdr_length(h, &n_len, &v_off, &v_len, TFW_H2_TRANS_INPLACE); - c_size = 2 + tfw_hpack_int_size(v_len, 0x7F); + tfw_http_hdr_split(h, &s_dummy, &s_val, true); + c_size = 2 + tfw_hpack_int_size(s_val.len, 0x7F); CHECK_REC_SPACE(); TFW_STR_FOR_EACH_CHUNK(c, &h_val, end) { if (c->flags & TFW_STR_VALUE) { @@ -1538,8 +1527,7 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr *rph, __save_hdr_304_off(ce, resp, field, TDB_OFF(db->hdr, p)); - n = tfw_cache_h2_copy_hdr(resp->pool, ce, &p, &trec, field, - &tot_len); + n = tfw_cache_h2_copy_hdr(ce, &p, &trec, field, &tot_len); if (unlikely(n < 0)) return n; } @@ -1619,7 +1607,6 @@ static long __cache_entry_size(TfwHttpResp *resp) { TfwStr host_val, *hdr, *hdr_end; - unsigned long n_len, v_off, v_len; TfwHttpReq *req = resp->req; long size, res_size = CE_BODY_SIZE; TfwStr *host = &req->h_tbl->tbl[TFW_HTTP_HDR_HOST]; @@ -1656,15 +1643,18 @@ __cache_entry_size(TfwHttpResp *resp) size = sizeof(TfwCStr); if (!TFW_STR_DUP(hdr)) { - tfw_h2_msg_hdr_length(hdr, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - size += tfw_h2_hdr_size(n_len, v_len, hdr->hpack_idx); + TfwStr s_nm = {}, s_val = {}; + + tfw_http_hdr_split(hdr, &s_nm, &s_val, true); + size += tfw_h2_hdr_size(s_nm.len, s_val.len, + hdr->hpack_idx); } else { TFW_STR_FOR_EACH_DUP(d, hdr, d_end) { + TfwStr s_nm = {}, s_val = {}; + size += sizeof(TfwCStr); - tfw_h2_msg_hdr_length(d, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - size += tfw_h2_hdr_size(n_len, v_len, + tfw_http_hdr_split(d, &s_nm, &s_val, true); + size += tfw_h2_hdr_size(s_nm.len, s_val.len, d->hpack_idx); } } diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index ebc85edc70..b9c8c69cc4 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -2413,6 +2413,36 @@ do { \ #undef SHIFT } +/* + * Copy the header part (i.e. name/value) into @out_buf from @h_field. + * Return pointer on the next position of @out_buf, after the copied data. + * Note that the size of prepared @out_buf must be not less than the + * length of the @h_field. + */ +static char * +tfw_hpack_write(const TfwStr *h_field, char *out_buf) +{ + const TfwStr *c, *end; + + T_DBG3("%s: enter, h_field->len=%lu,\n", __func__, h_field->len); + + if (WARN_ON_ONCE(TFW_STR_EMPTY(h_field))) + return out_buf; + + TFW_STR_FOR_EACH_CHUNK(c, h_field, end) { + if (!c->len) + continue; + + T_DBG3("%s: c->len=%lu, c->data='%.*s'\n", __func__, c->len, + (int)c->len, c->data); + + memcpy_fast(out_buf, c->data, c->len); + out_buf += c->len; + } + + return out_buf; +} + /* * Left rotation of red-black tree. */ @@ -2945,17 +2975,19 @@ tfw_hpack_rbuf_commit(TfwHPackETbl *__restrict tbl, * headers will be evicted from the index table. */ static int -tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, +tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, TfwStr *__restrict hdr, TfwHPackNodeIter *__restrict place, TfwH2TransOp op) { + char *ptr; unsigned long node_size, hdr_len; unsigned short new_size, node_len; unsigned short cur_size = tbl->size, window = tbl->window; - unsigned long nm_len, val_off, val_len; TfwHPackNode *del_list[HPACK_MAX_ENC_EVICTION] = {}; + TfwStr s_nm = {}, s_val = {}; TfwHPackETblIter it = {}; - hdr_len = tfw_h2_msg_hdr_length(hdr, &nm_len, &val_off, &val_len, op); + hdr_len = tfw_http_hdr_split(hdr, &s_nm, &s_val, + op == TFW_H2_TRANS_INPLACE); WARN_ON_ONCE(cur_size > window || window > HPACK_ENC_TABLE_MAX_SIZE); if ((node_size = hdr_len + HPACK_ENTRY_OVERHEAD) > window) { @@ -3050,7 +3082,9 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, it.last->hdr_len = hdr_len; it.last->rindex = ++tbl->idx_acc; - tfw_h2_msg_hdr_write(hdr, nm_len, val_off, val_len, it.last->hdr); + ptr = tfw_hpack_write(&s_nm, it.last->hdr); + tfw_hpack_write(&s_val, ptr); + tfw_hpack_rbuf_commit(tbl, del_list, place, &it); WARN_ON_ONCE(tbl->rb_len > tbl->size); @@ -3066,7 +3100,7 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, */ static TfwHPackETblRes tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, - const TfwStr *__restrict hdr, + TfwStr *__restrict hdr, unsigned short *__restrict out_index, unsigned long *__restrict flags, TfwH2TransOp op) @@ -3560,7 +3594,7 @@ tfw_hpack_hdr_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, if (!hdr || WARN_ON_ONCE(TFW_STR_PLAIN(hdr) || TFW_STR_DUP(hdr))) return -EINVAL; - tfw_http_hdr_split(hdr, &s_name, &s_val); + tfw_http_hdr_split(hdr, &s_name, &s_val, true); if (unlikely(!name_indexed)) { TfwHPackInt nlen; diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 0d6f088509..3cebeadb0d 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3819,39 +3819,74 @@ tfw_h2_set_stale_warn(TfwHttpResp *resp) } /* - * Split header in two parts: name and value, evicting ':' and OWS. + * Split header in two parts: name and value, evicting ':' and OWS. Return + * the resulting length of both parts. * - * NOTE: use this function with caution since it changes the underlying - * @TfwStr descriptors of @hdr; thus, this procedure should be used - * only in cases when the headers descriptors will not be needed any more - * (e.g. during message final transformation/adjusting, just before - * forwarding), or when the descriptors has been copied previously (e.g. - * via @tfw_strcpy_desc() function). + * NOTE: this function is intended for response processing only (during + * HTTP/1.1=>HTTP/2 transformation), since the response HTTP parser + * supports splitting the header name, colon, LWS, value and RWS into + * different chunks. */ -void -tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) +unsigned long +tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out, bool inplace) { - bool name_found = false, val_found = false; - unsigned long tail, last_tail = 0, hdr_tail = 0; + unsigned long hdr_tail = 0; TfwStr *chunk, *end, *last_chunk = NULL; + bool name_found = false, val_found = false; BUG_ON(!TFW_STR_EMPTY(name_out) || !TFW_STR_EMPTY(val_out)); + if (WARN_ON_ONCE(TFW_STR_PLAIN(hdr))) + return 0; + + if (TFW_STR_EMPTY(hdr)) + return 0; + + if (!inplace) { + unsigned long off = 0; + /* + * During headers addition (or message expansion) the source + * @hdr must have the following chunk structure (without the + * OWS): + * + * { name [S_DLM] value1 [value2 [value3 ...]] }. + * + */ + *name_out = *hdr->chunks; + + chunk = TFW_STR_CHUNK(hdr, 1); + if (WARN_ON_ONCE(!chunk)) + return 0; + + if (chunk->len == SLEN(S_DLM) + && *(short *)chunk->data == *(short *)S_DLM) + { + off = SLEN(S_DLM); + chunk = TFW_STR_CHUNK(hdr, 2); + if (WARN_ON_ONCE(!chunk)) + return 0; + } + + val_out->chunks = chunk; + val_out->nchunks = hdr->chunks + hdr->nchunks - chunk; + val_out->len = hdr->len - name_out->len - off; + + return hdr->len - off; + } + name_out->chunks = hdr->chunks; TFW_STR_FOR_EACH_CHUNK(chunk, hdr, end) { - unsigned long idx; - if (!chunk->len) continue; if (!name_found) { - ++name_out->nchunks; - name_out->len += chunk->len; - if (chunk->data[chunk->len - 1] == ':') { - --name_out->len; - TFW_STR_LAST(name_out)->len -= 1; + if (chunk->data[0] == ':') { + WARN_ON_ONCE(chunk->len != 1); name_found = true; + } else { + ++name_out->nchunks; + name_out->len += chunk->len; } continue; } @@ -3861,8 +3896,7 @@ tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) * value; thus, we can skip length of the entire (LWS) chunks. */ if (!val_found) { - if (unlikely(chunk->data[0] == ' ' - || chunk->data[0] == '\t')) + if (unlikely(chunk->flags & TFW_STR_OWS)) continue; val_out->chunks = chunk; @@ -3871,71 +3905,33 @@ tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) val_out->len += chunk->len; - /* - * Skip OWS after the header value (RWS); accumulate the length - * in @tail for RWS cutting off (if this is not the end chunk, - * @tail will be reset). + /* + * Skip OWS after the header value (RWS) - they must be in + * separate chunks too. */ - tail = 0; - idx = chunk->len - 1; - while (chunk->data[idx] == ' ' - || chunk->data[idx] == '\t') - { - ++tail; - if (unlikely(!idx)) - break; - --idx; - } - - if (unlikely(tail == chunk->len)) { - hdr_tail += tail; + if (unlikely(chunk->flags & TFW_STR_OWS)) { + hdr_tail += chunk->len; } else { last_chunk = chunk; - last_tail = hdr_tail = tail; + hdr_tail = 0; } } /* The header value is empty. */ if (unlikely(!val_found)) - return; + return name_out->len; if (WARN_ON_ONCE(!last_chunk)) - return; + return 0; - T_DBG3("%s: hdr_tail=%lu, val_out->len=%lu, last_tail=%lu," - " last_chunk->len=%lu, last_chunk->data='%.*s'\n", __func__, - hdr_tail, val_out->len, last_tail, last_chunk->len, - (int)last_chunk->len, last_chunk->data); + T_DBG3("%s: hdr_tail=%lu, val_out->len=%lu, last_chunk->len=%lu," + " last_chunk->data='%.*s'\n", __func__, hdr_tail, val_out->len, + last_chunk->len, (int)last_chunk->len, last_chunk->data); - val_out->nchunks = chunk - val_out->chunks; + val_out->nchunks = last_chunk - val_out->chunks + 1; val_out->len -= hdr_tail; - last_chunk->len -= last_tail; -} -/* - * Split header in two parts: name and value, with descriptors copying (see - * description of @tfw_http_hdr_split() for details). - * - * TODO: this procedure should be evicted after HTTP/1.1-parser extending - * (and @tfw_http_hdr_split()/@tfw_h2_msg_hdr_length procedures upgrading) in - * order to split headers into separate chunks for name, ':', LWS, value, and - * RWS - during the HTTP-message parsing stage. - */ -int -tfw_http_hdr_split_cp(TfwPool *pool, TfwStr *hdr, TfwStr *name_out, - TfwStr *val_out) -{ - TfwStr *h; - - BUG_ON(TFW_STR_DUP(hdr)); - - h = tfw_strdup_desc(pool, hdr); - if (unlikely(!h)) - return -ENOMEM; - - tfw_http_hdr_split(h, name_out, val_out); - - return 0; + return name_out->len + val_out->len; } unsigned long diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index 944ddfb4e0..da733440e1 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -683,9 +683,8 @@ void tfw_http_send_resp(TfwHttpReq *req, int status, const char *reason); /* Helper functions */ char *tfw_http_msg_body_dup(const char *filename, size_t *len); -void tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out); -int tfw_http_hdr_split_cp(TfwPool *pool, TfwStr *hdr, TfwStr *name_out, - TfwStr *val_out); +unsigned long tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out, + bool inplace); unsigned long tfw_h2_hdr_size(unsigned long n_len, unsigned long v_len, unsigned short st_index); TfwHdrModsDesc *tfw_http_find_desc(const TfwStr *hdr, const TfwHdrMods *h_mods); diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 004fce0623..cfc74cfcff 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -168,13 +168,6 @@ tfw_http_msg_req_spec_hid(const TfwStr *hdr) /** * Fills @val with second part of special HTTP header containing the header * value. - * - * TODO: this function should be brought to a uniform look with other similar - * procedures( @tfw_http_hdr_split() and @tfw_h2_msg_hdr_length()), more - * likely - aggregated into one general-purpose procedure, after HTTP/1.1-parser - * extending (see also TODO-comment for @tfw_http_hdr_split_cp() procedure); for - * now this procedure leaves RWS in the resulting @val and this is not correct - * in context of HTTP/2<=>HTTP/1.1 comparisons, transformations etc. */ void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client) @@ -1301,173 +1294,6 @@ __tfw_http_msg_alloc(int type, bool full) return hm; } -/** - * Determination length of the header's real part (for details see comment - * for @tfw_h2_msg_hdr_write() below) to store it in the encoder dynamic - * index. - */ -unsigned long -tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, - unsigned long *val_off, unsigned long *val_len, - TfwH2TransOp op) -{ - const TfwStr *chunk, *end; - unsigned long tail, hdr_tail = 0, hdr_len = 0; - bool name_found = false, val_found = false; - - *name_len = *val_off = *val_len = 0; - - if (TFW_STR_EMPTY(hdr)) - return 0; - - if (op != TFW_H2_TRANS_INPLACE) { - /* - * During headers addition (or message expansion) the source - * @hdr must have the following chunk structure (without the - * OWS): - * - * { name [S_DLM] value1 [value2 [value3 ...]] }. - * - */ - chunk = TFW_STR_CHUNK(hdr, 1); - if (WARN_ON_ONCE(!chunk)) - return 0; - - if (chunk->len == SLEN(S_DLM) - && *(short *)chunk->data == *(short *)S_DLM) - { - *val_off = SLEN(S_DLM); - } - - hdr_len = hdr->len; - *name_len = TFW_STR_CHUNK(hdr, 0)->len; - *val_len = hdr_len - *name_len - *val_off; - - return hdr_len - *val_off; - } - - TFW_STR_FOR_EACH_CHUNK(chunk, hdr, end) { - unsigned long idx; - - if (!chunk->len) - continue; - - hdr_len += chunk->len; - if (!name_found) { - *name_len += chunk->len; - if (chunk->data[chunk->len - 1] == ':') { - --*name_len; - name_found = true; - } - continue; - } - /* - * Skip OWS before the header value (LWS) during HTTP/2 header's - * real length calculation. LWS is always in the separate chunks - * between the name and value; thus, we can skip length of the - * entire (LWS) chunks. - */ - if (!val_found) { - if (unlikely(chunk->data[0] == ' ' - || chunk->data[0] == '\t')) - { - *val_off += chunk->len; - continue; - } - /* - * The colon must not be included into HTTP/2 header, - * thus, it should be counted in the value offset. - */ - ++*val_off; - val_found = true; - } - /* - * Skip OWS after the header value (RWS); accumulate the length - * in @tail for RWS cutting off (if this is not the end chunk, - * @tail will be reset). - */ - tail = 0; - idx = chunk->len - 1; - while (chunk->data[idx] == ' ' - || chunk->data[idx] == '\t') - { - ++tail; - if (unlikely(!idx)) - break; - --idx; - } - - if (unlikely(tail == chunk->len)) - hdr_tail += tail; - else - hdr_tail = tail; - } - - WARN_ON_ONCE(!name_found); - - *val_len = hdr_len - *name_len - *val_off - hdr_tail; - - T_DBG3("%s: name_len=%lu, val_off=%lu, val_len=%lu, hdr_tail=%lu," - " hdr_len=%lu\n", __func__, *name_len, *val_off, *val_len, - hdr_tail, hdr_len); - - return *name_len + *val_len; -} - -/** - * Copy the real part of header (i.e. the header in HTTP/2 form - without name - * colon and OWS) into @out_buf from @hdr; @nm_len is the real length of header - * name, @val_len - the real length of header value, and @val_off - the offset - * between header name and value (i.e. the part occupied by colon and OWS); OWS - * in the end of header's value are also skipped and will not be included into - * header's copied part. Note that the size of prepared @out_buf must be not - * less than sum of @nm_len and @val_len. - */ -void -tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, - unsigned long val_off, unsigned long val_len, - char *out_buf) -{ - const TfwStr *c, *end; - - T_DBG3("%s: enter, nm_len=%lu, val_off=%lu, val_len=%lu\n", __func__, - nm_len, val_off, val_len); - - BUG_ON(!nm_len); - TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { - unsigned long len; - - if (!c->len) - continue; - - len = 0; - if (nm_len) { - len = min(nm_len, c->len); - nm_len -= len; - } - - if (!nm_len) { - if (val_off) { - WARN_ON_ONCE(val_off < c->len - len); - val_off -= c->len - len; - } - else if (val_len && !len) { - len = min(val_len, c->len); - val_len -= len; - } - } - - if (!len) - continue; - - T_DBG3("%s: len=%lu, c->data='%.*s'\n", __func__, len, (int)len, - c->data); - - memcpy_fast(out_buf, c->data, len); - out_buf += len; - } -} - int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, const TfwStr *src, unsigned int *start_off) diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index b4e96b4ff3..417175681e 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -182,13 +182,6 @@ int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, const TfwStr *src, unsigned int *start_off); int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name); int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name); -unsigned long tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, - unsigned long *val_off, - unsigned long *val_len, - TfwH2TransOp op); -void tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, - unsigned long val_off, unsigned long val_len, - char *out_buf); int tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, const char *stop); diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index 6fef26c549..10c96921b5 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -230,6 +230,17 @@ do { \ } \ } while (0) +#define __FSM_MOVE_hdr_fixup(to, n) \ +do { \ + __msg_hdr_chunk_fixup(p, n); \ + p += n; \ + if (unlikely(__data_off(p) >= len)) { \ + parser->state = &&to; \ + __FSM_EXIT(TFW_POSTPONE); \ + } \ + goto to; \ +} while (0) + /* * __FSM_I_* macros are intended to help with parsing of message * header values. That is done with separate, nested, or interior @@ -356,12 +367,16 @@ __FSM_STATE(st, cold) { \ __FSM_JMP(RGen_HdrOtherN); \ } -/* As above, but reads OWS through transitional state. */ +/* + * As above, but reads OWS through transitional state. Note, that header + * name, colon, LWS and value are stored in different chunks. + */ #define __FSM_TX_AF_OWS(st, st_next) \ __FSM_STATE(st, cold) { \ if (likely(c == ':')) { \ + __msg_hdr_chunk_fixup(data, __data_off(p)); \ parser->_i_st = &&st_next; \ - __FSM_MOVE(RGen_LWS); \ + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); \ } \ /* It should be checked in st_fallback if `c` is allowed */ \ __FSM_JMP(RGen_HdrOtherN); \ @@ -371,9 +386,10 @@ __FSM_STATE(st, cold) { \ #define __FSM_TX_AF_OWS_HP(st, st_next, hp_idx) \ __FSM_STATE(st, cold) { \ if (likely(c == ':')) { \ + __msg_hdr_chunk_fixup(data, __data_off(p)); \ parser->_i_st = &&st_next; \ __msg_hdr_set_hpack_index(hp_idx); \ - __FSM_MOVE(RGen_LWS); \ + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); \ } \ /* It should be checked in st_fallback if `c` is allowed */ \ __FSM_JMP(RGen_HdrOtherN); \ @@ -839,6 +855,9 @@ mark_trailer_hdr(TfwHttpMsg *hm, TfwStr *hdr) return CSTR_POSTPONE; \ } +#define TRY_STR_fixup(str, curr_st, next_st) \ + TRY_STR_LAMBDA_fixup(str, &parser->hdr, { }, curr_st, next_st) + /* * Headers EOL processing. Allow only LF and CRLF as a newline delimiters. * @@ -1013,8 +1032,10 @@ __FSM_STATE(st_curr) { \ __FSM_STATE(RGen_HdrOtherN) { \ __FSM_MATCH_MOVE(token, RGen_HdrOtherN, 0); \ if (likely(*(p + __fsm_sz) == ':')) { \ + __msg_hdr_chunk_fixup(data, __data_off(p + __fsm_sz)); \ parser->_i_st = &&RGen_HdrOtherV; \ - __FSM_MOVE_n(RGen_LWS, __fsm_sz + 1); \ + p += __fsm_sz; \ + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); \ } \ TFW_PARSER_BLOCK(RGen_HdrOtherN); \ } \ @@ -1216,21 +1237,22 @@ __FSM_STATE(RGen_BodyCR, __VA_ARGS__) { \ */ #define RGEN_OWS() \ __FSM_STATE(RGen_LWS, hot) { \ - /* Store header name, LWS and value in different chunks. */ \ - __msg_hdr_chunk_fixup(data, p - data); \ __fsm_sz = __data_remain(p); \ __fsm_n = parse_ows(p, __fsm_sz); \ T_DBG3("parse LWS: __fsm_n=%d, __fsm_sz=%lu, len=%lu," \ " off=%lu\n", __fsm_n, __fsm_sz, len, __data_off(p)); \ if (__fsm_n == CSTR_POSTPONE) { \ __msg_hdr_chunk_fixup(p, __fsm_sz); \ + __msg_chunk_flags(TFW_STR_OWS); \ p += __fsm_sz; \ parser->state = &&RGen_LWS; \ __FSM_EXIT(TFW_POSTPONE); \ } \ BUG_ON(__fsm_n < 0); \ - if (__fsm_n) \ + if (__fsm_n) { \ __msg_hdr_chunk_fixup(p, __fsm_n); \ + __msg_chunk_flags(TFW_STR_OWS); \ + } \ parser->state = parser->_i_st; \ parser->_i_st = NULL; \ p += __fsm_n; \ @@ -1827,7 +1849,8 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, * to a message body (i.e., chunking an already * chunked message is not allowed). RFC 7230 3.3.1. */ - TRY_STR("chunked", I_TransEncodTok, I_TransEncodChunked); + TRY_STR_fixup(&TFW_STR_STRING("chunked"), I_TransEncodTok, + I_TransEncodChunked); TRY_STR_INIT(); __FSM_I_JMP(I_TransEncodOther); } @@ -1842,7 +1865,6 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, __FSM_I_JMP(I_TransEncodOther); } - /* * RFC 7230 3.3.1: * If any transfer coding @@ -1857,10 +1879,13 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, * compress; */ __FSM_STATE(I_TransEncodOther) { - __FSM_I_MATCH_MOVE(token, I_TransEncodOther); + __FSM_I_MATCH_MOVE_fixup(token, I_TransEncodOther, 0); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_I_MOVE_n(I_EoT, __fsm_sz + 1); + if (IS_WS(c) || c == ',') { + __msg_hdr_chunk_fixup(p, __fsm_sz); + p += __fsm_sz; + __FSM_I_JMP(I_EoT); + } if (IS_CRLF(c)) { if (unlikely(test_bit(TFW_HTTP_B_CHUNKED, msg->flags))) { @@ -1870,6 +1895,7 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, __set_bit(TFW_HTTP_B_CHUNKED_APPLIED, msg->flags); } + __msg_hdr_chunk_fixup(p, __fsm_sz); return __data_off(p + __fsm_sz); } return CSTR_NEQ; @@ -1877,8 +1903,10 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, /* End of term. */ __FSM_STATE(I_EoT) { - if (IS_WS(c) || c == ',') - __FSM_I_MOVE(I_EoT); + if (c == ',') + __FSM_I_MOVE_fixup(I_EoT, 1, 0); + if (IS_WS(c)) + __FSM_I_MOVE_fixup(I_EoT, 1, TFW_STR_OWS); if (IS_TOKEN(c)) __FSM_I_JMP(I_TransEncodTok); if (IS_CRLF(c)) @@ -2417,7 +2445,7 @@ __parse_etag(TfwHttpMsg *hm, unsigned char *data, size_t len) /* End of token */ __FSM_STATE(I_EoT) { if (IS_WS(c)) - __FSM_I_MOVE_fixup(I_EoT, 1, 0); + __FSM_I_MOVE_fixup(I_EoT, 1, TFW_STR_OWS); if (IS_CRLF(c)) return __data_off(p); if ((TFW_CONN_TYPE(hm->conn) & Conn_Clnt) && c == ',') @@ -2427,7 +2455,7 @@ __parse_etag(TfwHttpMsg *hm, unsigned char *data, size_t len) __FSM_STATE(I_EoL) { if (IS_WS(c)) - __FSM_I_MOVE_fixup(I_EoL, 1, 0); + __FSM_I_MOVE_fixup(I_EoL, 1, TFW_STR_OWS); if (IS_CRLF(c)) return __data_off(p); return CSTR_NEQ; @@ -2897,7 +2925,8 @@ __parse_pragma(TfwHttpMsg *hm, unsigned char *data, size_t len) __FSM_START(parser->_i_st); __FSM_STATE(I_Pragma) { - TRY_STR("no-cache", I_Pragma, I_Pragma_NoCache); + TRY_STR_fixup(&TFW_STR_STRING("no-cache"), I_Pragma, + I_Pragma_NoCache); TRY_STR_INIT(); __FSM_I_JMP(I_Pragma_Ext); } @@ -2905,24 +2934,32 @@ __parse_pragma(TfwHttpMsg *hm, unsigned char *data, size_t len) __FSM_STATE(I_Pragma_NoCache) { if (IS_WS(c) || c == ',' || IS_CRLF(c)) msg->cache_ctl.flags |= TFW_HTTP_CC_PRAGMA_NO_CACHE; - __FSM_I_JMP(I_Pragma_Ext); + /* Fall through. */ } __FSM_STATE(I_Pragma_Ext) { /* Verify and just skip the extensions. */ - __FSM_I_MATCH_MOVE(qetoken, I_Pragma_Ext); + __FSM_I_MATCH_MOVE_fixup(qetoken, I_Pragma_Ext, 0); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_I_MOVE_n(I_EoT, __fsm_sz + 1); - if (IS_CRLF(c)) + if (IS_WS(c) || c == ',') { + __msg_hdr_chunk_fixup(p, __fsm_sz); + p += __fsm_sz; + __FSM_I_JMP(I_EoT); + } + if (IS_CRLF(c)) { + __msg_hdr_chunk_fixup(p, __fsm_sz); return __data_off(p + __fsm_sz); + } + return CSTR_NEQ; } /* End of term. */ __FSM_STATE(I_EoT) { - if (IS_WS(c) || c == ',') - __FSM_I_MOVE(I_EoT); + if (IS_WS(c)) + __FSM_I_MOVE_fixup(I_EoT, 1, TFW_STR_OWS); + if (c == ',') + __FSM_I_MOVE_fixup(I_EoT, 1, 0); if (IS_CRLF(c)) return __data_off(p); __FSM_I_JMP(I_Pragma_Ext); @@ -3033,7 +3070,8 @@ __parse_keep_alive(TfwHttpMsg *hm, unsigned char *data, size_t len) __FSM_START(parser->_i_st); __FSM_STATE(I_KeepAlive) { - TRY_STR("timeout=", I_KeepAlive, I_KeepAliveTO); + TRY_STR_fixup(&TFW_STR_STRING("timeout="), I_KeepAlive, + I_KeepAliveTO); TRY_STR_INIT(); __FSM_I_JMP(I_KeepAliveExt); } @@ -3042,12 +3080,14 @@ __parse_keep_alive(TfwHttpMsg *hm, unsigned char *data, size_t len) __fsm_sz = __data_remain(p); __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); + __msg_hdr_chunk_fixup(p, __fsm_sz); if (__fsm_n < 0) return __fsm_n; hm->keep_alive = parser->_acc; parser->_acc = 0; - __FSM_I_MOVE_n(I_EoT, __fsm_n); + __msg_hdr_chunk_fixup(p, __fsm_n); + p += __fsm_n; + __FSM_I_JMP(I_EoT); } /* @@ -3055,21 +3095,28 @@ __parse_keep_alive(TfwHttpMsg *hm, unsigned char *data, size_t len) * max=N */ __FSM_STATE(I_KeepAliveExt) { - __FSM_I_MATCH_MOVE(qetoken, I_KeepAliveExt); + __FSM_I_MATCH_MOVE_fixup(qetoken, I_KeepAliveExt, 0); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_I_MOVE_n(I_EoT, __fsm_sz + 1); - if (IS_CRLF(c)) + if (IS_WS(c) || c == ',') { + __msg_hdr_chunk_fixup(p, __fsm_sz); + p += __fsm_sz; + __FSM_I_JMP(I_EoT); + } + if (IS_CRLF(c)) { + __msg_hdr_chunk_fixup(p, __fsm_sz); return __data_off(p + __fsm_sz); + } return CSTR_NEQ; } /* End of term. */ __FSM_STATE(I_EoT) { - if (IS_WS(c) || c == ',') - __FSM_I_MOVE(I_EoT); + if (c == ',') + __FSM_I_MOVE_fixup(I_EoT, 1, 0); + if (IS_WS(c)) + __FSM_I_MOVE_fixup(I_EoT, 1, TFW_STR_OWS); if (c == '=') - __FSM_I_MOVE(I_KeepAliveExt); + __FSM_I_MOVE_fixup(I_KeepAliveExt, 1, 0); if (IS_TOKEN(c)) __FSM_I_JMP(I_KeepAlive); if (IS_CRLF(c)) @@ -3605,8 +3652,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 6)) == 't' && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Req_HdrAcceptV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 14) && C8_INT_LCM(p + 1, 'u', 't', 'h', 'o', @@ -3614,8 +3663,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C4_INT_LCM(p + 9, 't', 'i', 'o', 'n') && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&Req_HdrAuthorizationV; - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrA); case 'c': @@ -3630,8 +3681,11 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, 't', 'r', 'o', 'l', ':'))) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 13)); parser->_i_st = &&Req_HdrCache_ControlV; - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 'n', 'e'): @@ -3639,8 +3693,11 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'n' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 10)); parser->_i_st = &&Req_HdrConnectionV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 't', 'e'): @@ -3655,8 +3712,11 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, if (likely(TFW_LC(*(p + 5)) == 'e' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 6)); parser->_i_st = &&Req_HdrCookieV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); default: @@ -3666,9 +3726,11 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'o', 's', 't', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Req_HdrHostV; parser->_hdr_tag = TFW_HTTP_HDR_HOST; - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrH); case 'i': @@ -3682,8 +3744,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 16)) == 'e' && *(p + 17) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 17)); parser->_i_st = &&Req_HdrIf_Modified_SinceV; - __FSM_MOVE_n(RGen_LWS, 18); + p += 17; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 14) && TFW_LC(*(p + 1)) == 'f' @@ -3694,8 +3758,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 12)) == 'h' && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&Req_HdrIf_None_MatchV; - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrI); case 'k': @@ -3706,8 +3772,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'e' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 10)); parser->_i_st = &&Req_HdrKeep_AliveV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrK); case 'p': @@ -3716,8 +3784,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 5)) == 'a' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Req_HdrPragmaV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrP); case 'r': @@ -3727,8 +3797,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 6)) == 'r' && *(p + 7) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 7)); parser->_i_st = &&Req_HdrRefererV; - __FSM_MOVE_n(RGen_LWS, 8); + p += 7; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrR); case 't': @@ -3740,8 +3812,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, 'd', 'i', 'n', 'g') && *(p + 17) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 17)); parser->_i_st = &&Req_HdrTransfer_EncodingV; - __FSM_MOVE_n(RGen_LWS, 18); + p += 17; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrT); case 'x': @@ -3754,8 +3828,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 8, 'd', 'e', 'd', '-', 'f', 'o', 'r', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 15)); parser->_i_st = &&Req_HdrX_Forwarded_ForV; - __FSM_MOVE_n(RGen_LWS, 16); + p += 15; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 14) && *(p + 1) == '-' @@ -3767,8 +3843,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 12) == 'd') && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 23) && *(p + 1) == '-' @@ -3782,8 +3860,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 16, 'v', 'e', 'r', 'r', 'i', 'd', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 22)); parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_LWS, 23); + p += 22; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 18) && *(p + 1) == '-' @@ -3794,8 +3874,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 10, 'v', 'e', 'r', 'r', 'i', 'd', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 17)); parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_LWS, 18); + p += 17; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrX); case 'u': @@ -3806,8 +3888,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 't' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 10)); parser->_i_st = &&Req_HdrUser_AgentV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrU); default: @@ -3824,16 +3908,20 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 5)) == 'h' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Req_HdrContent_LengthV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrContent_L); case 't': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'y', 'p', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Req_HdrContent_TypeV; - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrContent_T); default: @@ -3879,20 +3967,20 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, __req_parse_if_msince); /* 'Keep-Alive:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrKeep_AliveV, msg, __parse_keep_alive, - TFW_HTTP_HDR_KEEP_ALIVE); + __TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrKeep_AliveV, msg, __parse_keep_alive, + TFW_HTTP_HDR_KEEP_ALIVE, 0); /* 'Pragma:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_RAWHDR_VAL(Req_HdrPragmaV, msg, __parse_pragma); + __TFW_HTTP_PARSE_RAWHDR_VAL(Req_HdrPragmaV, msg, __parse_pragma, 0); /* 'Referer:*OWS' is read, process field-value. */ TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrRefererV, msg, __req_parse_referer, TFW_HTTP_HDR_REFERER); /* 'Transfer-Encoding:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrTransfer_EncodingV, msg, - __req_parse_transfer_encoding, - TFW_HTTP_HDR_TRANSFER_ENCODING); + __TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrTransfer_EncodingV, msg, + __req_parse_transfer_encoding, + TFW_HTTP_HDR_TRANSFER_ENCODING, 0); /* 'X-Forwarded-For:*OWS' is read, process field-value. */ __TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrX_Forwarded_ForV, msg, @@ -8069,17 +8157,19 @@ __resp_parse_age(TfwHttpResp *resp, unsigned char *data, size_t len) __fsm_sz = __data_remain(p); __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); + __msg_hdr_chunk_fixup(p, __fsm_sz); if (__fsm_n < 0) return __fsm_n; resp->cache_ctl.age = parser->_acc; parser->_acc = 0; - __FSM_I_MOVE_n(Resp_I_EoL, __fsm_n); + __msg_hdr_chunk_fixup(p, __fsm_n); + p += __fsm_n; + /* Fall through. */ } __FSM_STATE(Resp_I_EoL) { if (IS_WS(c)) - __FSM_I_MOVE(Resp_I_EoL); + __FSM_I_MOVE_fixup(Resp_I_EoL, 1, TFW_STR_OWS); if (IS_CRLF(c)) { resp->cache_ctl.flags |= TFW_HTTP_CC_HDR_AGE; return __data_off(p); @@ -8119,8 +8209,10 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } __FSM_STATE(Resp_I_CC_m) { - TRY_STR("max-age=", Resp_I_CC_m, Resp_I_CC_MaxAgeV); - TRY_STR_LAMBDA("must-revalidate", { + TRY_STR_fixup(&TFW_STR_STRING("max-age="), Resp_I_CC_m, + Resp_I_CC_MaxAgeV); + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("must-revalidate"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_MUST_REVAL; }, Resp_I_CC_m, Resp_I_Flag); TRY_STR_INIT(); @@ -8128,13 +8220,16 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } __FSM_STATE(Resp_I_CC_n) { - TRY_STR_LAMBDA("no-cache", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("no-cache"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_NO_CACHE; }, Resp_I_CC_n, Resp_I_Flag); - TRY_STR_LAMBDA("no-store", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("no-store"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_NO_STORE; }, Resp_I_CC_n, Resp_I_Flag); - TRY_STR_LAMBDA("no-transform", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("no-transform"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_NO_TRANSFORM; }, Resp_I_CC_n, Resp_I_Flag); TRY_STR_INIT(); @@ -8142,13 +8237,14 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } __FSM_STATE(Resp_I_CC_p) { - TRY_STR_LAMBDA("public", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("public"), &parser->hdr, { parser->_acc = TFW_HTTP_CC_PUBLIC; }, Resp_I_CC_p, Resp_I_Flag); - TRY_STR_LAMBDA("private", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("private"), &parser->hdr, { parser->_acc = TFW_HTTP_CC_PRIVATE; }, Resp_I_CC_p, Resp_I_Flag); - TRY_STR_LAMBDA("proxy-revalidate", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("proxy-revalidate"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_PROXY_REVAL; }, Resp_I_CC_p, Resp_I_Flag); TRY_STR_INIT(); @@ -8166,7 +8262,8 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } __FSM_STATE(Resp_I_CC_s) { - TRY_STR("s-maxage=", Resp_I_CC_s, Resp_I_CC_SMaxAgeV); + TRY_STR_fixup(&TFW_STR_STRING("s-maxage="), Resp_I_CC_s, + Resp_I_CC_SMaxAgeV); TRY_STR_INIT(); __FSM_I_JMP(Resp_I_Ext); } @@ -8179,7 +8276,7 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) __fsm_sz = __data_remain(p); __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); + __msg_hdr_chunk_fixup(p, __fsm_sz); if (__fsm_n < 0) { if (__fsm_n != CSTR_BADLEN) return __fsm_n; @@ -8187,7 +8284,7 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } resp->cache_ctl.max_age = parser->_acc; resp->cache_ctl.flags |= TFW_HTTP_CC_MAX_AGE; - __FSM_I_MOVE_n(Resp_I_EoT, __fsm_n); + __FSM_I_MOVE_fixup(Resp_I_EoT, __fsm_n, 0); } __FSM_STATE(Resp_I_CC_SMaxAgeV) { @@ -8198,7 +8295,7 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) __fsm_sz = __data_remain(p); __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); + __msg_hdr_chunk_fixup(p, __fsm_sz); if (__fsm_n < 0) { if (__fsm_n != CSTR_BADLEN) return __fsm_n; @@ -8206,17 +8303,21 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } resp->cache_ctl.s_maxage = parser->_acc; resp->cache_ctl.flags |= TFW_HTTP_CC_S_MAXAGE; - __FSM_I_MOVE_n(Resp_I_EoT, __fsm_n); + __FSM_I_MOVE_fixup(Resp_I_EoT, __fsm_n, 0); } __FSM_STATE(Resp_I_Ext) { /* TODO: process cache extensions. */ - __FSM_I_MATCH_MOVE(qetoken, Resp_I_Ext); + __FSM_I_MATCH_MOVE_fixup(qetoken, Resp_I_Ext, 0); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_I_MOVE_n(Resp_I_EoT, __fsm_sz + 1); + if (IS_WS(c) || c == ',') { + __msg_hdr_chunk_fixup(p, __fsm_sz); + p += __fsm_sz; + __FSM_I_JMP(Resp_I_EoT); + } if (IS_CRLF(c)) { parser->_acc = 0; + __msg_hdr_chunk_fixup(p, __fsm_sz); return __data_off(p + __fsm_sz); } return CSTR_NEQ; @@ -8224,8 +8325,11 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) /* End of term. */ __FSM_STATE(Resp_I_EoT) { - if (IS_WS(c) || c == ',') - __FSM_I_MOVE(Resp_I_EoT); + if (c == ',') + __FSM_I_MOVE_fixup(Resp_I_EoT, 1, 0); + + if (IS_WS(c)) + __FSM_I_MOVE_fixup(Resp_I_EoT, 1, TFW_STR_OWS); parser->_acc = 0; /* reinit for next token */ @@ -8235,7 +8339,7 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) * no-cache and private fields, so just skip '=[token]*'. */ if (c == '=') - __FSM_I_MOVE(Resp_I_Ext); + __FSM_I_MOVE_fixup(Resp_I_Ext, 1, 0); if (IS_TOKEN(c)) __FSM_I_JMP(Resp_I_CC); if (IS_CRLF(c)) @@ -8664,9 +8768,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, '-', 'o', 'r', 'i') && C4_INT3_LCM(p + 24, 'g', 'i', 'n', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 27)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(20); - __FSM_MOVE_n(RGen_LWS, 28); + p += 27; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 14) && C8_INT_LCM(p + 1, 'c', 'c', 'e', 'p', @@ -8674,24 +8780,30 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C4_INT_LCM(p + 9, 'n', 'g', 'e', 's') && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(18); - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 6) && C4_INT_LCM(p + 1, 'l', 'l', 'o', 'w') && *(p + 5) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 5)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(22); - __FSM_MOVE_n(RGen_LWS, 6); + p += 5; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 4) && C4_INT3_LCM(p, 'a', 'g', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 3)); parser->_i_st = &&Resp_HdrAgeV; __msg_hdr_set_hpack_index(21); - __FSM_MOVE_n(RGen_LWS, 4); + p += 3; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrA); case 'c': @@ -8706,9 +8818,12 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 't', 'r', 'o', 'l', ':'))) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 13)); parser->_i_st = &&Resp_HdrCache_CtrlV; __msg_hdr_set_hpack_index(24); - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 'n', 'e'): @@ -8716,8 +8831,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'n' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 10)); parser->_i_st = &&Resp_HdrConnectionV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 't', 'e'): @@ -8735,26 +8853,32 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'a', 't', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Resp_HdrDateV; __msg_hdr_set_hpack_index(33); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrD); case 'e': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 't', 'a', 'g', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Resp_HdrEtagV; __msg_hdr_set_hpack_index(34); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 8) && C8_INT7_LCM(p, 'e', 'x', 'p', 'i', 'r', 'e', 's', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 7)); parser->_i_st = &&Resp_HdrExpiresV; __msg_hdr_set_hpack_index(36); - __FSM_MOVE_n(RGen_LWS, 8); + p += 7; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrE); case 'k': @@ -8765,8 +8889,10 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'e' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 10)); parser->_i_st = &&Resp_HdrKeep_AliveV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrK); case 'l': @@ -8777,24 +8903,30 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'f', 'i', 'e', 'd') && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&Resp_HdrLast_ModifiedV; __msg_hdr_set_hpack_index(44); - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 9) && C8_INT7_LCM(p + 1, 'o', 'c', 'a', 't', 'i', 'o', 'n', ':'))) { + __msg_hdr_chunk_fixup(data,__data_off(p + 8)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(46); - __FSM_MOVE_n(RGen_LWS, 9); + p += 8; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'i', 'n', 'k', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(45); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrL); case 'p': @@ -8806,17 +8938,21 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 17)) == 'e' && *(p + 18) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 18)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(48); - __FSM_MOVE_n(RGen_LWS, 19); + p += 18; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 7)) && C4_INT_LCM(p + 1, 'r', 'a', 'g', 'm') && TFW_LC(*(p + 5)) == 'a' && *(p + 6) == ':') { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Resp_HdrPragmaV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrP); case 'r': @@ -8825,9 +8961,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'y', '-', 'a', 'f') && C4_INT3_LCM(p + 8, 't', 'e', 'r', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 11)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(53); - __FSM_MOVE_n(RGen_LWS, 12); + p += 11; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrR); case 's': @@ -8840,9 +8978,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'r', 'i', 't', 'y') && *(p + 25) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 25)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(56); - __FSM_MOVE_n(RGen_LWS, 26); + p += 25; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 11) && C8_INT_LCM(p + 1, 'e', 't', '-', 'c', @@ -8851,18 +8991,22 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'e' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 10)); parser->_i_st = &&Resp_HdrSet_CookieV; __msg_hdr_set_hpack_index(55); - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 7) && C4_INT_LCM(p + 1, 'e', 'r', 'v', 'e') && TFW_LC(*(p + 5)) == 'r' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Resp_HdrServerV; __msg_hdr_set_hpack_index(54); - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrS); case 't': @@ -8874,24 +9018,30 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'd', 'i', 'n', 'g') && *(p + 17) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 17)); parser->_i_st = &&Resp_HdrTransfer_EncodingV; - __FSM_MOVE_n(RGen_LWS, 18); + p += 17; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrT); case 'v': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'a', 'r', 'y', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(59); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 4) && C4_INT3_LCM(p, 'v', 'i', 'a', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 3)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(60); - __FSM_MOVE_n(RGen_LWS, 4); + p += 3; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrV); case 'w': @@ -8901,9 +9051,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 9, 'n', 't', 'i', 'c', 'a', 't', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 16)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(61); - __FSM_MOVE_n(RGen_LWS, 17); + p += 16; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrW); default: @@ -8920,9 +9072,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'o', 's', 'i', 't') && C4_INT3_LCM(p + 8, 'i', 'o', 'n', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 11)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(25); - __FSM_MOVE_n(RGen_LWS, 12); + p += 11; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_D); case 'e': @@ -8930,9 +9084,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 1, 'n', 'c', 'o', 'd', 'i', 'n', 'g', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 8)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(26); - __FSM_MOVE_n(RGen_LWS, 9); + p += 8; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_E); case 'l': @@ -8940,16 +9096,22 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, if (C8_INT7_LCM(p + 1, 'a', 'n', 'g', 'u', 'a', 'g', 'e', ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 8)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(27); - __FSM_MOVE_n(RGen_LWS, 9); + p += 8; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (C8_INT7_LCM(p + 1, 'o', 'c', 'a', 't', 'i', 'o', 'n', ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 8)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(29); - __FSM_MOVE_n(RGen_LWS, 9); + p += 8; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } } if (likely(__data_available(p, 7) @@ -8957,9 +9119,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 5)) == 'h' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Resp_HdrContent_LengthV; __msg_hdr_set_hpack_index(28); - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_L); case 'r': @@ -8967,18 +9131,22 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C4_INT_LCM(p + 1, 'a', 'n', 'g', 'e') && *(p + 5) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 5)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(30); - __FSM_MOVE_n(RGen_LWS, 6); + p += 5; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_R); case 't': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'y', 'p', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Resp_HdrContent_TypeV; __msg_hdr_set_hpack_index(31); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_T); default: @@ -8987,11 +9155,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, } /* 'Age:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrAgeV, resp, __resp_parse_age); + __TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrAgeV, resp, __resp_parse_age, 0); /* 'Cache-Control:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrCache_CtrlV, resp, - __resp_parse_cache_control); + __TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrCache_CtrlV, resp, + __resp_parse_cache_control, 0); /* 'Connection:*OWS' is read, process field-value. */ TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrConnectionV, msg, __parse_connection, @@ -9018,24 +9186,24 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrExpiresV, msg, __resp_parse_expires); /* 'Keep-Alive:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrKeep_AliveV, msg, __parse_keep_alive, - TFW_HTTP_HDR_KEEP_ALIVE); + __TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrKeep_AliveV, msg, __parse_keep_alive, + TFW_HTTP_HDR_KEEP_ALIVE, 0); /* 'Last-Modified:*OWS' is read, process field-value. */ TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrLast_ModifiedV, msg, __resp_parse_if_modified); /* 'Pragma:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrPragmaV, msg, __parse_pragma); + __TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrPragmaV, msg, __parse_pragma, 0); /* 'Server:*OWS' is read, process field-value. */ TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrServerV, resp, __resp_parse_server, TFW_HTTP_HDR_SERVER); /* 'Transfer-Encoding:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrTransfer_EncodingV, msg, - __resp_parse_transfer_encoding, - TFW_HTTP_HDR_TRANSFER_ENCODING); + __TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrTransfer_EncodingV, msg, + __resp_parse_transfer_encoding, + TFW_HTTP_HDR_TRANSFER_ENCODING, 0); /* 'Set-Cookie:*OWS' is read, process field-value. */ __TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrSet_CookieV, resp, diff --git a/tempesta_fw/str.h b/tempesta_fw/str.h index 7273a5ccce..28b42d8526 100644 --- a/tempesta_fw/str.h +++ b/tempesta_fw/str.h @@ -215,6 +215,9 @@ size_t tfw_ultohex(unsigned long ai, char *buf, unsigned int len); */ #define TFW_STR_HDR_VALUE 0x80 +/* The chunk contains only WS characters. */ +#define TFW_STR_OWS 0x100 + /* * @ptr - pointer to string data or array of nested strings; * @skb - socket buffer containing the string data; diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index b0c72ea7ef..45f3c82b9b 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -1205,8 +1205,9 @@ TEST(hpack, enc_huffman) TEST(hpack, enc_table_hdr_write) { - char *buf; - unsigned long hdr_len, n_len, v_off, v_len; + char *buf, *ptr; + unsigned long hdr_len; + TfwStr s_nm = {}, s_val = {}; #define HDR_NAME_1 "x-forwarded-for" #define HDR_VALUE_1 "test.com, foo.com, example.com" @@ -1219,103 +1220,116 @@ TEST(hpack, enc_table_hdr_write) #define HDR_NAME_5 "custom-key" #define HDR_VALUE_5 "custom-example-value" - TFW_STR(s1, HDR_NAME_1 ":"); - TFW_STR(s1_lws, " "); - TFW_STR(s1_value, HDR_VALUE_1 " "); + TFW_STR(col, ":"); + + TFW_STR(s1, HDR_NAME_1); + TFW_STR(s1_lws, " "); + TFW_STR(s1_value, HDR_VALUE_1); + TFW_STR(s1_rws, " "); const char *t_s1 = HDR_NAME_1 HDR_VALUE_1; - unsigned long off1 = s1_lws->len + 1; unsigned long t_s1_len = strlen(t_s1); - TFW_STR(s2, HDR_NAME_2 ":"); + TFW_STR(s2, HDR_NAME_2); TFW_STR(s2_value, HDR_VALUE_2); const char *t_s2 = HDR_NAME_2 HDR_VALUE_2; - unsigned long off2 = 1; unsigned long t_s2_len = strlen(t_s2); - TFW_STR(s3, HDR_NAME_3 ":"); + TFW_STR(s3, HDR_NAME_3); TFW_STR(s3_lws, "\t "); - TFW_STR(s3_value, HDR_VALUE_3 " "); + TFW_STR(s3_value, HDR_VALUE_3); + TFW_STR(s3_rws, " "); const char *t_s3 = HDR_NAME_3 HDR_VALUE_3; - unsigned long off3 = s3_lws->len + 1; unsigned long t_s3_len = strlen(t_s3); - TFW_STR(s4, HDR_NAME_4 ":"); + TFW_STR(s4, HDR_NAME_4); TFW_STR(s4_lws, " "); - TFW_STR(s4_value, HDR_VALUE_4 "\t\t \t"); + TFW_STR(s4_value, HDR_VALUE_4); + TFW_STR(s4_rws, "\t\t \t"); const char *t_s4 = HDR_NAME_4 HDR_VALUE_4; - unsigned long off4 = s4_lws->len + 1; unsigned long t_s4_len = strlen(t_s4); - TFW_STR(s5, HDR_NAME_5 ":"); + TFW_STR(s5, HDR_NAME_5); TFW_STR(s5_lws, "\t\t\t"); - TFW_STR(s5_value, HDR_VALUE_5 "\t\t\t\t"); + TFW_STR(s5_value, HDR_VALUE_5); + TFW_STR(s5_rws, "\t\t\t\t"); const char *t_s5 = HDR_NAME_5 HDR_VALUE_5; - unsigned long off5 = s5_lws->len + 1; unsigned long t_s5_len = strlen(t_s5); - collect_compound_str(s1, s1_lws, 0); + collect_compound_str(s1, col, 0); + collect_compound_str(s1, s1_lws, TFW_STR_OWS); collect_compound_str(s1, s1_value, 0); + collect_compound_str(s1, s1_rws, TFW_STR_OWS); + collect_compound_str(s2, col, 0); collect_compound_str(s2, s2_value, 0); - collect_compound_str(s3, s3_lws, 0); + collect_compound_str(s3, col, 0); + collect_compound_str(s3, s3_lws, TFW_STR_OWS); collect_compound_str(s3, s3_value, 0); - collect_compound_str(s4, s4_lws, 0); + collect_compound_str(s3, s3_rws, TFW_STR_OWS); + collect_compound_str(s4, col, 0); + collect_compound_str(s4, s4_lws, TFW_STR_OWS); collect_compound_str(s4, s4_value, 0); - collect_compound_str(s5, s5_lws, 0); + collect_compound_str(s4, s4_rws, TFW_STR_OWS); + collect_compound_str(s5, col, 0); + collect_compound_str(s5, s5_lws, TFW_STR_OWS); collect_compound_str(s5, s5_value, 0); + collect_compound_str(s5, s5_rws, TFW_STR_OWS); - hdr_len = tfw_h2_msg_hdr_length(s1, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_1)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_1)); - EXPECT_EQ(v_off, off1); + hdr_len = tfw_http_hdr_split(s1, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_1)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_1)); EXPECT_EQ(hdr_len, t_s1_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s1, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s1, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s2, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_2)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_2)); - EXPECT_EQ(v_off, off2); + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + hdr_len = tfw_http_hdr_split(s2, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_2)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_2)); EXPECT_EQ(hdr_len, t_s2_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s2, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s2, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s3, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_3)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_3)); - EXPECT_EQ(v_off, off3); + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + hdr_len = tfw_http_hdr_split(s3, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_3)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_3)); EXPECT_EQ(hdr_len, t_s3_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s3, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s3, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s4, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_4)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_4)); - EXPECT_EQ(v_off, off4); + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + hdr_len = tfw_http_hdr_split(s4, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_4)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_4)); EXPECT_EQ(hdr_len, t_s4_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s4, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s4, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s5, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_5)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_5)); - EXPECT_EQ(v_off, off5); + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + hdr_len = tfw_http_hdr_split(s5, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_5)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_5)); EXPECT_EQ(hdr_len, t_s5_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s5, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s5, buf, hdr_len)); #undef HDR_NAME_1 @@ -1345,28 +1359,37 @@ TEST(hpack, enc_table_index) #define HDR_NAME_3 "test-example-key" #define HDR_VALUE_3 "custom-example-value" - TFW_STR(s1, HDR_NAME_1 ":"); + TFW_STR(col, ":"); + + TFW_STR(s1, HDR_NAME_1); TFW_STR(s1_lws, " \t "); - TFW_STR(s1_value, HDR_VALUE_1 " "); + TFW_STR(s1_value, HDR_VALUE_1); + TFW_STR(s1_rws, " "); const char *t_s1 = HDR_NAME_1 HDR_VALUE_1; unsigned long t_s1_len = strlen(t_s1); - TFW_STR(s2, HDR_NAME_2 ":"); + TFW_STR(s2, HDR_NAME_2); TFW_STR(s2_value, HDR_VALUE_2); const char *t_s2 = HDR_NAME_2 HDR_VALUE_2; unsigned long t_s2_len = strlen(t_s2); - TFW_STR(s3, HDR_NAME_3 ":"); + TFW_STR(s3, HDR_NAME_3); TFW_STR(s3_lws, "\t \t\t\t"); - TFW_STR(s3_value, HDR_VALUE_3 "\t\t\t\t "); + TFW_STR(s3_value, HDR_VALUE_3); + TFW_STR(s3_rws, "\t\t\t\t "); const char *t_s3 = HDR_NAME_3 HDR_VALUE_3; unsigned long t_s3_len = strlen(t_s3); - collect_compound_str(s1, s1_lws, 0); + collect_compound_str(s1, col, 0); + collect_compound_str(s1, s1_lws, TFW_STR_OWS); collect_compound_str(s1, s1_value, 0); + collect_compound_str(s1, s1_rws, TFW_STR_OWS); + collect_compound_str(s2, col, 0); collect_compound_str(s2, s2_value, 0); - collect_compound_str(s3, s3_lws, 0); + collect_compound_str(s3, col, 0); + collect_compound_str(s3, s3_lws, TFW_STR_OWS); collect_compound_str(s3, s3_value, 0); + collect_compound_str(s3, s3_rws, TFW_STR_OWS); tbl = &ctx.hpack.enc_tbl; @@ -1470,21 +1493,27 @@ TEST(hpack, enc_table_rbtree) #define HDR_NAME_5 "test-foo-name" #define HDR_VALUE_5 "test-foo-value" - TFW_STR(s1, HDR_NAME_1 ":"); + TFW_STR(col, ":"); + TFW_STR(s1, HDR_NAME_1); TFW_STR(s1_value, HDR_VALUE_1); - TFW_STR(s2, HDR_NAME_2 ":"); + TFW_STR(s2, HDR_NAME_2); TFW_STR(s2_value, HDR_VALUE_2); - TFW_STR(s3, HDR_NAME_3 ":"); + TFW_STR(s3, HDR_NAME_3); TFW_STR(s3_value, HDR_VALUE_3); - TFW_STR(s4, HDR_NAME_4 ":"); + TFW_STR(s4, HDR_NAME_4); TFW_STR(s4_value, HDR_VALUE_4); - TFW_STR(s5, HDR_NAME_5 ":"); + TFW_STR(s5, HDR_NAME_5); TFW_STR(s5_value, HDR_VALUE_5); + collect_compound_str(s1, col, 0); collect_compound_str(s1, s1_value, 0); + collect_compound_str(s2, col, 0); collect_compound_str(s2, s2_value, 0); + collect_compound_str(s3, col, 0); collect_compound_str(s3, s3_value, 0); + collect_compound_str(s4, col, 0); collect_compound_str(s4, s4_value, 0); + collect_compound_str(s5, col, 0); collect_compound_str(s5, s5_value, 0); tbl = &ctx.hpack.enc_tbl; diff --git a/tempesta_fw/t/unit/test_http_parser.c b/tempesta_fw/t/unit/test_http_parser.c index de102cdac9..24b317c5c3 100644 --- a/tempesta_fw/t/unit/test_http_parser.c +++ b/tempesta_fw/t/unit/test_http_parser.c @@ -1763,7 +1763,8 @@ TEST(http_parser, cookie) unsigned int flags; const char *str; } kv[] = { - { 0, "Cookie: " }, + { 0, "Cookie:" }, + { TFW_STR_OWS, " " }, { TFW_STR_NAME, "session=" }, { TFW_STR_VALUE, "42" }, { 0, "; " }, @@ -1858,7 +1859,9 @@ TEST(http_parser, set_cookie) TfwStr *s_parsed = &resp->h_tbl->tbl[TFW_HTTP_HDR_SET_COOKIE]; TfwStr s_expected = { .chunks = (TfwStr []) { - { .data = "Set-Cookie: " , .len = 12 }, + { .data = "Set-Cookie:" , .len = 11 }, + { .data = " " , .len = 1, + .flags = TFW_STR_OWS }, { .data = "sessionid=" , .len = 10, .flags = TFW_STR_NAME }, { .data = "38afes7a8" , .len = 9, @@ -1866,7 +1869,7 @@ TEST(http_parser, set_cookie) { .data = "; HttpOnly; Path=/" , .len = 18 } }, .len = 49, - .nchunks = 4 + .nchunks = 5 }; test_string_split(&s_expected, s_parsed); } @@ -1881,7 +1884,9 @@ TEST(http_parser, set_cookie) TfwStr *s_parsed = &resp->h_tbl->tbl[TFW_HTTP_HDR_SET_COOKIE]; TfwStr s_expected = { .chunks = (TfwStr []) { - { .data = "Set-Cookie: " , .len = 12 }, + { .data = "Set-Cookie:" , .len = 11 }, + { .data = " " , .len = 1, + .flags = TFW_STR_OWS }, { .data = "sessionid=" , .len = 10, .flags = TFW_STR_NAME }, { .data = "\"38afes7a8\"" , .len = 11, @@ -1889,7 +1894,7 @@ TEST(http_parser, set_cookie) { .data = "; HttpOnly; Path=/" , .len = 18 } }, .len = 51, - .nchunks = 4 + .nchunks = 5 }; test_string_split(&s_expected, s_parsed); } @@ -1904,7 +1909,9 @@ TEST(http_parser, set_cookie) TfwStr *s_parsed = &resp->h_tbl->tbl[TFW_HTTP_HDR_SET_COOKIE]; TfwStr s_expected = { .chunks = (TfwStr []) { - { .data = "Set-Cookie: " , .len = 12 }, + { .data = "Set-Cookie:" , .len = 11 }, + { .data = " " , .len = 1, + .flags = TFW_STR_OWS }, { .data = "id=" , .len = 3, .flags = TFW_STR_NAME }, { .data = "a3fWa" , .len = 5, @@ -1914,7 +1921,7 @@ TEST(http_parser, set_cookie) .len = 57 } }, .len = 77, - .nchunks = 4 + .nchunks = 5 }; test_string_split(&s_expected, s_parsed); } @@ -1928,7 +1935,9 @@ TEST(http_parser, set_cookie) TfwStr *s_parsed = &resp->h_tbl->tbl[TFW_HTTP_HDR_SET_COOKIE]; TfwStr s_expected = { .chunks = (TfwStr []) { - { .data = "Set-Cookie: " , .len = 12 }, + { .data = "Set-Cookie:" , .len = 11 }, + { .data = " " , .len = 1, + .flags = TFW_STR_OWS }, { .data = "__Host-id=" , .len = 10, .flags = TFW_STR_NAME }, { .data = "1" , .len = 1, @@ -1937,7 +1946,7 @@ TEST(http_parser, set_cookie) .len = 36 } }, .len = 59, - .nchunks = 4 + .nchunks = 5 }; test_string_split(&s_expected, s_parsed); } From 2d7bb75081fd95dd3aa945cbf660e838f79c88f5 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Sun, 1 Mar 2020 01:30:46 +0300 Subject: [PATCH 57/64] HTTP/2: fix duplicate headers processing during response forwarding (#309). --- tempesta_fw/http.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 3cebeadb0d..5fb5beb922 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -4006,16 +4006,18 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) for (i = mit->curr; i < map->count; ++i) { int k; + TfwStr *first; unsigned short hid = map->index[i].idx; unsigned short d_num = map->index[i].d_idx; TfwStr *tgt = &ht->tbl[hid]; - TfwStr *first = TFW_STR_CHUNK(tgt, 0); TfwHdrModsDesc *f_desc = NULL; const TfwStr *val; if (TFW_STR_DUP(tgt)) tgt = TFW_STR_CHUNK(tgt, d_num); + first = TFW_STR_CHUNK(tgt, 0); + if (WARN_ON_ONCE(!tgt || TFW_STR_EMPTY(tgt) || TFW_STR_DUP(tgt))) From b5e71ac0f3273277ea4b6e9559d752a461dfea0f Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Mon, 2 Mar 2020 00:23:54 +0300 Subject: [PATCH 58/64] HTTP/2: add header name tracking into the headers' compare procedure (#309). --- tempesta_fw/hpack.c | 3 +- tempesta_fw/http.c | 20 +----- tempesta_fw/http.h | 1 - tempesta_fw/http_msg.c | 120 +++++++++++++------------------- tempesta_fw/http_msg.h | 5 +- tempesta_fw/t/unit/test_hpack.c | 84 ++++++++++++++-------- 6 files changed, 108 insertions(+), 125 deletions(-) diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index b9c8c69cc4..34d2b20188 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -1822,8 +1822,7 @@ do { \ for (; i < h_mods->sz; ++i) { TfwHdrModsDesc *d = &h_mods->hdrs[i]; - if (!__hdr_name_cmp(&dc_iter->hdr_data, - TFW_STR_CHUNK(d->hdr, 0))) + if (!__hdr_name_cmp(&dc_iter->hdr_data, d->hdr)) { dc_iter->desc = d; break; diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 5fb5beb922..09d0f31c9d 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3104,7 +3104,7 @@ __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) } } else { - hid = __h2_hdr_lookup(hm, TFW_STR_CHUNK(hdr, 0)); + hid = __h2_hdr_lookup(hm, hdr); if (hid == ht->off && !s_val) /* * The raw header not found, and there is nothing @@ -4035,8 +4035,7 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) if ((hid < TFW_HTTP_HDR_RAW && hid == desc->hid) || (hid >= TFW_HTTP_HDR_RAW - && !__hdr_name_cmp(tgt, - TFW_STR_CHUNK(desc->hdr, 0)))) + && !__hdr_name_cmp(tgt, desc->hdr))) { f_desc = desc; break; @@ -5870,21 +5869,6 @@ tfw_http_req_key_calc(TfwHttpReq *req) } EXPORT_SYMBOL(tfw_http_req_key_calc); -TfwHdrModsDesc * -tfw_http_find_desc(const TfwStr *hdr, const TfwHdrMods *h_mods) -{ - int i; - - for (i = 0; i < h_mods->sz; ++i) { - TfwHdrModsDesc *desc = &h_mods->hdrs[i]; - - if (!__hdr_name_cmp(hdr, TFW_STR_CHUNK(desc->hdr, 0))) - return desc; - } - - return NULL; -} - static TfwConnHooks http_conn_hooks = { .conn_init = tfw_http_conn_init, .conn_repair = tfw_http_conn_repair, diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index da733440e1..f04cbc41f0 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -687,6 +687,5 @@ unsigned long tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out, bool inplace); unsigned long tfw_h2_hdr_size(unsigned long n_len, unsigned long v_len, unsigned short st_index); -TfwHdrModsDesc *tfw_http_find_desc(const TfwStr *hdr, const TfwHdrMods *h_mods); #endif /* __TFW_HTTP_H__ */ diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index cfc74cfcff..469dc77e03 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -260,35 +260,6 @@ __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client) } EXPORT_SYMBOL(__http_msg_hdr_val); -void -__h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name) -{ - const TfwStr *c, *end; - - if (unlikely(TFW_STR_EMPTY(hdr))) { - TFW_STR_INIT(out_name); - return; - } - - BUG_ON(TFW_STR_DUP(hdr)); - BUG_ON(TFW_STR_EMPTY(hdr)); - - *out_name = *hdr; - - if (unlikely(TFW_STR_PLAIN(hdr))) { - WARN_ON_ONCE(hdr->flags & TFW_STR_HDR_VALUE); - return; - } - - TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { - if (c->flags & TFW_STR_HDR_VALUE) { - out_name->len -= c->len; - out_name->nchunks--; - } - } -} -EXPORT_SYMBOL(__h2_msg_hdr_name); - void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val) { @@ -380,33 +351,36 @@ __hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) } /** - * Special procedure comparing specified name against the header in HTTP/2 - * or HTTP/1.1 format. + * Special procedure comparing the name or HPACK static index of @cmp_hdr (can + * be in HTTP/2 or HTTP/1.1 format) against the header @hdr which also can be + * in HTTP/2 or HTTP/1.1 format. */ -int -__hdr_name_cmp(const TfwStr *hdr, const TfwStr *name) +bool +__hdr_name_cmp(const TfwStr *hdr, const TfwStr *cmp_hdr) { long n; int i1, i2, off1, off2; const TfwStr *c1, *c2; BUG_ON(hdr->flags & TFW_STR_DUPLICATE); - BUG_ON(!name->len); + BUG_ON(!cmp_hdr->len); + + if (cmp_hdr->hpack_idx && cmp_hdr->hpack_idx == hdr->hpack_idx) + return 0; if (unlikely(!hdr->len)) - return -name->len; + return 1; i1 = i2 = 0; off1 = off2 = 0; - n = min(hdr->len, name->len); + n = min(hdr->len, cmp_hdr->len); c1 = TFW_STR_CHUNK(hdr, 0); - c2 = TFW_STR_CHUNK(name, 0); + c2 = TFW_STR_CHUNK(cmp_hdr, 0); while (n) { int cn = min(c1->len - off1, c2->len - off2); - int r = tfw_cstricmp(c1->data + off1, - c2->data + off2, cn); - if (r) - return r; + + if (tfw_cstricmp(c1->data + off1, c2->data + off2, cn)) + return 1; n -= cn; if (cn == c1->len - off1) { @@ -419,36 +393,44 @@ __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name) if (cn == c2->len - off2) { off2 = 0; ++i2; - c2 = TFW_STR_CHUNK(name, i2); + c2 = TFW_STR_CHUNK(cmp_hdr, i2); } else { off2 += cn; } - BUG_ON(n && (!c1 || !c2)); - } - /* Only name is contained in the header. */ - if (hdr->len == name->len) - return 0; + BUG_ON(n && (!c1 || !c2)); - if (hdr->len > name->len) { /* - * If the header is of HTTP/2 format, the end of name must match - * the end of the chunk, and the following value must have - * appropriate flag. + * Regardless of the header format (HTTP/2 or HTTP/1.1), the end + * of the name must match the end of the chunk, and the following + * chunk must contain value with appropriate flag (or it must + * contain just a single colon in case of HTTP/1.1-header). */ - if (!off1 - && (c1->flags & TFW_STR_HDR_VALUE) - && !(TFW_STR_CHUNK(hdr, i1 - 1)->flags & TFW_STR_HDR_VALUE)) - return 0; - /* - * If this is the HTTP/1.1-format header, the value must begin - * after the colon. - */ - if (*(c1->data + off1) == ':') - return 0; + if (!off2) { + const TfwStr *prev_c1; + /* + * If @c2 or @c1 is NULL, then only name is contained in + * the @cmp_hdr or @hdr respectively. + */ + if (c2 + && !(c2->flags & TFW_STR_HDR_VALUE) + && *c2->data != ':') + continue; + + prev_c1 = TFW_STR_CHUNK(hdr, i1 - 1); + + if (!off1 + && !(prev_c1->flags & TFW_STR_HDR_VALUE) + && (!c1 + || c1->flags & TFW_STR_HDR_VALUE + || *c1->data == ':')) + return 0; + + return 1; + } } - return (long)hdr->len - (long)name->len; + return 1; } /** @@ -457,7 +439,7 @@ __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name) * headers in HTTP/2 or HTTP/1.1 format. */ int -__h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name) +__h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) { unsigned int id; TfwHttpHdrTbl *ht = hm->h_tbl; @@ -470,7 +452,7 @@ __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name) */ if (h->flags & TFW_STR_DUPLICATE) h = TFW_STR_CHUNK(h, 0); - if (!__hdr_name_cmp(h, h_name)) + if (!__hdr_name_cmp(h, hdr)) break; } @@ -548,15 +530,7 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) * Both the headers, the new one and existing one, can already be * compound. */ - if (TFW_MSG_H2(hm)) { - TfwStr h_name; - - __h2_msg_hdr_name(&parser->hdr, &h_name); - id = __h2_hdr_lookup(hm, &h_name); - } - else { - id = __hdr_lookup(hm, &parser->hdr); - } + id = __h2_hdr_lookup(hm, &parser->hdr); /* Allocate some more room if not enough to store the header. */ if (unlikely(id == ht->size)) { diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 417175681e..f7f87c5040 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -57,7 +57,6 @@ __tfw_http_msg_set_str_data(TfwStr *str, void *data, struct sk_buff *skb) __tfw_http_msg_set_str_data(str, data, \ ss_skb_peek_tail(&hm->msg.skb_head)) -void __h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name); void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val); void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client); @@ -180,8 +179,8 @@ int tfw_http_msg_grow_hdr_tbl(TfwHttpMsg *hm); void tfw_http_msg_free(TfwHttpMsg *m); int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, const TfwStr *src, unsigned int *start_off); -int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name); -int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name); +int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *cmp_hdr); +int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr); int tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, const char *stop); diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index 45f3c82b9b..c410f6a140 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -92,6 +92,34 @@ test_h2_teardown(void) free_all_str(); } +static void +test_h2_hdr_name(TfwStr *hdr, TfwStr *out_name) +{ + const TfwStr *c, *end; + + if (unlikely(TFW_STR_EMPTY(hdr))) { + TFW_STR_INIT(out_name); + return; + } + + BUG_ON(TFW_STR_DUP(hdr)); + BUG_ON(TFW_STR_EMPTY(hdr)); + + *out_name = *hdr; + + if (unlikely(TFW_STR_PLAIN(hdr))) { + WARN_ON_ONCE(hdr->flags & TFW_STR_HDR_VALUE); + return; + } + + TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { + if (c->flags & TFW_STR_HDR_VALUE) { + out_name->len -= c->len; + out_name->nchunks--; + } + } +} + TEST(hpack, dec_table_static) { TfwHPack *hp; @@ -186,21 +214,21 @@ TEST(hpack, dec_table_dynamic) hp = &ctx.hpack; - __h2_msg_hdr_name(s1, &h_name); + test_h2_hdr_name(s1, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s1; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - __h2_msg_hdr_name(s2, &h_name); + test_h2_hdr_name(s2, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_X_FORWARDED_FOR; *it->parsed_hdr = *s2; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - __h2_msg_hdr_name(s3, &h_name); + test_h2_hdr_name(s3, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; @@ -254,14 +282,14 @@ TEST(hpack, dec_table_dynamic_inc) hp = &ctx.hpack; - __h2_msg_hdr_name(s1, &h_name); + test_h2_hdr_name(s1, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s1; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - __h2_msg_hdr_name(s2, &h_name); + test_h2_hdr_name(s2, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; @@ -278,14 +306,14 @@ TEST(hpack, dec_table_dynamic_inc) if (entry) EXPECT_TRUE(tfw_strcmp(entry->hdr, s1) == 0); - __h2_msg_hdr_name(s3, &h_name); + test_h2_hdr_name(s3, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_CACHE_CONTROL; *it->parsed_hdr = *s3; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - __h2_msg_hdr_name(s4, &h_name); + test_h2_hdr_name(s4, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; @@ -302,7 +330,7 @@ TEST(hpack, dec_table_dynamic_inc) if (entry) EXPECT_TRUE(tfw_strcmp(entry->hdr, s3) == 0); - __h2_msg_hdr_name(s5, &h_name); + test_h2_hdr_name(s5, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; @@ -415,7 +443,7 @@ TEST(hpack, dec_table_wrap) EXPECT_NOT_NULL(l_entry->hdr); EXPECT_NOT_NULL(t_entry->hdr); if (l_entry->hdr) { - __h2_msg_hdr_name(t_entry->hdr, &h_name); + test_h2_hdr_name(t_entry->hdr, &h_name); EXPECT_TRUE(tfw_strcmp(&h_name, l_entry->hdr) == 0); } } @@ -524,7 +552,7 @@ TEST(hpack, dec_raw) EXPECT_NE(ht, test_req->h_tbl); ht = test_req->h_tbl; - __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW], &h_name); + test_h2_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW], &h_name); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -537,7 +565,7 @@ TEST(hpack, dec_raw) strlen(test_value1), 0)); } - __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_name); + test_h2_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_name); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -550,7 +578,7 @@ TEST(hpack, dec_raw) strlen(test_value2), 0)); } - __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_name); + test_h2_hdr_name(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_name); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -747,7 +775,7 @@ TEST(hpack, dec_indexed) EXPECT_EQ(hdr->nchunks, 3); if (hdr->nchunks == 3) { dup = hdr->chunks; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -760,7 +788,7 @@ TEST(hpack, dec_indexed) test_len_val1, 0)); } dup = hdr->chunks + 1; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -773,7 +801,7 @@ TEST(hpack, dec_indexed) test_len_val1, 0)); } dup = hdr->chunks + 2; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -792,7 +820,7 @@ TEST(hpack, dec_indexed) EXPECT_EQ(hdr->nchunks, 2); if (hdr->nchunks == 2) { dup = hdr->chunks; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -805,7 +833,7 @@ TEST(hpack, dec_indexed) test_len_val2, 0)); } dup = hdr->chunks + 1; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -820,7 +848,7 @@ TEST(hpack, dec_indexed) } hdr = &ht->tbl[TFW_HTTP_HDR_HOST]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -834,7 +862,7 @@ TEST(hpack, dec_indexed) } hdr = &ht->tbl[TFW_HTTP_HDR_REFERER]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -863,7 +891,7 @@ TEST(hpack, dec_indexed) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -884,7 +912,7 @@ TEST(hpack, dec_indexed) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -905,7 +933,7 @@ TEST(hpack, dec_indexed) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1047,7 +1075,7 @@ TEST(hpack, dec_huffman) * into the headers table. */ hdr = &ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1061,7 +1089,7 @@ TEST(hpack, dec_huffman) } hdr = &ht->tbl[TFW_HTTP_HDR_RAW]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1075,7 +1103,7 @@ TEST(hpack, dec_huffman) } hdr = &ht->tbl[TFW_HTTP_HDR_RAW + 1]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1096,7 +1124,7 @@ TEST(hpack, dec_huffman) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1117,7 +1145,7 @@ TEST(hpack, dec_huffman) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1138,7 +1166,7 @@ TEST(hpack, dec_huffman) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); From 4f0cdd1fdc00b06effca744f83b7c1507c6e5b10 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Mon, 2 Mar 2020 16:48:29 +0300 Subject: [PATCH 59/64] HTTP/2: corrections according review comments (#309). --- tempesta_fw/cache.c | 34 +++++++++++++++++----------------- tempesta_fw/http.c | 5 ++--- tempesta_fw/http.h | 3 --- tempesta_fw/http_msg.c | 2 +- tempesta_fw/http_msg.h | 6 ++++++ tempesta_fw/http_stream.c | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) diff --git a/tempesta_fw/cache.c b/tempesta_fw/cache.c index 069b68effe..2d57adf02b 100644 --- a/tempesta_fw/cache.c +++ b/tempesta_fw/cache.c @@ -183,6 +183,8 @@ static struct task_struct *cache_mgr_thr; #endif static DEFINE_PER_CPU(TfwWorkTasklet, cache_wq); +#define RESP_BUF_LEN 128 + static DEFINE_PER_CPU(char[RESP_BUF_LEN], g_c_buf); static TfwStr g_crlf = { .data = S_CRLF, .len = SLEN(S_CRLF) }; @@ -619,9 +621,9 @@ tfw_cache_h2_decode_write(TDB *db, TdbVRec **trec, TfwHttpResp *resp, return r; } -static inline int -tfw_cache_h2_set_status(TDB *db, TfwCacheEntry *ce, TfwHttpResp *resp, - TdbVRec **trec, char **p, unsigned long *acc_len) +static int +tfw_cache_set_status(TDB *db, TfwCacheEntry *ce, TfwHttpResp *resp, + TdbVRec **trec, char **p, unsigned long *acc_len) { int r; TfwMsgIter *it = &resp->mit.iter; @@ -685,8 +687,8 @@ tfw_cache_h2_set_status(TDB *db, TfwCacheEntry *ce, TfwHttpResp *resp, * Write HTTP header to skb data. */ static int -tfw_cache_h2_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwHdrMods *hmods, - TdbVRec **trec, char **p, unsigned long *acc_len) +tfw_cache_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwHdrMods *hmods, + TdbVRec **trec, char **p, unsigned long *acc_len) { tfw_cache_write_actor_t *write_actor; TfwCStr *s = (TfwCStr *)*p; @@ -707,7 +709,7 @@ tfw_cache_h2_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwHdrMods *hmods, r = write_actor(db, trec, resp, p, s->len, &dc_iter); if (likely(!r)) *acc_len += dc_iter.acc_len; - goto out; + return r; } /* Process duplicated headers. */ @@ -722,7 +724,6 @@ tfw_cache_h2_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwHdrMods *hmods, *acc_len += dc_iter.acc_len;; } -out: return r; } @@ -786,8 +787,7 @@ tfw_cache_send_304(TfwHttpReq *req, TfwCacheEntry *ce) trec = tdb_next_rec_chunk(db, trec); BUG_ON(!trec); - if (tfw_cache_h2_build_resp_hdr(db, resp, NULL, &trec, &p, - &h_len)) + if (tfw_cache_build_resp_hdr(db, resp, NULL, &trec, &p, &h_len)) goto err_setup; } @@ -1845,7 +1845,7 @@ tfw_cache_purge_method(TfwHttpReq *req) * See do_tcp_sendpages() as reference. */ static int -tfw_cache_h2_build_resp_body(TDB *db, TdbVRec *trec, TfwMsgIter *it, char *p) +tfw_cache_build_resp_body(TDB *db, TdbVRec *trec, TfwMsgIter *it, char *p) { int r; @@ -1968,8 +1968,8 @@ tfw_cache_set_hdr_age(TfwHttpResp *resp, TfwCacheEntry *ce) * TODO use iterator and passed skbs to be called from net_tx_action. */ static TfwHttpResp * -tfw_cache_h2_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, time_t lifetime, - unsigned int stream_id) +tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, time_t lifetime, + unsigned int stream_id) { int h; TfwMsgIter *it; @@ -2007,12 +2007,12 @@ tfw_cache_h2_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, time_t lifetime, goto free; } - if (tfw_cache_h2_set_status(db, ce, resp, &trec, &p, &h_len)) + if (tfw_cache_set_status(db, ce, resp, &trec, &p, &h_len)) goto free; for (h = TFW_HTTP_HDR_REGULAR; h < ce->hdr_num; ++h) { - if (tfw_cache_h2_build_resp_hdr(db, resp, h_mods, &trec, &p, - &h_len)) + if (tfw_cache_build_resp_hdr(db, resp, h_mods, &trec, &p, + &h_len)) goto free; } @@ -2101,7 +2101,7 @@ tfw_cache_h2_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, time_t lifetime, /* Fill skb with body from cache for HTTP/2 or HTTP/1.1 response. */ BUG_ON(p != TDB_PTR(db->hdr, ce->body)); if (ce->body_len) { - if (tfw_cache_h2_build_resp_body(db, trec, it, p)) + if (tfw_cache_build_resp_body(db, trec, it, p)) goto free; if (!TFW_MSG_H2(req) && test_bit(TFW_HTTP_B_CHUNKED, resp->flags) @@ -2159,7 +2159,7 @@ cache_req_process_node(TfwHttpReq *req, tfw_http_cache_cb_t action) } } - resp = tfw_cache_h2_build_resp(req, ce, lifetime, id); + resp = tfw_cache_build_resp(req, ce, lifetime, id); /* * The stream of HTTP/2-request should be closed here since we have * successfully created the resulting response from cache and will diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 09d0f31c9d..71aac9f515 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -118,6 +118,8 @@ T_WARN("%s, status %d: %s\n", \ msg, status, addr_str)) +#define RESP_BUF_LEN 128 + static DEFINE_PER_CPU(char[RESP_BUF_LEN], g_buf); int ghprio; /* GFSM hook priority. */ @@ -3961,9 +3963,6 @@ tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods, TfwHttpTransIter *mit = &resp->mit; TfwH2TransOp op = cache ? TFW_H2_TRANS_EXPAND : TFW_H2_TRANS_ADD; - if (!h_mods) - return 0; - if (!h_mods) return 0; diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index f04cbc41f0..b17b0e4633 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -300,9 +300,6 @@ enum { ((!hmmsg->conn || TFW_CONN_TYPE(hmmsg->conn) & Conn_Srv) && \ hmmsg->pair && TFW_MSG_H2(hmmsg->pair)) -#define H2_STAT_VAL_LEN 3 -#define RESP_BUF_LEN 128 - /** * The structure to hold data for an HTTP error response. * An error response is sent later in an unlocked queue context. diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 469dc77e03..bba5e0ce93 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -355,7 +355,7 @@ __hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) * be in HTTP/2 or HTTP/1.1 format) against the header @hdr which also can be * in HTTP/2 or HTTP/1.1 format. */ -bool +int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *cmp_hdr) { long n; diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index f7f87c5040..37e3028e60 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -37,6 +37,12 @@ #define SLEN(s) (sizeof(s) - 1) +/* + * The size of the buffer to store the value for ':status' pseudo-header + * of HTTP/2-response. + */ +#define H2_STAT_VAL_LEN 3 + TfwStr *tfw_http_msg_make_hdr(TfwPool *pool, const char *name, const char *val); unsigned int tfw_http_msg_resp_spec_hid(const TfwStr *hdr); unsigned int tfw_http_msg_req_spec_hid(const TfwStr *hdr); diff --git a/tempesta_fw/http_stream.c b/tempesta_fw/http_stream.c index 6a0f8357c6..f98c186fbc 100644 --- a/tempesta_fw/http_stream.c +++ b/tempesta_fw/http_stream.c @@ -134,7 +134,7 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, } /* * Received RST_STREAM frame immediately moves stream into the - * final 'closed' state, while the the sent RST_STREAM moves stream + * final 'closed' state, while the sent RST_STREAM moves stream * into the intermediate 'locally closed' state. */ else if (type == HTTP2_RST_STREAM) { From bb502aedfd7e02240822a4c5a16795ba0a5a38d0 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sun, 1 Mar 2020 17:53:28 +0500 Subject: [PATCH 60/64] h2 parser: reduce number of compare operators during headers parsing --- tempesta_fw/http_parser.c | 426 +++++++++++++++++++------------------- 1 file changed, 213 insertions(+), 213 deletions(-) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index e0a3e8e3d1..df530181a0 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -6771,264 +6771,218 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, tfw_http_msg_hdr_open(msg, p); - switch (c) { - case ':': - if (likely(__data_available(p, 7) - && C4_INT(p + 1, 'm', 'e', 't', 'h') - && *(p + 5) == 'o' - && *(p + 6) == 'd')) - { + /* Ensure we have enough data for largest match. */ + if (unlikely(!__data_available(p, 4))) + __FSM_JMP(Req_Hdr); + /* + * Some successful matches cause drop action instead of move: + * - All allowed pseudo headers are listed here, no need to + * fallback to slow path on partial matches. + * - RFC 7540 Section 8.1.2.2: Messages with connection-specific + * headers must be treated as malformed. + */ + + switch (PI(p)) { + + /* :authority */ + case TFW_CHAR4_INT(':', 'a', 'u', 't'): + if (unlikely(!__data_available(p, 10))) + __FSM_H2_NEXT_n(Req_HdrPsAut, 4); + if (C8_INT(p + 2, 'u', 't', 'h', 'o', 'r', 'i', 't', 'y')) + __FSM_H2_FIN(Req_HdrPsAuthorityV, 10, + TFW_TAG_HDR_H2_AUTHORITY); + __FSM_H2_DROP(RGen_Hdr); + /* :method */ + case TFW_CHAR4_INT(':', 'm', 'e', 't'): + if (unlikely(!__data_available(p, 7))) + __FSM_H2_NEXT_n(Req_HdrPsMet, 4); + if (C4_INT(p + 3, 't', 'h', 'o', 'd')) __FSM_H2_FIN(Req_HdrPsMethodV, 7, TFW_TAG_HDR_H2_METHOD); - } - if (likely(__data_available(p, 7) - && C4_INT(p + 1, 's', 'c', 'h', 'e') - && *(p + 5) == 'm' - && *(p + 6) == 'e')) - { + __FSM_H2_DROP(RGen_Hdr); + /* :scheme */ + case TFW_CHAR4_INT(':', 's', 'c', 'h'): + if (unlikely(!__data_available(p, 7))) + __FSM_H2_NEXT_n(Req_HdrPsSch, 4); + if (C4_INT(p + 3, 'h', 'e', 'm', 'e')) __FSM_H2_FIN(Req_HdrPsSchemeV, 7, TFW_TAG_HDR_H2_SCHEME); - } - if (likely(__data_available(p, 10) - && C8_INT(p + 1, 'a', 'u', 't', 'h', - 'o', 'r', 'i', 't') - && *(p + 9) == 'y')) - { - __FSM_H2_FIN(Req_HdrPsAuthorityV, 10, - TFW_TAG_HDR_H2_AUTHORITY); - } - if (likely(__data_available(p, 5) - && C4_INT(p + 1, 'p', 'a', 't', 'h'))) - { + __FSM_H2_DROP(RGen_Hdr); + /* :path */ + case TFW_CHAR4_INT(':', 'p', 'a', 't'): + if (unlikely(!__data_available(p, 5))) + __FSM_H2_NEXT_n(Req_HdrPsPat, 4); + if (*(p + 4) == 'h') __FSM_H2_FIN(Req_HdrPsPathV, 5, TFW_TAG_HDR_H2_PATH); - } - __FSM_H2_NEXT(Req_HdrPseudo); - case 'a': - if (likely(__data_available(p, 6) - && C4_INT(p + 1, 'c', 'c', 'e', 'p') - && *(p + 5) == 't')) - { + __FSM_H2_DROP(RGen_Hdr); + /* accept */ + case TFW_CHAR4_INT('a', 'c', 'c', 'e'): + if (unlikely(!__data_available(p, 6))) + __FSM_H2_NEXT_n(Req_HdrAcce, 4); + if (C4_INT(p + 2, 'c', 'e', 'p', 't')) __FSM_H2_FIN(Req_HdrAcceptV, 6, TFW_TAG_HDR_ACCEPT); - } - if (likely(__data_available(p, 13) - && C8_INT(p + 1, 'u', 't', 'h', 'o', - 'r', 'i', 'z', 'a') - && C4_INT(p + 9, 't', 'i', 'o', 'n'))) + __FSM_H2_OTHER_n(4); + /* authorization */ + case TFW_CHAR4_INT('a', 'u', 't', 'h'): + if (unlikely(!__data_available(p, 13))) + __FSM_H2_NEXT_n(Req_HdrAuth, 4); + if(C8_INT(p + 4, 'o', 'r', 'i', 'z', 'a', 't', 'i', 'o') + && *(p + 12) == 'n') { __FSM_H2_FIN(Req_HdrAuthorizationV, 13, TFW_TAG_HDR_AUTHORIZATION); } - __FSM_H2_NEXT(Req_HdrA); - case 'c': + __FSM_H2_OTHER_n(4); + /* cache-control */ + case TFW_CHAR4_INT('c', 'a', 'c', 'h'): if (unlikely(!__data_available(p, 13))) - __FSM_H2_NEXT(Req_HdrC); - switch (PI(p + 1)) { - case TFW_CHAR4_INT('a', 'c', 'h', 'e'): - if (likely(C8_INT(p + 5, '-', 'c', 'o', 'n', - 't', 'r', 'o', 'l'))) - { - __FSM_H2_FIN(Req_HdrCache_ControlV, 13, - TFW_TAG_HDR_CACHE_CONTROL); - } - __FSM_H2_OTHER_n(5); - case TFW_CHAR4_INT('o', 'n', 'n', 'e'): - if (likely(C4_INT(p + 5, 'c', 't', 'i', 'o') - && *(p + 9) == 'n')) - { - /* - * Messages with connection-specific - * headers must be treated as malformed - * in HTTP/2 (see RFC 7540 section - * 8.1.2.2 for details). - */ - __FSM_H2_DROP(Req_HdrConnection); - } - __FSM_H2_OTHER_n(5); - case TFW_CHAR4_INT('o', 'n', 't', 'e'): - if (likely(*(p + 5) == 'n' - && *(p + 6) == 't' - && *(p + 7) == '-')) - { - __FSM_H2_NEXT_n(Req_HdrContent_, 8); - } - __FSM_H2_OTHER_n(5); - case TFW_CHAR4_INT('o', 'o', 'k', 'i'): - if (likely(*(p + 5) == 'e')) - { - __FSM_H2_FIN(Req_HdrCookieV, 6, - TFW_TAG_HDR_COOKIE); - } - __FSM_H2_OTHER_n(5); - default: - __FSM_H2_OTHER(); - } - case 'h': - if (likely(__data_available(p, 4) - && *(p + 1) == 'o' - && *(p + 2) == 's' - && *(p + 3) == 't')) + __FSM_H2_NEXT_n(Req_HdrCach, 4); + if (C8_INT(p + 4, 'e', '-', 'c', 'o', 'n', 't', 'r', 'o') + && *(p + 12) == 'l') { - __FSM_H2_FIN(Req_HdrHostV, 4, - TFW_TAG_HDR_HOST); + __FSM_H2_FIN(Req_HdrCache_ControlV, 13, + TFW_TAG_HDR_CACHE_CONTROL); } - __FSM_H2_NEXT(Req_HdrH); - case 'i': - if (likely(__data_available(p, 17) - && *(p + 1) == 'f' - && *(p + 2) == '-' - && C8_INT(p + 3, 'm', 'o', 'd', 'i', - 'f', 'i', 'e', 'd') - && *(p + 11) == '-' - && C4_INT(p + 12, 's', 'i', 'n', 'c') - && *(p + 16) == 'e')) + __FSM_H2_OTHER_n(4); + /* connection */ + case TFW_CHAR4_INT('c', 'o', 'n', 'n'): + if (unlikely(!__data_available(p, 9))) + __FSM_H2_NEXT_n(Req_HdrConn, 4); + if (C8_INT(p + 1, 'o', 'n', 'n', 'e', 'c', 't', 'i', 'n')) + __FSM_H2_DROP(Req_HdrConnection); + __FSM_H2_OTHER_n(4); + /* content-* */ + case TFW_CHAR4_INT('c', 'o', 'n', 't'): + if (unlikely(!__data_available(p, 14))) + __FSM_H2_NEXT_n(Req_HdrCont, 4); + if (C8_INT(p + 4, 'e', 'n', 't', '-', 't', 'y', 'p', 'e')) + __FSM_H2_FIN(Req_HdrContent_TypeV, 12, + TFW_TAG_HDR_CONTENT_TYPE); + if (C8_INT(p + 4, 'e', 'n', 't', '-', 'l', 'e', 'n', 'g') + && C4_INT(p + 10, 'n', 'g', 't', 'h')) + __FSM_H2_FIN(Req_HdrContent_LengthV, 14, + TFW_TAG_HDR_CONTENT_LENGTH); + __FSM_H2_OTHER_n(4); + /* cookie */ + case TFW_CHAR4_INT('c', 'o', 'o', 'k'): + if (unlikely(!__data_available(p, 6))) + __FSM_H2_NEXT_n(Req_HdrCook, 4); + if (C4_INT(p + 2, 'o', 'k', 'i', 'e')) + __FSM_H2_FIN(Req_HdrCookieV, 6, + TFW_TAG_HDR_COOKIE); + __FSM_H2_OTHER_n(4); + /* host */ + case TFW_CHAR4_INT('h', 'o', 's', 't'): + __FSM_H2_FIN(Req_HdrHostV, 4, TFW_TAG_HDR_HOST); + /* if-modified-since */ + case TFW_CHAR4_INT('i', 'f', '-', 'm'): + if (unlikely(!__data_available(p, 17))) + __FSM_H2_NEXT_n(Req_HdrIf_M, 4); + if (C8_INT(p + 4, 'o', 'd', 'i', 'f', 'i', 'e', 'd', '-') + && C8_INT(p + 9, 'e', 'd', '-', 's', 'i', 'n', 'c', + 'e')) { __FSM_H2_FIN(Req_HdrIf_Modified_SinceV, 17, TFW_TAG_HDR_IF_MODIFIED_SINCE); } - if (likely(__data_available(p, 13) - && *(p + 1) == 'f' - && *(p + 2) == '-' - && C4_INT(p + 3, 'n', 'o', 'n', 'e') - && *(p + 7) == '-' - && C4_INT(p + 8, 'm', 'a', 't', 'c') - && *(p + 12) == 'h')) + __FSM_H2_OTHER_n(4); + /* if-none-match */ + case TFW_CHAR4_INT('i', 'f', '-', 'n'): + if (unlikely(!__data_available(p, 13))) + __FSM_H2_NEXT_n(Req_HdrIf_N, 4); + if (C8_INT(p + 4, 'o', 'n', 'e', '-', 'm', 'a', 't', 'c') + && *(p + 12) == 'h') { __FSM_H2_FIN(Req_HdrIf_None_MatchV, 13, TFW_TAG_HDR_IF_NONE_MATCH); } - __FSM_H2_NEXT(Req_HdrI); - case 'k': - if (likely(__data_available(p, 10) - && C4_INT(p, 'k', 'e', 'e', 'p') - && *(p + 4) == '-' - && C4_INT(p + 5, 'a', 'l', 'i', 'v') - && *(p + 9) == 'e')) - { + __FSM_H2_OTHER_n(4); + /* keep-alive */ + case TFW_CHAR4_INT('k', 'e', 'e', 'p'): + if (unlikely(!__data_available(p, 10))) + __FSM_H2_NEXT_n(Req_HdrKeep, 4); + if (C8_INT(p + 2, 'e', 'p', '-', 'a', 'l', 'i', 'v', 'e')) __FSM_H2_DROP(Req_HdrKeep_Alive); - } - __FSM_H2_NEXT(Req_HdrK); - case 'p': - if (likely(__data_available(p, 6) - && C4_INT(p + 1, 'r', 'a', 'g', 'm') - && *(p + 5) == 'a')) - { + __FSM_H2_OTHER_n(4); + /* pragma */ + case TFW_CHAR4_INT('p', 'r', 'a', 'g'): + if (unlikely(!__data_available(p, 6))) + __FSM_H2_NEXT_n(Req_HdrPrag, 4); + if (C4_INT(p + 2, 'a', 'g', 'm', 'a')) __FSM_H2_FIN(Req_HdrPragmaV, 6, TFW_TAG_HDR_PRAGMA); - } - __FSM_H2_NEXT(Req_HdrP); - case 'r': - if (likely(__data_available(p, 7) - && C4_INT(p + 1, 'e', 'f', 'e', 'r') - && *(p + 5) == 'e' - && *(p + 6) == 'r')) - { - __FSM_H2_FIN(Req_HdrRefererV, 7, - TFW_TAG_HDR_REFERER); - } - __FSM_H2_NEXT(Req_HdrR); - case 't': - if (likely(__data_available(p, 17) - && C8_INT(p, 't', 'r', 'a', 'n', - 's', 'f', 'e', 'r') - && *(p + 8) == '-' - && C8_INT(p + 9, 'e', 'n', 'c', 'o', - 'd', 'i', 'n', 'g'))) + __FSM_H2_OTHER_n(4); + /* transfer-encoding */ + case TFW_CHAR4_INT('t', 'r', 'a', 'n'): + if (unlikely(!__data_available(p, 17))) + __FSM_H2_NEXT_n(Req_HdrTran, 4); + if (C8_INT(p + 1, 'r', 'a', 'n', 's', 'f', 'e', 'r', '-') + && C8_INT(p + 9, 'e', 'n', 'c', 'o', 'd', 'i', 'n', + 'g')) { __FSM_H2_DROP(Req_HdrTransfer_Encoding); } - __FSM_H2_NEXT(Req_HdrT); - case 'u': - if (likely(__data_available(p, 10) - && C4_INT(p, 'u', 's', 'e', 'r') - && *(p + 4) == '-' - && C4_INT(p + 5, 'a', 'g', 'e', 'n') - && *(p + 9) == 't')) - { + __FSM_H2_OTHER_n(4); + /* referer */ + case TFW_CHAR4_INT('r', 'e', 'f', 'e'): + if (unlikely(!__data_available(p, 7))) + __FSM_H2_NEXT_n(Req_HdrRefe, 4); + if (C4_INT(p + 3, 'e', 'r', 'e', 'r')) + __FSM_H2_FIN(Req_HdrRefererV, 7, + TFW_TAG_HDR_REFERER); + __FSM_H2_OTHER_n(4); + /* user-agent */ + case TFW_CHAR4_INT('u', 's', 'e', 'r'): + if (unlikely(!__data_available(p, 10))) + __FSM_H2_NEXT_n(Req_HdrUser, 4); + if (C8_INT(p + 2, 'e', 'r', '-', 'a', 'g', 'e', 'n', 't')) __FSM_H2_FIN(Req_HdrUser_AgentV, 10, TFW_TAG_HDR_USER_AGENT); - } - __FSM_H2_NEXT(Req_HdrU); - case 'x': - if (likely(__data_available(p, 15) - && C8_INT(p + 1, '-', 'f', 'o', 'r', 'w', - 'a', 'r', 'd') - && C4_INT(p + 9, 'e', 'd', '-', 'f') - && *(p + 13) == 'o' - && *(p + 14) == 'r')) + __FSM_H2_OTHER_n(4); + /* x-forwarded-for */ + case TFW_CHAR4_INT('x', '-', 'f', 'o'): + if (unlikely(!__data_available(p, 15))) + __FSM_H2_NEXT_n(Req_HdrX_Fo, 4); + if (C8_INT(p + 4, 'r', 'w', 'a', 'r', 'd', 'e', 'd', '-') + && C4_INT(p + 11, '-', 'f', 'o', 'r')) { __FSM_H2_FIN(Req_HdrX_Forwarded_ForV, 15, TFW_TAG_HDR_X_FORWARDED_FOR); } - if (likely(__data_available(p, 14) - && *(p + 1) == '-' - && *(p + 7) == '-' - /* Safe match: '-' is checked above. */ - && C8_INT_LCM(p, 'x', '-', 'h', 't', - 't', 'p', '-', 'm') - && C4_INT_LCM(p + 8, 'e', 't', 'h', 'o') - && TFW_LC(*(p + 12) == 'd') - && *(p + 10) == ':')) + __FSM_H2_OTHER_n(4); + /* x-method-override family. */ + case TFW_CHAR4_INT('x', '-', 'h', 't'): + if (unlikely(!__data_available(p, 22))) + __FSM_H2_NEXT_n(Req_HdrX_Ht, 4); + if (C8_INT(p + 4, 't', 'p', '-', 'm', 'e', 't', 'h', 'o') + && C8_INT(p + 12, 'd', '-', 'o', 'v','e', 'r', 'r', + 'i') + && C4_INT(p + 18, 'r', 'i', 'd', 'e')) { - __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 14, + __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 22, TFW_TAG_HDR_RAW); } - if (likely(__data_available(p, 23) - && *(p + 1) == '-' - && *(p + 7) == '-' - && *(p + 14) == '-' - /* Safe match: '-' is checked above. */ - && C8_INT_LCM(p, 'x', '-', 'h', 't', - 't', 'p', '-', 'm') - && C8_INT_LCM(p + 8, 'e', 't', 'h', 'o', - 'd', '-', 'o', 'v') - && C8_INT7_LCM(p + 16, 'v', 'e', 'r', 'r', - 'i', 'd', 'e', ':'))) + if (C8_INT(p + 4, 't', 'p', '-', 'm', 'e', 't', 'h', 'o') + && *(p + 12) == 'd') { - __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 23, + __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 13, TFW_TAG_HDR_RAW); } - if (likely(__data_available(p, 18) - && *(p + 1) == '-' - && *(p + 9) == '-' - /* Safe match: '-' is checked above. */ - && C8_INT_LCM(p + 2, 'm', 'e', 't', 'h', - 'o', 'd', '-', 'o') - && C8_INT7_LCM(p + 10, 'v', 'e', 'r', 'r', - 'i', 'd', 'e', ':'))) + __FSM_H2_OTHER_n(4); + case TFW_CHAR4_INT('x', '-', 'm', 'e'): + if (unlikely(!__data_available(p, 17))) + __FSM_H2_NEXT_n(Req_HdrX_Me, 4); + if (C8_INT(p + 4, 't', 'h', 'o', 'd', '-', 'o', 'v', 'e') + && C8_INT(p + 9, 'o', 'v', 'e', 'r', 'r', 'i', 'd', + 'e')) { - - __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 18, + __FSM_H2_FIN(Req_HdrX_Method_OverrideV, 17, TFW_TAG_HDR_RAW); } - __FSM_H2_NEXT(Req_HdrX); - default: - __FSM_JMP(RGen_HdrOtherN); - } - } + __FSM_H2_OTHER_n(4); - __FSM_STATE(Req_HdrContent_) { - switch (c) { - case 'l': - if (likely(__data_available(p, 6) - && C4_INT(p + 1, 'e', 'n', 'g', 't') - && *(p + 5) == 'h')) - { - __FSM_H2_FIN(Req_HdrContent_LengthV, 6, - TFW_TAG_HDR_CONTENT_LENGTH); - } - __FSM_H2_NEXT(Req_HdrContent_L); - case 't': - if (likely(__data_available(p, 4) - && *(p + 1) == 'y' - && *(p + 2) == 'p' - && *(p + 3) == 'e')) - { - __FSM_H2_FIN(Req_HdrContent_TypeV, 4, - TFW_TAG_HDR_CONTENT_TYPE); - } - __FSM_H2_NEXT(Req_HdrContent_T); default: __FSM_JMP(RGen_HdrOtherN); } @@ -7036,6 +6990,12 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, __FSM_STATE(RGen_HdrOtherN) { __fsm_n = __data_remain(p); + /* + * TODO: RFC 7540, Section 8.1.2: + * A request or response containing uppercase header field + * names MUST be treated as malformed. + * We should use here lower-case matching function. + */ __fsm_sz = tfw_match_token(p, __fsm_n); if (unlikely(__fsm_sz != __fsm_n)) __FSM_H2_DROP(RGen_HdrOtherN); @@ -7299,6 +7259,35 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, /* Improbable states of (pseudo-)header names processing. */ + __FSM_STATE(Req_Hdr, cold) { + switch (c) { + case ':': + __FSM_H2_NEXT(Req_HdrPseudo); + case 'a': + __FSM_H2_NEXT(Req_HdrA); + case 'c': + __FSM_H2_NEXT(Req_HdrC); + case 'h': + __FSM_H2_NEXT(Req_HdrH); + case 'i': + __FSM_H2_NEXT(Req_HdrI); + case 'k': + __FSM_H2_NEXT(Req_HdrK); + case 'p': + __FSM_H2_NEXT(Req_HdrP); + case 'r': + __FSM_H2_NEXT(Req_HdrR); + case 't': + __FSM_H2_NEXT(Req_HdrT); + case 'u': + __FSM_H2_NEXT(Req_HdrU); + case 'x': + __FSM_H2_NEXT(Req_HdrX); + default: + __FSM_H2_DROP(Req_HdrPseudo); + } + } + __FSM_STATE(Req_HdrPseudo, cold) { switch (c) { case 'a': @@ -7430,6 +7419,17 @@ tfw_h2_parse_req_hdr(unsigned char *data, unsigned long len, TfwHttpReq *req, __FSM_H2_TX_AF(Req_HdrConten, 't', Req_HdrContent); __FSM_H2_TX_AF(Req_HdrContent, '-', Req_HdrContent_); + __FSM_STATE(Req_HdrContent_, cold) { + switch (c) { + case 'l': + __FSM_H2_NEXT(Req_HdrContent_L); + case 't': + __FSM_H2_NEXT(Req_HdrContent_T); + default: + __FSM_JMP(RGen_HdrOtherN); + } + } + __FSM_H2_TX_AF(Req_HdrContent_L, 'e', Req_HdrContent_Le); __FSM_H2_TX_AF(Req_HdrContent_Le, 'n', Req_HdrContent_Len); __FSM_H2_TX_AF(Req_HdrContent_Len, 'g', Req_HdrContent_Leng); From 2585ef932715c07c44e985d055262ba85fa4f230 Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Sun, 1 Mar 2020 17:53:39 +0500 Subject: [PATCH 61/64] If listening for http/1.1 mode print performance warning --- tempesta_fw/sock_clnt.c | 30 +++++++++++++++++++++--------- tempesta_fw/tls.c | 4 +++- tempesta_fw/tls.h | 2 +- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/tempesta_fw/sock_clnt.c b/tempesta_fw/sock_clnt.c index df7cd5c018..41ac7b95df 100644 --- a/tempesta_fw/sock_clnt.c +++ b/tempesta_fw/sock_clnt.c @@ -473,10 +473,11 @@ tfw_sock_check_lst(TfwServer *srv) static int tfw_cfgop_listen(TfwCfgSpec *cs, TfwCfgEntry *ce) { - int r; + int r, type = TFW_FSM_HTTP; int port; TfwAddr addr; const char *in_str = NULL; + bool deprecated = true; if (tfw_cfg_check_val_n(ce, 1) || ce->attr_n > 1) goto parse_err; @@ -507,25 +508,36 @@ tfw_cfgop_listen(TfwCfgSpec *cs, TfwCfgEntry *ce) goto parse_err; if (!ce->attr_n) - return tfw_listen_sock_add(&addr, TFW_FSM_HTTP); + goto done; in_str = tfw_cfg_get_attr(ce, "proto", NULL); if (!in_str) goto parse_err; - if (!strcasecmp(in_str, "http")) { - return tfw_listen_sock_add(&addr, TFW_FSM_HTTP); - } + if (!strcasecmp(in_str, "http")) + goto done; - if (!tfw_tls_cfg_alpn_protos(in_str)) { - tfw_tls_cfg_require(); - return tfw_listen_sock_add(&addr, TFW_FSM_HTTPS); - } + type = TFW_FSM_HTTPS; + if (!tfw_tls_cfg_alpn_protos(in_str, &deprecated)) + goto done; parse_err: T_ERR_NL("Unable to parse 'listen' value: '%s'\n", in_str ? in_str : "Invalid directive format"); return -EINVAL; + +done: + if (deprecated) { + static int warn = 1; + if (warn) + T_WARN_NL("Listening for HTTP/1.1 protocol is " + "compatibility mode and may lack of " + "performance.\n"); + warn = 0; + } + if (type == TFW_FSM_HTTPS) + tfw_tls_cfg_require(); + return tfw_listen_sock_add(&addr, type); } static int diff --git a/tempesta_fw/tls.c b/tempesta_fw/tls.c index 9a30bfac29..9f857ea8b0 100644 --- a/tempesta_fw/tls.c +++ b/tempesta_fw/tls.c @@ -822,7 +822,7 @@ tfw_tls_match_any_sni_to_dflt(bool match) } int -tfw_tls_cfg_alpn_protos(const char *cfg_str) +tfw_tls_cfg_alpn_protos(const char *cfg_str, bool *deprecated) { ttls_alpn_proto *protos; @@ -841,11 +841,13 @@ do { \ if (!strcasecmp(cfg_str, "https")) { PROTO_INIT(0, HTTP1); + *deprecated = true; return 0; } if (!strcasecmp(cfg_str, "h2")) { PROTO_INIT(0, HTTP2); + *deprecated = false; return 0; } diff --git a/tempesta_fw/tls.h b/tempesta_fw/tls.h index 936024cac1..b399185891 100644 --- a/tempesta_fw/tls.h +++ b/tempesta_fw/tls.h @@ -41,7 +41,7 @@ enum { void tfw_tls_cfg_require(void); void tfw_tls_cfg_configured(bool global); void tfw_tls_match_any_sni_to_dflt(bool match); -int tfw_tls_cfg_alpn_protos(const char *cfg_str); +int tfw_tls_cfg_alpn_protos(const char *cfg_str, bool *deprecated); void tfw_tls_free_alpn_protos(void); int tfw_tls_encrypt(struct sock *sk, struct sk_buff *skb, unsigned int limit); From a841f0bf834028bcb563ce7aafba8cd06429ae7c Mon Sep 17 00:00:00 2001 From: Ivan Koveshnikov Date: Tue, 3 Mar 2020 15:16:41 +0500 Subject: [PATCH 62/64] Bump copyrights --- tempesta_fw/http_parser.c | 2 +- tempesta_fw/sock_clnt.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index df530181a0..bf8ae59772 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -2,7 +2,7 @@ * Tempesta FW * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015-2019 Tempesta Technologies, Inc. + * Copyright (C) 2015-2020 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by diff --git a/tempesta_fw/sock_clnt.c b/tempesta_fw/sock_clnt.c index 41ac7b95df..2af15f622a 100644 --- a/tempesta_fw/sock_clnt.c +++ b/tempesta_fw/sock_clnt.c @@ -4,7 +4,7 @@ * TCP/IP stack hooks and socket routines to handle client traffic. * * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com). - * Copyright (C) 2015-2018 Tempesta Technologies, Inc. + * Copyright (C) 2015-2020 Tempesta Technologies, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by From 4f430329f0c677974298aa16835cce9b9a6b8170 Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Wed, 4 Mar 2020 10:21:59 +0300 Subject: [PATCH 63/64] HTTP/2: evict redundant space during response status-line creation from cache (#309). --- tempesta_fw/cache.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tempesta_fw/cache.c b/tempesta_fw/cache.c index 2d57adf02b..f28eec0e20 100644 --- a/tempesta_fw/cache.c +++ b/tempesta_fw/cache.c @@ -646,11 +646,10 @@ tfw_cache_set_status(TDB *db, TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr s_line = { .chunks = (TfwStr []){ { .data = S_0, .len = SLEN(S_0) }, - { .data = buf, .len = H2_STAT_VAL_LEN}, - { .data = " ", .len = 1 } + { .data = buf, .len = H2_STAT_VAL_LEN} }, - .len = SLEN(S_0) + H2_STAT_VAL_LEN + 1, - .nchunks = 3 + .len = SLEN(S_0) + H2_STAT_VAL_LEN, + .nchunks = 2 }; if (!tfw_ultoa(ce->resp_status, __TFW_STR_CH(&s_line, 1)->data, From 616496802bf1b6f2a710b5654a718cd1b4bc906a Mon Sep 17 00:00:00 2001 From: Alexander Ostapenko Date: Wed, 4 Mar 2020 13:32:50 +0300 Subject: [PATCH 64/64] HTTP/2: corrections according additional review comments (#309). --- tempesta_fw/cache.c | 2 +- tempesta_fw/http.c | 2 +- tempesta_fw/http_msg.c | 26 ++++++++++++++++++++------ tempesta_fw/http_msg.h | 2 +- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/tempesta_fw/cache.c b/tempesta_fw/cache.c index f28eec0e20..3293ef236e 100644 --- a/tempesta_fw/cache.c +++ b/tempesta_fw/cache.c @@ -1145,7 +1145,7 @@ tfw_cache_h2_copy_hdr(TfwCacheEntry *ce, char **p, TdbVRec **trec, TfwStr *hdr, st_index = hdr->hpack_idx; h_len = tfw_h2_hdr_size(s_nm.len, s_val.len, st_index); - /* Don't split short stprings. */ + /* Don't split short strings. */ if (sizeof(TfwCStr) + h_len <= L1_CACHE_BYTES) n += h_len; } diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 71aac9f515..8c29ea3562 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -3106,7 +3106,7 @@ __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) } } else { - hid = __h2_hdr_lookup(hm, hdr); + hid = __http_hdr_lookup(hm, hdr); if (hid == ht->off && !s_val) /* * The raw header not found, and there is nothing diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index bba5e0ce93..bc02a4bc62 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -166,8 +166,12 @@ tfw_http_msg_req_spec_hid(const TfwStr *hdr) } /** - * Fills @val with second part of special HTTP header containing the header - * value. + * Fills @val with second part of special HTTP/1.1 header containing the + * header value. + * + * TODO: with the current HTTP-parser implementation (parsing header name, + * colon, LWS and value into different chunks) this procedure can be + * simplified to avoid the usage of predefined header arrays. */ void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client) @@ -338,9 +342,19 @@ tfw_http_msg_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) * Certain header fields are strictly singular and may not be repeated in * an HTTP message. Duplicate of a singular header fields is a bug worth * blocking the whole HTTP message. + * + * TODO: with the current HTTP-parser implementation (parsing header name, + * colon, LWS and value into different chunks) we can avoid slow string + * matcher, which is used in @tfw_http_msg_hdr_lookup(), and can compare + * strings just by chunks (including searching the stop character) for both + * HTTP/2 and HTTP/1.1 formatted headers (see @__hdr_name_cmp() below). + * Thus, @__h1_hdr_lookup() and @tfw_http_msg_hdr_lookup() procedures should + * be unified to @__hdr_name_cmp() and @__http_hdr_lookup() in order to + * substitute current mess of multiple partially duplicated procedures with + * one simple interface. */ static inline unsigned int -__hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) +__h1_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) { unsigned int id = tfw_http_msg_hdr_lookup(hm, hdr); @@ -439,7 +453,7 @@ __hdr_name_cmp(const TfwStr *hdr, const TfwStr *cmp_hdr) * headers in HTTP/2 or HTTP/1.1 format. */ int -__h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) +__http_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) { unsigned int id; TfwHttpHdrTbl *ht = hm->h_tbl; @@ -530,7 +544,7 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) * Both the headers, the new one and existing one, can already be * compound. */ - id = __h2_hdr_lookup(hm, &parser->hdr); + id = __http_hdr_lookup(hm, &parser->hdr); /* Allocate some more room if not enough to store the header. */ if (unlikely(id == ht->size)) { @@ -827,7 +841,7 @@ tfw_http_msg_hdr_xfrm_str(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid, /* Not found, nothing to delete. */ return 0; } else { - hid = __hdr_lookup(hm, hdr); + hid = __h1_hdr_lookup(hm, hdr); if (hid == ht->off && !s_val) /* Not found, nothing to delete. */ return 0; diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 37e3028e60..24b93c2ebc 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -186,7 +186,7 @@ void tfw_http_msg_free(TfwHttpMsg *m); int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, const TfwStr *src, unsigned int *start_off); int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *cmp_hdr); -int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr); +int __http_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr); int tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, const char *stop);