Skip to content

Commit

Permalink
Provide the count of non-idempotent reqs in a server conn. (#419)
Browse files Browse the repository at this point in the history 
Non-idempotent requests make up an internal @nip_queue within the
server's fwd_queue. @nipcnt keeps the count of those requests in
@nip_queue that can be used by schedulers when making decision.
Special care is taken for the case where a non-idempotent request
is followed by another requests, which re-enables pipelining of
those messages.
  • Loading branch information
@keshonok
keshonok committed Oct 14, 2016
1 parent da254e3 commit 05a0a7c
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 28 deletions.
10 changes: 6 additions & 4 deletions tempesta_fw/connection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,21 +77,26 @@ enum {
* @state - connection processing state;
* @list - member in the list of connections with @peer;
* @msg_queue - queue of messages to be sent over the connection;
* @nip_queue - queue of non-idempotent messages within @msg_queue;
* @msg_qlock - lock for accessing @msg_queue;
* @refcnt - number of users of the connection structure instance;
* @nipcnt - number of non-idempotent requests in the connection;
* @timer - The keep-alive/retry timer for the connection;
* @msg - message that is currently being processed;
* @msg_sent - message that was sent last in the connection;
* @peer - TfwClient or TfwServer handler;
* @sk - an appropriate sock handler;
* @destructor - called when a connection is destroyed;
*/
typedef struct {
SsProto proto;
TfwGState state;
struct list_head list;
struct list_head msg_queue;
struct list_head nip_queue;
spinlock_t msg_qlock;
atomic_t refcnt;
unsigned long flags;
atomic_t nipcnt;
struct timer_list timer;
TfwMsg *msg;
TfwMsg *msg_sent;
Expand All @@ -104,9 +109,6 @@ typedef struct {

#define TFW_CONN_TYPE(c) ((c)->proto.type)

/* Connection flags. */
#define TFW_CONN_FWD_HOLD 0x0001 /* Hold sending messages */

/**
* TLS hardened connection.
*/
Expand Down
94 changes: 70 additions & 24 deletions tempesta_fw/http.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,45 @@ tfw_http_send_504(TfwHttpReq *req)
return tfw_http_send_resp(req, &rh, __TFW_STR_CH(&rh, 1));
}

static inline bool
tfw_http_req_is_nonidempotent(TfwHttpReq *req)
{
return (req->flags & TFW_HTTP_NON_IDEMP);
}

static inline void
__tfw_http_req_flip_nonidempotent(TfwConnection *srv_conn, TfwHttpReq *req)
{
list_del_init(&req->nip_list);
atomic_dec(&srv_conn->nipcnt);
}

/*
* Flip a non-idempotent request. If @req in server connection @srv_conn
* is non-idempotent, then make it idempotent.
*/
static inline void
tfw_http_req_flip_nonidempotent(TfwConnection *srv_conn, TfwHttpReq *req)
{
if (!list_empty(&req->nip_list))
__tfw_http_req_flip_nonidempotent(srv_conn, req);
}

/*
* If a request on the list of non-idempotent requests in server
* connection @srv_conn had become an idempotent request, then flip it
* and make it idempotent.
*/
static inline void
tfw_http_conn_flip_nonidempotent(TfwConnection *srv_conn)
{
TfwHttpReq *req, *tmp;

list_for_each_entry_safe(req, tmp, &srv_conn->nip_queue, nip_list)
if (!tfw_http_req_is_nonidempotent(req))
__tfw_http_req_flip_nonidempotent(srv_conn, req);
}

/*
* Allocate a new HTTP message structure, and link it with
* the connection structure. Increment the number of users
Expand All @@ -385,9 +424,10 @@ tfw_http_conn_msg_alloc(TfwConnection *conn)
hm->conn = conn;
tfw_connection_get(conn);

if (TFW_CONN_TYPE(conn) & Conn_Clnt) {
TFW_INC_STAT_BH(clnt.rx_messages);
} else {
if (TFW_CONN_TYPE(conn) & Conn_Clnt) {
INIT_LIST_HEAD(&((TfwHttpReq *)hm)->nip_list);
TFW_INC_STAT_BH(clnt.rx_messages);
} else {
TfwHttpReq *req;

spin_lock(&conn->msg_qlock);
Expand All @@ -396,7 +436,7 @@ tfw_http_conn_msg_alloc(TfwConnection *conn)
spin_unlock(&conn->msg_qlock);
if (req && (req->method == TFW_HTTP_METH_HEAD))
hm->flags |= TFW_HTTP_VOID_BODY;
TFW_INC_STAT_BH(serv.rx_messages);
TFW_INC_STAT_BH(serv.rx_messages);
}

return (TfwMsg *)hm;
Expand All @@ -407,6 +447,10 @@ tfw_http_req_destruct(void *msg)
{
TfwHttpReq *req = msg;

BUG_ON(!list_empty(&req->msg.seq_list));
BUG_ON(!list_empty(&req->msg.fwd_list));
BUG_ON(!list_empty(&req->nip_list));

if (req->sess)
tfw_http_sess_put(req->sess);
}
Expand Down Expand Up @@ -446,6 +490,10 @@ tfw_http_conn_msg_free(TfwHttpMsg *hm)
static int
tfw_http_conn_init(TfwConnection *conn)
{
if (TFW_CONN_TYPE(conn) & Conn_Srv) {
atomic_set(&conn->nipcnt, 0);
INIT_LIST_HEAD(&conn->nip_queue);
}
tfw_gfsm_state_init(&conn->state, conn, TFW_HTTP_FSM_INIT);
return 0;
}
Expand All @@ -466,24 +514,20 @@ static void
tfw_http_conn_release(TfwConnection *srv_conn)
{
TfwHttpReq *req, *tmp;
struct list_head zap_queue;
struct list_head *fwd_queue = &srv_conn->msg_queue;

TFW_DBG2("%s: conn = %p\n", __func__, srv_conn);
BUG_ON(!(TFW_CONN_TYPE(srv_conn) & Conn_Srv));

INIT_LIST_HEAD(&zap_queue);

spin_lock(&srv_conn->msg_qlock);
list_splice_tail_init(&srv_conn->msg_queue, &zap_queue);
spin_unlock(&srv_conn->msg_qlock);

list_for_each_entry_safe(req, tmp, &zap_queue, msg.fwd_list) {
list_for_each_entry_safe(req, tmp, fwd_queue, msg.fwd_list) {
BUG_ON(req->conn && (req->conn == srv_conn));
list_del_init(&req->msg.fwd_list);
tfw_http_req_flip_nonidempotent(srv_conn, req);
tfw_http_send_404(req);
TFW_INC_STAT_BH(clnt.msgs_otherr);
}
INIT_LIST_HEAD(&srv_conn->msg_queue);
BUG_ON(atomic_read(&srv_conn->nipcnt) != 0);
BUG_ON(!list_empty(&srv_conn->nip_queue));
}

/*
Expand Down Expand Up @@ -796,12 +840,6 @@ tfw_http_adjust_resp(TfwHttpResp *resp, TfwHttpReq *req)
TFW_HTTP_HDR_SERVER, 0);
}

static inline bool
tfw_http_req_is_nonidempotent(TfwHttpReq *req)
{
return (req->flags & TFW_HTTP_NON_IDEMP);
}

/*
* Tell if the server connection's forwarding queue is on hold.
* It's on hold it the request that was sent last was non-idempotent.
Expand Down Expand Up @@ -909,6 +947,7 @@ __tfw_http_req_fwd_many(TfwConnection *srv_conn, struct list_head *err_queue)
/* Stop sending if the request is non-idempotent. */
if (tfw_http_req_is_nonidempotent(req))
break;
tfw_http_req_flip_nonidempotent(srv_conn, req);
}
}

Expand Down Expand Up @@ -961,6 +1000,10 @@ tfw_http_req_fwd(TfwConnection *srv_conn, TfwHttpReq *req)
spin_lock(&srv_conn->msg_qlock);
drained = __tfw_http_conn_drained(srv_conn);
list_add_tail(&req->msg.fwd_list, &srv_conn->msg_queue);
if (tfw_http_req_is_nonidempotent(req)) {
list_add_tail(&req->nip_list, &srv_conn->nip_queue);
atomic_inc(&srv_conn->nipcnt);
}
if (__tfw_http_conn_on_hold(srv_conn)) {
spin_unlock(&srv_conn->msg_qlock);
TFW_DBG2("%s: Server connection is on hold: conn=[%p]\n",
Expand All @@ -976,6 +1019,7 @@ tfw_http_req_fwd(TfwConnection *srv_conn, TfwHttpReq *req)
}
if (tfw_connection_send(srv_conn, (TfwMsg *)req)) {
list_del_init(&req->msg.fwd_list);
tfw_http_req_flip_nonidempotent(srv_conn, req);
spin_unlock(&srv_conn->msg_qlock);
TFW_DBG2("%s: Error sending to server connection: "
"conn=[%p] req=[%p]\n", __func__, srv_conn, req);
Expand Down Expand Up @@ -1448,7 +1492,7 @@ tfw_http_resp_cache_cb(TfwHttpReq *req, TfwHttpResp *resp)
static TfwHttpReq *
tfw_http_popreq(TfwHttpMsg *hmresp)
{
TfwMsg *msg;
TfwHttpReq *req;
TfwConnection *srv_conn = hmresp->conn;
struct list_head *fwd_queue = &srv_conn->msg_queue;

Expand All @@ -1462,10 +1506,12 @@ tfw_http_popreq(TfwHttpMsg *hmresp)
TFW_INC_STAT_BH(serv.msgs_otherr);
return NULL;
}
msg = list_first_entry(fwd_queue, TfwMsg, fwd_list);
list_del_init(&msg->fwd_list);
if (srv_conn->msg_sent == msg)
req = list_first_entry(fwd_queue, TfwHttpReq, msg.fwd_list);
list_del_init(&req->msg.fwd_list);
if (srv_conn->msg_sent == (TfwMsg *)req)
srv_conn->msg_sent = NULL;
tfw_http_req_flip_nonidempotent(srv_conn, req);
tfw_http_conn_flip_nonidempotent(srv_conn);
/*
* If the server connection is no longer on hold, and the queue
* is not drained, then forward pending requests to the server.
Expand All @@ -1476,7 +1522,7 @@ tfw_http_popreq(TfwHttpMsg *hmresp)
else
spin_unlock(&srv_conn->msg_qlock);

return (TfwHttpReq *)msg;
return req;
}

/*
Expand Down
2 changes: 2 additions & 0 deletions tempesta_fw/http.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ typedef struct {
* @userinfo - userinfo in URI, not mandatory.
* @host - host in URI, may differ from Host header;
* @uri_path - path + query + fragment from URI (RFC3986.3);
* @nip_list - member in the queue of non-idempotent requests;
* @method - HTTP request method, one of GET/PORT/HEAD/etc;
* @node - NUMA node where request is serviced;
* @frang_st - current state of FRANG classifier;
Expand All @@ -320,6 +321,7 @@ typedef struct {
TfwStr userinfo;
TfwStr host;
TfwStr uri_path;
struct list_head nip_list;
unsigned char method;
unsigned short node;
unsigned int frang_st;
Expand Down

0 comments on commit 05a0a7c

Please sign in to comment.