Skip to content

Commit

Permalink
Merge #181
Browse files Browse the repository at this point in the history
  • Loading branch information
@jfeather-amd
jfeather-amd committed Nov 29, 2023
2 parents 46a49d3 + 433a1d7 commit 47c5d50
Show file tree
Hide file tree
Showing 7 changed files with 284 additions and 95 deletions.
238 changes: 157 additions & 81 deletions src/include/ci/internal/ip_timestamp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,98 +81,173 @@ ci_rx_pkt_timestamp_nic(const ci_ip_pkt_fmt* pkt,
}

static inline void
ci_rx_pkt_timestamp_cpacket(const ci_ip_pkt_fmt* pkt,
struct onload_timestamp* ts_out)
ci_rx_pkt_timestamp_cpacket(struct onload_timestamp* ts_out,
const char *buf_end, const char *pkt_end)
{
ts_out->sec = 0;
/* These fields are appended to the packet, after the IP payload, the
* original Ethernet FCS, and any extension tags. A new FCS was added to
* the end to make a valid packet, then checked and removed by the NIC.
*
* We look for this at the very end of the packet, assuming that nothing
* else has added any other trailer after it. If there are multiple cpacket
* trailers, then we will take the last (most recent) one.
*/
struct cpacket {
uint32_t sec;
uint32_t nsec;
uint8_t flags;
uint16_t dev;
uint8_t port;
} __attribute__((packed));

/* fixme: fragmented packets will need different treatment */
if(CI_LIKELY( OO_PP_IS_NULL(pkt->frag_next) )) {

/* These fields are appended to the packet, after the IP payload, the
* original Ethernet FCS, and any extension tags. A new FCS was added to
* the end to make a valid packet, then checked and removed by the NIC.
*
* We look for this at the very end of the packet, assuming that nothing
* else has added any other trailer after it. If there are multiple cpacket
* trailers, then we will take the last (most recent) one.
*/
struct cpacket {
uint32_t sec;
uint32_t nsec;
uint8_t flags;
uint16_t dev;
uint8_t port;
} __attribute__((packed));

const char* buf_end = PKT_START(pkt) + pkt->pay_len;
const char* pkt_end = (char*)RX_PKT_IPX_HDR(pkt) + RX_PKT_PAYLOAD_LEN(pkt);
ci_assert_ge(buf_end, pkt_end);

if( buf_end >= pkt_end + sizeof(struct cpacket) ) {
const struct cpacket* cp = (const struct cpacket*)buf_end - 1;
ts_out->sec = ntohl(cp->sec);
ts_out->nsec = ntohl(cp->nsec);
ts_out->nsec_frac = 0;

/* If extensions are present, search for a sub-ns timestamp */
if( cp->flags & 0x2 ) {
/* This points to the last byte of the current tag */
const uint8_t* ext = (const uint8_t*)cp - 1;
const uint8_t* end = (const uint8_t*)pkt_end;

for( ;; ) {
int tag = *ext;
int type = tag & 0x1f;
int len;

/* Optimise for the expected case of a single sub-ns timestamp tag.
*
* NOTE: it's possible that a malformed trailer could cause us to read
* a garbage value here, and we do not check for that. That should be
* the worst possible failure since, however badly formed the trailer,
* we will always calculate a non-zero length below and eventually
* reach the packet data and bail out. */
if(CI_LIKELY( type == 0x01 )) {
ts_out->nsec_frac = ext[-1] | (ext[-2] << 8) | (ext[-3] << 16);
break;
}

/* Check the flag that indicates this is the last extension */
if( tag & 0x20 )
break;

/* Extract the length depending on tag type */
if( type == 0x1f )
/* secondary tag: 10 bits excluding first and second words */
len = ((tag >> 6) | (ext[-1] << 2)) + 2;
else
/* primary tag: 2 bits excluding first word */
len = (tag >> 6) + 1;

/* length field gives 32-bit words */
len *= 4;

/* Bail out if a bogus length takes us out of range */
if( ext - end < len )
break;

ext -= len;
}
const struct cpacket* cp = (const struct cpacket*)buf_end - 1;

if( buf_end - pkt_end < sizeof(struct cpacket) )
return;

ts_out->sec = ntohl(cp->sec);
ts_out->nsec = ntohl(cp->nsec);
ts_out->nsec_frac = 0;

/* If extensions are present, search for a sub-ns timestamp */
if( cp->flags & 0x2 ) {
/* This points to the last byte of the current tag */
const uint8_t* ext = (const uint8_t*)cp - 1;
const uint8_t* end = (const uint8_t*)pkt_end;

for( ;; ) {
int tag = *ext;
int type = tag & 0x1f;
int len;

/* Optimise for the expected case of a single sub-ns timestamp tag.
*
* NOTE: it's possible that a malformed trailer could cause us to read
* a garbage value here, and we do not check for that. That should be
* the worst possible failure since, however badly formed the trailer,
* we will always calculate a non-zero length below and eventually
* reach the packet data and bail out. */
if(CI_LIKELY( type == 0x01 )) {
ts_out->nsec_frac = ext[-1] | (ext[-2] << 8) | (ext[-3] << 16);
break;
}

/* Check the flag that indicates this is the last extension */
if( tag & 0x20 )
break;

/* Extract the length depending on tag type */
if( type == 0x1f )
/* secondary tag: 10 bits excluding first and second words */
len = ((tag >> 6) | (ext[-1] << 2)) + 2;
else
/* primary tag: 2 bits excluding first word */
len = (tag >> 6) + 1;

/* length field gives 32-bit words */
len *= 4;

/* Bail out if a bogus length takes us out of range */
if( ext - end < len )
break;

ext -= len;
}
}
}

static inline void
ci_rx_pkt_timestamp(const ci_ip_pkt_fmt* pkt, struct onload_timestamp* ts_out, int src)
ci_rx_pkt_timestamp_ttag(struct onload_timestamp* ts_out,
const char *buf_end, const char *pkt_end)
{
struct ttag {
uint8_t ts[6]; /* 48b nsecs */
} __attribute__((packed));

const struct ttag* ttag = (const struct ttag*)buf_end - 1;
const uint64_t nsec_per_sec = 1000ULL * 1000 * 1000;
uint64_t ts;

if( buf_end - pkt_end < sizeof(struct ttag) )
return;

ts = ((uint64_t)ttag->ts[0] << 40) | ((uint64_t)ttag->ts[1] << 32) |
((uint64_t)ttag->ts[2] << 24) | ((uint64_t)ttag->ts[3] << 16) |
((uint64_t)ttag->ts[4] << 8) | ((uint64_t)ttag->ts[5]);

ts_out->sec = ts / nsec_per_sec;
ts_out->nsec = ts - (ts_out->sec * nsec_per_sec);
ts_out->nsec_frac = 0;
}

static inline void
ci_rx_pkt_timestamp_brcm(struct onload_timestamp* ts_out,
const char *buf_end, const char *pkt_end)
{
struct brcm_trailer {
uint8_t ts[6]; /* 48b timestamp: 18b sec + 30b nsec */

uint8_t reserved;
uint8_t origin[3];
/* Followed by FCS, already removed by the NIC for this packet.
* With multiple trailers, each ends in such a 4B reserved field.
*/
} __attribute__((packed));

const struct brcm_trailer* tt = (const struct brcm_trailer*)buf_end - 1;

if( buf_end - pkt_end < sizeof(struct brcm_trailer) )
return;

ts_out->sec = ((uint32_t)(tt->ts[0]) << 10) |
((uint32_t)(tt->ts[1]) << 2) |
(tt->ts[2] >> 6);
ts_out->nsec = ((tt->ts[2] & 0x3F) << 24) |
((uint32_t)(tt->ts[3]) << 16) |
((uint32_t)(tt->ts[4]) << 8) |
tt->ts[5];
ts_out->nsec_frac = 0;
}

static inline void
ci_rx_pkt_timestamp_trailer(const ci_ip_pkt_fmt* pkt,
struct onload_timestamp* ts_out, int format)
{
const char *buf_end, *pkt_end;

ts_out->sec = 0;

/* fixme: fragmented packets will need different treatment */
if(CI_UNLIKELY( !OO_PP_IS_NULL(pkt->frag_next) ))
return;

buf_end = PKT_START(pkt) + pkt->pay_len;
pkt_end = (char*)RX_PKT_IPX_HDR(pkt) + RX_PKT_PAYLOAD_LEN(pkt);
ci_assert_ge(buf_end, pkt_end);

switch( format ) {
case CITP_RX_TIMESTAMPING_TRAILER_FORMAT_CPACKET:
ci_rx_pkt_timestamp_cpacket(ts_out, buf_end, pkt_end);
break;
case CITP_RX_TIMESTAMPING_TRAILER_FORMAT_TTAG:
ci_rx_pkt_timestamp_ttag(ts_out, buf_end, pkt_end);
break;
case CITP_RX_TIMESTAMPING_TRAILER_FORMAT_BRCM:
ci_rx_pkt_timestamp_brcm(ts_out, buf_end, pkt_end);
break;
}
}

static inline void
ci_rx_pkt_timestamp(const ci_ip_pkt_fmt* pkt, struct onload_timestamp* ts_out,
int src, int format)
{
switch( src ) {
case CITP_RX_TIMESTAMPING_SOURCE_NIC:
ci_rx_pkt_timestamp_nic(pkt, ts_out);
break;
case CITP_RX_TIMESTAMPING_SOURCE_CPACKET:
ci_rx_pkt_timestamp_cpacket(pkt, ts_out);
case CITP_RX_TIMESTAMPING_SOURCE_TRAILER:
ci_rx_pkt_timestamp_trailer(pkt, ts_out, format);
break;
default:
ts_out->sec = 0;
Expand All @@ -181,10 +256,11 @@ ci_rx_pkt_timestamp(const ci_ip_pkt_fmt* pkt, struct onload_timestamp* ts_out, i
}

static inline void
ci_rx_pkt_timespec(const ci_ip_pkt_fmt* pkt, ef_timespec* ts_out, int src)
ci_rx_pkt_timespec(const ci_ip_pkt_fmt* pkt, ef_timespec* ts_out, int src,
int trailer_type)
{
struct onload_timestamp ts;
ci_rx_pkt_timestamp(pkt, &ts, src);
ci_rx_pkt_timestamp(pkt, &ts, src, trailer_type);
onload_timestamp_to_timespec(&ts, ts_out);
}

Expand Down
17 changes: 14 additions & 3 deletions src/include/ci/internal/opts_netif_def.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,12 +346,23 @@ CI_CFG_OPT("EF_RX_TIMESTAMPING", rx_timestamping, ci_uint32,
2, , 0, 0, 3, count)

#define CITP_RX_TIMESTAMPING_SOURCE_NIC 0
#define CITP_RX_TIMESTAMPING_SOURCE_CPACKET 1
#define CITP_RX_TIMESTAMPING_SOURCE_TRAILER 1
CI_CFG_OPT("EF_RX_TIMESTAMPING_ORDERING", rx_timestamping_ordering, ci_uint32,
"Select the source of timestamps to use to order received packets.\n"
" nic - use hardware timestamps generated by the NIC\n"
" cpacket - use cPacket timestamps on received packets\n",
2, , 0, 0, 1, oneof:nic;cpacket)
" trailer - use trailer timestamps on received packets\n"
" cPacket - synonym for trailer (legacy)\n",
1, , 0, 0, 2, oneof:nic;trailer;cpacket)

#define CITP_RX_TIMESTAMPING_TRAILER_FORMAT_CPACKET 0
#define CITP_RX_TIMESTAMPING_TRAILER_FORMAT_TTAG 1
#define CITP_RX_TIMESTAMPING_TRAILER_FORMAT_BRCM 2
CI_CFG_OPT("EF_RX_TIMESTAMPING_TRAILER_FORMAT", rx_timestamping_trailer_fmt, ci_uint32,
"Select the format of timestamps received as a packet trailer.\n"
" cpacket - use cPacket trailer timestamp format\n"
" ttag - use TTAG trailer timestamp format\n"
" brcm - use Broadcom trailer timestamp format\n",
2, , 0, 0, 2, oneof:nic;cpacket;ttag;brcm)

CI_CFG_OPT("EF_TX_TIMESTAMPING", tx_timestamping, ci_uint32,
"Control of hardware timestamping of transmitted packets, possible values:\n"
Expand Down
3 changes: 2 additions & 1 deletion src/lib/transport/ip/ip_cmsg.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,8 @@ void ip_cmsg_recv_timestamping(ci_netif *ni, const ci_ip_pkt_fmt *pkt,
}
if( flags & ONLOAD_TIMESTAMPING_FLAG_RX_CPACKET ) {
ci_assert_lt(n, ONLOAD_TIMESTAMPING_FLAG_RX_COUNT);
ci_rx_pkt_timestamp_cpacket(pkt, &ts[n++]);
ci_rx_pkt_timestamp_trailer(pkt, &ts[n++],
NI_OPTS(ni).rx_timestamping_trailer_fmt);
}

ci_put_cmsg(cmsg_state, SOL_SOCKET, ONLOAD_SO_TIMESTAMPING,
Expand Down
9 changes: 8 additions & 1 deletion src/lib/transport/ip/netif_init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1176,9 +1176,16 @@ void ci_netif_config_opts_getenv(ci_netif_config_opts* opts)
if( (s = getenv("EF_RX_TIMESTAMPING")) )
opts->rx_timestamping = atoi(s);

static const char* const timestamping_opts[] = { "nic", "cpacket", 0 };
static const char* const timestamping_opts[] = { "nic", "trailer", "cpacket", 0 };
opts->rx_timestamping_ordering =
parse_enum(opts, "EF_RX_TIMESTAMPING_ORDERING", timestamping_opts, "nic");
/* cpacket (2) is a synonym for trailer (1) */
if (opts->rx_timestamping_ordering > 1)
opts->rx_timestamping_ordering--;

static const char* const ts_trailer_formats[] = { "cpacket", "ttag", "brcm", 0 };
opts->rx_timestamping_trailer_fmt =
parse_enum(opts, "EF_RX_TIMESTAMPING_TRAILER_FORMAT", ts_trailer_formats, "cpacket");

if( (s = getenv("EF_TX_TIMESTAMPING")) )
opts->tx_timestamping = atoi(s);
Expand Down
3 changes: 2 additions & 1 deletion src/lib/transport/unix/tcp_fd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2166,7 +2166,8 @@ citp_tcp_ordered_data(citp_fdinfo* fdi, struct timespec* limit,
do {
struct timespec stamp;
ci_rx_pkt_timespec(pkt, &stamp,
NI_OPTS(epi->sock.netif).rx_timestamping_ordering);
NI_OPTS(epi->sock.netif).rx_timestamping_ordering,
NI_OPTS(epi->sock.netif).rx_timestamping_trailer_fmt);

if( citp_timespec_compare(&stamp, limit) < 1 ) {
*bytes_out += oo_offbuf_left(&pkt->buf);
Expand Down
3 changes: 2 additions & 1 deletion src/lib/transport/unix/udp_fd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -715,7 +715,8 @@ citp_udp_ordered_data(citp_fdinfo* fdi, struct timespec* limit,
do {
struct timespec stamp;
ci_rx_pkt_timespec(pkt, &stamp,
NI_OPTS(epi->sock.netif).rx_timestamping_ordering);
NI_OPTS(epi->sock.netif).rx_timestamping_ordering,
NI_OPTS(epi->sock.netif).rx_timestamping_trailer_fmt);

if( citp_timespec_compare(&stamp, limit) < 1 ) {
/* We have data before the limit, add on the number of readable bytes. */
Expand Down
Loading

0 comments on commit 47c5d50

Please sign in to comment.