From 750040a4b789bf1f510be4140ee3022697bd5244 Mon Sep 17 00:00:00 2001 From: udippant Date: Tue, 27 Aug 2019 22:01:05 -0700 Subject: [PATCH] add an ability to run simulation against loaded katran's bpf program (#41) Summary: i've added an ability to run simulation for specified 5 tuple: user would be able to provide 5 tuple (src/dst address + src/dst ports + protocol number) and would receive address of the real, where this 5 tuple is going to be sent. this could be used for debug/troubleshooting scenarios (e.g. some flows works slow/does not work at all and we want to figure out which real is responsible for serving em). all this simulation is being run against loaded BPF program (so if you have features, like src based routing enable, it would work for em out of box)) added dedicated section to katran_tester which implements integration tests against this new freature Pull Request resolved: https://github.com/facebookincubator/katran/pull/41 Pulled By: udippant Test Plan: Imported from GitHub, without a `Test Plan:` line. Unit test Reviewed By: yangchi Differential Revision: D17076371 fbshipit-source-id: c12affd44b914233f40c7595693117d0b6b03752 --- katran/lib/CMakeLists.txt | 10 ++ katran/lib/KatranLb.cpp | 10 ++ katran/lib/KatranLb.h | 11 ++ katran/lib/KatranSimulator.cpp | 241 +++++++++++++++++++++++++++ katran/lib/KatranSimulator.h | 54 ++++++ katran/lib/testing/katran_tester.cpp | 88 ++++++++++ 6 files changed, 414 insertions(+) create mode 100644 katran/lib/KatranSimulator.cpp create mode 100644 katran/lib/KatranSimulator.h diff --git a/katran/lib/CMakeLists.txt b/katran/lib/CMakeLists.txt index 34b62b61b..d96315f7d 100644 --- a/katran/lib/CMakeLists.txt +++ b/katran/lib/CMakeLists.txt @@ -115,6 +115,15 @@ set(PCAPWRITER_DEPS target_link_libraries(pcapwriter "${PCAPWRITER_DEPS}") +add_library(katransimulator STATIC + KatranSimulator.h + KatranSimulator.cpp +) + +target_link_libraries(katransimulator + bpfadapter +) + add_library(katranlb STATIC KatranEventReader.h KatranEventReader.cpp @@ -134,6 +143,7 @@ target_link_libraries(katranlb chhelpers iphelpers pcapwriter + katransimulator "${GFLAGS}" "${PTHREAD}" "-Wl,--end-group" diff --git a/katran/lib/KatranLb.cpp b/katran/lib/KatranLb.cpp index 82892c8ff..53256468c 100644 --- a/katran/lib/KatranLb.cpp +++ b/katran/lib/KatranLb.cpp @@ -38,6 +38,7 @@ constexpr int kMaxForwardingCores = 128; constexpr int kFirstElem = 0; constexpr int kError = -1; constexpr uint32_t kMaxQuicId = 65535; // 2^16-1 +constexpr folly::StringPiece kEmptyString = ""; } // namespace KatranLb::KatranLb(const KatranConfig& config) @@ -1202,6 +1203,15 @@ std::unordered_map KatranLb::getHealthcheckersDst() { return hcs; } +const std::string KatranLb::getRealForFlow(const KatranFlow& flow) { + if (!progsLoaded_) { + LOG(ERROR) << "bpf programs are not loaded"; + return kEmptyString.data(); + } + auto sim = KatranSimulator(getKatranProgFd()); + return sim.getRealForFlow(flow); +} + bool KatranLb::updateVipMap( const ModifyAction action, const VipKey& vip, diff --git a/katran/lib/KatranLb.h b/katran/lib/KatranLb.h index c4233c5dc..8c04c3189 100644 --- a/katran/lib/KatranLb.h +++ b/katran/lib/KatranLb.h @@ -30,6 +30,7 @@ #include "katran/lib/CHHelpers.h" #include "katran/lib/IpHelpers.h" #include "katran/lib/KatranLbStructs.h" +#include "katran/lib/KatranSimulator.h" #include "katran/lib/Vip.h" namespace katran { @@ -485,6 +486,16 @@ class KatranLb { return lbStats_; } + /** + * @param KatranFlow 5 tuple which describes a flow + * @return string address of the real. + * + * getRealForFlow functions returns address of the real where specified + * 5 tuple is going to be sent. + * returns empty string if given 5 tuple does not belong to a configured vip + */ + const std::string getRealForFlow(const KatranFlow& flow); + private: /** * update vipmap(add or remove vip) in forwarding plane diff --git a/katran/lib/KatranSimulator.cpp b/katran/lib/KatranSimulator.cpp new file mode 100644 index 000000000..a96f9f4cf --- /dev/null +++ b/katran/lib/KatranSimulator.cpp @@ -0,0 +1,241 @@ +#include "katran/lib/KatranSimulator.h" + +#include +#include +#include + +#include "katran/lib/BpfAdapter.h" + +extern "C" { +#include +#include +#include +#include +#include +#include +} + +namespace katran { + +namespace { +constexpr uint16_t kMaxXdpPcktSize = 4096; +constexpr uint16_t kTestPacketSize = 512; +constexpr int kTestRepeatCount = 1; +constexpr uint8_t kDefaultTtl = 64; +constexpr uint8_t kIPv6AddrSize = 16; +constexpr folly::StringPiece kEmptyString = ""; +} // namespace + +namespace { + +void createV4Packet( + const folly::IPAddress& src, + const folly::IPAddress& dst, + std::unique_ptr& buf, + uint8_t proto, + uint16_t size) { + auto ehdr = reinterpret_cast(buf->writableData()); + auto iph = reinterpret_cast( + buf->writableData() + sizeof(struct ethhdr)); + ehdr->h_proto = htons(ETH_P_IP); + iph->ihl = 5; + iph->version = 4; + iph->frag_off = 0; + iph->protocol = proto; + iph->check = 0; + iph->tos = 0; + iph->tot_len = htons(size); + iph->daddr = dst.asV4().toLong(); + iph->saddr = src.asV4().toLong(); + iph->ttl = kDefaultTtl; +} + +void createV6Packet( + const folly::IPAddress& src, + const folly::IPAddress& dst, + std::unique_ptr& buf, + uint8_t proto, + uint16_t size) { + auto ehdr = reinterpret_cast(buf->writableData()); + auto ip6h = reinterpret_cast( + buf->writableData() + sizeof(struct ethhdr)); + ehdr->h_proto = htons(ETH_P_IPV6); + ip6h->version = 6; + ip6h->priority = 0; + std::memset(ip6h->flow_lbl, 0, sizeof(ip6h->flow_lbl)); + ip6h->nexthdr = proto; + ip6h->payload_len = htons(size - sizeof(struct ipv6hdr)); + ip6h->hop_limit = kDefaultTtl; + std::memcpy( + ip6h->daddr.s6_addr16, dst.asV6().toBinary().data(), kIPv6AddrSize); + std::memcpy( + ip6h->saddr.s6_addr16, src.asV6().toBinary().data(), kIPv6AddrSize); +} + +void createTcpHeader( + std::unique_ptr& buf, + uint16_t srcPort, + uint16_t dstPort, + uint16_t offset) { + auto tcph = reinterpret_cast(buf->writableData() + offset); + std::memset(tcph, 0, sizeof(struct tcphdr)); + tcph->source = htons(srcPort); + tcph->dest = htons(dstPort); + tcph->syn = 1; +} + +void createUdpHeader( + std::unique_ptr& buf, + uint16_t srcPort, + uint16_t dstPort, + uint16_t offset, + uint16_t size) { + auto udph = reinterpret_cast(buf->writableData() + offset); + std::memset(udph, 0, sizeof(struct udphdr)); + udph->source = htons(srcPort); + udph->dest = htons(dstPort); + udph->len = size; +} + +const std::string toV4String(uint32_t addr) { + return folly::IPAddressV4::fromLong(addr).str(); +} + +const std::string toV6String(uint8_t const* v6) { + folly::ByteRange bytes(v6, kIPv6AddrSize); + return folly::IPAddressV6::fromBinary(bytes).str(); +} + +std::string getPcktDst(std::unique_ptr& pckt) { + if (pckt->computeChainDataLength() < sizeof(struct ethhdr)) { + LOG(ERROR) << "resulting packet is invalid"; + return kEmptyString.data(); + } + const struct ethhdr* ehdr = + reinterpret_cast(pckt->data()); + if (ehdr->h_proto == htons(ETH_P_IP)) { + if (pckt->computeChainDataLength() < + (sizeof(struct ethhdr) + sizeof(struct iphdr))) { + LOG(ERROR) << "resulting ipv4 packet is invalid"; + return kEmptyString.data(); + } + const struct iphdr* iph = reinterpret_cast( + pckt->data() + sizeof(struct ethhdr)); + return toV4String(iph->daddr); + } else { + if (pckt->computeChainDataLength() < + (sizeof(struct ethhdr) + sizeof(struct ipv6hdr))) { + LOG(ERROR) << "resulting ipv6 packet is invalid"; + return kEmptyString.data(); + } + const struct ipv6hdr* ip6h = reinterpret_cast( + pckt->data() + sizeof(struct ethhdr)); + return toV6String(ip6h->daddr.s6_addr); + } +} + +std::unique_ptr createPacketFromFlow(const KatranFlow& flow) { + int offset = sizeof(struct ethhdr); + bool is_tcp = true; + bool is_v4 = true; + size_t l3hdr_len; + + auto srcExp = folly::IPAddress::tryFromString(flow.src); + auto dstExp = folly::IPAddress::tryFromString(flow.dst); + if (srcExp.hasError() || dstExp.hasError()) { + LOG(ERROR) << "malformed src or dst ip address. src: " << flow.src + << " dst: " << flow.dst; + return nullptr; + } + auto src = srcExp.value(); + auto dst = dstExp.value(); + if (src.family() != dst.family()) { + LOG(ERROR) << "src and dst must have same address family"; + return nullptr; + } + auto pckt = folly::IOBuf::create(kTestPacketSize); + if (!pckt) { + LOG(ERROR) << "cannot allocate IOBuf"; + return pckt; + } + if (src.family() == AF_INET) { + l3hdr_len = sizeof(struct iphdr); + } else { + is_v4 = false; + l3hdr_len = sizeof(struct ipv6hdr); + } + offset += l3hdr_len; + switch (flow.proto) { + case IPPROTO_TCP: + break; + case IPPROTO_UDP: + is_tcp = false; + break; + default: + LOG(ERROR) << "unsupported protocol: " << flow.proto + << " must be either TCP or UDP"; + return nullptr; + } + pckt->append(kTestPacketSize); + auto payload_size = kTestPacketSize - sizeof(struct ethhdr); + if (is_v4) { + createV4Packet(src, dst, pckt, flow.proto, payload_size); + } else { + createV6Packet(src, dst, pckt, flow.proto, payload_size); + } + payload_size -= l3hdr_len; + if (is_tcp) { + createTcpHeader(pckt, flow.srcPort, flow.dstPort, offset); + } else { + createUdpHeader(pckt, flow.srcPort, flow.dstPort, offset, payload_size); + } + return pckt; +} + +} // namespace + +KatranSimulator::KatranSimulator(int progFd) : progFd_(progFd) {} + +KatranSimulator::~KatranSimulator() {} + +std::unique_ptr KatranSimulator::runSimulation( + std::unique_ptr pckt) { + auto rpckt = folly::IOBuf::create(kMaxXdpPcktSize); + if (!rpckt) { + LOG(ERROR) << "not able to allocate memory for resulting packet"; + return rpckt; + } + uint32_t output_pckt_size{0}; + uint32_t prog_ret_val{0}; + auto res = BpfAdapter::testXdpProg( + progFd_, + kTestRepeatCount, + pckt->writableData(), + pckt->computeChainDataLength(), + rpckt->writableData(), + &output_pckt_size, + &prog_ret_val); + if (res < 0) { + LOG(ERROR) << "failed to run simulator"; + return nullptr; + } + if (prog_ret_val != XDP_TX) { + return nullptr; + } + rpckt->append(output_pckt_size); + return rpckt; +} + +const std::string KatranSimulator::getRealForFlow(const KatranFlow& flow) { + auto pckt = createPacketFromFlow(flow); + if (!pckt) { + return kEmptyString.data(); + } + auto rpckt = runSimulation(std::move(pckt)); + if (!rpckt) { + return kEmptyString.data(); + } + return getPcktDst(rpckt); +} + +} // namespace katran diff --git a/katran/lib/KatranSimulator.h b/katran/lib/KatranSimulator.h new file mode 100644 index 000000000..264506ffb --- /dev/null +++ b/katran/lib/KatranSimulator.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + +namespace katran { + +/** + * struct that contains all the fields that uniquely identifies a flow + */ +struct KatranFlow { + // source ip address of the packet + std::string src; + // destination ip address of the packet + std::string dst; + uint16_t srcPort; + uint16_t dstPort; + // protocol number (e.g. 6 for TCP, 17 for UDP) + uint8_t proto; +}; + +/** + * KatranSimulator allows end user to simulate what is going to happen + * with specified packet after it is processed by katran load balancer. + * For e.g. where (address of the real) this packet is going to be sent + */ +class KatranSimulator final { + public: + /** + * @param int progFd descriptor of katran xdp program + */ + explicit KatranSimulator(int progFd); + ~KatranSimulator(); + + /** + * @param KatranFlow& flow that we are interested in + * @return string ip address of the real (or empty string if packet will not + * be sent) + * + * getRealForFlow helps to determines where specific flow is going to be sent + * by returning ip address of the real + */ + const std::string getRealForFlow(const KatranFlow& flow); + + private: + // runSimulation takes packet (in iobuf represenation) and + // run it through katran bpf program. It returns a modified pckt, if the + // result was XDP_TX or nullptr otherwise. + std::unique_ptr runSimulation( + std::unique_ptr pckt); + int progFd_; +}; +} // namespace katran diff --git a/katran/lib/testing/katran_tester.cpp b/katran/lib/testing/katran_tester.cpp index ae5f74b00..364568f3f 100644 --- a/katran/lib/testing/katran_tester.cpp +++ b/katran/lib/testing/katran_tester.cpp @@ -102,6 +102,93 @@ void addQuicMappings(katran::KatranLb& lb) { lb.modifyQuicRealsMapping(action, qreals); } +void testSimulator(katran::KatranLb& lb) { + // udp, v4 vip v4 real + auto real = lb.getRealForFlow(katran::KatranFlow{ + .src = "172.16.0.1", + .dst = "10.200.1.1", + .srcPort = 31337, + .dstPort = 80, + .proto = kUdp, + }); + if (real != "10.0.0.2") { + VLOG(2) << "real: " << real; + LOG(INFO) << "simulation is incorrect for v4 real and v4 udp vip"; + } + // tcp, v4 vip v4 real + real = lb.getRealForFlow(katran::KatranFlow{ + .src = "172.16.0.1", + .dst = "10.200.1.1", + .srcPort = 31337, + .dstPort = 80, + .proto = kTcp, + }); + if (real != "10.0.0.2") { + VLOG(2) << "real: " << real; + LOG(INFO) << "simulation is incorrect for v4 real and v4 tcp vip"; + } + // tcp, v4 vip v6 real + real = lb.getRealForFlow(katran::KatranFlow{ + .src = "172.16.0.1", + .dst = "10.200.1.3", + .srcPort = 31337, + .dstPort = 80, + .proto = kTcp, + }); + if (real != "fc00::2") { + VLOG(2) << "real: " << real; + LOG(INFO) << "simulation is incorrect for v6 real and v4 tcp vip"; + } + // tcp, v6 vip v6 real + real = lb.getRealForFlow(katran::KatranFlow{ + .src = "fc00:2::1", + .dst = "fc00:1::1", + .srcPort = 31337, + .dstPort = 80, + .proto = kTcp, + }); + if (real != "fc00::3") { + VLOG(2) << "real: " << real; + LOG(INFO) << "simulation is incorrect for v6 real and v6 tcp vip"; + } + // non existing vip + real = lb.getRealForFlow(katran::KatranFlow{ + .src = "fc00:2::1", + .dst = "fc00:1::2", + .srcPort = 31337, + .dstPort = 80, + .proto = kTcp, + }); + if (!real.empty()) { + VLOG(2) << "real: " << real; + LOG(INFO) << "incorrect real for non existing vip"; + } + // malformed flow #1 + real = lb.getRealForFlow(katran::KatranFlow{ + .src = "10.0.0.1", + .dst = "fc00:1::1", + .srcPort = 31337, + .dstPort = 80, + .proto = kTcp, + }); + if (!real.empty()) { + VLOG(2) << "real: " << real; + LOG(INFO) << "incorrect real for malformed flow #1"; + } + // malformed flow #2 + real = lb.getRealForFlow(katran::KatranFlow{ + .src = "aaaa", + .dst = "bbbb", + .srcPort = 31337, + .dstPort = 80, + .proto = kTcp, + }); + if (!real.empty()) { + VLOG(2) << "real: " << real; + LOG(INFO) << "incorrect real for malformed flow #2"; + } +} + void prepareLbData(katran::KatranLb& lb) { lb.restartKatranMonitor(kMonitorLimit); katran::VipKey vip; @@ -302,6 +389,7 @@ int main(int argc, char** argv) { } else if (FLAGS_test_from_fixtures) { tester.testFromFixture(); testLbCounters(lb); + testSimulator(lb); if (FLAGS_optional_tests) { prepareOptionalLbData(lb); LOG(INFO) << "Running optional tests. they could fail if requirements "