diff --git a/orchagent/muxorch.cpp b/orchagent/muxorch.cpp index 3fffd18946..368acb553d 100644 --- a/orchagent/muxorch.cpp +++ b/orchagent/muxorch.cpp @@ -163,6 +163,34 @@ static sai_status_t remove_route(IpPrefix &pfx) return status; } +/** + * @brief sets the given route to point to the given nexthop + * @param pfx IpPrefix of the route + * @param nexthop NextHopKey of the nexthop + * @return SAI_STATUS_SUCCESS on success + */ +static sai_status_t set_route(const IpPrefix& pfx, sai_object_id_t next_hop_id) +{ + /* set route entry to point to nh */ + sai_route_entry_t route_entry; + sai_attribute_t route_attr; + + route_entry.vr_id = gVirtualRouterId; + route_entry.switch_id = gSwitchId; + copy(route_entry.destination, pfx); + + route_attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID; + route_attr.value.oid = next_hop_id; + + sai_status_t status = sai_route_api->set_route_entry_attribute(&route_entry, &route_attr); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to set route entry %s nh %" PRIx64 " rv:%d", + pfx.to_string().c_str(), next_hop_id, status); + } + return status; +} + static sai_object_id_t create_tunnel( const IpAddress* p_dst_ip, const IpAddress* p_src_ip, @@ -534,9 +562,11 @@ bool MuxCable::isIpInSubnet(IpAddress ip) bool MuxCable::nbrHandler(bool enable, bool update_rt) { + bool ret; if (enable) { - return nbr_handler_->enable(update_rt); + ret = nbr_handler_->enable(update_rt); + updateRoutes(); } else { @@ -546,9 +576,10 @@ bool MuxCable::nbrHandler(bool enable, bool update_rt) SWSS_LOG_INFO("Null NH object id, retry for %s", peer_ip4_.to_string().c_str()); return false; } - - return nbr_handler_->disable(tnh); + updateRoutes(); + ret = nbr_handler_->disable(tnh); } + return ret; } void MuxCable::updateNeighbor(NextHopKey nh, bool add) @@ -562,6 +593,29 @@ void MuxCable::updateNeighbor(NextHopKey nh, bool add) else if (mux_name_ == mux_orch_->getNexthopMuxName(nh)) { mux_orch_->removeNexthop(nh); + nbr_handler_->removeNeighborRoutes(nh); + } + updateRoutes(); +} + +/** + * @brief updates all routes pointing to the cables neighbor list + */ +void MuxCable::updateRoutes() +{ + MuxNeighbor neighbors = nbr_handler_->getNeighbors(); + string alias = nbr_handler_->getAlias(); + for (auto nh = neighbors.begin(); nh != neighbors.end(); nh ++) + { + std::set routes; + NextHopKey nhkey(nh->first, alias); + if (gRouteOrch->getRoutesForNexthop(routes, nhkey)) + { + for (auto rt = routes.begin(); rt != routes.end(); rt++) + { + mux_orch_->updateRoute(rt->prefix, false); + } + } } } @@ -749,6 +803,45 @@ bool MuxNbrHandler::disable(sai_object_id_t tnh) return true; } +/** + * @brief removes routes programmed by updateRoute + * @param nh NextHopKey that points to neighbor for which to remove routes + */ +void MuxNbrHandler::removeNeighborRoutes(NextHopKey nh) +{ + std::set routes; + NextHopGroupKey nhg_key; + NextHopGroupEntry nhg_entry; + + if (gRouteOrch->getRoutesForNexthop(routes, nh)) + { + for (auto rt = routes.begin(); rt != routes.end(); rt++) + { + SWSS_LOG_INFO("Removing multi-nexthop route for %s", rt->prefix.to_string().c_str()); + return false; + /* get nexthop group key from syncd */ + nhg_key = gRouteOrch->getSyncdRouteNhgKey(gVirtualRouterId, rt->prefix); + + /* check for cross-mux neighbors */ + if (nhg_key.getSize() > 1) + { + sai_route_entry_t route_entry; + route_entry.switch_id = gSwitchId; + route_entry.vr_id = gVirtualRouterId; + copy(route_entry.destination, rt->prefix); + subnet(route_entry.destination, route_entry.destination); + + sai_status_t status = sai_route_api->remove_route_entry(&route_entry); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove route %s, rv:%d", + rt->prefix.to_string().c_str(), status); + } + } + } + } +} + sai_object_id_t MuxNbrHandler::getNextHopId(const NextHopKey nhKey) { auto it = neighbors_.find(nhKey.ip_address); @@ -973,6 +1066,91 @@ sai_object_id_t MuxOrch::getNextHopTunnelId(std::string tunnelKey, IpAddress& ip return it->second.nh_id; } +/** + * @brief updates the given route to point to a single active NH or tunnel + * @param pfx IpPrefix of route to update + * @param remove bool only true when route is getting removed + */ +void MuxOrch::updateRoute(const IpPrefix &pfx, bool remove) +{ + NextHopGroupKey nhg_key; + NextHopGroupEntry nhg_entry; + + if (remove) + { + mux_multi_nh_route_tb.erase(pfx); + return; + } + + /* get nexthop group key from syncd */ + nhg_key = gRouteOrch->getSyncdRouteNhgKey(gVirtualRouterId, pfx); + + /* check for cross-mux neighbors */ + if (nhg_key.getSize() > 1) + { + std::set nextHops; + sai_object_id_t next_hop_id; + sai_status_t status; + bool active_found = false; + + /* get nexthops from nexthop group */ + nextHops = nhg_key.getNextHops(); + + auto it = mux_multi_nh_route_tb.find(pfx); + if (it != mux_multi_nh_route_tb.end()) + { + MuxCable* cable = findMuxCableInSubnet(it->second.ip_address); + if (cable == nullptr || cable->isActive()) + { + SWSS_LOG_NOTICE("Route %s pointing to active neighbor %s", + pfx.to_string().c_str(), it->second.to_string().c_str()); + return; + } + } + + SWSS_LOG_NOTICE("Updating route %s pointing to Mux nexthops %s", + pfx.to_string().c_str(), nhg_key.to_string().c_str()); + + for (auto it = nextHops.begin(); it != nextHops.end(); it++) + { + NextHopKey nexthop = *it; + MuxCable* cable = findMuxCableInSubnet(nexthop.ip_address); + if (cable == nullptr || (cable->isActive() && !active_found)) + { + next_hop_id = gNeighOrch->getLocalNextHopId(nexthop); + /* set route entry to point to nh */ + status = set_route(pfx, next_hop_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to set route entry %s to nexthop %s", + pfx.to_string().c_str(), nexthop.to_string().c_str()); + continue; + } + SWSS_LOG_INFO("setting route %s with nexthop %s %" PRIx64 "", + pfx.to_string().c_str(), nexthop.to_string().c_str(), next_hop_id); + mux_multi_nh_route_tb[pfx] = nexthop; + active_found = true; + break; + } + } + + if (!active_found) + { + next_hop_id = getNextHopTunnelId(MUX_TUNNEL, mux_peer_switch_); + /* no active nexthop found, point to first */ + SWSS_LOG_NOTICE("No Active neighbors found, setting route %s to point to tun", + pfx.getIp().to_string().c_str()); + status = set_route(pfx, next_hop_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to set route entry %s to tunnel", + pfx.getIp().to_string().c_str()); + } + mux_multi_nh_route_tb.erase(pfx); + } + } +} + MuxCable* MuxOrch::findMuxCableInSubnet(IpAddress ip) { for (auto it = mux_cable_tb_.begin(); it != mux_cable_tb_.end(); it++) @@ -1201,6 +1379,35 @@ void MuxOrch::removeNexthop(NextHopKey nh) mux_nexthop_tb_.erase(nh); } +/** + * @brief checks if mux nexthop tb contains nexthop + * @param nexthop NextHopKey + * @return true if a mux contains the nexthop + */ +bool MuxOrch::containsNextHop(const NextHopKey& nexthop) +{ + return mux_nexthop_tb_.find(nexthop) != mux_nexthop_tb_.end(); +} + +/** + * @brief checks if a given nexthop group belongs to a mux + * @param nextHops NextHopGroupKey + * @return true if a mux contains any of the nexthops in the group + * false if none of the nexthops belong to a mux + */ +bool MuxOrch::isMuxNexthops(const NextHopGroupKey& nextHops) +{ + const std::set s_nexthops = nextHops.getNextHops(); + for (auto it = s_nexthops.begin(); it != s_nexthops.end(); it ++) + { + if (this->containsNextHop(*it)) + { + return true; + } + } + return false; +} + string MuxOrch::getNexthopMuxName(NextHopKey nh) { if (mux_nexthop_tb_.find(nh) == mux_nexthop_tb_.end()) diff --git a/orchagent/muxorch.h b/orchagent/muxorch.h index f4126979f6..41362fc2e8 100644 --- a/orchagent/muxorch.h +++ b/orchagent/muxorch.h @@ -71,7 +71,11 @@ class MuxNbrHandler bool disable(sai_object_id_t); void update(NextHopKey nh, sai_object_id_t, bool = true, MuxState = MuxState::MUX_STATE_INIT); + void removeNeighborRoutes(NextHopKey nh); + sai_object_id_t getNextHopId(const NextHopKey); + MuxNeighbor getNeighbors() const { return neighbors_; }; + string getAlias() const { return alias_; }; private: inline void updateTunnelRoute(NextHopKey, bool = true); @@ -102,6 +106,7 @@ class MuxCable bool isIpInSubnet(IpAddress ip); void updateNeighbor(NextHopKey nh, bool add); + void updateRoutes(); sai_object_id_t getNextHopId(const NextHopKey nh) { return nbr_handler_->getNextHopId(nh); @@ -158,6 +163,7 @@ typedef std::unique_ptr MuxCable_T; typedef std::map MuxCableTb; typedef std::map MuxTunnelNHs; typedef std::map NextHopTb; +typedef std::map MuxRouteTb; class MuxCfgRequest : public Request { @@ -195,6 +201,8 @@ class MuxOrch : public Orch2, public Observer, public Subject void addNexthop(NextHopKey, string = ""); void removeNexthop(NextHopKey); + bool containsNextHop(const NextHopKey&); + bool isMuxNexthops(const NextHopGroupKey&); string getNexthopMuxName(NextHopKey); sai_object_id_t getNextHopId(const NextHopKey&); @@ -202,6 +210,8 @@ class MuxOrch : public Orch2, public Observer, public Subject bool removeNextHopTunnel(std::string tunnelKey, IpAddress& ipAddr); sai_object_id_t getNextHopTunnelId(std::string tunnelKey, IpAddress& ipAddr); + void updateRoute(const IpPrefix &pfx, bool remove); + private: virtual bool addOperation(const Request& request); virtual bool delOperation(const Request& request); @@ -241,6 +251,9 @@ class MuxOrch : public Orch2, public Observer, public Subject MuxTunnelNHs mux_tunnel_nh_; NextHopTb mux_nexthop_tb_; + /* contains reference of programmed routes by updateRoute */ + MuxRouteTb mux_multi_nh_route_tb; + handler_map handler_map_; TunnelDecapOrch *decap_orch_; diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 247d945d1c..e92cbb51b1 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -6,6 +6,7 @@ #include "cbf/cbfnhgorch.h" #include "logger.h" #include "flowcounterrouteorch.h" +#include "muxorch.h" #include "swssnet.h" #include "crmorch.h" #include "directory.h" @@ -258,6 +259,24 @@ bool RouteOrch::hasNextHopGroup(const NextHopGroupKey& nexthops) const return m_syncdNextHopGroups.find(nexthops) != m_syncdNextHopGroups.end(); } +/** + * @brief checks if given nexthop is in a nexthop group + * @param nexthop NextHopKey + * @returns true if nexthop is in a nexthop group + */ +bool RouteOrch::inNextHopGroup(const NextHopKey& nexthop, NextHopGroupKey& nhgKey) +{ + for (auto it = m_syncdNextHopGroups.begin(); it != m_syncdNextHopGroups.end(); it++) + { + if (it->second.nhopgroup_members.find(nexthop) != it->second.nhopgroup_members.end()) + { + nhgKey = it->first; + return true; + } + } + return false; +} + sai_object_id_t RouteOrch::getNextHopGroupId(const NextHopGroupKey& nexthops) { assert(hasNextHopGroup(nexthops)); @@ -1558,6 +1577,19 @@ bool RouteOrch::updateNextHopRoutes(const NextHopKey& nextHop, uint32_t& numRout return true; } + /* Check if nexthop is mux nexthop */ + MuxOrch* mux_orch = gDirectory.get(); + NextHopGroupKey nhg_key; + if (inNextHopGroup(nextHop, nhg_key) && mux_orch->isMuxNexthops(nhg_key)) + { + /* multiple mux nexthop case: + * skip for now, muxOrch::updateRoute() will handle route + */ + SWSS_LOG_INFO("NH %s is in mux nexthop group, skipping.", + nextHop.ip_address.to_string().c_str()); + return true; + } + sai_route_entry_t route_entry; sai_attribute_t route_attr; sai_object_id_t next_hop_id; @@ -1593,6 +1625,24 @@ bool RouteOrch::updateNextHopRoutes(const NextHopKey& nextHop, uint32_t& numRout return true; } +/** + * @brief returns a route prefix associated with nexthopkey + * @param routeKeys empty set of routekeys to populate + * @param nexthopKey nexthop key to lookup + * @return true if found, false if not found. + */ +bool RouteOrch::getRoutesForNexthop(std::set& routeKeys, const NextHopKey& nexthopKey) +{ + auto it = m_nextHops.find(nexthopKey); + + if (it != m_nextHops.end()) + { + routeKeys = it->second; + } + + return it != m_nextHops.end(); +} + void RouteOrch::addTempRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) { SWSS_LOG_ENTER(); @@ -2215,6 +2265,9 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); } + m_syncdRoutes[vrf_id][ipPrefix] = nextHops; + + MuxOrch* mux_orch = gDirectory.get(); if (ctx.nhg_index.empty() && nextHops.getSize() == 1 && !nextHops.is_overlay_nexthop() && !nextHops.is_srv6_nexthop()) { RouteKey r_key = { vrf_id, ipPrefix }; @@ -2223,6 +2276,19 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey { addNextHopRoute(nexthop, r_key); } + } else if (mux_orch->isMuxNexthops(nextHops)) + { + RouteKey routekey = { vrf_id, ipPrefix }; + auto nexthop_list = nextHops.getNextHops(); + for (auto nh = nexthop_list.begin(); nh != nexthop_list.end(); nh++) + { + if (!nh->ip_address.isZero()) + { + addNextHopRoute(*nh, routekey); + } + } + // update routes to reflect mux state + mux_orch->updateRoute(ipPrefix, false); } if (ipPrefix.isDefaultRoute()) @@ -2399,11 +2465,25 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) decreaseNextHopRefCount(it_route->second.nhg_key); auto ol_nextHops = it_route->second.nhg_key; - + MuxOrch* mux_orch = gDirectory.get(); if (it_route->second.nhg_key.getSize() > 1 && m_syncdNextHopGroups[it_route->second.nhg_key].ref_count == 0) { m_bulkNhgReducedRefCnt.emplace(it_route->second.nhg_key, 0); + if (mux_orch->isMuxNexthops(ol_nextHops)) + { + SWSS_LOG_NOTICE("Remove mux Nexthop %s", ol_nextHops.to_string().c_str()); + RouteKey routekey = { vrf_id, ipPrefix }; + auto nexthop_list = ol_nextHops.getNextHops(); + for (auto nh = nexthop_list.begin(); nh != nexthop_list.end(); nh++) + { + if (!nh->ip_address.isZero()) + { + removeNextHopRoute(*nh, routekey); + } + } + mux_orch->updateRoute(ipPrefix, true); + } } else if (ol_nextHops.is_overlay_nexthop()) { diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index cc89005d7a..f26701ca17 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -186,7 +186,9 @@ class RouteOrch : public Orch, public Subject RouteOrch(DBConnector *db, vector &tableNames, SwitchOrch *switchOrch, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch, FgNhgOrch *fgNhgOrch, Srv6Orch *srv6Orch); bool hasNextHopGroup(const NextHopGroupKey&) const; + bool inNextHopGroup(const NextHopKey&, NextHopGroupKey& nhgKey); sai_object_id_t getNextHopGroupId(const NextHopGroupKey&); + NextHopGroupEntry getNextHopGroupEntry(const NextHopGroupKey&); void attach(Observer *, const IpAddress&, sai_object_id_t vrf_id = gVirtualRouterId); void detach(Observer *, const IpAddress&, sai_object_id_t vrf_id = gVirtualRouterId); @@ -201,6 +203,7 @@ class RouteOrch : public Orch, public Subject void addNextHopRoute(const NextHopKey&, const RouteKey&); void removeNextHopRoute(const NextHopKey&, const RouteKey&); bool updateNextHopRoutes(const NextHopKey&, uint32_t&); + bool getRoutesForNexthop(std::set&, const NextHopKey&); bool validnexthopinNextHopGroup(const NextHopKey&, uint32_t&); bool invalidnexthopinNextHopGroup(const NextHopKey&, uint32_t&); diff --git a/tests/test_mux.py b/tests/test_mux.py index 8313980130..6eda6af7e3 100644 --- a/tests/test_mux.py +++ b/tests/test_mux.py @@ -15,6 +15,7 @@ def create_fvs(**kwargs): class TestMuxTunnelBase(): APP_MUX_CABLE = "MUX_CABLE_TABLE" APP_NEIGH_TABLE = "NEIGH_TABLE" + APP_ROUTE_TABLE = "ROUTE_TABLE" APP_TUNNEL_DECAP_TABLE_NAME = "TUNNEL_DECAP_TABLE" APP_TUNNEL_ROUTE_TABLE_NAME = "TUNNEL_ROUTE_TABLE" ASIC_TUNNEL_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL" @@ -23,6 +24,7 @@ class TestMuxTunnelBase(): ASIC_VRF_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_VIRTUAL_ROUTER" ASIC_NEIGH_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY" ASIC_NEXTHOP_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP" + ASIC_NHG_MEMBER_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER" ASIC_ROUTE_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" ASIC_FDB_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_FDB_ENTRY" ASIC_SWITCH_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_SWITCH" @@ -166,6 +168,26 @@ def get_vlan_rif_oid(self, asicdb): break return vlan_oid + + def get_nexthop_oid(self, asicdb, nexthop): + # gets nexthop oid + nexthop_keys = asicdb.get_keys(self.ASIC_NEXTHOP_TABLE) + + nexthop_oid = '' + for nexthop_key in nexthop_keys: + entry = asicdb.get_entry(self.ASIC_NEXTHOP_TABLE, nexthop_key) + if entry["SAI_NEXT_HOP_ATTR_IP"] == nexthop: + nexthop_oid = nexthop_key + break + + return nexthop_oid + + def get_route_nexthop_oid(self, route_key, asicdb): + # gets nexthop oid + entry = asicdb.get_entry(self.ASIC_ROUTE_TABLE, route_key) + assert 'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID' in entry + + return entry['SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID'] def check_tunnel_route_in_app_db(self, dvs, destinations, expected=True): appdb = dvs.get_app_db() @@ -245,6 +267,18 @@ def check_nexthop_group_in_asic_db(self, asicdb, key, num_tnl_nh=0): assert num_tnl_nh == count + def check_route_nexthop(self, dvs_route, asicdb, route, nexthop, tunnel=False): + route_key = dvs_route.check_asicdb_route_entries([route]) + route_nexthop_oid = self.get_route_nexthop_oid(route_key[0], asicdb) + + if tunnel: + assert route_nexthop_oid == nexthop + return + + nexthop_oid = self.get_nexthop_oid(asicdb, nexthop) + + assert route_nexthop_oid == nexthop_oid + def add_neighbor(self, dvs, ip, mac): if ip_address(ip).version == 6: dvs.runcmd("ip -6 neigh replace " + ip + " lladdr " + mac + " dev Vlan1000") @@ -273,6 +307,26 @@ def del_fdb(self, dvs, mac): time.sleep(1) + def add_route(self, dvs, route, nexthops, ifaces=[]): + apdb = dvs.get_app_db() + nexthop_str = ",".join(nexthops) + if len(ifaces) == 0: + ifaces = [self.VLAN_1000 for k in range(len(nexthops))] + iface_str = ",".join(ifaces) + ps = swsscommon.ProducerStateTable(apdb.db_connection, self.APP_ROUTE_TABLE) + fvs = swsscommon.FieldValuePairs( + [ + ("nexthop", nexthop_str), + ("ifname", iface_str) + ] + ) + ps.set(route, fvs) + + def del_route(self, dvs, route): + apdb = dvs.get_app_db() + ps = swsscommon.ProducerStateTable(apdb.db_connection, self.APP_ROUTE_TABLE) + ps._del(route) + def create_and_test_neighbor(self, confdb, appdb, asicdb, dvs, dvs_route): self.set_mux_state(appdb, "Ethernet0", "active") @@ -450,92 +504,214 @@ def create_and_test_route(self, appdb, asicdb, dvs, dvs_route): self.set_mux_state(appdb, "Ethernet4", "active") dvs_route.check_asicdb_deleted_route_entries([rtprefix]) - # Test ECMP routes - - self.set_mux_state(appdb, "Ethernet0", "active") - self.set_mux_state(appdb, "Ethernet4", "active") - - rtprefix = "5.6.7.0/24" - - dvs_route.check_asicdb_deleted_route_entries([rtprefix]) - - ps = swsscommon.ProducerStateTable(pdb.db_connection, "ROUTE_TABLE") + def multi_nexthop_test(self, dvs, dvs_route, asicdb, appdb, route, neighbors, macs): + mux_ports = ["Ethernet0", "Ethernet4"] - fvs = swsscommon.FieldValuePairs( - [ - ("nexthop", self.SERV1_IPV4 + "," + self.SERV2_IPV4), - ("ifname", "Vlan1000,Vlan1000") - ] - ) + # Set state to active for initial state + for port in mux_ports: + self.set_mux_state(appdb, port, "active") - ps.set(rtprefix, fvs) - - # Check if route was propagated to ASIC DB - rtkeys = dvs_route.check_asicdb_route_entries([rtprefix]) - - # Check for nexthop group and validate nexthop group member in asic db - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0]) - - # Step: 1 - Change one NH to standby and verify ecmp route - self.set_mux_state(appdb, "Ethernet0", "standby") - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 1) - - # Step: 2 - Change the other NH to standby and verify ecmp route - self.set_mux_state(appdb, "Ethernet4", "standby") - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 2) - - # Step: 3 - Change one NH to back to Active and verify ecmp route - self.set_mux_state(appdb, "Ethernet0", "active") - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 1) - - # Step: 4 - Change the other NH to Active and verify ecmp route - self.set_mux_state(appdb, "Ethernet4", "active") - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0]) - - ps._del(rtprefix) - - # Test IPv6 ECMP routes and start with standby config - self.set_mux_state(appdb, "Ethernet0", "standby") - self.set_mux_state(appdb, "Ethernet4", "standby") - - rtprefix = "2020::/64" - - dvs_route.check_asicdb_deleted_route_entries([rtprefix]) - - ps = swsscommon.ProducerStateTable(pdb.db_connection, "ROUTE_TABLE") - - fvs = swsscommon.FieldValuePairs( - [ - ("nexthop", self.SERV1_IPV6 + "," + self.SERV2_IPV6), - ("ifname", "tun0,tun0") - ] - ) - - ps.set(rtprefix, fvs) - - # Check if route was propagated to ASIC DB - rtkeys = dvs_route.check_asicdb_route_entries([rtprefix]) - - # Check for nexthop group and validate nexthop group member in asic db - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 2) + # add neighbors for initial state + for i,neighbor in enumerate(neighbors): + self.add_neighbor(dvs, neighbor, macs[i]) + + try: + # toggle between states and add route in various combos of state + print("Testing add/remove/update of route") + starting_states = [(ACTIVE, ACTIVE), (ACTIVE, STANDBY), (STANDBY, ACTIVE), (STANDBY, STANDBY)] + for start in starting_states: + print("Adding route with %s: %s and %s: %s" % (mux_ports[0], start[0], mux_ports[1], start[1])) + self.set_mux_state(appdb, mux_ports[0], start[0]) + self.set_mux_state(appdb, mux_ports[1], start[1]) + self.add_route(dvs, route, neighbors) + if start[0] == "active": + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[0]) + elif start[1] == "active": + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[1]) + else: + self.check_route_nexthop(dvs_route, asicdb, route, tunnel_nh_id, True) + + print("Testing fdb update in %s, %s for %s" % (start[0], start[1], neighbors[0])) + # move neighbor 1 + self.add_neighbor(dvs, neighbors[0], "00:aa:bb:cc:dd:ee") + if start[0] == "active": + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[0]) + elif start[1] == "active": + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[1]) + else: + self.check_route_nexthop(dvs_route, asicdb, route, tunnel_nh_id, True) + + # move neighbor 1 back + self.add_neighbor(dvs, neighbors[0], macs[i]) + + print("Testing fdb update in %s, %s for %s" % (start[0], start[1], neighbors[1])) + # move neighbor 2 + self.add_neighbor(dvs, neighbors[0], "00:aa:bb:cc:dd:ee") + if start[0] == "active": + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[0]) + elif start[1] == "active": + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[1]) + else: + self.check_route_nexthop(dvs_route, asicdb, route, tunnel_nh_id, True) + + # move neighbor 2 back + self.add_neighbor(dvs, neighbors[0], macs[i]) + + self.del_route(dvs, route) + + + # toggle mux states to check setState actions + print("Testing toggling state") + starting_states = [(ACTIVE, ACTIVE), (ACTIVE, STANDBY), (STANDBY, ACTIVE), (STANDBY, STANDBY)] + for start in starting_states: + self.set_mux_state(appdb, mux_ports[0], start[0]) + self.set_mux_state(appdb, mux_ports[1], start[1]) + self.add_route(dvs, route, neighbors) + + for toggle_index,port in enumerate(mux_ports): + keep_index = (toggle_index + 1) % 2 + + print("keeping %s as %s while toggling %s from %s" % \ + (mux_ports[keep_index], start[keep_index], mux_ports[toggle_index], start[toggle_index])) + + if start[toggle_index] == ACTIVE: + print("setting %s to %s" % (mux_ports[toggle_index], STANDBY)) + self.set_mux_state(appdb, mux_ports[toggle_index], STANDBY) + if start[keep_index] == ACTIVE: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[keep_index]) + else: + self.check_route_nexthop(dvs_route, asicdb, route, tunnel_nh_id, True) + + print("setting %s to %s" % (mux_ports[toggle_index], ACTIVE)) + self.set_mux_state(appdb, mux_ports[toggle_index], ACTIVE) + if start[keep_index] == ACTIVE: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[keep_index]) + else: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[toggle_index]) + else: + print("setting %s to %s" % (mux_ports[toggle_index], ACTIVE)) + self.set_mux_state(appdb, mux_ports[toggle_index], ACTIVE) + if start[keep_index] == ACTIVE: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[keep_index]) + else: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[toggle_index]) + + print("setting %s to %s" % (mux_ports[toggle_index], STANDBY)) + self.set_mux_state(appdb, mux_ports[toggle_index], STANDBY) + if start[keep_index] == ACTIVE: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[keep_index]) + else: + self.check_route_nexthop(dvs_route, asicdb, route, tunnel_nh_id, True) + self.del_route(dvs, route) + + for neighbor in neighbors: + self.del_neighbor(dvs, neighbor) + + print("Testing add/remove of neighbors in mux nexthop group") + starting_states = [(ACTIVE, ACTIVE), (ACTIVE, STANDBY), (STANDBY, ACTIVE), (STANDBY, STANDBY)] + for start in starting_states: + print("Testing add/remove of neighbors in %s, %s" % start) + self.set_mux_state(appdb, mux_ports[0], start[0]) + self.set_mux_state(appdb, mux_ports[1], start[1]) + self.add_route(dvs, route, neighbors) + + # add first neighbor + self.add_neighbor(dvs, neighbors[0], macs[0]) + if start[0] == ACTIVE: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[0]) + + # add second neighbor + self.add_neighbor(dvs, neighbors[1], macs[1]) + time.sleep(1) + if start[0] == ACTIVE: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[0]) + elif start[1] == ACTIVE: + self.check_route_nexthop(dvs_route, asicdb, route, neighbors[1]) + else: + self.check_route_nexthop(dvs_route, asicdb, route, tunnel_nh_id, True) - # Step: 1 - Change one NH to active and verify ecmp route - self.set_mux_state(appdb, "Ethernet0", "active") - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 1) + # remove first neighbor + self.del_route(dvs,route) + for neighbor in neighbors: + self.del_neighbor(dvs, neighbor) - # Step: 2 - Change the other NH to active and verify ecmp route - self.set_mux_state(appdb, "Ethernet4", "active") - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0]) + finally: + # Cleanup + for port in mux_ports: + self.set_mux_state(appdb, port, "active") + self.del_route(dvs,route) + for neighbor in neighbors: + self.del_neighbor(dvs, neighbor) + + def create_and_test_multi_nexthop_routes(self, dvs, dvs_route, appdb, macs, asicdb): + ''' + Tests case where there are multiple nexthops tied to a route + If the nexthops are tied to a mux, then only the first active neighbor will be programmed + If not, the route should point to a regular ECMP group + ''' - # Step: 3 - Change one NH to back to standby and verify ecmp route - self.set_mux_state(appdb, "Ethernet0", "standby") - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 1) + route_ipv4 = "2.3.4.0/24" + route_ipv6 = "2023::/64" + ipv4_neighbors = [self.SERV1_IPV4, self.SERV2_IPV4] + ipv6_neighbors = [self.SERV1_IPV6, self.SERV2_IPV6] - # Step: 4 - Change the other NH to standby and verify ecmp route - self.set_mux_state(appdb, "Ethernet4", "standby") - self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 2) + self.multi_nexthop_test(dvs, dvs_route, asicdb, appdb, route_ipv4, ipv4_neighbors, macs) + self.multi_nexthop_test(dvs, dvs_route, asicdb, appdb, route_ipv6, ipv6_neighbors, macs) - ps._del(rtprefix) + try: + # neighbor not tied to mux cable case + non_mux_ipv4 = ["11.11.11.11", "12.12.12.12"] + non_mux_ipv6 = ["2222::100", "2222::101"] + non_mux_macs = ["00:aa:bb:cc:dd:ee", "00:aa:bb:cc:dd:ff"] + print("Testing neighbors that are not tied to a mux cable") + + for i in range(2): + self.add_neighbor(dvs, non_mux_ipv4[i], non_mux_macs[i]) + self.add_neighbor(dvs, non_mux_ipv6[i], non_mux_macs[i]) + + self.add_route(dvs, route_ipv4, non_mux_ipv4) + self.add_route(dvs, route_ipv6, non_mux_ipv6) + + # Check for route pointing to first neighbor + self.check_route_nexthop(dvs_route, asicdb, route_ipv4, non_mux_ipv4[0]) + self.check_route_nexthop(dvs_route, asicdb, route_ipv6, non_mux_ipv6[0]) + + # Cleanup + self.del_route(dvs, route_ipv4) + self.del_route(dvs, route_ipv6) + for i in range(2): + self.del_neighbor(dvs, non_mux_ipv4[i]) + self.del_neighbor(dvs, non_mux_ipv6[i]) + + # neighbor not in mux cable case + non_mux_ipv4 = ["11.11.11.11", "12.12.12.12"] + non_mux_ipv6 = ["2222::100", "2222::101"] + non_mux_macs = ["00:aa:bb:cc:dd:ee", "00:aa:bb:cc:dd:ff"] + print("Testing neighbors that are not tied to a mux cable") + for i in range(2): + self.add_neighbor(dvs, non_mux_ipv4[i], non_mux_macs[i]) + self.add_neighbor(dvs, non_mux_ipv6[i], non_mux_macs[i]) + + self.add_route(dvs, route_ipv4, non_mux_ipv4) + self.add_route(dvs, route_ipv6, non_mux_ipv6) + + # Check for route pointing to first neighbor + self.check_route_nexthop(dvs_route, asicdb, route_ipv4, non_mux_ipv4[0]) + self.check_route_nexthop(dvs_route, asicdb, route_ipv6, non_mux_ipv6[0]) + + # Cleanup + self.del_route(dvs, route_ipv4) + self.del_route(dvs, route_ipv6) + for i in range(2): + self.del_neighbor(dvs, non_mux_ipv4[i]) + self.del_neighbor(dvs, non_mux_ipv6[i]) + finally: + # Cleanup + self.del_route(dvs, route_ipv4) + self.del_route(dvs, route_ipv6) + for i in range(2): + self.del_neighbor(dvs, non_mux_ipv4[i]) + self.del_neighbor(dvs, non_mux_ipv6[i]) def create_and_test_NH_routes(self, appdb, asicdb, dvs, dvs_route, mac): ''' @@ -1181,6 +1357,13 @@ def test_NH(self, dvs, dvs_route, intf_fdb_map, setup_peer_switch, setup_tunnel, self.create_and_test_NH_routes(appdb, asicdb, dvs, dvs_route, mac) + def test_multi_nexthop(self, dvs, dvs_route, intf_fdb_map, neighbor_cleanup, testlog): + appdb = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) + asicdb = dvs.get_asic_db() + macs = [intf_fdb_map["Ethernet0"], intf_fdb_map["Ethernet4"]] + + self.create_and_test_multi_nexthop_routes(dvs, dvs_route, appdb, macs, asicdb) + def test_acl(self, dvs, dvs_acl, testlog): """ test acl and mux state change """