From 5d228bb30954fae999a23f0ebd7d81c359764766 Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Thu, 7 Sep 2017 11:25:06 -0700 Subject: [PATCH] flush peerdb entries on network delete peerDB was never being flushed on network delete leaveing behind stale entries Signed-off-by: Flavio Crisciani --- drivers/overlay/encryption.go | 1 - drivers/overlay/joinleave.go | 19 ++++++++----------- drivers/overlay/ov_network.go | 18 +++--------------- drivers/overlay/peerdb.go | 32 +++++++++++++++++++++++++++----- 4 files changed, 38 insertions(+), 32 deletions(-) diff --git a/drivers/overlay/encryption.go b/drivers/overlay/encryption.go index f12d7a8c67..802d7bc36d 100644 --- a/drivers/overlay/encryption.go +++ b/drivers/overlay/encryption.go @@ -21,7 +21,6 @@ import ( const ( r = 0xD0C4E3 - timeout = 30 pktExpansion = 26 // SPI(4) + SeqN(4) + IV(8) + PadLength(1) + NextHeader(1) + ICV(8) ) diff --git a/drivers/overlay/joinleave.go b/drivers/overlay/joinleave.go index 11edf43765..b97cc88f05 100644 --- a/drivers/overlay/joinleave.go +++ b/drivers/overlay/joinleave.go @@ -68,7 +68,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, ep.ifName = containerIfName - if err := d.writeEndpointToStore(ep); err != nil { + if err = d.writeEndpointToStore(ep); err != nil { return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err) } @@ -86,7 +86,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, return err } - if err := sbox.AddInterface(overlayIfName, "veth", + if err = sbox.AddInterface(overlayIfName, "veth", sbox.InterfaceOptions().Master(s.brName)); err != nil { return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err) } @@ -100,7 +100,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, return err } - if err := nlh.LinkSetHardwareAddr(veth, ep.mac); err != nil { + if err = nlh.LinkSetHardwareAddr(veth, ep.mac); err != nil { return fmt.Errorf("could not set mac address (%v) to the container interface: %v", ep.mac, err) } @@ -108,7 +108,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, if sub == s { continue } - if err := jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil { + if err = jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil { logrus.Errorf("Adding subnet %s static route in network %q failed\n", s.subnetIP, n.id) } } @@ -122,7 +122,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, d.peerAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), false, false, true) - if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil { + if err = d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil { logrus.Warn(err) } @@ -224,6 +224,7 @@ func (d *driver) Leave(nid, eid string) error { return types.InternalMaskableErrorf("could not find endpoint with id %s", eid) } + logrus.Errorf("The channel is valid:%t", d.notifyCh != nil) if d.notifyCh != nil { d.notifyCh <- ovNotify{ action: "leave", @@ -232,13 +233,9 @@ func (d *driver) Leave(nid, eid string) error { } } - n.leaveSandbox() - - // if err := d.checkEncryption(nid, nil, 0, true, false); err != nil { - // logrus.Warn(err) - // } - d.peerDelete(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true) + n.leaveSandbox() + return nil } diff --git a/drivers/overlay/ov_network.go b/drivers/overlay/ov_network.go index 419f22d03d..ee00a4bf0b 100644 --- a/drivers/overlay/ov_network.go +++ b/drivers/overlay/ov_network.go @@ -251,8 +251,9 @@ func (d *driver) DeleteNetwork(nid string) error { if err := d.deleteEndpointFromStore(ep); err != nil { logrus.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err) } - } + // flush the peerDB entries + d.peerFlush(nid) d.deleteNetwork(nid) vnis, err := n.releaseVxlanID() @@ -505,11 +506,7 @@ func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) erro vxlanIfaceOption := make([]osl.IfaceOption, 1) vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName)) Ifaces[vxlanName+"+vxlan"] = vxlanIfaceOption - err = sbox.Restore(Ifaces, nil, nil, nil) - if err != nil { - return err - } - return nil + return sbox.Restore(Ifaces, nil, nil, nil) } func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error { @@ -1063,15 +1060,6 @@ func (n *network) contains(ip net.IP) bool { return false } -func (n *network) getSubnetforIPAddr(ip net.IP) *subnet { - for _, s := range n.subnets { - if s.subnetIP.Contains(ip) { - return s - } - } - return nil -} - // getSubnetforIP returns the subnet to which the given IP belongs func (n *network) getSubnetforIP(ip *net.IPNet) *subnet { for _, s := range n.subnets { diff --git a/drivers/overlay/peerdb.go b/drivers/overlay/peerdb.go index 7779f3fd59..b6e7be080a 100644 --- a/drivers/overlay/peerdb.go +++ b/drivers/overlay/peerdb.go @@ -58,11 +58,13 @@ func (p *peerEntryDB) UnMarshalDB() peerEntry { } type peerMap struct { + // set of peerEntry, note they have to be objects and not pointers to maintain the proper equality checks mp common.SetMatrix sync.Mutex } type peerNetworkMap struct { + // map with key peerKey mp map[string]*peerMap sync.Mutex } @@ -253,6 +255,7 @@ const ( peerOperationINIT peerOperationType = iota peerOperationADD peerOperationDELETE + peerOperationFLUSH ) type peerOperation struct { @@ -283,6 +286,8 @@ func (d *driver) peerOpRoutine(ctx context.Context, ch chan *peerOperation) { err = d.peerAddOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.l2Miss, op.l3Miss, true, op.localPeer) case peerOperationDELETE: err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.localPeer) + case peerOperationFLUSH: + err = d.peerFlushOp(op.networkID) } if err != nil { logrus.Warnf("Peer operation failed:%s op:%v", err, op) @@ -315,7 +320,6 @@ func (d *driver) peerInitOp(nid string) error { func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, peerMac net.HardwareAddr, vtep net.IP, l2Miss, l3Miss, localPeer bool) { - callerName := common.CallerName(1) d.peerOpCh <- &peerOperation{ opType: peerOperationADD, networkID: nid, @@ -327,7 +331,7 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, l2Miss: l2Miss, l3Miss: l3Miss, localPeer: localPeer, - callerName: callerName, + callerName: common.CallerName(1), } } @@ -410,7 +414,6 @@ func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, peerMac net.HardwareAddr, vtep net.IP, localPeer bool) { - callerName := common.CallerName(1) d.peerOpCh <- &peerOperation{ opType: peerOperationDELETE, networkID: nid, @@ -419,7 +422,7 @@ func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMas peerIPMask: peerIPMask, peerMac: peerMac, vtepIP: vtep, - callerName: callerName, + callerName: common.CallerName(1), localPeer: localPeer, } } @@ -447,7 +450,7 @@ func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPM return nil } - if err := d.checkEncryption(nid, vtep, 0, false, false); err != nil { + if err := d.checkEncryption(nid, vtep, 0, localPeer, false); err != nil { logrus.Warn(err) } @@ -481,6 +484,25 @@ func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPM return d.peerAddOp(nid, peerEntry.eid, peerIP, peerEntry.peerIPMask, peerKey.peerMac, peerEntry.vtep, false, false, false, peerEntry.isLocal) } +func (d *driver) peerFlush(nid string) { + d.peerOpCh <- &peerOperation{ + opType: peerOperationFLUSH, + networkID: nid, + callerName: common.CallerName(1), + } +} + +func (d *driver) peerFlushOp(nid string) error { + d.peerDb.Lock() + defer d.peerDb.Unlock() + _, ok := d.peerDb.mp[nid] + if !ok { + return fmt.Errorf("Unable to find the peerDB for nid:%s", nid) + } + delete(d.peerDb.mp, nid) + return nil +} + func (d *driver) pushLocalDb() { d.peerDbWalk(func(nid string, pKey *peerKey, pEntry *peerEntry) bool { if pEntry.isLocal {