Skip to content

Commit

Permalink
Assign egress IPs to dummy device instead of external device (#2345)
Browse files Browse the repository at this point in the history 
Instead of assigning Egress IPs to the device that is connected to the
external network and relying on persisting assigned IPs to files, this
patch creates a dummy device and uses it to hold the external IPs
configured by antrea-agent. From function's perspective, it's totally
same regardless of the device the IPs are assigned to. The advantages of
using a dummy device are as below:

1. It doesn't need to persist assigned IPs to files to know which IPs
were configured by antrea-agent.

2. It avoids touching user or system managed network device, and is easy
to exclude the device from being managed by network manager tools.

3. It is more friendly for troubleshooting.

Besides, this patch fixes an issue that old Egress IPs were not
unassigned from the Node when updating Egress's EgressIP, and makes
IPAssigner more generic so that it can be used by other features that
need assign IPs to the system.

It also adds e2e tests that covers basic workflow of CRUD of Egress and
failover scenario.

Signed-off-by: Quan Tian <[email protected]>
  • Loading branch information
@tnqn
tnqn authored Jul 9, 2021
1 parent 652b322 commit 1588b16
Show file tree
Hide file tree
Showing 10 changed files with 655 additions and 305 deletions.
96 changes: 53 additions & 43 deletions pkg/agent/controller/egress/egress_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
Expand Down Expand Up @@ -66,7 +67,8 @@ const (
egressIPIndex = "egressIP"
externalIPPoolIndex = "externalIPPool"

DefaultEgressRunDir = "/var/run/antrea/egress"
// egressDummyDevice is the dummy device that holds the Egress IPs configured to the system by antrea-agent.
egressDummyDevice = "antrea-egress0"
)

var emptyWatch = watch.NewEmptyWatch()
Expand Down Expand Up @@ -173,7 +175,7 @@ func NewEgressController(
localIPDetector: localIPDetector,
idAllocator: newIDAllocator(minEgressMark, maxEgressMark),
}
ipAssigner, err := ipassigner.NewIPAssigner(nodeIP, DefaultEgressRunDir)
ipAssigner, err := ipassigner.NewIPAssigner(nodeIP, egressDummyDevice)
if err != nil {
return nil, fmt.Errorf("initializing egressIP assigner failed: %v", err)
}
Expand Down Expand Up @@ -276,20 +278,9 @@ func (c *EgressController) Run(stopCh <-chan struct{}) {
return
}

go c.cluster.Run(stopCh)
c.removeStaleEgressIPs()

// The Egress has been deleted but assigned IP has not been deleted, so agent should delete those IPs when it starts.
for egressName := range c.ipAssigner.AssignedIPs() {
if _, err := c.egressLister.Get(egressName); err != nil {
if errors.IsNotFound(err) {
if err := c.ipAssigner.UnassignEgressIP(egressName); err != nil {
klog.ErrorS(err, "Unassign EgressIP failed")
}
} else {
klog.ErrorS(err, "Get Egress error", "egressName", egressName)
}
}
}
go c.cluster.Run(stopCh)

go wait.NonSlidingUntil(c.watchEgressGroup, 5*time.Second, stopCh)

Expand All @@ -299,6 +290,25 @@ func (c *EgressController) Run(stopCh <-chan struct{}) {
<-stopCh
}

// removeStaleEgressIPs unassigns stale Egress IPs that shouldn't be present on this Node.
// These Egresses were either deleted from the Kubernetes API or migrated to other Nodes when the agent on this Node
// was not running.
func (c *EgressController) removeStaleEgressIPs() {
desiredLocalEgressIPs := sets.NewString()
egresses, _ := c.egressLister.List(labels.Everything())
for _, egress := range egresses {
if egress.Spec.EgressIP != "" && egress.Spec.ExternalIPPool != "" && egress.Status.EgressNode == c.nodeName {
desiredLocalEgressIPs.Insert(egress.Spec.EgressIP)
}
}
actualLocalEgressIPs := c.ipAssigner.AssignedIPs()
for ip := range actualLocalEgressIPs.Difference(desiredLocalEgressIPs) {
if err := c.ipAssigner.UnassignIP(ip); err != nil {
klog.ErrorS(err, "Failed to clean up stale Egress IP", "ip", ip)
}
}
}

// worker is a long-running function that will continually call the processNextWorkItem function in
// order to read and process a message on the workqueue.
func (c *EgressController) worker() {
Expand Down Expand Up @@ -534,7 +544,7 @@ func (c *EgressController) syncEgress(egressName string) error {

egress, err := c.egressLister.Get(egressName)
if err != nil {
// The Egress has been removed, clean up it.
// The Egress has been removed, clean it up.
if errors.IsNotFound(err) {
eState, exist := c.getEgressState(egressName)
// The Egress hasn't been installed, do nothing.
Expand All @@ -544,38 +554,11 @@ func (c *EgressController) syncEgress(egressName string) error {
if err := c.uninstallEgress(egressName, eState); err != nil {
return err
}
// Unassign the Egress IP (assigned by agent) from the local Node.
if err := c.ipAssigner.UnassignEgressIP(egressName); err != nil {
return err
}
return nil
}
return err
}

localNodeSelected, err := c.cluster.ShouldSelectEgress(egress)
if err != nil {
return err
}
if localNodeSelected {
// Assign Egress IP to the local Node.
if !c.localIPDetector.IsLocalIP(egress.Spec.EgressIP) {
err := c.ipAssigner.AssignEgressIP(egress.Spec.EgressIP, egressName)
if err != nil {
return err
}
if err := c.updateEgressStatus(egress, c.nodeName); err != nil {
return err
}
klog.InfoS("Assigned Egress IP", "Egress", egressName, "ip", egress.Spec.EgressIP, "nodeName", c.nodeName)
}
} else {
// Unassign the Egress IP (assigned by agent) from the local Node.
if err := c.ipAssigner.UnassignEgressIP(egressName); err != nil {
return err
}
}

eState, exist := c.getEgressState(egressName)
// If the EgressIP changes, uninstalls this Egress first.
if exist && eState.egressIP != egress.Spec.EgressIP {
Expand All @@ -584,10 +567,33 @@ func (c *EgressController) syncEgress(egressName string) error {
}
exist = false
}
// Do not proceed if EgressIP is empty.
if egress.Spec.EgressIP == "" {
return nil
}
if !exist {
eState = c.newEgressState(egressName, egress.Spec.EgressIP)
}

localNodeSelected, err := c.cluster.ShouldSelectEgress(egress)
if err != nil {
return err
}
if localNodeSelected {
// Ensure the Egress IP is assigned to the system.
if err := c.ipAssigner.AssignIP(egress.Spec.EgressIP); err != nil {
return err
}
if err := c.updateEgressStatus(egress, c.nodeName); err != nil {
return err
}
} else {
// Unassign the Egress IP from the local Node if it was assigned by the agent.
if err := c.ipAssigner.UnassignIP(egress.Spec.EgressIP); err != nil {
return err
}
}

// Realize the latest EgressIP and get the desired mark.
mark, err := c.realizeEgressIP(egressName, egress.Spec.EgressIP)
if err != nil {
Expand Down Expand Up @@ -666,6 +672,10 @@ func (c *EgressController) uninstallEgress(egressName string, eState *egressStat
if err := c.unrealizeEgressIP(egressName, eState.egressIP); err != nil {
return err
}
// Unassign the Egress IP from the local Node if it was assigned by the agent.
if err := c.ipAssigner.UnassignIP(eState.egressIP); err != nil {
return err
}
// Remove the Egress's state.
c.deleteEgressState(egressName)
return nil
Expand Down
60 changes: 31 additions & 29 deletions pkg/agent/controller/egress/egress_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,17 +180,17 @@ func TestSyncEgress(t *testing.T) {
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)

mockOFClient.EXPECT().UninstallSNATMarkFlows(uint32(1))
mockRouteClient.EXPECT().DeleteSNATRule(uint32(1))
mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1))
mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(2))
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)

mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(0))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(0))
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
},
},
{
Expand Down Expand Up @@ -221,17 +221,17 @@ func TestSyncEgress(t *testing.T) {
expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) {
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeRemoteEgressIP1), uint32(0))
mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)

mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1))
mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(2))
mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)

mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeRemoteEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(1))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeRemoteEgressIP1), uint32(1))
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)
},
},
{
Expand Down Expand Up @@ -263,19 +263,20 @@ func TestSyncEgress(t *testing.T) {
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)

mockOFClient.EXPECT().UninstallSNATMarkFlows(uint32(1))
mockRouteClient.EXPECT().DeleteSNATRule(uint32(1))
mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1))
mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(2))
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP2)

mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP2), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP2), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP2), uint32(1))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP2), uint32(1))
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP2)
},
},
{
Expand Down Expand Up @@ -307,17 +308,18 @@ func TestSyncEgress(t *testing.T) {
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)

mockOFClient.EXPECT().UninstallSNATMarkFlows(uint32(1))
mockRouteClient.EXPECT().DeleteSNATRule(uint32(1))
mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1))
mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(2))
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)

mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(0))
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)
},
},
{
Expand Down Expand Up @@ -347,17 +349,18 @@ func TestSyncEgress(t *testing.T) {
expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) {
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeRemoteEgressIP1), uint32(0))
mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)

mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1))
mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(2))
mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)

mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
},
},
{
Expand Down Expand Up @@ -389,13 +392,14 @@ func TestSyncEgress(t *testing.T) {
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)

mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP2), uint32(2))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP2), uint32(2))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP2), uint32(2))
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressB")
mockIPAssigner.EXPECT().UnassignEgressIP("egressB")
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP2)

mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP2)
},
},
{
Expand Down Expand Up @@ -428,9 +432,7 @@ func TestSyncEgress(t *testing.T) {
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(1))
mockIPAssigner.EXPECT().UnassignEgressIP("egressA")
mockIPAssigner.EXPECT().UnassignEgressIP("egressB")
mockIPAssigner.EXPECT().UnassignEgressIP("egressB")
mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1).Times(3)
},
},
}
Expand Down Expand Up @@ -530,19 +532,19 @@ func TestSyncOverlappingEgress(t *testing.T) {
c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1))
c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1))
c.mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1))
c.mockIPAssigner.EXPECT().UnassignEgressIP(egress1.Name)
c.mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
err := c.syncEgress(egress1.Name)
assert.NoError(t, err)

// egress2's IP is not local and pod1 has enforced egress1, so only one Pod SNAT flow is expected.
c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(0))
c.mockIPAssigner.EXPECT().UnassignEgressIP(egress2.Name)
c.mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)
err = c.syncEgress(egress2.Name)
assert.NoError(t, err)

// egress3 shares the same IP as egress1 and pod2 has enforced egress1, so only one Pod SNAT flow is expected.
c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(4), net.ParseIP(fakeLocalEgressIP1), uint32(1))
c.mockIPAssigner.EXPECT().UnassignEgressIP(egress3.Name)
c.mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
err = c.syncEgress(egress3.Name)
assert.NoError(t, err)

Expand All @@ -556,7 +558,7 @@ func TestSyncOverlappingEgress(t *testing.T) {
_, err := c.egressLister.Get(egress1.Name)
return err != nil, nil
}))
c.mockIPAssigner.EXPECT().UnassignEgressIP(egress1.Name)
c.mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
err = c.syncEgress(egress1.Name)
assert.NoError(t, err)
require.Equal(t, 2, c.queue.Len())
Expand All @@ -571,21 +573,21 @@ func TestSyncOverlappingEgress(t *testing.T) {

// pod1 is expected to enforce egress2.
c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0))
c.mockIPAssigner.EXPECT().UnassignEgressIP(egress2.Name)
c.mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)
err = c.syncEgress(egress2.Name)
assert.NoError(t, err)

// pod2 is expected to enforce egress3.
c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1))
c.mockIPAssigner.EXPECT().UnassignEgressIP(egress3.Name)
c.mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
err = c.syncEgress(egress3.Name)
assert.NoError(t, err)

// After deleting egress2, pod1 and pod3 no longer enforces any Egress.
c.mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1))
c.mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(3))
c.crdClient.CrdV1alpha2().Egresses().Delete(context.TODO(), egress2.Name, metav1.DeleteOptions{})
c.mockIPAssigner.EXPECT().UnassignEgressIP(egress2.Name)
c.mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1)
assert.NoError(t, wait.Poll(time.Millisecond*100, time.Second, func() (bool, error) {
_, err := c.egressLister.Get(egress2.Name)
return err != nil, nil
Expand All @@ -600,7 +602,7 @@ func TestSyncOverlappingEgress(t *testing.T) {
c.mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(2))
c.mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(4))
c.crdClient.CrdV1alpha2().Egresses().Delete(context.TODO(), egress3.Name, metav1.DeleteOptions{})
c.mockIPAssigner.EXPECT().UnassignEgressIP(egress3.Name)
c.mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1)
assert.NoError(t, wait.Poll(time.Millisecond*100, time.Second, func() (bool, error) {
_, err := c.egressLister.Get(egress3.Name)
return err != nil, nil
Expand Down
13 changes: 9 additions & 4 deletions pkg/agent/controller/egress/ipassigner/ip_assigner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,14 @@

package ipassigner

// IPAssigner provides methods to assign or unassign egressIP.
import "k8s.io/apimachinery/pkg/util/sets"

// IPAssigner provides methods to assign or unassign IP.
type IPAssigner interface {
AssignEgressIP(egressIP, egressName string) error
UnassignEgressIP(egressName string) error
AssignedIPs() (ips map[string]string)
// AssignIP ensures the provided IP is assigned to the system.
AssignIP(ip string) error
// UnassignIP ensures the provided IP is not assigned to the system.
UnassignIP(ip string) error
// AssignedIPs return the IPs that are assigned to the system by this IPAssigner.
AssignedIPs() sets.String
}
Loading

0 comments on commit 1588b16

Please sign in to comment.