From 4ab021d79b7e56886e8683dafbb4f140fe2cc172 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 30 Sep 2015 18:42:55 -0400 Subject: [PATCH 001/259] move to p2p dir --- p2p/net/swarm/addr/addr.go | 275 +++++++++++++++++ p2p/net/swarm/addr/addr_test.go | 232 ++++++++++++++ p2p/net/swarm/dial_test.go | 440 ++++++++++++++++++++++++++ p2p/net/swarm/peers_test.go | 72 +++++ p2p/net/swarm/simul_test.go | 75 +++++ p2p/net/swarm/swarm.go | 321 +++++++++++++++++++ p2p/net/swarm/swarm_addr.go | 39 +++ p2p/net/swarm/swarm_addr_test.go | 123 ++++++++ p2p/net/swarm/swarm_conn.go | 141 +++++++++ p2p/net/swarm/swarm_dial.go | 493 ++++++++++++++++++++++++++++++ p2p/net/swarm/swarm_listen.go | 142 +++++++++ p2p/net/swarm/swarm_net.go | 172 +++++++++++ p2p/net/swarm/swarm_net_test.go | 77 +++++ p2p/net/swarm/swarm_notif_test.go | 210 +++++++++++++ p2p/net/swarm/swarm_stream.go | 54 ++++ p2p/net/swarm/swarm_test.go | 330 ++++++++++++++++++++ 16 files changed, 3196 insertions(+) create mode 100644 p2p/net/swarm/addr/addr.go create mode 100644 p2p/net/swarm/addr/addr_test.go create mode 100644 p2p/net/swarm/dial_test.go create mode 100644 p2p/net/swarm/peers_test.go create mode 100644 p2p/net/swarm/simul_test.go create mode 100644 p2p/net/swarm/swarm.go create mode 100644 p2p/net/swarm/swarm_addr.go create mode 100644 p2p/net/swarm/swarm_addr_test.go create mode 100644 p2p/net/swarm/swarm_conn.go create mode 100644 p2p/net/swarm/swarm_dial.go create mode 100644 p2p/net/swarm/swarm_listen.go create mode 100644 p2p/net/swarm/swarm_net.go create mode 100644 p2p/net/swarm/swarm_net_test.go create mode 100644 p2p/net/swarm/swarm_notif_test.go create mode 100644 p2p/net/swarm/swarm_stream.go create mode 100644 p2p/net/swarm/swarm_test.go diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go new file mode 100644 index 0000000000..649d54b3c0 --- /dev/null +++ b/p2p/net/swarm/addr/addr.go @@ -0,0 +1,275 @@ +package addrutil + +import ( + "fmt" + + logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +var log = logging.Logger("p2p/net/swarm/addr") + +// SupportedTransportStrings is the list of supported transports for the swarm. +// These are strings of encapsulated multiaddr protocols. E.g.: +// /ip4/tcp +var SupportedTransportStrings = []string{ + "/ip4/tcp", + "/ip6/tcp", + // "/ip4/udp/utp", disabled because the lib is broken + // "/ip6/udp/utp", disabled because the lib is broken + // "/ip4/udp/udt", disabled because the lib doesnt work on arm + // "/ip6/udp/udt", disabled because the lib doesnt work on arm +} + +// SupportedTransportProtocols is the list of supported transports for the swarm. +// These are []ma.Protocol lists. Populated at runtime from SupportedTransportStrings +var SupportedTransportProtocols = [][]ma.Protocol{} + +func init() { + // initialize SupportedTransportProtocols + transports := make([][]ma.Protocol, len(SupportedTransportStrings)) + for _, s := range SupportedTransportStrings { + t, err := ma.ProtocolsWithString(s) + if err != nil { + panic(err) // important to fix this in the codebase + } + transports = append(transports, t) + } + SupportedTransportProtocols = transports +} + +// FilterAddrs is a filter that removes certain addresses, according to filter. +// if filter returns true, the address is kept. +func FilterAddrs(a []ma.Multiaddr, filter func(ma.Multiaddr) bool) []ma.Multiaddr { + b := make([]ma.Multiaddr, 0, len(a)) + for _, addr := range a { + if filter(addr) { + b = append(b, addr) + } + } + return b +} + +// FilterUsableAddrs removes certain addresses +// from a list. the addresses removed are those known NOT +// to work with our network. Namely, addresses with UTP. +func FilterUsableAddrs(a []ma.Multiaddr) []ma.Multiaddr { + return FilterAddrs(a, func(m ma.Multiaddr) bool { + return AddrUsable(m, false) + }) +} + +// AddrOverNonLocalIP returns whether the addr uses a non-local ip link +func AddrOverNonLocalIP(a ma.Multiaddr) bool { + split := ma.Split(a) + if len(split) < 1 { + return false + } + if manet.IsIP6LinkLocal(split[0]) { + return false + } + return true +} + +// AddrUsable returns whether our network can use this addr. +// We only use the transports in SupportedTransportStrings, +// and we do not link local addresses. Loopback is ok +// as we need to be able to connect to multiple ipfs nodes +// in the same machine. +func AddrUsable(a ma.Multiaddr, partial bool) bool { + if a == nil { + return false + } + + if !AddrOverNonLocalIP(a) { + return false + } + + // test the address protocol list is in SupportedTransportProtocols + matches := func(supported, test []ma.Protocol) bool { + if len(test) > len(supported) { + return false + } + + // when partial, it's ok if test < supported. + if !partial && len(supported) != len(test) { + return false + } + + for i := range test { + if supported[i].Code != test[i].Code { + return false + } + } + return true + } + + transport := a.Protocols() + for _, supported := range SupportedTransportProtocols { + if matches(supported, transport) { + return true + } + } + + return false +} + +// ResolveUnspecifiedAddress expands an unspecified ip addresses (/ip4/0.0.0.0, /ip6/::) to +// use the known local interfaces. If ifaceAddr is nil, we request interface addresses +// from the network stack. (this is so you can provide a cached value if resolving many addrs) +func ResolveUnspecifiedAddress(resolve ma.Multiaddr, ifaceAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { + // split address into its components + split := ma.Split(resolve) + + // if first component (ip) is not unspecified, use it as is. + if !manet.IsIPUnspecified(split[0]) { + return []ma.Multiaddr{resolve}, nil + } + + out := make([]ma.Multiaddr, 0, len(ifaceAddrs)) + for _, ia := range ifaceAddrs { + // must match the first protocol to be resolve. + if ia.Protocols()[0].Code != resolve.Protocols()[0].Code { + continue + } + + split[0] = ia + joined := ma.Join(split...) + out = append(out, joined) + log.Debug("adding resolved addr:", resolve, joined, out) + } + if len(out) < 1 { + return nil, fmt.Errorf("failed to resolve: %s", resolve) + } + return out, nil +} + +// ResolveUnspecifiedAddresses expands unspecified ip addresses (/ip4/0.0.0.0, /ip6/::) to +// use the known local interfaces. +func ResolveUnspecifiedAddresses(unspecAddrs, ifaceAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { + + // todo optimize: only fetch these if we have a "any" addr. + if len(ifaceAddrs) < 1 { + var err error + ifaceAddrs, err = InterfaceAddresses() + if err != nil { + return nil, err + } + // log.Debug("InterfaceAddresses:", ifaceAddrs) + } + + var outputAddrs []ma.Multiaddr + for _, a := range unspecAddrs { + // unspecified? + resolved, err := ResolveUnspecifiedAddress(a, ifaceAddrs) + if err != nil { + continue // optimistic. if we cant resolve anything, we'll know at the bottom. + } + // log.Debug("resolved:", a, resolved) + outputAddrs = append(outputAddrs, resolved...) + } + + if len(outputAddrs) < 1 { + return nil, fmt.Errorf("failed to specify addrs: %s", unspecAddrs) + } + + log.Event(context.TODO(), "interfaceListenAddresses", func() logging.Loggable { + var addrs []string + for _, addr := range outputAddrs { + addrs = append(addrs, addr.String()) + } + return logging.Metadata{"addresses": addrs} + }()) + + log.Debug("ResolveUnspecifiedAddresses:", unspecAddrs, ifaceAddrs, outputAddrs) + return outputAddrs, nil +} + +// InterfaceAddresses returns a list of addresses associated with local machine +// Note: we do not return link local addresses. IP loopback is ok, because we +// may be connecting to other nodes in the same machine. +func InterfaceAddresses() ([]ma.Multiaddr, error) { + maddrs, err := manet.InterfaceMultiaddrs() + if err != nil { + return nil, err + } + log.Debug("InterfaceAddresses: from manet:", maddrs) + + var out []ma.Multiaddr + for _, a := range maddrs { + if !AddrUsable(a, true) { // partial + // log.Debug("InterfaceAddresses: skipping unusable:", a) + continue + } + + out = append(out, a) + } + + log.Debug("InterfaceAddresses: usable:", out) + return out, nil +} + +// AddrInList returns whether or not an address is part of a list. +// this is useful to check if NAT is happening (or other bugs?) +func AddrInList(addr ma.Multiaddr, list []ma.Multiaddr) bool { + for _, addr2 := range list { + if addr.Equal(addr2) { + return true + } + } + return false +} + +// AddrIsShareableOnWAN returns whether the given address should be shareable on the +// wide area network (wide internet). +func AddrIsShareableOnWAN(addr ma.Multiaddr) bool { + s := ma.Split(addr) + if len(s) < 1 { + return false + } + a := s[0] + if manet.IsIPLoopback(a) || manet.IsIP6LinkLocal(a) || manet.IsIPUnspecified(a) { + return false + } + return manet.IsThinWaist(a) +} + +// WANShareableAddrs filters addresses based on whether they're shareable on WAN +func WANShareableAddrs(inp []ma.Multiaddr) []ma.Multiaddr { + return FilterAddrs(inp, AddrIsShareableOnWAN) +} + +// Subtract filters out all addrs in b from a +func Subtract(a, b []ma.Multiaddr) []ma.Multiaddr { + return FilterAddrs(a, func(m ma.Multiaddr) bool { + for _, bb := range b { + if m.Equal(bb) { + return false + } + } + return true + }) +} + +// CheckNATWarning checks if our observed addresses differ. if so, +// informs the user that certain things might not work yet +func CheckNATWarning(observed, expected ma.Multiaddr, listen []ma.Multiaddr) { + if observed.Equal(expected) { + return + } + + if !AddrInList(observed, listen) { // probably a nat + log.Warningf(natWarning, observed, listen) + } +} + +const natWarning = `Remote peer observed our address to be: %s +The local addresses are: %s +Thus, connection is going through NAT, and other connections may fail. + +IPFS NAT traversal is still under development. Please bug us on github or irc to fix this. +Baby steps: http://jbenet.static.s3.amazonaws.com/271dfcf/baby-steps.gif +` diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go new file mode 100644 index 0000000000..eb843ffc09 --- /dev/null +++ b/p2p/net/swarm/addr/addr_test.go @@ -0,0 +1,232 @@ +package addrutil + +import ( + "testing" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" +) + +func newMultiaddr(t *testing.T, s string) ma.Multiaddr { + maddr, err := ma.NewMultiaddr(s) + if err != nil { + t.Fatal(err) + } + return maddr +} + +func TestFilterAddrs(t *testing.T) { + + bad := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/1.2.3.4/udp/1234"), // unreliable + newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/sctp/1234"), // not in manet + newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/utp"), // utp is broken + newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/udt"), // udt is broken on arm + newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), // link local + newMultiaddr(t, "/ip6/fe80::100/tcp/1234"), // link local + } + + good := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), + newMultiaddr(t, "/ip6/::1/tcp/1234"), + } + + goodAndBad := append(good, bad...) + + // test filters + + for _, a := range bad { + if AddrUsable(a, false) { + t.Errorf("addr %s should be unusable", a) + } + if AddrUsable(a, true) { + t.Errorf("addr %s should be unusable", a) + } + } + + for _, a := range good { + if !AddrUsable(a, false) { + t.Errorf("addr %s should be usable", a) + } + if !AddrUsable(a, true) { + t.Errorf("addr %s should be usable", a) + } + } + + subtestAddrsEqual(t, FilterUsableAddrs(bad), []ma.Multiaddr{}) + subtestAddrsEqual(t, FilterUsableAddrs(good), good) + subtestAddrsEqual(t, FilterUsableAddrs(goodAndBad), good) +} + +func subtestAddrsEqual(t *testing.T, a, b []ma.Multiaddr) { + if len(a) != len(b) { + t.Error(t) + } + + in := func(addr ma.Multiaddr, l []ma.Multiaddr) bool { + for _, addr2 := range l { + if addr.Equal(addr2) { + return true + } + } + return false + } + + for _, aa := range a { + if !in(aa, b) { + t.Errorf("%s not in %s", aa, b) + } + } +} + +func TestInterfaceAddrs(t *testing.T) { + addrs, err := InterfaceAddresses() + if err != nil { + t.Fatal(err) + } + + if len(addrs) < 1 { + t.Error("no addresses") + } + + for _, a := range addrs { + if manet.IsIP6LinkLocal(a) { + t.Error("should not return ip link local addresses", a) + } + } + + if len(addrs) < 1 { + t.Error("no good interface addrs") + } +} + +func TestResolvingAddrs(t *testing.T) { + + unspec := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/0.0.0.0/tcp/1234"), + newMultiaddr(t, "/ip4/1.2.3.4/tcp/1234"), + newMultiaddr(t, "/ip6/::/tcp/1234"), + newMultiaddr(t, "/ip6/::100/tcp/1234"), + } + + iface := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/127.0.0.1"), + newMultiaddr(t, "/ip4/10.20.30.40"), + newMultiaddr(t, "/ip6/::1"), + newMultiaddr(t, "/ip6/::f"), + } + + spec := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), + newMultiaddr(t, "/ip4/10.20.30.40/tcp/1234"), + newMultiaddr(t, "/ip4/1.2.3.4/tcp/1234"), + newMultiaddr(t, "/ip6/::1/tcp/1234"), + newMultiaddr(t, "/ip6/::f/tcp/1234"), + newMultiaddr(t, "/ip6/::100/tcp/1234"), + } + + actual, err := ResolveUnspecifiedAddresses(unspec, iface) + if err != nil { + t.Fatal(err) + } + + for i, a := range actual { + if !a.Equal(spec[i]) { + t.Error(a, " != ", spec[i]) + } + } + + ip4u := []ma.Multiaddr{newMultiaddr(t, "/ip4/0.0.0.0")} + ip4i := []ma.Multiaddr{newMultiaddr(t, "/ip4/1.2.3.4")} + + ip6u := []ma.Multiaddr{newMultiaddr(t, "/ip6/::")} + ip6i := []ma.Multiaddr{newMultiaddr(t, "/ip6/::1")} + + if _, err := ResolveUnspecifiedAddress(ip4u[0], ip6i); err == nil { + t.Fatal("should have failed") + } + if _, err := ResolveUnspecifiedAddress(ip6u[0], ip4i); err == nil { + t.Fatal("should have failed") + } + + if _, err := ResolveUnspecifiedAddresses(ip6u, ip4i); err == nil { + t.Fatal("should have failed") + } + if _, err := ResolveUnspecifiedAddresses(ip4u, ip6i); err == nil { + t.Fatal("should have failed") + } + +} + +func TestWANShareable(t *testing.T) { + + wanok := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/1.2.3.4/tcp/1234"), + newMultiaddr(t, "/ip6/abcd::1/tcp/1234"), + } + + wanbad := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), + newMultiaddr(t, "/ip4/0.0.0.0/tcp/1234"), + newMultiaddr(t, "/ip6/::1/tcp/1234"), + newMultiaddr(t, "/ip6/::/tcp/1234"), + newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), + newMultiaddr(t, "/ip6/fe80::/tcp/1234"), + } + + for _, a := range wanok { + if !AddrIsShareableOnWAN(a) { + t.Error("should be true", a) + } + } + + for _, a := range wanbad { + if AddrIsShareableOnWAN(a) { + t.Error("should be false", a) + } + } + + wanok2 := WANShareableAddrs(wanok) + if len(wanok) != len(wanok2) { + t.Error("should be the same") + } + + wanbad2 := WANShareableAddrs(wanbad) + if len(wanbad2) != 0 { + t.Error("should be zero") + } +} + +func TestSubtract(t *testing.T) { + + a := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), + newMultiaddr(t, "/ip4/0.0.0.0/tcp/1234"), + newMultiaddr(t, "/ip6/::1/tcp/1234"), + newMultiaddr(t, "/ip6/::/tcp/1234"), + newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), + newMultiaddr(t, "/ip6/fe80::/tcp/1234"), + } + + b := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), + newMultiaddr(t, "/ip6/::1/tcp/1234"), + newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), + } + + c1 := []ma.Multiaddr{ + newMultiaddr(t, "/ip4/0.0.0.0/tcp/1234"), + newMultiaddr(t, "/ip6/::/tcp/1234"), + newMultiaddr(t, "/ip6/fe80::/tcp/1234"), + } + + c2 := Subtract(a, b) + if len(c1) != len(c2) { + t.Error("should be the same") + } + for i, ca := range c1 { + if !c2[i].Equal(ca) { + t.Error("should be the same", ca, c2[i]) + } + } +} diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go new file mode 100644 index 0000000000..0ed10f4d30 --- /dev/null +++ b/p2p/net/swarm/dial_test.go @@ -0,0 +1,440 @@ +package swarm + +import ( + "net" + "sync" + "testing" + "time" + + addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" + peer "github.com/ipfs/go-ipfs/p2p/peer" + + testutil "github.com/ipfs/go-ipfs/util/testutil" + ci "github.com/ipfs/go-ipfs/util/testutil/ci" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +func acceptAndHang(l net.Listener) { + conns := make([]net.Conn, 0, 10) + for { + c, err := l.Accept() + if err != nil { + break + } + if c != nil { + conns = append(conns, c) + } + } + for _, c := range conns { + c.Close() + } +} + +func TestSimultDials(t *testing.T) { + // t.Skip("skipping for another test") + t.Parallel() + + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + + // connect everyone + { + var wg sync.WaitGroup + connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + // copy for other peer + log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr) + s.peers.AddAddr(dst, addr, peer.TempAddrTTL) + if _, err := s.Dial(ctx, dst); err != nil { + t.Fatal("error swarm dialing to peer", err) + } + wg.Done() + } + + ifaceAddrs0, err := swarms[0].InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + ifaceAddrs1, err := swarms[1].InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + + log.Info("Connecting swarms simultaneously.") + for i := 0; i < 10; i++ { // connect 10x for each. + wg.Add(2) + go connect(swarms[0], swarms[1].local, ifaceAddrs1[0]) + go connect(swarms[1], swarms[0].local, ifaceAddrs0[0]) + } + wg.Wait() + } + + // should still just have 1, at most 2 connections :) + c01l := len(swarms[0].ConnectionsToPeer(swarms[1].local)) + if c01l > 2 { + t.Error("0->1 has", c01l) + } + c10l := len(swarms[1].ConnectionsToPeer(swarms[0].local)) + if c10l > 2 { + t.Error("1->0 has", c10l) + } + + for _, s := range swarms { + s.Close() + } +} + +func newSilentPeer(t *testing.T) (peer.ID, ma.Multiaddr, net.Listener) { + dst := testutil.RandPeerIDFatal(t) + lst, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + addr, err := manet.FromNetAddr(lst.Addr()) + if err != nil { + t.Fatal(err) + } + addrs := []ma.Multiaddr{addr} + addrs, err = addrutil.ResolveUnspecifiedAddresses(addrs, nil) + if err != nil { + t.Fatal(err) + } + t.Log("new silent peer:", dst, addrs[0]) + return dst, addrs[0], lst +} + +func TestDialWait(t *testing.T) { + // t.Skip("skipping for another test") + t.Parallel() + + ctx := context.Background() + swarms := makeSwarms(ctx, t, 1) + s1 := swarms[0] + defer s1.Close() + + s1.dialT = time.Millisecond * 300 // lower timeout for tests. + if ci.IsRunning() { + s1.dialT = time.Second + } + + // dial to a non-existent peer. + s2p, s2addr, s2l := newSilentPeer(t) + go acceptAndHang(s2l) + defer s2l.Close() + s1.peers.AddAddr(s2p, s2addr, peer.PermanentAddrTTL) + + before := time.Now() + if c, err := s1.Dial(ctx, s2p); err == nil { + defer c.Close() + t.Fatal("error swarm dialing to unknown peer worked...", err) + } else { + t.Log("correctly got error:", err) + } + duration := time.Now().Sub(before) + + dt := s1.dialT + if duration < dt*dialAttempts { + t.Error("< DialTimeout * dialAttempts not being respected", duration, dt*dialAttempts) + } + if duration > 2*dt*dialAttempts { + t.Error("> 2*DialTimeout * dialAttempts not being respected", duration, 2*dt*dialAttempts) + } + + if !s1.backf.Backoff(s2p) { + t.Error("s2 should now be on backoff") + } +} + +func TestDialBackoff(t *testing.T) { + // t.Skip("skipping for another test") + if ci.IsRunning() { + t.Skip("travis and jenkins will never have fun with this test") + } + + t.Parallel() + + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + s1 := swarms[0] + s2 := swarms[1] + defer s1.Close() + defer s2.Close() + + s1.dialT = time.Second // lower timeout for tests. + s2.dialT = time.Second // lower timeout for tests. + + s2addrs, err := s2.InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + s1.peers.AddAddrs(s2.local, s2addrs, peer.PermanentAddrTTL) + + // dial to a non-existent peer. + s3p, s3addr, s3l := newSilentPeer(t) + go acceptAndHang(s3l) + defer s3l.Close() + s1.peers.AddAddr(s3p, s3addr, peer.PermanentAddrTTL) + + // in this test we will: + // 1) dial 10x to each node. + // 2) all dials should hang + // 3) s1->s2 should succeed. + // 4) s1->s3 should not (and should place s3 on backoff) + // 5) disconnect entirely + // 6) dial 10x to each node again + // 7) s3 dials should all return immediately (except 1) + // 8) s2 dials should all hang, and succeed + // 9) last s3 dial ends, unsuccessful + + dialOnlineNode := func(dst peer.ID, times int) <-chan bool { + ch := make(chan bool) + for i := 0; i < times; i++ { + go func() { + if _, err := s1.Dial(ctx, dst); err != nil { + t.Error("error dialing", dst, err) + ch <- false + } else { + ch <- true + } + }() + } + return ch + } + + dialOfflineNode := func(dst peer.ID, times int) <-chan bool { + ch := make(chan bool) + for i := 0; i < times; i++ { + go func() { + if c, err := s1.Dial(ctx, dst); err != nil { + ch <- false + } else { + t.Error("succeeded in dialing", dst) + ch <- true + c.Close() + } + }() + } + return ch + } + + { + // 1) dial 10x to each node. + N := 10 + s2done := dialOnlineNode(s2.local, N) + s3done := dialOfflineNode(s3p, N) + + // when all dials should be done by: + dialTimeout1x := time.After(s1.dialT) + // dialTimeout1Ax := time.After(s1.dialT * 2) // dialAttempts) + dialTimeout10Ax := time.After(s1.dialT * 2 * 10) // dialAttempts * 10) + + // 2) all dials should hang + select { + case <-s2done: + t.Error("s2 should not happen immediately") + case <-s3done: + t.Error("s3 should not happen yet") + case <-time.After(time.Millisecond): + // s2 may finish very quickly, so let's get out. + } + + // 3) s1->s2 should succeed. + for i := 0; i < N; i++ { + select { + case r := <-s2done: + if !r { + t.Error("s2 should not fail") + } + case <-s3done: + t.Error("s3 should not happen yet") + case <-dialTimeout1x: + t.Error("s2 took too long") + } + } + + select { + case <-s2done: + t.Error("s2 should have no more") + case <-s3done: + t.Error("s3 should not happen yet") + case <-dialTimeout1x: // let it pass + } + + // 4) s1->s3 should not (and should place s3 on backoff) + // N-1 should finish before dialTimeout1x * 2 + for i := 0; i < N; i++ { + select { + case <-s2done: + t.Error("s2 should have no more") + case r := <-s3done: + if r { + t.Error("s3 should not succeed") + } + case <-(dialTimeout1x): + if i < (N - 1) { + t.Fatal("s3 took too long") + } + t.Log("dialTimeout1x * 1.3 hit for last peer") + case <-dialTimeout10Ax: + t.Fatal("s3 took too long") + } + } + + // check backoff state + if s1.backf.Backoff(s2.local) { + t.Error("s2 should not be on backoff") + } + if !s1.backf.Backoff(s3p) { + t.Error("s3 should be on backoff") + } + + // 5) disconnect entirely + + for _, c := range s1.Connections() { + c.Close() + } + for i := 0; i < 100 && len(s1.Connections()) > 0; i++ { + <-time.After(time.Millisecond) + } + if len(s1.Connections()) > 0 { + t.Fatal("s1 conns must exit") + } + } + + { + // 6) dial 10x to each node again + N := 10 + s2done := dialOnlineNode(s2.local, N) + s3done := dialOfflineNode(s3p, N) + + // when all dials should be done by: + dialTimeout1x := time.After(s1.dialT) + // dialTimeout1Ax := time.After(s1.dialT * 2) // dialAttempts) + dialTimeout10Ax := time.After(s1.dialT * 2 * 10) // dialAttempts * 10) + + // 7) s3 dials should all return immediately (except 1) + for i := 0; i < N-1; i++ { + select { + case <-s2done: + t.Error("s2 should not succeed yet") + case r := <-s3done: + if r { + t.Error("s3 should not succeed") + } + case <-dialTimeout1x: + t.Fatal("s3 took too long") + } + } + + // 8) s2 dials should all hang, and succeed + for i := 0; i < N; i++ { + select { + case r := <-s2done: + if !r { + t.Error("s2 should succeed") + } + // case <-s3done: + case <-(dialTimeout1x): + t.Fatal("s3 took too long") + } + } + + // 9) the last s3 should return, failed. + select { + case <-s2done: + t.Error("s2 should have no more") + case r := <-s3done: + if r { + t.Error("s3 should not succeed") + } + case <-dialTimeout10Ax: + t.Fatal("s3 took too long") + } + + // check backoff state (the same) + if s1.backf.Backoff(s2.local) { + t.Error("s2 should not be on backoff") + } + if !s1.backf.Backoff(s3p) { + t.Error("s3 should be on backoff") + } + + } +} + +func TestDialBackoffClears(t *testing.T) { + // t.Skip("skipping for another test") + t.Parallel() + + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + s1 := swarms[0] + s2 := swarms[1] + defer s1.Close() + defer s2.Close() + s1.dialT = time.Millisecond * 300 // lower timeout for tests. + s2.dialT = time.Millisecond * 300 // lower timeout for tests. + if ci.IsRunning() { + s1.dialT = 2 * time.Second + s2.dialT = 2 * time.Second + } + + // use another address first, that accept and hang on conns + _, s2bad, s2l := newSilentPeer(t) + go acceptAndHang(s2l) + defer s2l.Close() + + // phase 1 -- dial to non-operational addresses + s1.peers.AddAddr(s2.local, s2bad, peer.PermanentAddrTTL) + + before := time.Now() + if c, err := s1.Dial(ctx, s2.local); err == nil { + t.Fatal("dialing to broken addr worked...", err) + defer c.Close() + } else { + t.Log("correctly got error:", err) + } + duration := time.Now().Sub(before) + + dt := s1.dialT + if duration < dt*dialAttempts { + t.Error("< DialTimeout * dialAttempts not being respected", duration, dt*dialAttempts) + } + if duration > 2*dt*dialAttempts { + t.Error("> 2*DialTimeout * dialAttempts not being respected", duration, 2*dt*dialAttempts) + } + + if !s1.backf.Backoff(s2.local) { + t.Error("s2 should now be on backoff") + } else { + t.Log("correctly added to backoff") + } + + // phase 2 -- add the working address. dial should succeed. + ifaceAddrs1, err := swarms[1].InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + s1.peers.AddAddrs(s2.local, ifaceAddrs1, peer.PermanentAddrTTL) + + before = time.Now() + if c, err := s1.Dial(ctx, s2.local); err != nil { + t.Fatal(err) + } else { + c.Close() + t.Log("correctly connected") + } + duration = time.Now().Sub(before) + + if duration >= dt { + // t.Error("took too long", duration, dt) + } + + if s1.backf.Backoff(s2.local) { + t.Error("s2 should no longer be on backoff") + } else { + t.Log("correctly cleared backoff") + } +} diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go new file mode 100644 index 0000000000..2f8b07ef0b --- /dev/null +++ b/p2p/net/swarm/peers_test.go @@ -0,0 +1,72 @@ +package swarm + +import ( + "testing" + + peer "github.com/ipfs/go-ipfs/p2p/peer" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +func TestPeers(t *testing.T) { + // t.Skip("skipping for another test") + + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + s1 := swarms[0] + s2 := swarms[1] + + connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + // TODO: make a DialAddr func. + s.peers.AddAddr(dst, addr, peer.PermanentAddrTTL) + // t.Logf("connections from %s", s.LocalPeer()) + // for _, c := range s.ConnectionsToPeer(dst) { + // t.Logf("connection from %s to %s: %v", s.LocalPeer(), dst, c) + // } + // t.Logf("") + if _, err := s.Dial(ctx, dst); err != nil { + t.Fatal("error swarm dialing to peer", err) + } + // t.Log(s.swarm.Dump()) + } + + s1GotConn := make(chan struct{}, 0) + s2GotConn := make(chan struct{}, 0) + s1.SetConnHandler(func(c *Conn) { + s1GotConn <- struct{}{} + }) + s2.SetConnHandler(func(c *Conn) { + s2GotConn <- struct{}{} + }) + + connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) + <-s2GotConn // have to wait here so the other side catches up. + connect(s2, s1.LocalPeer(), s1.ListenAddresses()[0]) + + for i := 0; i < 100; i++ { + connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) + connect(s2, s1.LocalPeer(), s1.ListenAddresses()[0]) + } + + for _, s := range swarms { + log.Infof("%s swarm routing table: %s", s.local, s.Peers()) + } + + test := func(s *Swarm) { + expect := 1 + actual := len(s.Peers()) + if actual != expect { + t.Errorf("%s has %d peers, not %d: %v", s.LocalPeer(), actual, expect, s.Peers()) + t.Log(s.swarm.Dump()) + } + actual = len(s.Connections()) + if actual != expect { + t.Errorf("%s has %d conns, not %d: %v", s.LocalPeer(), actual, expect, s.Connections()) + t.Log(s.swarm.Dump()) + } + } + + test(s1) + test(s2) +} diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go new file mode 100644 index 0000000000..dfb775c971 --- /dev/null +++ b/p2p/net/swarm/simul_test.go @@ -0,0 +1,75 @@ +package swarm + +import ( + "sync" + "testing" + "time" + + peer "github.com/ipfs/go-ipfs/p2p/peer" + ci "github.com/ipfs/go-ipfs/util/testutil/ci" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +func TestSimultOpen(t *testing.T) { + // t.Skip("skipping for another test") + t.Parallel() + + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + + // connect everyone + { + var wg sync.WaitGroup + connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + // copy for other peer + log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr) + s.peers.AddAddr(dst, addr, peer.PermanentAddrTTL) + if _, err := s.Dial(ctx, dst); err != nil { + t.Fatal("error swarm dialing to peer", err) + } + wg.Done() + } + + log.Info("Connecting swarms simultaneously.") + wg.Add(2) + go connect(swarms[0], swarms[1].local, swarms[1].ListenAddresses()[0]) + go connect(swarms[1], swarms[0].local, swarms[0].ListenAddresses()[0]) + wg.Wait() + } + + for _, s := range swarms { + s.Close() + } +} + +func TestSimultOpenMany(t *testing.T) { + // t.Skip("very very slow") + + addrs := 20 + rounds := 10 + if ci.IsRunning() { + addrs = 10 + rounds = 5 + } + SubtestSwarm(t, addrs, rounds) +} + +func TestSimultOpenFewStress(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + // t.Skip("skipping for another test") + t.Parallel() + + msgs := 40 + swarms := 2 + rounds := 10 + // rounds := 100 + + for i := 0; i < rounds; i++ { + SubtestSwarm(t, swarms, msgs) + <-time.After(10 * time.Millisecond) + } +} diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go new file mode 100644 index 0000000000..512369013c --- /dev/null +++ b/p2p/net/swarm/swarm.go @@ -0,0 +1,321 @@ +// package swarm implements a connection muxer with a pair of channels +// to synchronize all network communication. +package swarm + +import ( + "fmt" + "sync" + "time" + + metrics "github.com/ipfs/go-ipfs/metrics" + inet "github.com/ipfs/go-ipfs/p2p/net" + filter "github.com/ipfs/go-ipfs/p2p/net/filter" + addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" + peer "github.com/ipfs/go-ipfs/p2p/peer" + logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" + pst "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-stream-muxer" + psy "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-stream-muxer/yamux" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess" + goprocessctx "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/context" + prom "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus" + mafilter "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/whyrusleeping/multiaddr-filter" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +var log = logging.Logger("swarm2") + +var PSTransport pst.Transport + +var peersTotal = prom.NewGaugeVec(prom.GaugeOpts{ + Namespace: "ipfs", + Subsystem: "p2p", + Name: "peers_total", + Help: "Number of connected peers", +}, []string{"peer_id"}) + +func init() { + tpt := *psy.DefaultTransport + tpt.MaxStreamWindowSize = 512 * 1024 + PSTransport = &tpt +} + +// Swarm is a connection muxer, allowing connections to other peers to +// be opened and closed, while still using the same Chan for all +// communication. The Chan sends/receives Messages, which note the +// destination or source Peer. +// +// Uses peerstream.Swarm +type Swarm struct { + swarm *ps.Swarm + local peer.ID + peers peer.Peerstore + connh ConnHandler + + dsync dialsync + backf dialbackoff + dialT time.Duration // mainly for tests + + notifmu sync.RWMutex + notifs map[inet.Notifiee]ps.Notifiee + + // filters for addresses that shouldnt be dialed + Filters *filter.Filters + + proc goprocess.Process + ctx context.Context + bwc metrics.Reporter +} + +// NewSwarm constructs a Swarm, with a Chan. +func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, + local peer.ID, peers peer.Peerstore, bwc metrics.Reporter) (*Swarm, error) { + + listenAddrs, err := filterAddrs(listenAddrs) + if err != nil { + return nil, err + } + + s := &Swarm{ + swarm: ps.NewSwarm(PSTransport), + local: local, + peers: peers, + ctx: ctx, + dialT: DialTimeout, + notifs: make(map[inet.Notifiee]ps.Notifiee), + bwc: bwc, + Filters: filter.NewFilters(), + } + + // configure Swarm + s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown) + s.SetConnHandler(nil) // make sure to setup our own conn handler. + + // setup swarm metrics + prom.MustRegisterOrGet(peersTotal) + s.Notify((*metricsNotifiee)(s)) + + return s, s.listen(listenAddrs) +} + +func (s *Swarm) teardown() error { + return s.swarm.Close() +} + +func (s *Swarm) AddAddrFilter(f string) error { + m, err := mafilter.NewMask(f) + if err != nil { + return err + } + + s.Filters.AddDialFilter(m) + return nil +} +func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { + if len(listenAddrs) > 0 { + filtered := addrutil.FilterUsableAddrs(listenAddrs) + if len(filtered) < 1 { + return nil, fmt.Errorf("swarm cannot use any addr in: %s", listenAddrs) + } + listenAddrs = filtered + } + return listenAddrs, nil +} + +func (s *Swarm) Listen(addrs ...ma.Multiaddr) error { + addrs, err := filterAddrs(addrs) + if err != nil { + return err + } + + return s.listen(addrs) +} + +// Process returns the Process of the swarm +func (s *Swarm) Process() goprocess.Process { + return s.proc +} + +// Context returns the context of the swarm +func (s *Swarm) Context() context.Context { + return s.ctx +} + +// Close stops the Swarm. +func (s *Swarm) Close() error { + return s.proc.Close() +} + +// StreamSwarm returns the underlying peerstream.Swarm +func (s *Swarm) StreamSwarm() *ps.Swarm { + return s.swarm +} + +// SetConnHandler assigns the handler for new connections. +// See peerstream. You will rarely use this. See SetStreamHandler +func (s *Swarm) SetConnHandler(handler ConnHandler) { + + // handler is nil if user wants to clear the old handler. + if handler == nil { + s.swarm.SetConnHandler(func(psconn *ps.Conn) { + s.connHandler(psconn) + }) + return + } + + s.swarm.SetConnHandler(func(psconn *ps.Conn) { + // sc is nil if closed in our handler. + if sc := s.connHandler(psconn); sc != nil { + // call the user's handler. in a goroutine for sync safety. + go handler(sc) + } + }) +} + +// SetStreamHandler assigns the handler for new streams. +// See peerstream. +func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) { + s.swarm.SetStreamHandler(func(s *ps.Stream) { + handler(wrapStream(s)) + }) +} + +// NewStreamWithPeer creates a new stream on any available connection to p +func (s *Swarm) NewStreamWithPeer(p peer.ID) (*Stream, error) { + // if we have no connections, try connecting. + if len(s.ConnectionsToPeer(p)) == 0 { + log.Debug("Swarm: NewStreamWithPeer no connections. Attempting to connect...") + if _, err := s.Dial(s.Context(), p); err != nil { + return nil, err + } + } + log.Debug("Swarm: NewStreamWithPeer...") + + st, err := s.swarm.NewStreamWithGroup(p) + return wrapStream(st), err +} + +// StreamsWithPeer returns all the live Streams to p +func (s *Swarm) StreamsWithPeer(p peer.ID) []*Stream { + return wrapStreams(ps.StreamsWithGroup(p, s.swarm.Streams())) +} + +// ConnectionsToPeer returns all the live connections to p +func (s *Swarm) ConnectionsToPeer(p peer.ID) []*Conn { + return wrapConns(ps.ConnsWithGroup(p, s.swarm.Conns())) +} + +// Connections returns a slice of all connections. +func (s *Swarm) Connections() []*Conn { + return wrapConns(s.swarm.Conns()) +} + +// CloseConnection removes a given peer from swarm + closes the connection +func (s *Swarm) CloseConnection(p peer.ID) error { + conns := s.swarm.ConnsWithGroup(p) // boom. + for _, c := range conns { + c.Close() + } + return nil +} + +// Peers returns a copy of the set of peers swarm is connected to. +func (s *Swarm) Peers() []peer.ID { + conns := s.Connections() + + seen := make(map[peer.ID]struct{}) + peers := make([]peer.ID, 0, len(conns)) + for _, c := range conns { + p := c.RemotePeer() + if _, found := seen[p]; found { + continue + } + + seen[p] = struct{}{} + peers = append(peers, p) + } + return peers +} + +// LocalPeer returns the local peer swarm is associated to. +func (s *Swarm) LocalPeer() peer.ID { + return s.local +} + +// notifyAll sends a signal to all Notifiees +func (s *Swarm) notifyAll(notify func(inet.Notifiee)) { + s.notifmu.RLock() + for f := range s.notifs { + go notify(f) + } + s.notifmu.RUnlock() +} + +// Notify signs up Notifiee to receive signals when events happen +func (s *Swarm) Notify(f inet.Notifiee) { + // wrap with our notifiee, to translate function calls + n := &ps2netNotifee{net: (*Network)(s), not: f} + + s.notifmu.Lock() + s.notifs[f] = n + s.notifmu.Unlock() + + // register for notifications in the peer swarm. + s.swarm.Notify(n) +} + +// StopNotify unregisters Notifiee fromr receiving signals +func (s *Swarm) StopNotify(f inet.Notifiee) { + s.notifmu.Lock() + n, found := s.notifs[f] + if found { + delete(s.notifs, f) + } + s.notifmu.Unlock() + + if found { + s.swarm.StopNotify(n) + } +} + +type ps2netNotifee struct { + net *Network + not inet.Notifiee +} + +func (n *ps2netNotifee) Connected(c *ps.Conn) { + n.not.Connected(n.net, inet.Conn((*Conn)(c))) +} + +func (n *ps2netNotifee) Disconnected(c *ps.Conn) { + n.not.Disconnected(n.net, inet.Conn((*Conn)(c))) +} + +func (n *ps2netNotifee) OpenedStream(s *ps.Stream) { + n.not.OpenedStream(n.net, inet.Stream((*Stream)(s))) +} + +func (n *ps2netNotifee) ClosedStream(s *ps.Stream) { + n.not.ClosedStream(n.net, inet.Stream((*Stream)(s))) +} + +type metricsNotifiee Swarm + +func (nn *metricsNotifiee) Connected(n inet.Network, v inet.Conn) { + peersTotalGauge(n.LocalPeer()).Set(float64(len(n.Conns()))) +} + +func (nn *metricsNotifiee) Disconnected(n inet.Network, v inet.Conn) { + peersTotalGauge(n.LocalPeer()).Set(float64(len(n.Conns()))) +} + +func (nn *metricsNotifiee) OpenedStream(n inet.Network, v inet.Stream) {} +func (nn *metricsNotifiee) ClosedStream(n inet.Network, v inet.Stream) {} +func (nn *metricsNotifiee) Listen(n inet.Network, a ma.Multiaddr) {} +func (nn *metricsNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {} + +func peersTotalGauge(id peer.ID) prom.Gauge { + return peersTotal.With(prom.Labels{"peer_id": id.Pretty()}) +} diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go new file mode 100644 index 0000000000..8f9360d9d0 --- /dev/null +++ b/p2p/net/swarm/swarm_addr.go @@ -0,0 +1,39 @@ +package swarm + +import ( + conn "github.com/ipfs/go-ipfs/p2p/net/conn" + addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" +) + +// ListenAddresses returns a list of addresses at which this swarm listens. +func (s *Swarm) ListenAddresses() []ma.Multiaddr { + listeners := s.swarm.Listeners() + addrs := make([]ma.Multiaddr, 0, len(listeners)) + for _, l := range listeners { + if l2, ok := l.NetListener().(conn.Listener); ok { + addrs = append(addrs, l2.Multiaddr()) + } + } + return addrs +} + +// InterfaceListenAddresses returns a list of addresses at which this swarm +// listens. It expands "any interface" addresses (/ip4/0.0.0.0, /ip6/::) to +// use the known local interfaces. +func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { + return addrutil.ResolveUnspecifiedAddresses(s.ListenAddresses(), nil) +} + +// checkNATWarning checks if our observed addresses differ. if so, +// informs the user that certain things might not work yet +func checkNATWarning(s *Swarm, observed ma.Multiaddr, expected ma.Multiaddr) { + listen, err := s.InterfaceListenAddresses() + if err != nil { + log.Debugf("Error retrieving swarm.InterfaceListenAddresses: %s", err) + return + } + + addrutil.CheckNATWarning(observed, expected, listen) +} diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go new file mode 100644 index 0000000000..b75b491c42 --- /dev/null +++ b/p2p/net/swarm/swarm_addr_test.go @@ -0,0 +1,123 @@ +package swarm + +import ( + "testing" + + metrics "github.com/ipfs/go-ipfs/metrics" + addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" + peer "github.com/ipfs/go-ipfs/p2p/peer" + testutil "github.com/ipfs/go-ipfs/util/testutil" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +func TestFilterAddrs(t *testing.T) { + + m := func(s string) ma.Multiaddr { + maddr, err := ma.NewMultiaddr(s) + if err != nil { + t.Fatal(err) + } + return maddr + } + + bad := []ma.Multiaddr{ + m("/ip4/1.2.3.4/udp/1234"), // unreliable + m("/ip4/1.2.3.4/udp/1234/sctp/1234"), // not in manet + m("/ip4/1.2.3.4/udp/1234/utp"), // utp is broken + m("/ip4/1.2.3.4/udp/1234/udt"), // udt is broken on arm + m("/ip6/fe80::1/tcp/0"), // link local + m("/ip6/fe80::100/tcp/1234"), // link local + } + + good := []ma.Multiaddr{ + m("/ip4/127.0.0.1/tcp/0"), + m("/ip6/::1/tcp/0"), + } + + goodAndBad := append(good, bad...) + + // test filters + + for _, a := range bad { + if addrutil.AddrUsable(a, true) { + t.Errorf("addr %s should be unusable", a) + } + } + + for _, a := range good { + if !addrutil.AddrUsable(a, true) { + t.Errorf("addr %s should be usable", a) + } + } + + subtestAddrsEqual(t, addrutil.FilterUsableAddrs(bad), []ma.Multiaddr{}) + subtestAddrsEqual(t, addrutil.FilterUsableAddrs(good), good) + subtestAddrsEqual(t, addrutil.FilterUsableAddrs(goodAndBad), good) + + // now test it with swarm + + id, err := testutil.RandPeerID() + if err != nil { + t.Fatal(err) + } + + ps := peer.NewPeerstore() + ctx := context.Background() + + if _, err := NewNetwork(ctx, bad, id, ps, metrics.NewBandwidthCounter()); err == nil { + t.Fatal("should have failed to create swarm") + } + + if _, err := NewNetwork(ctx, goodAndBad, id, ps, metrics.NewBandwidthCounter()); err != nil { + t.Fatal("should have succeeded in creating swarm", err) + } +} + +func subtestAddrsEqual(t *testing.T, a, b []ma.Multiaddr) { + if len(a) != len(b) { + t.Error(t) + } + + in := func(addr ma.Multiaddr, l []ma.Multiaddr) bool { + for _, addr2 := range l { + if addr.Equal(addr2) { + return true + } + } + return false + } + + for _, aa := range a { + if !in(aa, b) { + t.Errorf("%s not in %s", aa, b) + } + } +} + +func TestDialBadAddrs(t *testing.T) { + + m := func(s string) ma.Multiaddr { + maddr, err := ma.NewMultiaddr(s) + if err != nil { + t.Fatal(err) + } + return maddr + } + + ctx := context.Background() + s := makeSwarms(ctx, t, 1)[0] + + test := func(a ma.Multiaddr) { + p := testutil.RandPeerIDFatal(t) + s.peers.AddAddr(p, a, peer.PermanentAddrTTL) + if _, err := s.Dial(ctx, p); err == nil { + t.Error("swarm should not dial: %s", m) + } + } + + test(m("/ip6/fe80::1")) // link local + test(m("/ip6/fe80::100")) // link local + test(m("/ip4/127.0.0.1/udp/1234/utp")) // utp +} diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go new file mode 100644 index 0000000000..126ace627a --- /dev/null +++ b/p2p/net/swarm/swarm_conn.go @@ -0,0 +1,141 @@ +package swarm + +import ( + "fmt" + + ic "github.com/ipfs/go-ipfs/p2p/crypto" + inet "github.com/ipfs/go-ipfs/p2p/net" + conn "github.com/ipfs/go-ipfs/p2p/net/conn" + peer "github.com/ipfs/go-ipfs/p2p/peer" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +// a Conn is a simple wrapper around a ps.Conn that also exposes +// some of the methods from the underlying conn.Conn. +// There's **five** "layers" to each connection: +// * 0. the net.Conn - underlying net.Conn (TCP/UDP/UTP/etc) +// * 1. the manet.Conn - provides multiaddr friendly Conn +// * 2. the conn.Conn - provides Peer friendly Conn (inc Secure channel) +// * 3. the peerstream.Conn - provides peerstream / spdysptream happiness +// * 4. the Conn - abstracts everyting out, exposing only key parts of underlying layers +// (I know, this is kinda crazy. it's more historical than a good design. though the +// layers do build up pieces of functionality. and they're all just io.RW :) ) +type Conn ps.Conn + +// ConnHandler is called when new conns are opened from remote peers. +// See peerstream.ConnHandler +type ConnHandler func(*Conn) + +func (c *Conn) StreamConn() *ps.Conn { + return (*ps.Conn)(c) +} + +func (c *Conn) RawConn() conn.Conn { + // righly panic if these things aren't true. it is an expected + // invariant that these Conns are all of the typewe expect: + // ps.Conn wrapping a conn.Conn + // if we get something else it is programmer error. + return (*ps.Conn)(c).NetConn().(conn.Conn) +} + +func (c *Conn) String() string { + return fmt.Sprintf("", c.RawConn()) +} + +// LocalMultiaddr is the Multiaddr on this side +func (c *Conn) LocalMultiaddr() ma.Multiaddr { + return c.RawConn().LocalMultiaddr() +} + +// LocalPeer is the Peer on our side of the connection +func (c *Conn) LocalPeer() peer.ID { + return c.RawConn().LocalPeer() +} + +// RemoteMultiaddr is the Multiaddr on the remote side +func (c *Conn) RemoteMultiaddr() ma.Multiaddr { + return c.RawConn().RemoteMultiaddr() +} + +// RemotePeer is the Peer on the remote side +func (c *Conn) RemotePeer() peer.ID { + return c.RawConn().RemotePeer() +} + +// LocalPrivateKey is the public key of the peer on this side +func (c *Conn) LocalPrivateKey() ic.PrivKey { + return c.RawConn().LocalPrivateKey() +} + +// RemotePublicKey is the public key of the peer on the remote side +func (c *Conn) RemotePublicKey() ic.PubKey { + return c.RawConn().RemotePublicKey() +} + +// NewSwarmStream returns a new Stream from this connection +func (c *Conn) NewSwarmStream() (*Stream, error) { + s, err := c.StreamConn().NewStream() + return wrapStream(s), err +} + +// NewStream returns a new Stream from this connection +func (c *Conn) NewStream() (inet.Stream, error) { + s, err := c.NewSwarmStream() + return inet.Stream(s), err +} + +func (c *Conn) Close() error { + return c.StreamConn().Close() +} + +func wrapConn(psc *ps.Conn) (*Conn, error) { + // grab the underlying connection. + if _, ok := psc.NetConn().(conn.Conn); !ok { + // this should never happen. if we see it ocurring it means that we added + // a Listener to the ps.Swarm that is NOT one of our net/conn.Listener. + return nil, fmt.Errorf("swarm connHandler: invalid conn (not a conn.Conn): %s", psc) + } + return (*Conn)(psc), nil +} + +// wrapConns returns a *Conn for all these ps.Conns +func wrapConns(conns1 []*ps.Conn) []*Conn { + conns2 := make([]*Conn, len(conns1)) + for i, c1 := range conns1 { + if c2, err := wrapConn(c1); err == nil { + conns2[i] = c2 + } + } + return conns2 +} + +// newConnSetup does the swarm's "setup" for a connection. returns the underlying +// conn.Conn this method is used by both swarm.Dial and ps.Swarm connHandler +func (s *Swarm) newConnSetup(ctx context.Context, psConn *ps.Conn) (*Conn, error) { + + // wrap with a Conn + sc, err := wrapConn(psConn) + if err != nil { + return nil, err + } + + // if we have a public key, make sure we add it to our peerstore! + // This is an important detail. Otherwise we must fetch the public + // key from the DHT or some other system. + if pk := sc.RemotePublicKey(); pk != nil { + s.peers.AddPubKey(sc.RemotePeer(), pk) + } + + // ok great! we can use it. add it to our group. + + // set the RemotePeer as a group on the conn. this lets us group + // connections in the StreamSwarm by peer, and get a streams from + // any available connection in the group (better multiconn): + // swarm.StreamSwarm().NewStreamWithGroup(remotePeer) + psConn.AddGroup(sc.RemotePeer()) + + return sc, nil +} diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go new file mode 100644 index 0000000000..a68d288059 --- /dev/null +++ b/p2p/net/swarm/swarm_dial.go @@ -0,0 +1,493 @@ +package swarm + +import ( + "errors" + "fmt" + "math/rand" + "net" + "sync" + "time" + + mconn "github.com/ipfs/go-ipfs/metrics/conn" + conn "github.com/ipfs/go-ipfs/p2p/net/conn" + addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" + peer "github.com/ipfs/go-ipfs/p2p/peer" + lgbl "github.com/ipfs/go-ipfs/util/eventlog/loggables" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" + process "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess" + processctx "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/context" + ratelimit "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/ratelimit" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +// Diagram of dial sync: +// +// many callers of Dial() synched w. dials many addrs results to callers +// ----------------------\ dialsync use earliest /-------------- +// -----------------------\ |----------\ /---------------- +// ------------------------>------------<------- >---------<----------------- +// -----------------------| \----x \---------------- +// ----------------------| \-----x \--------------- +// any may fail if no addr at end +// retry dialAttempt x + +var ( + ErrDialBackoff = errors.New("dial backoff") + ErrDialFailed = errors.New("dial attempt failed") + ErrDialToSelf = errors.New("dial to self attempted") +) + +// dialAttempts governs how many times a goroutine will try to dial a given peer. +// Note: this is down to one, as we have _too many dials_ atm. To add back in, +// add loop back in Dial(.) +const dialAttempts = 1 + +// DialTimeout is the amount of time each dial attempt has. We can think about making +// this larger down the road, or putting more granular timeouts (i.e. within each +// subcomponent of Dial) +var DialTimeout time.Duration = time.Second * 10 + +// dialsync is a small object that helps manage ongoing dials. +// this way, if we receive many simultaneous dial requests, one +// can do its thing, while the rest wait. +// +// this interface is so would-be dialers can just: +// +// for { +// c := findConnectionToPeer(peer) +// if c != nil { +// return c +// } +// +// // ok, no connections. should we dial? +// if ok, wait := dialsync.Lock(peer); !ok { +// <-wait // can optionally wait +// continue +// } +// defer dialsync.Unlock(peer) +// +// c := actuallyDial(peer) +// return c +// } +// +type dialsync struct { + // ongoing is a map of tickets for the current peers being dialed. + // this way, we dont kick off N dials simultaneously. + ongoing map[peer.ID]chan struct{} + lock sync.Mutex +} + +// Lock governs the beginning of a dial attempt. +// If there are no ongoing dials, it returns true, and the client is now +// scheduled to dial. Every other goroutine that calls startDial -- with +//the same dst -- will block until client is done. The client MUST call +// ds.Unlock(p) when it is done, to unblock the other callers. +// The client is not reponsible for achieving a successful dial, only for +// reporting the end of the attempt (calling ds.Unlock(p)). +// +// see the example below `dialsync` +func (ds *dialsync) Lock(dst peer.ID) (bool, chan struct{}) { + ds.lock.Lock() + if ds.ongoing == nil { // init if not ready + ds.ongoing = make(map[peer.ID]chan struct{}) + } + wait, found := ds.ongoing[dst] + if !found { + ds.ongoing[dst] = make(chan struct{}) + } + ds.lock.Unlock() + + if found { + return false, wait + } + + // ok! you're signed up to dial! + return true, nil +} + +// Unlock releases waiters to a dial attempt. see Lock. +// if Unlock(p) is called without calling Lock(p) first, Unlock panics. +func (ds *dialsync) Unlock(dst peer.ID) { + ds.lock.Lock() + wait, found := ds.ongoing[dst] + if !found { + panic("called dialDone with no ongoing dials to peer: " + dst.Pretty()) + } + delete(ds.ongoing, dst) // remove ongoing dial + close(wait) // release everyone else + ds.lock.Unlock() +} + +// dialbackoff is a struct used to avoid over-dialing the same, dead peers. +// Whenever we totally time out on a peer (all three attempts), we add them +// to dialbackoff. Then, whenevers goroutines would _wait_ (dialsync), they +// check dialbackoff. If it's there, they don't wait and exit promptly with +// an error. (the single goroutine that is actually dialing continues to +// dial). If a dial is successful, the peer is removed from backoff. +// Example: +// +// for { +// if ok, wait := dialsync.Lock(p); !ok { +// if backoff.Backoff(p) { +// return errDialFailed +// } +// <-wait +// continue +// } +// defer dialsync.Unlock(p) +// c, err := actuallyDial(p) +// if err != nil { +// dialbackoff.AddBackoff(p) +// continue +// } +// dialbackoff.Clear(p) +// } +// +type dialbackoff struct { + entries map[peer.ID]struct{} + lock sync.RWMutex +} + +func (db *dialbackoff) init() { + if db.entries == nil { + db.entries = make(map[peer.ID]struct{}) + } +} + +// Backoff returns whether the client should backoff from dialing +// peeer p +func (db *dialbackoff) Backoff(p peer.ID) bool { + db.lock.Lock() + db.init() + _, found := db.entries[p] + db.lock.Unlock() + return found +} + +// AddBackoff lets other nodes know that we've entered backoff with +// peer p, so dialers should not wait unnecessarily. We still will +// attempt to dial with one goroutine, in case we get through. +func (db *dialbackoff) AddBackoff(p peer.ID) { + db.lock.Lock() + db.init() + db.entries[p] = struct{}{} + db.lock.Unlock() +} + +// Clear removes a backoff record. Clients should call this after a +// successful Dial. +func (db *dialbackoff) Clear(p peer.ID) { + db.lock.Lock() + db.init() + delete(db.entries, p) + db.lock.Unlock() +} + +// Dial connects to a peer. +// +// The idea is that the client of Swarm does not need to know what network +// the connection will happen over. Swarm can use whichever it choses. +// This allows us to use various transport protocols, do NAT traversal/relay, +// etc. to achive connection. +func (s *Swarm) Dial(ctx context.Context, p peer.ID) (*Conn, error) { + var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + if p == s.local { + log.Event(ctx, "swarmDialSelf", logdial) + return nil, ErrDialToSelf + } + + return s.gatedDialAttempt(ctx, p) +} + +func (s *Swarm) bestConnectionToPeer(p peer.ID) *Conn { + cs := s.ConnectionsToPeer(p) + for _, conn := range cs { + if conn != nil { // dump out the first one we find. (TODO pick better) + return conn + } + } + return nil +} + +// gatedDialAttempt is an attempt to dial a node. It is gated by the swarm's +// dial synchronization systems: dialsync and dialbackoff. +func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) { + var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + defer log.EventBegin(ctx, "swarmDialAttemptSync", logdial).Done() + + // check if we already have an open connection first + conn := s.bestConnectionToPeer(p) + if conn != nil { + return conn, nil + } + + // check if there's an ongoing dial to this peer + if ok, wait := s.dsync.Lock(p); ok { + // ok, we have been charged to dial! let's do it. + // if it succeeds, dial will add the conn to the swarm itself. + + defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done() + ctxT, cancel := context.WithTimeout(ctx, s.dialT) + conn, err := s.dial(ctxT, p) + cancel() + s.dsync.Unlock(p) + log.Debugf("dial end %s", conn) + if err != nil { + log.Event(ctx, "swarmDialBackoffAdd", logdial) + s.backf.AddBackoff(p) // let others know to backoff + + // ok, we failed. try again. (if loop is done, our error is output) + return nil, fmt.Errorf("dial attempt failed: %s", err) + } + log.Event(ctx, "swarmDialBackoffClear", logdial) + s.backf.Clear(p) // okay, no longer need to backoff + return conn, nil + + } else { + // we did not dial. we must wait for someone else to dial. + + // check whether we should backoff first... + if s.backf.Backoff(p) { + log.Event(ctx, "swarmDialBackoff", logdial) + return nil, ErrDialBackoff + } + + defer log.EventBegin(ctx, "swarmDialWait", logdial).Done() + select { + case <-wait: // wait for that other dial to finish. + + // see if it worked, OR we got an incoming dial in the meantime... + conn := s.bestConnectionToPeer(p) + if conn != nil { + return conn, nil + } + return nil, ErrDialFailed + case <-ctx.Done(): // or we may have to bail... + return nil, ctx.Err() + } + } +} + +// dial is the actual swarm's dial logic, gated by Dial. +func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { + var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + if p == s.local { + log.Event(ctx, "swarmDialDoDialSelf", logdial) + return nil, ErrDialToSelf + } + defer log.EventBegin(ctx, "swarmDialDo", logdial).Done() + logdial["dial"] = "failure" // start off with failure. set to "success" at the end. + + sk := s.peers.PrivKey(s.local) + logdial["encrypted"] = (sk != nil) // log wether this will be an encrypted dial or not. + if sk == nil { + // fine for sk to be nil, just log. + log.Debug("Dial not given PrivateKey, so WILL NOT SECURE conn.") + } + + // get our own addrs. try dialing out from our listener addresses (reusing ports) + // Note that using our peerstore's addresses here is incorrect, as that would + // include observed addresses. TODO: make peerstore's address book smarter. + localAddrs := s.ListenAddresses() + if len(localAddrs) == 0 { + log.Debug("Dialing out with no local addresses.") + } + + // get remote peer addrs + remoteAddrs := s.peers.Addrs(p) + // make sure we can use the addresses. + remoteAddrs = addrutil.FilterUsableAddrs(remoteAddrs) + // drop out any addrs that would just dial ourselves. use ListenAddresses + // as that is a more authoritative view than localAddrs. + ila, _ := s.InterfaceListenAddresses() + remoteAddrs = addrutil.Subtract(remoteAddrs, ila) + remoteAddrs = addrutil.Subtract(remoteAddrs, s.peers.Addrs(s.local)) + + log.Debugf("%s swarm dialing %s -- local:%s remote:%s", s.local, p, s.ListenAddresses(), remoteAddrs) + if len(remoteAddrs) == 0 { + err := errors.New("peer has no addresses") + logdial["error"] = err + return nil, err + } + + remoteAddrs = s.filterAddrs(remoteAddrs) + if len(remoteAddrs) == 0 { + err := errors.New("all adresses for peer have been filtered out") + logdial["error"] = err + return nil, err + } + + // open connection to peer + d := &conn.Dialer{ + Dialer: manet.Dialer{ + Dialer: net.Dialer{ + Timeout: s.dialT, + }, + }, + LocalPeer: s.local, + LocalAddrs: localAddrs, + PrivateKey: sk, + Wrapper: func(c manet.Conn) manet.Conn { + return mconn.WrapConn(s.bwc, c) + }, + } + + // try to get a connection to any addr + connC, err := s.dialAddrs(ctx, d, p, remoteAddrs) + if err != nil { + logdial["error"] = err + return nil, err + } + logdial["netconn"] = lgbl.NetConn(connC) + + // ok try to setup the new connection. + defer log.EventBegin(ctx, "swarmDialDoSetup", logdial, lgbl.NetConn(connC)).Done() + swarmC, err := dialConnSetup(ctx, s, connC) + if err != nil { + logdial["error"] = err + connC.Close() // close the connection. didn't work out :( + return nil, err + } + + logdial["dial"] = "success" + return swarmC, nil +} + +func (s *Swarm) dialAddrs(ctx context.Context, d *conn.Dialer, p peer.ID, remoteAddrs []ma.Multiaddr) (conn.Conn, error) { + + // try to connect to one of the peer's known addresses. + // we dial concurrently to each of the addresses, which: + // * makes the process faster overall + // * attempts to get the fastest connection available. + // * mitigates the waste of trying bad addresses + log.Debugf("%s swarm dialing %s %s", s.local, p, remoteAddrs) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() // cancel work when we exit func + + foundConn := make(chan struct{}) + conns := make(chan conn.Conn, len(remoteAddrs)) + errs := make(chan error, len(remoteAddrs)) + + // dialSingleAddr is used in the rate-limited async thing below. + dialSingleAddr := func(addr ma.Multiaddr) { + connC, err := s.dialAddr(ctx, d, p, addr) + + // check parent still wants our results + select { + case <-foundConn: + if connC != nil { + connC.Close() + } + return + default: + } + + if err != nil { + errs <- err + } else if connC == nil { + errs <- fmt.Errorf("failed to dial %s %s", p, addr) + } else { + conns <- connC + } + } + + // this whole thing is in a goroutine so we can use foundConn + // to end early. + go func() { + // rate limiting just in case. at most 10 addrs at once. + limiter := ratelimit.NewRateLimiter(process.Background(), 10) + limiter.Go(func(worker process.Process) { + // permute addrs so we try different sets first each time. + for _, i := range rand.Perm(len(remoteAddrs)) { + select { + case <-foundConn: // if one of them succeeded already + break + case <-worker.Closing(): // our context was cancelled + break + default: + } + + workerAddr := remoteAddrs[i] // shadow variable to avoid race + limiter.LimitedGo(func(worker process.Process) { + dialSingleAddr(workerAddr) + }) + } + }) + + processctx.CloseAfterContext(limiter, ctx) + }() + + // wair fot the results. + exitErr := fmt.Errorf("failed to dial %s", p) + for i := 0; i < len(remoteAddrs); i++ { + select { + case exitErr = <-errs: // + log.Debug("dial error: ", exitErr) + case connC := <-conns: + // take the first + return asap + close(foundConn) + return connC, nil + } + } + return nil, exitErr +} + +func (s *Swarm) dialAddr(ctx context.Context, d *conn.Dialer, p peer.ID, addr ma.Multiaddr) (conn.Conn, error) { + log.Debugf("%s swarm dialing %s %s", s.local, p, addr) + + connC, err := d.Dial(ctx, addr, p) + if err != nil { + return nil, fmt.Errorf("%s --> %s dial attempt failed: %s", s.local, p, err) + } + + // if the connection is not to whom we thought it would be... + remotep := connC.RemotePeer() + if remotep != p { + connC.Close() + return nil, fmt.Errorf("misdial to %s through %s (got %s)", p, addr, remotep) + } + + // if the connection is to ourselves... + // this can happen TONS when Loopback addrs are advertized. + // (this should be caught by two checks above, but let's just make sure.) + if remotep == s.local { + connC.Close() + return nil, fmt.Errorf("misdial to %s through %s (got self)", p, addr) + } + + // success! we got one! + return connC, nil +} + +func (s *Swarm) filterAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { + var out []ma.Multiaddr + for _, a := range addrs { + if !s.Filters.AddrBlocked(a) { + out = append(out, a) + } + } + return out +} + +// dialConnSetup is the setup logic for a connection from the dial side. it +// needs to add the Conn to the StreamSwarm, then run newConnSetup +func dialConnSetup(ctx context.Context, s *Swarm, connC conn.Conn) (*Conn, error) { + + psC, err := s.swarm.AddConn(connC) + if err != nil { + // connC is closed by caller if we fail. + return nil, fmt.Errorf("failed to add conn to ps.Swarm: %s", err) + } + + // ok try to setup the new connection. (newConnSetup will add to group) + swarmC, err := s.newConnSetup(ctx, psC) + if err != nil { + psC.Close() // we need to make sure psC is Closed. + return nil, err + } + + return swarmC, err +} diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go new file mode 100644 index 0000000000..d1bcb07521 --- /dev/null +++ b/p2p/net/swarm/swarm_listen.go @@ -0,0 +1,142 @@ +package swarm + +import ( + "fmt" + + mconn "github.com/ipfs/go-ipfs/metrics/conn" + inet "github.com/ipfs/go-ipfs/p2p/net" + conn "github.com/ipfs/go-ipfs/p2p/net/conn" + addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" + lgbl "github.com/ipfs/go-ipfs/util/eventlog/loggables" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" + ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + multierr "github.com/ipfs/go-ipfs/thirdparty/multierr" +) + +// Open listeners for each network the swarm should listen on +func (s *Swarm) listen(addrs []ma.Multiaddr) error { + + for _, addr := range addrs { + if !addrutil.AddrUsable(addr, true) { + return fmt.Errorf("cannot use addr: %s", addr) + } + } + + retErr := multierr.New() + + // listen on every address + for i, addr := range addrs { + err := s.setupListener(addr) + if err != nil { + if retErr.Errors == nil { + retErr.Errors = make([]error, len(addrs)) + } + retErr.Errors[i] = err + log.Debugf("Failed to listen on: %s - %s", addr, err) + } + } + + if retErr.Errors != nil { + return retErr + } + return nil +} + +// Listen for new connections on the given multiaddr +func (s *Swarm) setupListener(maddr ma.Multiaddr) error { + + // TODO rethink how this has to work. (jbenet) + // + // resolved, err := resolveUnspecifiedAddresses([]ma.Multiaddr{maddr}) + // if err != nil { + // return err + // } + // for _, a := range resolved { + // s.peers.AddAddr(s.local, a) + // } + + sk := s.peers.PrivKey(s.local) + if sk == nil { + // may be fine for sk to be nil, just log a warning. + log.Warning("Listener not given PrivateKey, so WILL NOT SECURE conns.") + } + log.Debugf("Swarm Listening at %s", maddr) + list, err := conn.Listen(s.Context(), maddr, s.local, sk) + if err != nil { + return err + } + + list.SetAddrFilters(s.Filters) + + if cw, ok := list.(conn.ListenerConnWrapper); ok { + cw.SetConnWrapper(func(c manet.Conn) manet.Conn { + return mconn.WrapConn(s.bwc, c) + }) + } + + // AddListener to the peerstream Listener. this will begin accepting connections + // and streams! + sl, err := s.swarm.AddListener(list) + if err != nil { + return err + } + log.Debugf("Swarm Listeners at %s", s.ListenAddresses()) + + // signal to our notifiees on successful conn. + s.notifyAll(func(n inet.Notifiee) { + n.Listen((*Network)(s), maddr) + }) + + // go consume peerstream's listen accept errors. note, these ARE errors. + // they may be killing the listener, and if we get _any_ we should be + // fixing this in our conn.Listener (to ignore them or handle them + // differently.) + go func(ctx context.Context, sl *ps.Listener) { + + // signal to our notifiees closing + defer s.notifyAll(func(n inet.Notifiee) { + n.ListenClose((*Network)(s), maddr) + }) + + for { + select { + case err, more := <-sl.AcceptErrors(): + if !more { + return + } + log.Warningf("swarm listener accept error: %s", err) + case <-ctx.Done(): + return + } + } + }(s.Context(), sl) + + return nil +} + +// connHandler is called by the StreamSwarm whenever a new connection is added +// here we configure it slightly. Note that this is sequential, so if anything +// will take a while do it in a goroutine. +// See https://godoc.org/github.com/jbenet/go-peerstream for more information +func (s *Swarm) connHandler(c *ps.Conn) *Conn { + ctx := context.Background() + // this context is for running the handshake, which -- when receiveing connections + // -- we have no bound on beyond what the transport protocol bounds it at. + // note that setup + the handshake are bounded by underlying io. + // (i.e. if TCP or UDP disconnects (or the swarm closes), we're done. + // Q: why not have a shorter handshake? think about an HTTP server on really slow conns. + // as long as the conn is live (TCP says its online), it tries its best. we follow suit.) + + sc, err := s.newConnSetup(ctx, c) + if err != nil { + log.Debug(err) + log.Event(ctx, "newConnHandlerDisconnect", lgbl.NetConn(c.NetConn()), lgbl.Error(err)) + c.Close() // boom. close it. + return nil + } + + return sc +} diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go new file mode 100644 index 0000000000..da9b52251e --- /dev/null +++ b/p2p/net/swarm/swarm_net.go @@ -0,0 +1,172 @@ +package swarm + +import ( + "fmt" + + peer "github.com/ipfs/go-ipfs/p2p/peer" + + metrics "github.com/ipfs/go-ipfs/metrics" + inet "github.com/ipfs/go-ipfs/p2p/net" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +// Network implements the inet.Network interface. +// It is simply a swarm, with a few different functions +// to implement inet.Network. +type Network Swarm + +// NewNetwork constructs a new network and starts listening on given addresses. +func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, + peers peer.Peerstore, bwc metrics.Reporter) (*Network, error) { + + s, err := NewSwarm(ctx, listen, local, peers, bwc) + if err != nil { + return nil, err + } + + return (*Network)(s), nil +} + +// DialPeer attempts to establish a connection to a given peer. +// Respects the context. +func (n *Network) DialPeer(ctx context.Context, p peer.ID) (inet.Conn, error) { + log.Debugf("[%s] network dialing peer [%s]", n.local, p) + sc, err := n.Swarm().Dial(ctx, p) + if err != nil { + return nil, err + } + + log.Debugf("network for %s finished dialing %s", n.local, p) + return inet.Conn(sc), nil +} + +// Process returns the network's Process +func (n *Network) Process() goprocess.Process { + return n.proc +} + +// Swarm returns the network's peerstream.Swarm +func (n *Network) Swarm() *Swarm { + return (*Swarm)(n) +} + +// LocalPeer the network's LocalPeer +func (n *Network) LocalPeer() peer.ID { + return n.Swarm().LocalPeer() +} + +// Peers returns the known peer IDs from the Peerstore +func (n *Network) Peers() []peer.ID { + return n.Swarm().Peers() +} + +// Peers returns the Peerstore, which tracks known peers +func (n *Network) Peerstore() peer.Peerstore { + return n.Swarm().peers +} + +// Conns returns the connected peers +func (n *Network) Conns() []inet.Conn { + conns1 := n.Swarm().Connections() + out := make([]inet.Conn, len(conns1)) + for i, c := range conns1 { + out[i] = inet.Conn(c) + } + return out +} + +// ConnsToPeer returns the connections in this Netowrk for given peer. +func (n *Network) ConnsToPeer(p peer.ID) []inet.Conn { + conns1 := n.Swarm().ConnectionsToPeer(p) + out := make([]inet.Conn, len(conns1)) + for i, c := range conns1 { + out[i] = inet.Conn(c) + } + return out +} + +// ClosePeer connection to peer +func (n *Network) ClosePeer(p peer.ID) error { + return n.Swarm().CloseConnection(p) +} + +// close is the real teardown function +func (n *Network) close() error { + return n.Swarm().Close() +} + +// Close calls the ContextCloser func +func (n *Network) Close() error { + return n.Swarm().proc.Close() +} + +// Listen tells the network to start listening on given multiaddrs. +func (n *Network) Listen(addrs ...ma.Multiaddr) error { + return n.Swarm().Listen(addrs...) +} + +// ListenAddresses returns a list of addresses at which this network listens. +func (n *Network) ListenAddresses() []ma.Multiaddr { + return n.Swarm().ListenAddresses() +} + +// InterfaceListenAddresses returns a list of addresses at which this network +// listens. It expands "any interface" addresses (/ip4/0.0.0.0, /ip6/::) to +// use the known local interfaces. +func (n *Network) InterfaceListenAddresses() ([]ma.Multiaddr, error) { + return n.Swarm().InterfaceListenAddresses() +} + +// Connectedness returns a state signaling connection capabilities +// For now only returns Connected || NotConnected. Expand into more later. +func (n *Network) Connectedness(p peer.ID) inet.Connectedness { + c := n.Swarm().ConnectionsToPeer(p) + if c != nil && len(c) > 0 { + return inet.Connected + } + return inet.NotConnected +} + +// NewStream returns a new stream to given peer p. +// If there is no connection to p, attempts to create one. +func (n *Network) NewStream(p peer.ID) (inet.Stream, error) { + log.Debugf("[%s] network opening stream to peer [%s]", n.local, p) + s, err := n.Swarm().NewStreamWithPeer(p) + if err != nil { + return nil, err + } + + return inet.Stream(s), nil +} + +// SetHandler sets the protocol handler on the Network's Muxer. +// This operation is threadsafe. +func (n *Network) SetStreamHandler(h inet.StreamHandler) { + n.Swarm().SetStreamHandler(h) +} + +// SetConnHandler sets the conn handler on the Network. +// This operation is threadsafe. +func (n *Network) SetConnHandler(h inet.ConnHandler) { + n.Swarm().SetConnHandler(func(c *Conn) { + h(inet.Conn(c)) + }) +} + +// String returns a string representation of Network. +func (n *Network) String() string { + return fmt.Sprintf("", n.LocalPeer()) +} + +// Notify signs up Notifiee to receive signals when events happen +func (n *Network) Notify(f inet.Notifiee) { + n.Swarm().Notify(f) +} + +// StopNotify unregisters Notifiee fromr receiving signals +func (n *Network) StopNotify(f inet.Notifiee) { + n.Swarm().StopNotify(f) +} diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go new file mode 100644 index 0000000000..b5f5c56996 --- /dev/null +++ b/p2p/net/swarm/swarm_net_test.go @@ -0,0 +1,77 @@ +package swarm_test + +import ( + "fmt" + "testing" + "time" + + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + inet "github.com/ipfs/go-ipfs/p2p/net" + testutil "github.com/ipfs/go-ipfs/p2p/test/util" +) + +// TestConnectednessCorrect starts a few networks, connects a few +// and tests Connectedness value is correct. +func TestConnectednessCorrect(t *testing.T) { + + ctx := context.Background() + + nets := make([]inet.Network, 4) + for i := 0; i < 4; i++ { + nets[i] = testutil.GenSwarmNetwork(t, ctx) + } + + // connect 0-1, 0-2, 0-3, 1-2, 2-3 + + dial := func(a, b inet.Network) { + testutil.DivulgeAddresses(b, a) + if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { + t.Fatalf("Failed to dial: %s", err) + } + } + + dial(nets[0], nets[1]) + dial(nets[0], nets[3]) + dial(nets[1], nets[2]) + dial(nets[3], nets[2]) + + // there's something wrong with dial, i think. it's not finishing + // completely. there must be some async stuff. + <-time.After(100 * time.Millisecond) + + // test those connected show up correctly + + // test connected + expectConnectedness(t, nets[0], nets[1], inet.Connected) + expectConnectedness(t, nets[0], nets[3], inet.Connected) + expectConnectedness(t, nets[1], nets[2], inet.Connected) + expectConnectedness(t, nets[3], nets[2], inet.Connected) + + // test not connected + expectConnectedness(t, nets[0], nets[2], inet.NotConnected) + expectConnectedness(t, nets[1], nets[3], inet.NotConnected) + + for _, n := range nets { + n.Close() + } +} + +func expectConnectedness(t *testing.T, a, b inet.Network, expected inet.Connectedness) { + es := "%s is connected to %s, but Connectedness incorrect. %s %s" + if a.Connectedness(b.LocalPeer()) != expected { + t.Errorf(es, a, b, printConns(a), printConns(b)) + } + + // test symmetric case + if b.Connectedness(a.LocalPeer()) != expected { + t.Errorf(es, b, a, printConns(b), printConns(a)) + } +} + +func printConns(n inet.Network) string { + s := fmt.Sprintf("Connections in %s:\n", n) + for _, c := range n.Conns() { + s = s + fmt.Sprintf("- %s\n", c) + } + return s +} diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go new file mode 100644 index 0000000000..9b0e3a9bab --- /dev/null +++ b/p2p/net/swarm/swarm_notif_test.go @@ -0,0 +1,210 @@ +package swarm + +import ( + "testing" + "time" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + + inet "github.com/ipfs/go-ipfs/p2p/net" + peer "github.com/ipfs/go-ipfs/p2p/peer" +) + +func TestNotifications(t *testing.T) { + ctx := context.Background() + swarms := makeSwarms(ctx, t, 5) + defer func() { + for _, s := range swarms { + s.Close() + } + }() + + timeout := 5 * time.Second + + // signup notifs + notifiees := make([]*netNotifiee, len(swarms)) + for i, swarm := range swarms { + n := newNetNotifiee() + swarm.Notify(n) + notifiees[i] = n + } + + connectSwarms(t, ctx, swarms) + + <-time.After(time.Millisecond) + // should've gotten 5 by now. + + // test everyone got the correct connection opened calls + for i, s := range swarms { + n := notifiees[i] + notifs := make(map[peer.ID][]inet.Conn) + for j, s2 := range swarms { + if i == j { + continue + } + + // this feels a little sketchy, but its probably okay + for len(s.ConnectionsToPeer(s2.LocalPeer())) != len(notifs[s2.LocalPeer()]) { + select { + case c := <-n.connected: + nfp := notifs[c.RemotePeer()] + notifs[c.RemotePeer()] = append(nfp, c) + case <-time.After(timeout): + t.Fatal("timeout") + } + } + } + + for p, cons := range notifs { + expect := s.ConnectionsToPeer(p) + if len(expect) != len(cons) { + t.Fatal("got different number of connections") + } + + for _, c := range cons { + var found bool + for _, c2 := range expect { + if c == c2 { + found = true + break + } + } + + if !found { + t.Fatal("connection not found!") + } + } + } + } + + complement := func(c inet.Conn) (*Swarm, *netNotifiee, *Conn) { + for i, s := range swarms { + for _, c2 := range s.Connections() { + if c.LocalMultiaddr().Equal(c2.RemoteMultiaddr()) && + c2.LocalMultiaddr().Equal(c.RemoteMultiaddr()) { + return s, notifiees[i], c2 + } + } + } + t.Fatal("complementary conn not found", c) + return nil, nil, nil + } + + testOCStream := func(n *netNotifiee, s inet.Stream) { + var s2 inet.Stream + select { + case s2 = <-n.openedStream: + t.Log("got notif for opened stream") + case <-time.After(timeout): + t.Fatal("timeout") + } + if s != s2 { + t.Fatal("got incorrect stream", s.Conn(), s2.Conn()) + } + + select { + case s2 = <-n.closedStream: + t.Log("got notif for closed stream") + case <-time.After(timeout): + t.Fatal("timeout") + } + if s != s2 { + t.Fatal("got incorrect stream", s.Conn(), s2.Conn()) + } + } + + streams := make(chan inet.Stream) + for _, s := range swarms { + s.SetStreamHandler(func(s inet.Stream) { + streams <- s + s.Close() + }) + } + + // open a streams in each conn + for i, s := range swarms { + for _, c := range s.Connections() { + _, n2, _ := complement(c) + + st1, err := c.NewStream() + if err != nil { + t.Error(err) + } else { + st1.Write([]byte("hello")) + st1.Close() + testOCStream(notifiees[i], st1) + st2 := <-streams + testOCStream(n2, st2) + } + } + } + + // close conns + for i, s := range swarms { + n := notifiees[i] + for _, c := range s.Connections() { + _, n2, c2 := complement(c) + c.Close() + c2.Close() + + var c3, c4 inet.Conn + select { + case c3 = <-n.disconnected: + case <-time.After(timeout): + t.Fatal("timeout") + } + if c != c3 { + t.Fatal("got incorrect conn", c, c3) + } + + select { + case c4 = <-n2.disconnected: + case <-time.After(timeout): + t.Fatal("timeout") + } + if c2 != c4 { + t.Fatal("got incorrect conn", c, c2) + } + } + } +} + +type netNotifiee struct { + listen chan ma.Multiaddr + listenClose chan ma.Multiaddr + connected chan inet.Conn + disconnected chan inet.Conn + openedStream chan inet.Stream + closedStream chan inet.Stream +} + +func newNetNotifiee() *netNotifiee { + return &netNotifiee{ + listen: make(chan ma.Multiaddr), + listenClose: make(chan ma.Multiaddr), + connected: make(chan inet.Conn), + disconnected: make(chan inet.Conn), + openedStream: make(chan inet.Stream), + closedStream: make(chan inet.Stream), + } +} + +func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr) { + nn.listen <- a +} +func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) { + nn.listenClose <- a +} +func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) { + nn.connected <- v +} +func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) { + nn.disconnected <- v +} +func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) { + nn.openedStream <- v +} +func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) { + nn.closedStream <- v +} diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go new file mode 100644 index 0000000000..66dd6353b7 --- /dev/null +++ b/p2p/net/swarm/swarm_stream.go @@ -0,0 +1,54 @@ +package swarm + +import ( + inet "github.com/ipfs/go-ipfs/p2p/net" + + ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" +) + +// a Stream is a wrapper around a ps.Stream that exposes a way to get +// our Conn and Swarm (instead of just the ps.Conn and ps.Swarm) +type Stream ps.Stream + +// Stream returns the underlying peerstream.Stream +func (s *Stream) Stream() *ps.Stream { + return (*ps.Stream)(s) +} + +// Conn returns the Conn associated with this Stream, as an inet.Conn +func (s *Stream) Conn() inet.Conn { + return s.SwarmConn() +} + +// SwarmConn returns the Conn associated with this Stream, as a *Conn +func (s *Stream) SwarmConn() *Conn { + return (*Conn)(s.Stream().Conn()) +} + +// Read reads bytes from a stream. +func (s *Stream) Read(p []byte) (n int, err error) { + return s.Stream().Read(p) +} + +// Write writes bytes to a stream, flushing for each call. +func (s *Stream) Write(p []byte) (n int, err error) { + return s.Stream().Write(p) +} + +// Close closes the stream, indicating this side is finished +// with the stream. +func (s *Stream) Close() error { + return s.Stream().Close() +} + +func wrapStream(pss *ps.Stream) *Stream { + return (*Stream)(pss) +} + +func wrapStreams(st []*ps.Stream) []*Stream { + out := make([]*Stream, len(st)) + for i, s := range st { + out[i] = wrapStream(s) + } + return out +} diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go new file mode 100644 index 0000000000..9193db0109 --- /dev/null +++ b/p2p/net/swarm/swarm_test.go @@ -0,0 +1,330 @@ +package swarm + +import ( + "bytes" + "fmt" + "io" + "net" + "sync" + "testing" + "time" + + metrics "github.com/ipfs/go-ipfs/metrics" + inet "github.com/ipfs/go-ipfs/p2p/net" + peer "github.com/ipfs/go-ipfs/p2p/peer" + testutil "github.com/ipfs/go-ipfs/util/testutil" + + ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" +) + +func EchoStreamHandler(stream inet.Stream) { + go func() { + defer stream.Close() + + // pull out the ipfs conn + c := stream.Conn() + log.Infof("%s ponging to %s", c.LocalPeer(), c.RemotePeer()) + + buf := make([]byte, 4) + + for { + if _, err := stream.Read(buf); err != nil { + if err != io.EOF { + log.Info("ping receive error:", err) + } + return + } + + if !bytes.Equal(buf, []byte("ping")) { + log.Infof("ping receive error: ping != %s %v", buf, buf) + return + } + + if _, err := stream.Write([]byte("pong")); err != nil { + log.Info("pond send error:", err) + return + } + } + }() +} + +func makeSwarms(ctx context.Context, t *testing.T, num int) []*Swarm { + swarms := make([]*Swarm, 0, num) + + for i := 0; i < num; i++ { + localnp := testutil.RandPeerNetParamsOrFatal(t) + + peerstore := peer.NewPeerstore() + peerstore.AddPubKey(localnp.ID, localnp.PubKey) + peerstore.AddPrivKey(localnp.ID, localnp.PrivKey) + + addrs := []ma.Multiaddr{localnp.Addr} + swarm, err := NewSwarm(ctx, addrs, localnp.ID, peerstore, metrics.NewBandwidthCounter()) + if err != nil { + t.Fatal(err) + } + + swarm.SetStreamHandler(EchoStreamHandler) + swarms = append(swarms, swarm) + } + + return swarms +} + +func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { + + var wg sync.WaitGroup + connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + // TODO: make a DialAddr func. + s.peers.AddAddr(dst, addr, peer.PermanentAddrTTL) + if _, err := s.Dial(ctx, dst); err != nil { + t.Fatal("error swarm dialing to peer", err) + } + wg.Done() + } + + log.Info("Connecting swarms simultaneously.") + for _, s1 := range swarms { + for _, s2 := range swarms { + if s2.local != s1.local { // don't connect to self. + wg.Add(1) + connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) // try the first. + } + } + } + wg.Wait() + + for _, s := range swarms { + log.Infof("%s swarm routing table: %s", s.local, s.Peers()) + } +} + +func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { + // t.Skip("skipping for another test") + + ctx := context.Background() + swarms := makeSwarms(ctx, t, SwarmNum) + + // connect everyone + connectSwarms(t, ctx, swarms) + + // ping/pong + for _, s1 := range swarms { + log.Debugf("-------------------------------------------------------") + log.Debugf("%s ping pong round", s1.local) + log.Debugf("-------------------------------------------------------") + + _, cancel := context.WithCancel(ctx) + got := map[peer.ID]int{} + errChan := make(chan error, MsgNum*len(swarms)) + streamChan := make(chan *Stream, MsgNum) + + // send out "ping" x MsgNum to every peer + go func() { + defer close(streamChan) + + var wg sync.WaitGroup + send := func(p peer.ID) { + defer wg.Done() + + // first, one stream per peer (nice) + stream, err := s1.NewStreamWithPeer(p) + if err != nil { + errChan <- err + return + } + + // send out ping! + for k := 0; k < MsgNum; k++ { // with k messages + msg := "ping" + log.Debugf("%s %s %s (%d)", s1.local, msg, p, k) + if _, err := stream.Write([]byte(msg)); err != nil { + errChan <- err + continue + } + } + + // read it later + streamChan <- stream + } + + for _, s2 := range swarms { + if s2.local == s1.local { + continue // dont send to self... + } + + wg.Add(1) + go send(s2.local) + } + wg.Wait() + }() + + // receive "pong" x MsgNum from every peer + go func() { + defer close(errChan) + count := 0 + countShouldBe := MsgNum * (len(swarms) - 1) + for stream := range streamChan { // one per peer + defer stream.Close() + + // get peer on the other side + p := stream.Conn().RemotePeer() + + // receive pings + msgCount := 0 + msg := make([]byte, 4) + for k := 0; k < MsgNum; k++ { // with k messages + + // read from the stream + if _, err := stream.Read(msg); err != nil { + errChan <- err + continue + } + + if string(msg) != "pong" { + errChan <- fmt.Errorf("unexpected message: %s", msg) + continue + } + + log.Debugf("%s %s %s (%d)", s1.local, msg, p, k) + msgCount++ + } + + got[p] = msgCount + count += msgCount + } + + if count != countShouldBe { + errChan <- fmt.Errorf("count mismatch: %d != %d", count, countShouldBe) + } + }() + + // check any errors (blocks till consumer is done) + for err := range errChan { + if err != nil { + t.Error(err.Error()) + } + } + + log.Debugf("%s got pongs", s1.local) + if (len(swarms) - 1) != len(got) { + t.Errorf("got (%d) less messages than sent (%d).", len(got), len(swarms)) + } + + for p, n := range got { + if n != MsgNum { + t.Error("peer did not get all msgs", p, n, "/", MsgNum) + } + } + + cancel() + <-time.After(10 * time.Millisecond) + } + + for _, s := range swarms { + s.Close() + } +} + +func TestSwarm(t *testing.T) { + // t.Skip("skipping for another test") + t.Parallel() + + // msgs := 1000 + msgs := 100 + swarms := 5 + SubtestSwarm(t, swarms, msgs) +} + +func TestConnHandler(t *testing.T) { + // t.Skip("skipping for another test") + t.Parallel() + + ctx := context.Background() + swarms := makeSwarms(ctx, t, 5) + + gotconn := make(chan struct{}, 10) + swarms[0].SetConnHandler(func(conn *Conn) { + gotconn <- struct{}{} + }) + + connectSwarms(t, ctx, swarms) + + <-time.After(time.Millisecond) + // should've gotten 5 by now. + + swarms[0].SetConnHandler(nil) + + expect := 4 + for i := 0; i < expect; i++ { + select { + case <-time.After(time.Second): + t.Fatal("failed to get connections") + case <-gotconn: + } + } + + select { + case <-gotconn: + t.Fatalf("should have connected to %d swarms", expect) + default: + } +} + +func TestAddrBlocking(t *testing.T) { + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + + swarms[0].SetConnHandler(func(conn *Conn) { + t.Fatal("no connections should happen!") + }) + + _, block, err := net.ParseCIDR("127.0.0.1/8") + if err != nil { + t.Fatal(err) + } + + swarms[1].Filters.AddDialFilter(block) + + swarms[1].peers.AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], peer.PermanentAddrTTL) + _, err = swarms[1].Dial(ctx, swarms[0].LocalPeer()) + if err == nil { + t.Fatal("dial should have failed") + } + + swarms[0].peers.AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], peer.PermanentAddrTTL) + _, err = swarms[0].Dial(ctx, swarms[1].LocalPeer()) + if err == nil { + t.Fatal("dial should have failed") + } +} + +func TestFilterBounds(t *testing.T) { + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + + conns := make(chan struct{}, 8) + swarms[0].SetConnHandler(func(conn *Conn) { + conns <- struct{}{} + }) + + // Address that we wont be dialing from + _, block, err := net.ParseCIDR("192.0.0.1/8") + if err != nil { + t.Fatal(err) + } + + // set filter on both sides, shouldnt matter + swarms[1].Filters.AddDialFilter(block) + swarms[0].Filters.AddDialFilter(block) + + connectSwarms(t, ctx, swarms) + + select { + case <-time.After(time.Second): + t.Fatal("should have gotten connection") + case <-conns: + t.Log("got connect") + } +} From 9bf24a5968e97f268ccc3bbf69e6bfee3fcfcede Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 30 Sep 2015 19:24:00 -0400 Subject: [PATCH 002/259] rewrote imports to p2p --- p2p/net/swarm/dial_test.go | 4 ++-- p2p/net/swarm/peers_test.go | 2 +- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm.go | 10 +++++----- p2p/net/swarm/swarm_addr.go | 4 ++-- p2p/net/swarm/swarm_addr_test.go | 6 +++--- p2p/net/swarm/swarm_conn.go | 8 ++++---- p2p/net/swarm/swarm_dial.go | 10 +++++----- p2p/net/swarm/swarm_listen.go | 10 +++++----- p2p/net/swarm/swarm_net.go | 6 +++--- p2p/net/swarm/swarm_net_test.go | 4 ++-- p2p/net/swarm/swarm_notif_test.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 6 +++--- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 0ed10f4d30..4d4ea1515e 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" - peer "github.com/ipfs/go-ipfs/p2p/peer" + addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" + peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-ipfs/util/testutil" ci "github.com/ipfs/go-ipfs/util/testutil/ci" diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 2f8b07ef0b..65756a6c2b 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -3,7 +3,7 @@ package swarm import ( "testing" - peer "github.com/ipfs/go-ipfs/p2p/peer" + peer "github.com/ipfs/go-libp2p/p2p/peer" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index dfb775c971..09be20a60a 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - peer "github.com/ipfs/go-ipfs/p2p/peer" ci "github.com/ipfs/go-ipfs/util/testutil/ci" + peer "github.com/ipfs/go-libp2p/p2p/peer" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 512369013c..b20b128930 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -7,12 +7,12 @@ import ( "sync" "time" - metrics "github.com/ipfs/go-ipfs/metrics" - inet "github.com/ipfs/go-ipfs/p2p/net" - filter "github.com/ipfs/go-ipfs/p2p/net/filter" - addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" - peer "github.com/ipfs/go-ipfs/p2p/peer" logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0" + inet "github.com/ipfs/go-libp2p/p2p/net" + filter "github.com/ipfs/go-libp2p/p2p/net/filter" + addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" + peer "github.com/ipfs/go-libp2p/p2p/peer" + metrics "github.com/ipfs/go-libp2p/util/metrics" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 8f9360d9d0..c3eaa3b001 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -1,8 +1,8 @@ package swarm import ( - conn "github.com/ipfs/go-ipfs/p2p/net/conn" - addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" + conn "github.com/ipfs/go-libp2p/p2p/net/conn" + addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index b75b491c42..fbfbcb8c3b 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -3,10 +3,10 @@ package swarm import ( "testing" - metrics "github.com/ipfs/go-ipfs/metrics" - addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" - peer "github.com/ipfs/go-ipfs/p2p/peer" testutil "github.com/ipfs/go-ipfs/util/testutil" + addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" + peer "github.com/ipfs/go-libp2p/p2p/peer" + metrics "github.com/ipfs/go-libp2p/util/metrics" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 126ace627a..dd9f308eef 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -3,10 +3,10 @@ package swarm import ( "fmt" - ic "github.com/ipfs/go-ipfs/p2p/crypto" - inet "github.com/ipfs/go-ipfs/p2p/net" - conn "github.com/ipfs/go-ipfs/p2p/net/conn" - peer "github.com/ipfs/go-ipfs/p2p/peer" + ic "github.com/ipfs/go-libp2p/p2p/crypto" + inet "github.com/ipfs/go-libp2p/p2p/net" + conn "github.com/ipfs/go-libp2p/p2p/net/conn" + peer "github.com/ipfs/go-libp2p/p2p/peer" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index a68d288059..d48d8a95f7 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -8,11 +8,11 @@ import ( "sync" "time" - mconn "github.com/ipfs/go-ipfs/metrics/conn" - conn "github.com/ipfs/go-ipfs/p2p/net/conn" - addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" - peer "github.com/ipfs/go-ipfs/p2p/peer" - lgbl "github.com/ipfs/go-ipfs/util/eventlog/loggables" + conn "github.com/ipfs/go-libp2p/p2p/net/conn" + addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" + peer "github.com/ipfs/go-libp2p/p2p/peer" + lgbl "github.com/ipfs/go-libp2p/util/eventlog/loggables" + mconn "github.com/ipfs/go-libp2p/util/metrics/conn" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index d1bcb07521..d41e173574 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -3,11 +3,11 @@ package swarm import ( "fmt" - mconn "github.com/ipfs/go-ipfs/metrics/conn" - inet "github.com/ipfs/go-ipfs/p2p/net" - conn "github.com/ipfs/go-ipfs/p2p/net/conn" - addrutil "github.com/ipfs/go-ipfs/p2p/net/swarm/addr" - lgbl "github.com/ipfs/go-ipfs/util/eventlog/loggables" + inet "github.com/ipfs/go-libp2p/p2p/net" + conn "github.com/ipfs/go-libp2p/p2p/net/conn" + addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" + lgbl "github.com/ipfs/go-libp2p/util/eventlog/loggables" + mconn "github.com/ipfs/go-libp2p/util/metrics/conn" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index da9b52251e..863ef5e64f 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -3,10 +3,10 @@ package swarm import ( "fmt" - peer "github.com/ipfs/go-ipfs/p2p/peer" + peer "github.com/ipfs/go-libp2p/p2p/peer" - metrics "github.com/ipfs/go-ipfs/metrics" - inet "github.com/ipfs/go-ipfs/p2p/net" + inet "github.com/ipfs/go-libp2p/p2p/net" + metrics "github.com/ipfs/go-libp2p/util/metrics" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess" diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index b5f5c56996..59c4892b87 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -6,8 +6,8 @@ import ( "time" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" - inet "github.com/ipfs/go-ipfs/p2p/net" - testutil "github.com/ipfs/go-ipfs/p2p/test/util" + inet "github.com/ipfs/go-libp2p/p2p/net" + testutil "github.com/ipfs/go-libp2p/p2p/test/util" ) // TestConnectednessCorrect starts a few networks, connects a few diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 9b0e3a9bab..9bbf43e529 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -7,8 +7,8 @@ import ( ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" - inet "github.com/ipfs/go-ipfs/p2p/net" - peer "github.com/ipfs/go-ipfs/p2p/peer" + inet "github.com/ipfs/go-libp2p/p2p/net" + peer "github.com/ipfs/go-libp2p/p2p/peer" ) func TestNotifications(t *testing.T) { diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 66dd6353b7..bcb188bc49 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -1,7 +1,7 @@ package swarm import ( - inet "github.com/ipfs/go-ipfs/p2p/net" + inet "github.com/ipfs/go-libp2p/p2p/net" ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" ) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 9193db0109..1d0479f2cb 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -9,10 +9,10 @@ import ( "testing" "time" - metrics "github.com/ipfs/go-ipfs/metrics" - inet "github.com/ipfs/go-ipfs/p2p/net" - peer "github.com/ipfs/go-ipfs/p2p/peer" testutil "github.com/ipfs/go-ipfs/util/testutil" + inet "github.com/ipfs/go-libp2p/p2p/net" + peer "github.com/ipfs/go-libp2p/p2p/peer" + metrics "github.com/ipfs/go-libp2p/util/metrics" ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" From 9b779001c2c9aa0a3c0760d962b184b484863b36 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 11 Nov 2015 16:15:47 -0800 Subject: [PATCH 003/259] extract from 0.4.0 --- p2p/net/swarm/addr/addr.go | 10 +- p2p/net/swarm/addr/addr_test.go | 4 +- p2p/net/swarm/dial_test.go | 58 +++++-- p2p/net/swarm/peers_test.go | 4 +- p2p/net/swarm/simul_test.go | 10 +- p2p/net/swarm/swarm.go | 71 +++++--- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 8 +- p2p/net/swarm/swarm_conn.go | 6 +- p2p/net/swarm/swarm_dial.go | 265 ++++++++++++++++++++---------- p2p/net/swarm/swarm_listen.go | 104 +++++++----- p2p/net/swarm/swarm_net.go | 8 +- p2p/net/swarm/swarm_net_test.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 4 +- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 17 +- 16 files changed, 379 insertions(+), 196 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 649d54b3c0..cec777a25e 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,14 +3,14 @@ package addrutil import ( "fmt" - logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0" + logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" + context "golang.org/x/net/context" ) -var log = logging.Logger("p2p/net/swarm/addr") +var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") // SupportedTransportStrings is the list of supported transports for the swarm. // These are strings of encapsulated multiaddr protocols. E.g.: diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index eb843ffc09..49eec13511 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,8 +3,8 @@ package addrutil import ( "testing" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" ) func newMultiaddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 4d4ea1515e..f121f97f05 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -2,6 +2,7 @@ package swarm import ( "net" + "sort" "sync" "testing" "time" @@ -9,12 +10,12 @@ import ( addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - testutil "github.com/ipfs/go-ipfs/util/testutil" - ci "github.com/ipfs/go-ipfs/util/testutil/ci" + testutil "util/testutil" + ci "util/testutil/ci" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" + context "golang.org/x/net/context" ) func acceptAndHang(l net.Listener) { @@ -419,18 +420,18 @@ func TestDialBackoffClears(t *testing.T) { } s1.peers.AddAddrs(s2.local, ifaceAddrs1, peer.PermanentAddrTTL) - before = time.Now() + if _, err := s1.Dial(ctx, s2.local); err == nil { + t.Fatal("should have failed to dial backed off peer") + } + + time.Sleep(baseBackoffTime) + if c, err := s1.Dial(ctx, s2.local); err != nil { t.Fatal(err) } else { c.Close() t.Log("correctly connected") } - duration = time.Now().Sub(before) - - if duration >= dt { - // t.Error("took too long", duration, dt) - } if s1.backf.Backoff(s2.local) { t.Error("s2 should no longer be on backoff") @@ -438,3 +439,38 @@ func TestDialBackoffClears(t *testing.T) { t.Log("correctly cleared backoff") } } + +func mkAddr(t *testing.T, s string) ma.Multiaddr { + a, err := ma.NewMultiaddr(s) + if err != nil { + t.Fatal(err) + } + + return a +} + +func TestAddressSorting(t *testing.T) { + u1 := mkAddr(t, "/ip4/152.12.23.53/udp/1234/utp") + u2l := mkAddr(t, "/ip4/127.0.0.1/udp/1234/utp") + local := mkAddr(t, "/ip4/127.0.0.1/tcp/1234") + norm := mkAddr(t, "/ip4/6.5.4.3/tcp/1234") + + l := AddrList{local, u1, u2l, norm} + sort.Sort(l) + + if !l[0].Equal(u2l) { + t.Fatal("expected utp local addr to be sorted first: ", l[0]) + } + + if !l[1].Equal(u1) { + t.Fatal("expected utp addr to be sorted second") + } + + if !l[2].Equal(local) { + t.Fatal("expected tcp localhost addr thid") + } + + if !l[3].Equal(norm) { + t.Fatal("expected normal addr last") + } +} diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 65756a6c2b..b107fbfe83 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,8 +5,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 09be20a60a..63e07ff8ac 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -1,15 +1,16 @@ package swarm import ( + "runtime" "sync" "testing" "time" - ci "github.com/ipfs/go-ipfs/util/testutil/ci" peer "github.com/ipfs/go-libp2p/p2p/peer" + ci "util/testutil/ci" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestSimultOpen(t *testing.T) { @@ -49,7 +50,8 @@ func TestSimultOpenMany(t *testing.T) { addrs := 20 rounds := 10 - if ci.IsRunning() { + if ci.IsRunning() || runtime.GOOS == "darwin" { + // osx has a limit of 256 file descriptors addrs = 10 rounds = 5 } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index b20b128930..245b7204f7 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -7,22 +7,26 @@ import ( "sync" "time" - logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0" + metrics "github.com/ipfs/go-libp2p/p2p/metrics" + mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" inet "github.com/ipfs/go-libp2p/p2p/net" + conn "github.com/ipfs/go-libp2p/p2p/net/conn" filter "github.com/ipfs/go-libp2p/p2p/net/filter" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" + transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" - metrics "github.com/ipfs/go-libp2p/util/metrics" - - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" - pst "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-stream-muxer" - psy "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-stream-muxer/yamux" - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess" - goprocessctx "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/context" - prom "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus" - mafilter "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/whyrusleeping/multiaddr-filter" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + pst "github.com/jbenet/go-stream-muxer" + psmss "github.com/jbenet/go-stream-muxer/multistream" + "github.com/jbenet/goprocess" + goprocessctx "github.com/jbenet/goprocess/context" + prom "github.com/prometheus/client_golang/prometheus" + mafilter "github.com/whyrusleeping/multiaddr-filter" + context "golang.org/x/net/context" + + logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" ) var log = logging.Logger("swarm2") @@ -37,9 +41,7 @@ var peersTotal = prom.NewGaugeVec(prom.GaugeOpts{ }, []string{"peer_id"}) func init() { - tpt := *psy.DefaultTransport - tpt.MaxStreamWindowSize = 512 * 1024 - PSTransport = &tpt + PSTransport = psmss.NewTransport() } // Swarm is a connection muxer, allowing connections to other peers to @@ -58,12 +60,19 @@ type Swarm struct { backf dialbackoff dialT time.Duration // mainly for tests + dialer *conn.Dialer + notifmu sync.RWMutex notifs map[inet.Notifiee]ps.Notifiee + transports []transport.Transport + // filters for addresses that shouldnt be dialed Filters *filter.Filters + // file descriptor rate limited + fdRateLimit chan struct{} + proc goprocess.Process ctx context.Context bwc metrics.Reporter @@ -78,15 +87,22 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, return nil, err } + wrap := func(c transport.Conn) transport.Conn { + return mconn.WrapConn(bwc, c) + } + s := &Swarm{ - swarm: ps.NewSwarm(PSTransport), - local: local, - peers: peers, - ctx: ctx, - dialT: DialTimeout, - notifs: make(map[inet.Notifiee]ps.Notifiee), - bwc: bwc, - Filters: filter.NewFilters(), + swarm: ps.NewSwarm(PSTransport), + local: local, + peers: peers, + ctx: ctx, + dialT: DialTimeout, + notifs: make(map[inet.Notifiee]ps.Notifiee), + transports: []transport.Transport{transport.NewTCPTransport()}, + bwc: bwc, + fdRateLimit: make(chan struct{}, concurrentFdDials), + Filters: filter.NewFilters(), + dialer: conn.NewDialer(local, peers.PrivKey(local), wrap), } // configure Swarm @@ -97,7 +113,12 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, prom.MustRegisterOrGet(peersTotal) s.Notify((*metricsNotifiee)(s)) - return s, s.listen(listenAddrs) + err = s.setupInterfaces(listenAddrs) + if err != nil { + return nil, err + } + + return s, nil } func (s *Swarm) teardown() error { @@ -130,7 +151,7 @@ func (s *Swarm) Listen(addrs ...ma.Multiaddr) error { return err } - return s.listen(addrs) + return s.setupInterfaces(addrs) } // Process returns the Process of the swarm diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index c3eaa3b001..039683bfb9 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -4,7 +4,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index fbfbcb8c3b..b1f167ae49 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -3,13 +3,13 @@ package swarm import ( "testing" - testutil "github.com/ipfs/go-ipfs/util/testutil" + metrics "github.com/ipfs/go-libp2p/p2p/metrics" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - metrics "github.com/ipfs/go-libp2p/util/metrics" + testutil "util/testutil" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index dd9f308eef..840a9b465c 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,9 +8,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + context "golang.org/x/net/context" ) // a Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index d48d8a95f7..4790eb0738 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -1,25 +1,21 @@ package swarm import ( + "bytes" "errors" "fmt" - "math/rand" - "net" + "sort" "sync" "time" + "github.com/jbenet/go-multiaddr-net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - lgbl "github.com/ipfs/go-libp2p/util/eventlog/loggables" - mconn "github.com/ipfs/go-libp2p/util/metrics/conn" - - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" - process "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess" - processctx "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/context" - ratelimit "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/ratelimit" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + lgbl "util/eventlog/loggables" + + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) // Diagram of dial sync: @@ -44,6 +40,9 @@ var ( // add loop back in Dial(.) const dialAttempts = 1 +// number of concurrent outbound dials over transports that consume file descriptors +const concurrentFdDials = 160 + // DialTimeout is the amount of time each dial attempt has. We can think about making // this larger down the road, or putting more granular timeouts (i.e. within each // subcomponent of Dial) @@ -115,6 +114,7 @@ func (ds *dialsync) Unlock(dst peer.ID) { if !found { panic("called dialDone with no ongoing dials to peer: " + dst.Pretty()) } + delete(ds.ongoing, dst) // remove ongoing dial close(wait) // release everyone else ds.lock.Unlock() @@ -145,44 +145,71 @@ func (ds *dialsync) Unlock(dst peer.ID) { // dialbackoff.Clear(p) // } // + type dialbackoff struct { - entries map[peer.ID]struct{} + entries map[peer.ID]*backoffPeer lock sync.RWMutex } +type backoffPeer struct { + tries int + until time.Time +} + func (db *dialbackoff) init() { if db.entries == nil { - db.entries = make(map[peer.ID]struct{}) + db.entries = make(map[peer.ID]*backoffPeer) } } // Backoff returns whether the client should backoff from dialing -// peeer p -func (db *dialbackoff) Backoff(p peer.ID) bool { +// peer p +func (db *dialbackoff) Backoff(p peer.ID) (backoff bool) { db.lock.Lock() + defer db.lock.Unlock() db.init() - _, found := db.entries[p] - db.lock.Unlock() - return found + bp, found := db.entries[p] + if found && time.Now().Before(bp.until) { + return true + } + + return false } +const baseBackoffTime = time.Second * 5 +const maxBackoffTime = time.Minute * 5 + // AddBackoff lets other nodes know that we've entered backoff with // peer p, so dialers should not wait unnecessarily. We still will // attempt to dial with one goroutine, in case we get through. func (db *dialbackoff) AddBackoff(p peer.ID) { db.lock.Lock() + defer db.lock.Unlock() db.init() - db.entries[p] = struct{}{} - db.lock.Unlock() + bp, ok := db.entries[p] + if !ok { + db.entries[p] = &backoffPeer{ + tries: 1, + until: time.Now().Add(baseBackoffTime), + } + return + } + + expTimeAdd := time.Second * time.Duration(bp.tries*bp.tries) + if expTimeAdd > maxBackoffTime { + expTimeAdd = maxBackoffTime + } + bp.until = time.Now().Add(baseBackoffTime + expTimeAdd) + bp.tries++ } // Clear removes a backoff record. Clients should call this after a // successful Dial. func (db *dialbackoff) Clear(p peer.ID) { db.lock.Lock() + defer db.lock.Unlock() db.init() delete(db.entries, p) - db.lock.Unlock() } // Dial connects to a peer. @@ -225,14 +252,20 @@ func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) // check if there's an ongoing dial to this peer if ok, wait := s.dsync.Lock(p); ok { + defer s.dsync.Unlock(p) + + // if this peer has been backed off, lets get out of here + if s.backf.Backoff(p) { + log.Event(ctx, "swarmDialBackoff", logdial) + return nil, ErrDialBackoff + } + // ok, we have been charged to dial! let's do it. // if it succeeds, dial will add the conn to the swarm itself. - defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done() ctxT, cancel := context.WithTimeout(ctx, s.dialT) conn, err := s.dial(ctxT, p) cancel() - s.dsync.Unlock(p) log.Debugf("dial end %s", conn) if err != nil { log.Event(ctx, "swarmDialBackoffAdd", logdial) @@ -287,14 +320,6 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { log.Debug("Dial not given PrivateKey, so WILL NOT SECURE conn.") } - // get our own addrs. try dialing out from our listener addresses (reusing ports) - // Note that using our peerstore's addresses here is incorrect, as that would - // include observed addresses. TODO: make peerstore's address book smarter. - localAddrs := s.ListenAddresses() - if len(localAddrs) == 0 { - log.Debug("Dialing out with no local addresses.") - } - // get remote peer addrs remoteAddrs := s.peers.Addrs(p) // make sure we can use the addresses. @@ -319,23 +344,8 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return nil, err } - // open connection to peer - d := &conn.Dialer{ - Dialer: manet.Dialer{ - Dialer: net.Dialer{ - Timeout: s.dialT, - }, - }, - LocalPeer: s.local, - LocalAddrs: localAddrs, - PrivateKey: sk, - Wrapper: func(c manet.Conn) manet.Conn { - return mconn.WrapConn(s.bwc, c) - }, - } - // try to get a connection to any addr - connC, err := s.dialAddrs(ctx, d, p, remoteAddrs) + connC, err := s.dialAddrs(ctx, p, remoteAddrs) if err != nil { logdial["error"] = err return nil, err @@ -355,7 +365,10 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return swarmC, nil } -func (s *Swarm) dialAddrs(ctx context.Context, d *conn.Dialer, p peer.ID, remoteAddrs []ma.Multiaddr) (conn.Conn, error) { +func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs []ma.Multiaddr) (conn.Conn, error) { + + // sort addresses so preferred addresses are dialed sooner + sort.Sort(AddrList(remoteAddrs)) // try to connect to one of the peer's known addresses. // we dial concurrently to each of the addresses, which: @@ -367,78 +380,89 @@ func (s *Swarm) dialAddrs(ctx context.Context, d *conn.Dialer, p peer.ID, remote ctx, cancel := context.WithCancel(ctx) defer cancel() // cancel work when we exit func - foundConn := make(chan struct{}) - conns := make(chan conn.Conn, len(remoteAddrs)) + conns := make(chan conn.Conn) errs := make(chan error, len(remoteAddrs)) // dialSingleAddr is used in the rate-limited async thing below. dialSingleAddr := func(addr ma.Multiaddr) { - connC, err := s.dialAddr(ctx, d, p, addr) + // rebind chans in scope so we can nil them out easily + connsout := conns + errsout := errs + + connC, err := s.dialAddr(ctx, p, addr) + if err != nil { + connsout = nil + } else if connC == nil { + // NOTE: this really should never happen + log.Errorf("failed to dial %s %s and got no error!", p, addr) + err = fmt.Errorf("failed to dial %s %s", p, addr) + connsout = nil + } else { + errsout = nil + } // check parent still wants our results select { - case <-foundConn: + case <-ctx.Done(): if connC != nil { connC.Close() } - return - default: - } - - if err != nil { - errs <- err - } else if connC == nil { - errs <- fmt.Errorf("failed to dial %s %s", p, addr) - } else { - conns <- connC + case errsout <- err: + case connsout <- connC: } } // this whole thing is in a goroutine so we can use foundConn // to end early. go func() { - // rate limiting just in case. at most 10 addrs at once. - limiter := ratelimit.NewRateLimiter(process.Background(), 10) - limiter.Go(func(worker process.Process) { - // permute addrs so we try different sets first each time. - for _, i := range rand.Perm(len(remoteAddrs)) { - select { - case <-foundConn: // if one of them succeeded already - break - case <-worker.Closing(): // our context was cancelled - break - default: - } - - workerAddr := remoteAddrs[i] // shadow variable to avoid race - limiter.LimitedGo(func(worker process.Process) { - dialSingleAddr(workerAddr) - }) + limiter := make(chan struct{}, 8) + for _, addr := range remoteAddrs { + // returns whatever ratelimiting is acceptable for workerAddr. + // may not rate limit at all. + rl := s.addrDialRateLimit(addr) + select { + case <-ctx.Done(): // our context was cancelled + return + case rl <- struct{}{}: + // take the token, move on + } + + select { + case <-ctx.Done(): // our context was cancelled + return + case limiter <- struct{}{}: + // take the token, move on } - }) - processctx.CloseAfterContext(limiter, ctx) + go func(rlc <-chan struct{}, a ma.Multiaddr) { + dialSingleAddr(a) + <-limiter + <-rlc + }(rl, addr) + } }() - // wair fot the results. + // wair for the results. exitErr := fmt.Errorf("failed to dial %s", p) - for i := 0; i < len(remoteAddrs); i++ { + for range remoteAddrs { select { case exitErr = <-errs: // log.Debug("dial error: ", exitErr) case connC := <-conns: // take the first + return asap - close(foundConn) return connC, nil + case <-ctx.Done(): + // break out and return error + break } } return nil, exitErr } -func (s *Swarm) dialAddr(ctx context.Context, d *conn.Dialer, p peer.ID, addr ma.Multiaddr) (conn.Conn, error) { +func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (conn.Conn, error) { log.Debugf("%s swarm dialing %s %s", s.local, p, addr) - connC, err := d.Dial(ctx, addr, p) + connC, err := s.dialer.Dial(ctx, addr, p) if err != nil { return nil, fmt.Errorf("%s --> %s dial attempt failed: %s", s.local, p, err) } @@ -491,3 +515,72 @@ func dialConnSetup(ctx context.Context, s *Swarm, connC conn.Conn) (*Conn, error return swarmC, err } + +// addrDialRateLimit returns a ratelimiting channel for dialing transport +// addrs like a. for example, tcp is fd-ratelimited. utp is not ratelimited. +func (s *Swarm) addrDialRateLimit(a ma.Multiaddr) chan struct{} { + if isFDCostlyTransport(a) { + return s.fdRateLimit + } + + // do not rate limit it at all + return make(chan struct{}, 1) +} + +func isFDCostlyTransport(a ma.Multiaddr) bool { + return isTCPMultiaddr(a) +} + +func isTCPMultiaddr(a ma.Multiaddr) bool { + p := a.Protocols() + return len(p) == 2 && (p[0].Name == "ip4" || p[0].Name == "ip6") && p[1].Name == "tcp" +} + +type AddrList []ma.Multiaddr + +func (al AddrList) Len() int { + return len(al) +} + +func (al AddrList) Swap(i, j int) { + al[i], al[j] = al[j], al[i] +} + +func (al AddrList) Less(i, j int) bool { + a := al[i] + b := al[j] + + // dial localhost addresses next, they should fail immediately + lba := manet.IsIPLoopback(a) + lbb := manet.IsIPLoopback(b) + if lba { + if !lbb { + return true + } + } + + // dial utp and similar 'non-fd-consuming' addresses first + fda := isFDCostlyTransport(a) + fdb := isFDCostlyTransport(b) + if !fda { + if fdb { + return true + } + + // if neither consume fd's, assume equal ordering + return false + } + + // if 'b' doesnt take a file descriptor + if !fdb { + return false + } + + // if 'b' is loopback and both take file descriptors + if lbb { + return false + } + + // for the rest, just sort by bytes + return bytes.Compare(a.Bytes(), b.Bytes()) > 0 +} diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index d41e173574..f3da2eeea7 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -3,68 +3,81 @@ package swarm import ( "fmt" + mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" - addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - lgbl "github.com/ipfs/go-libp2p/util/eventlog/loggables" - mconn "github.com/ipfs/go-libp2p/util/metrics/conn" - - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - manet "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" - ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" - multierr "github.com/ipfs/go-ipfs/thirdparty/multierr" + transport "github.com/ipfs/go-libp2p/p2p/net/transport" + lgbl "util/eventlog/loggables" + + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + context "golang.org/x/net/context" ) -// Open listeners for each network the swarm should listen on -func (s *Swarm) listen(addrs []ma.Multiaddr) error { +// Open listeners and reuse-dialers for the given addresses +func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { + errs := make([]error, len(addrs)) + var succeeded int + for i, a := range addrs { + tpt := s.transportForAddr(a) + if tpt == nil { + errs[i] = fmt.Errorf("no transport for address: %s", a) + continue + } - for _, addr := range addrs { - if !addrutil.AddrUsable(addr, true) { - return fmt.Errorf("cannot use addr: %s", addr) + d, err := tpt.Dialer(a, transport.TimeoutOpt(DialTimeout), transport.ReusePorts) + if err != nil { + errs[i] = err + continue } - } - retErr := multierr.New() + s.dialer.AddDialer(d) - // listen on every address - for i, addr := range addrs { - err := s.setupListener(addr) + list, err := tpt.Listen(a) if err != nil { - if retErr.Errors == nil { - retErr.Errors = make([]error, len(addrs)) - } - retErr.Errors[i] = err - log.Debugf("Failed to listen on: %s - %s", addr, err) + errs[i] = err + continue } + + err = s.addListener(list) + if err != nil { + errs[i] = err + continue + } + succeeded++ } - if retErr.Errors != nil { - return retErr + for i, e := range errs { + if e != nil { + log.Warning("listen on %s failed: %s", addrs[i], errs[i]) + } } + if succeeded == 0 && len(addrs) > 0 { + return fmt.Errorf("failed to listen on any addresses: %s", errs) + } + return nil } -// Listen for new connections on the given multiaddr -func (s *Swarm) setupListener(maddr ma.Multiaddr) error { +func (s *Swarm) transportForAddr(a ma.Multiaddr) transport.Transport { + for _, t := range s.transports { + if t.Matches(a) { + return t + } + } + + return nil +} - // TODO rethink how this has to work. (jbenet) - // - // resolved, err := resolveUnspecifiedAddresses([]ma.Multiaddr{maddr}) - // if err != nil { - // return err - // } - // for _, a := range resolved { - // s.peers.AddAddr(s.local, a) - // } +func (s *Swarm) addListener(tptlist transport.Listener) error { sk := s.peers.PrivKey(s.local) if sk == nil { // may be fine for sk to be nil, just log a warning. log.Warning("Listener not given PrivateKey, so WILL NOT SECURE conns.") } - log.Debugf("Swarm Listening at %s", maddr) - list, err := conn.Listen(s.Context(), maddr, s.local, sk) + + list, err := conn.WrapTransportListener(s.Context(), tptlist, s.local, sk) if err != nil { return err } @@ -72,11 +85,15 @@ func (s *Swarm) setupListener(maddr ma.Multiaddr) error { list.SetAddrFilters(s.Filters) if cw, ok := list.(conn.ListenerConnWrapper); ok { - cw.SetConnWrapper(func(c manet.Conn) manet.Conn { + cw.SetConnWrapper(func(c transport.Conn) transport.Conn { return mconn.WrapConn(s.bwc, c) }) } + return s.addConnListener(list) +} + +func (s *Swarm) addConnListener(list conn.Listener) error { // AddListener to the peerstream Listener. this will begin accepting connections // and streams! sl, err := s.swarm.AddListener(list) @@ -85,6 +102,8 @@ func (s *Swarm) setupListener(maddr ma.Multiaddr) error { } log.Debugf("Swarm Listeners at %s", s.ListenAddresses()) + maddr := list.Multiaddr() + // signal to our notifiees on successful conn. s.notifyAll(func(n inet.Notifiee) { n.Listen((*Network)(s), maddr) @@ -107,7 +126,7 @@ func (s *Swarm) setupListener(maddr ma.Multiaddr) error { if !more { return } - log.Warningf("swarm listener accept error: %s", err) + log.Errorf("swarm listener accept error: %s", err) case <-ctx.Done(): return } @@ -138,5 +157,8 @@ func (s *Swarm) connHandler(c *ps.Conn) *Conn { return nil } + // if a peer dials us, remove from dial backoff. + s.backf.Clear(sc.RemotePeer()) + return sc } diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 863ef5e64f..91c4208370 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -5,12 +5,12 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" + metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - metrics "github.com/ipfs/go-libp2p/util/metrics" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + "github.com/jbenet/goprocess" + context "golang.org/x/net/context" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 59c4892b87..647c53c74e 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + context "golang.org/x/net/context" inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/p2p/test/util" ) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 9bbf43e529..0d976e81d0 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" inet "github.com/ipfs/go-libp2p/p2p/net" peer "github.com/ipfs/go-libp2p/p2p/peer" diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index bcb188bc49..7965d27439 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,7 +3,7 @@ package swarm import ( inet "github.com/ipfs/go-libp2p/p2p/net" - ps "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" + ps "github.com/jbenet/go-peerstream" ) // a Stream is a wrapper around a ps.Stream that exposes a way to get diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 1d0479f2cb..2c98e5e854 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -9,13 +9,13 @@ import ( "testing" "time" - testutil "github.com/ipfs/go-ipfs/util/testutil" + metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" peer "github.com/ipfs/go-libp2p/p2p/peer" - metrics "github.com/ipfs/go-libp2p/util/metrics" + testutil "util/testutil" - ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func EchoStreamHandler(stream inet.Stream) { @@ -237,6 +237,15 @@ func TestSwarm(t *testing.T) { SubtestSwarm(t, swarms, msgs) } +func TestBasicSwarm(t *testing.T) { + // t.Skip("skipping for another test") + t.Parallel() + + msgs := 1 + swarms := 2 + SubtestSwarm(t, swarms, msgs) +} + func TestConnHandler(t *testing.T) { // t.Skip("skipping for another test") t.Parallel() From 5070870e7bda75c083e1dd2ab83a6e969018858c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 15 Nov 2015 11:56:01 -0800 Subject: [PATCH 004/259] more vendoring --- p2p/net/swarm/addr/addr.go | 4 ++-- p2p/net/swarm/addr/addr_test.go | 4 ++-- p2p/net/swarm/dial_test.go | 4 ++-- p2p/net/swarm/peers_test.go | 2 +- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm.go | 4 ++-- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_dial.go | 4 ++-- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_net.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_test.go | 2 +- 14 files changed, 19 insertions(+), 19 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index cec777a25e..cdb7058590 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -5,8 +5,8 @@ import ( logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + manet "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 49eec13511..2015b974cd 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,8 +3,8 @@ package addrutil import ( "testing" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + manet "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net" ) func newMultiaddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index f121f97f05..77f827c333 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -13,8 +13,8 @@ import ( testutil "util/testutil" ci "util/testutil/ci" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + manet "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index b107fbfe83..cc8e14328f 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,7 +5,7 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 63e07ff8ac..c046ac2e45 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -9,7 +9,7 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" ci "util/testutil/ci" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 245b7204f7..d09b62ad50 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -16,14 +16,14 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "github.com/jbenet/go-multiaddr" + mafilter "QmVdADza4QFVAR9xqAxRQjt9vTZJ6UrVLgBstKua1Xg7he/multiaddr-filter" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" ps "github.com/jbenet/go-peerstream" pst "github.com/jbenet/go-stream-muxer" psmss "github.com/jbenet/go-stream-muxer/multistream" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" prom "github.com/prometheus/client_golang/prometheus" - mafilter "github.com/whyrusleeping/multiaddr-filter" context "golang.org/x/net/context" logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 039683bfb9..87ee37bf17 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -4,7 +4,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index b1f167ae49..78685a4d38 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -8,7 +8,7 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "util/testutil" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 840a9b465c..c97a47dfcb 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,7 +8,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" ps "github.com/jbenet/go-peerstream" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 4790eb0738..47caa69d65 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -8,13 +8,13 @@ import ( "sync" "time" - "github.com/jbenet/go-multiaddr-net" + "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" lgbl "util/eventlog/loggables" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index f3da2eeea7..394d5c2179 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -9,7 +9,7 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" lgbl "util/eventlog/loggables" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" ps "github.com/jbenet/go-peerstream" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 91c4208370..3c5e69225b 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -8,7 +8,7 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" "github.com/jbenet/goprocess" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 0d976e81d0..3d2ef86d7b 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" context "golang.org/x/net/context" inet "github.com/ipfs/go-libp2p/p2p/net" diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 2c98e5e854..c6e4a981d1 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -14,7 +14,7 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "util/testutil" - ma "github.com/jbenet/go-multiaddr" + ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" context "golang.org/x/net/context" ) From 21e5c03e6894b48d1b42951f7735ae78250506b6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 15 Nov 2015 18:28:35 -0800 Subject: [PATCH 005/259] fixes for sha3 --- p2p/net/swarm/addr/addr.go | 4 ++-- p2p/net/swarm/addr/addr_test.go | 4 ++-- p2p/net/swarm/dial_test.go | 4 ++-- p2p/net/swarm/peers_test.go | 2 +- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm.go | 4 ++-- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_dial.go | 4 ++-- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_net.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_test.go | 2 +- 14 files changed, 19 insertions(+), 19 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index cdb7058590..c0c4851c26 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -5,8 +5,8 @@ import ( logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" - manet "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net" + manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 2015b974cd..21d645724c 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,8 +3,8 @@ package addrutil import ( "testing" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" - manet "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net" + manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ) func newMultiaddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 77f827c333..632ceb7af8 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -13,8 +13,8 @@ import ( testutil "util/testutil" ci "util/testutil/ci" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" - manet "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net" + manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index cc8e14328f..29099de46f 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,7 +5,7 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index c046ac2e45..7adc022f8f 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -9,7 +9,7 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" ci "util/testutil/ci" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index d09b62ad50..2fcd275675 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -16,8 +16,8 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" - mafilter "QmVdADza4QFVAR9xqAxRQjt9vTZJ6UrVLgBstKua1Xg7he/multiaddr-filter" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + mafilter "QmYWqTn1i8yv9QRDzGPJ2yRudKzYCaC5Aqasbm8vwaG92E/multiaddr-filter" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ps "github.com/jbenet/go-peerstream" pst "github.com/jbenet/go-stream-muxer" psmss "github.com/jbenet/go-stream-muxer/multistream" diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 87ee37bf17..6149eb66e3 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -4,7 +4,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 78685a4d38..0b0d6fffa5 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -8,7 +8,7 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "util/testutil" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index c97a47dfcb..be95a4282b 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,7 +8,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ps "github.com/jbenet/go-peerstream" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 47caa69d65..2bedebc2c8 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -8,13 +8,13 @@ import ( "sync" "time" - "QmanZCL6SXRfafiUEMCBLq2QR171uQSdXQ8YAdHXLd8Cwr/go-multiaddr-net" + "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" lgbl "util/eventlog/loggables" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 394d5c2179..ee8ee91366 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -9,7 +9,7 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" lgbl "util/eventlog/loggables" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ps "github.com/jbenet/go-peerstream" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 3c5e69225b..972afc1d02 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -8,7 +8,7 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" "github.com/jbenet/goprocess" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 3d2ef86d7b..2ba852d188 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" context "golang.org/x/net/context" inet "github.com/ipfs/go-libp2p/p2p/net" diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index c6e4a981d1..f9ba35ed04 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -14,7 +14,7 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "util/testutil" - ma "QmaA6aDzeHjZiuqBtgYRz8ZXb1qMCoyMHgyDjBEYQniUKF/go-multiaddr" + ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" context "golang.org/x/net/context" ) From 7d2dd014c8f44d012c78722b328e578d13c5f0ba Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 15 Nov 2015 18:42:27 -0800 Subject: [PATCH 006/259] vendor in notifier --- p2p/net/swarm/addr/addr.go | 2 +- p2p/net/swarm/dial_test.go | 2 +- p2p/net/swarm/peers_test.go | 2 +- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm.go | 6 +++--- p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_dial.go | 2 +- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_net.go | 4 ++-- p2p/net/swarm/swarm_net_test.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_test.go | 2 +- 13 files changed, 16 insertions(+), 16 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index c0c4851c26..857b0f7289 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -6,8 +6,8 @@ import ( logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - context "golang.org/x/net/context" ) var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 632ceb7af8..027e7945cb 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -14,8 +14,8 @@ import ( ci "util/testutil/ci" manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - context "golang.org/x/net/context" ) func acceptAndHang(l net.Listener) { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 29099de46f..0068b37901 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,8 +5,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - context "golang.org/x/net/context" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 7adc022f8f..549dad4ff8 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -9,8 +9,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" ci "util/testutil/ci" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - context "golang.org/x/net/context" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 2fcd275675..e5424802f7 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -16,15 +16,15 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" + "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess" + goprocessctx "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess/context" mafilter "QmYWqTn1i8yv9QRDzGPJ2yRudKzYCaC5Aqasbm8vwaG92E/multiaddr-filter" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ps "github.com/jbenet/go-peerstream" pst "github.com/jbenet/go-stream-muxer" psmss "github.com/jbenet/go-stream-muxer/multistream" - "github.com/jbenet/goprocess" - goprocessctx "github.com/jbenet/goprocess/context" prom "github.com/prometheus/client_golang/prometheus" - context "golang.org/x/net/context" logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" ) diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 0b0d6fffa5..ec5c6576fa 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -8,8 +8,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "util/testutil" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - context "golang.org/x/net/context" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index be95a4282b..7b9c920fae 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,9 +8,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ps "github.com/jbenet/go-peerstream" - context "golang.org/x/net/context" ) // a Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 2bedebc2c8..3c7aa01d31 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -14,8 +14,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" lgbl "util/eventlog/loggables" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - context "golang.org/x/net/context" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index ee8ee91366..8355815869 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -9,9 +9,9 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" lgbl "util/eventlog/loggables" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ps "github.com/jbenet/go-peerstream" - context "golang.org/x/net/context" ) // Open listeners and reuse-dialers for the given addresses diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 972afc1d02..36416fa394 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -8,9 +8,9 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" + "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - "github.com/jbenet/goprocess" - context "golang.org/x/net/context" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 647c53c74e..2db22fbb35 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - context "golang.org/x/net/context" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/p2p/test/util" ) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 2ba852d188..1180d641ff 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - context "golang.org/x/net/context" inet "github.com/ipfs/go-libp2p/p2p/net" peer "github.com/ipfs/go-libp2p/p2p/peer" diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index f9ba35ed04..27331ade57 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -14,8 +14,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "util/testutil" + context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - context "golang.org/x/net/context" ) func EchoStreamHandler(stream inet.Stream) { From a82cbe7b2ff22d2f93e4b616d1de197bb539c3f4 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 15 Nov 2015 19:59:59 -0800 Subject: [PATCH 007/259] move testutil up --- p2p/net/swarm/dial_test.go | 4 ++-- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 027e7945cb..128ddb5827 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -10,8 +10,8 @@ import ( addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - testutil "util/testutil" - ci "util/testutil/ci" + testutil "github.com/ipfs/go-libp2p/testutil" + ci "github.com/ipfs/go-libp2p/testutil/ci" manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 549dad4ff8..ec988bcd6c 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,7 +7,7 @@ import ( "time" peer "github.com/ipfs/go-libp2p/p2p/peer" - ci "util/testutil/ci" + ci "github.com/ipfs/go-libp2p/testutil/ci" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index ec5c6576fa..b397a07f62 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -6,7 +6,7 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - testutil "util/testutil" + testutil "github.com/ipfs/go-libp2p/testutil" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 27331ade57..670ea26592 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -12,7 +12,7 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" peer "github.com/ipfs/go-libp2p/p2p/peer" - testutil "util/testutil" + testutil "github.com/ipfs/go-libp2p/testutil" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" From 8cf47173c898399496f4af16b42708428d598ed0 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 15 Nov 2015 21:04:44 -0800 Subject: [PATCH 008/259] remove multiple multihash deps --- p2p/net/swarm/addr/addr.go | 2 +- p2p/net/swarm/addr/addr_test.go | 2 +- p2p/net/swarm/dial_test.go | 2 +- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_dial.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 857b0f7289..00b8ed1c29 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -5,7 +5,7 @@ import ( logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" - manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + manet "QmRCPT5WRph8aWXmaT2Rfn6ac98YRUUJnNURpD3hNAWp4f/go-multiaddr-net" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ) diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 21d645724c..8a0bc9c35a 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,7 +3,7 @@ package addrutil import ( "testing" - manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + manet "QmRCPT5WRph8aWXmaT2Rfn6ac98YRUUJnNURpD3hNAWp4f/go-multiaddr-net" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 128ddb5827..f915808b75 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -13,7 +13,7 @@ import ( testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" - manet "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + manet "QmRCPT5WRph8aWXmaT2Rfn6ac98YRUUJnNURpD3hNAWp4f/go-multiaddr-net" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index e5424802f7..b2b6b83835 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -18,7 +18,7 @@ import ( "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess" goprocessctx "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess/context" - mafilter "QmYWqTn1i8yv9QRDzGPJ2yRudKzYCaC5Aqasbm8vwaG92E/multiaddr-filter" + mafilter "QmXE4GFk66B5Ts362YWZosjiZLpP4QDmBTBadRZFffZ58U/multiaddr-filter" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ps "github.com/jbenet/go-peerstream" diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 3c7aa01d31..df3b89d821 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "QmU5s159q8cZuM1f9Vqti4LHu6y8zyVc5dxv2py81sdp6Q/go-multiaddr-net" + "QmRCPT5WRph8aWXmaT2Rfn6ac98YRUUJnNURpD3hNAWp4f/go-multiaddr-net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" From 46df570e0eb2151b54ff332a3fefa318fa9fc5da Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 15 Nov 2015 21:15:23 -0800 Subject: [PATCH 009/259] vendor go-ipfs-util --- p2p/net/swarm/swarm_dial.go | 2 +- p2p/net/swarm/swarm_listen.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index df3b89d821..11b3f20463 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -12,7 +12,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - lgbl "util/eventlog/loggables" + lgbl "github.com/ipfs/go-libp2p/loggables" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 8355815869..4dd95a579d 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -7,7 +7,7 @@ import ( inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" transport "github.com/ipfs/go-libp2p/p2p/net/transport" - lgbl "util/eventlog/loggables" + lgbl "github.com/ipfs/go-libp2p/loggables" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" From 66d6d96cebf0bcdbb27a8190fd6b59b23e56a63b Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 16 Nov 2015 16:41:23 -0800 Subject: [PATCH 010/259] update multiaddr-filter --- p2p/net/swarm/swarm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index b2b6b83835..5fc02a1a16 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -18,7 +18,7 @@ import ( "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess" goprocessctx "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess/context" - mafilter "QmXE4GFk66B5Ts362YWZosjiZLpP4QDmBTBadRZFffZ58U/multiaddr-filter" + mafilter "QmYhewVqJhkgEsm3AYVUbpT14q2P9V4Xb7np9JXKua6y7A/multiaddr-filter" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" ps "github.com/jbenet/go-peerstream" From f4661d9a5bb7a4d077f23c2c41c99d2a26421478 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 16 Nov 2015 16:47:49 -0800 Subject: [PATCH 011/259] Vendor in go-peerstream --- p2p/net/swarm/swarm.go | 6 +++--- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_listen.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 5fc02a1a16..a6dfa61639 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -16,14 +16,14 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" + pst "QmPxuHs2NQjz16gnvndgkzHkm5PjtqbB5rwoSpLusBkQ7Q/go-stream-muxer" + psmss "QmPxuHs2NQjz16gnvndgkzHkm5PjtqbB5rwoSpLusBkQ7Q/go-stream-muxer/multistream" "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess" goprocessctx "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess/context" + ps "QmTgxFwS1nDK126fH5XPnLFcxcDFsxKbPPnCBwyRWNAjDX/go-peerstream" mafilter "QmYhewVqJhkgEsm3AYVUbpT14q2P9V4Xb7np9JXKua6y7A/multiaddr-filter" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - ps "github.com/jbenet/go-peerstream" - pst "github.com/jbenet/go-stream-muxer" - psmss "github.com/jbenet/go-stream-muxer/multistream" prom "github.com/prometheus/client_golang/prometheus" logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 7b9c920fae..e07e82f41f 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,9 +8,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" + ps "QmTgxFwS1nDK126fH5XPnLFcxcDFsxKbPPnCBwyRWNAjDX/go-peerstream" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - ps "github.com/jbenet/go-peerstream" ) // a Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 4dd95a579d..b986876825 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -3,15 +3,15 @@ package swarm import ( "fmt" + lgbl "github.com/ipfs/go-libp2p/loggables" mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" transport "github.com/ipfs/go-libp2p/p2p/net/transport" - lgbl "github.com/ipfs/go-libp2p/loggables" + ps "QmTgxFwS1nDK126fH5XPnLFcxcDFsxKbPPnCBwyRWNAjDX/go-peerstream" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - ps "github.com/jbenet/go-peerstream" ) // Open listeners and reuse-dialers for the given addresses diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 7965d27439..d52962ae83 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,7 +3,7 @@ package swarm import ( inet "github.com/ipfs/go-libp2p/p2p/net" - ps "github.com/jbenet/go-peerstream" + ps "QmTgxFwS1nDK126fH5XPnLFcxcDFsxKbPPnCBwyRWNAjDX/go-peerstream" ) // a Stream is a wrapper around a ps.Stream that exposes a way to get From 9a7fdeb38a4a03c75202d6b044984cf28c5a75f2 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 16 Nov 2015 17:25:35 -0800 Subject: [PATCH 012/259] remove prometheus dep --- p2p/net/swarm/swarm.go | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index a6dfa61639..3a37d30b88 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -24,7 +24,6 @@ import ( mafilter "QmYhewVqJhkgEsm3AYVUbpT14q2P9V4Xb7np9JXKua6y7A/multiaddr-filter" context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - prom "github.com/prometheus/client_golang/prometheus" logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" ) @@ -33,13 +32,6 @@ var log = logging.Logger("swarm2") var PSTransport pst.Transport -var peersTotal = prom.NewGaugeVec(prom.GaugeOpts{ - Namespace: "ipfs", - Subsystem: "p2p", - Name: "peers_total", - Help: "Number of connected peers", -}, []string{"peer_id"}) - func init() { PSTransport = psmss.NewTransport() } @@ -109,10 +101,6 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown) s.SetConnHandler(nil) // make sure to setup our own conn handler. - // setup swarm metrics - prom.MustRegisterOrGet(peersTotal) - s.Notify((*metricsNotifiee)(s)) - err = s.setupInterfaces(listenAddrs) if err != nil { return nil, err @@ -321,22 +309,3 @@ func (n *ps2netNotifee) OpenedStream(s *ps.Stream) { func (n *ps2netNotifee) ClosedStream(s *ps.Stream) { n.not.ClosedStream(n.net, inet.Stream((*Stream)(s))) } - -type metricsNotifiee Swarm - -func (nn *metricsNotifiee) Connected(n inet.Network, v inet.Conn) { - peersTotalGauge(n.LocalPeer()).Set(float64(len(n.Conns()))) -} - -func (nn *metricsNotifiee) Disconnected(n inet.Network, v inet.Conn) { - peersTotalGauge(n.LocalPeer()).Set(float64(len(n.Conns()))) -} - -func (nn *metricsNotifiee) OpenedStream(n inet.Network, v inet.Stream) {} -func (nn *metricsNotifiee) ClosedStream(n inet.Network, v inet.Stream) {} -func (nn *metricsNotifiee) Listen(n inet.Network, a ma.Multiaddr) {} -func (nn *metricsNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {} - -func peersTotalGauge(id peer.ID) prom.Gauge { - return peersTotal.With(prom.Labels{"peer_id": id.Pretty()}) -} From a120be4ee50459697721c086d27edb478a248d4a Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 18 Nov 2015 11:47:51 -0800 Subject: [PATCH 013/259] migrate to gx namespace --- p2p/net/swarm/addr/addr.go | 8 ++++---- p2p/net/swarm/addr/addr_test.go | 4 ++-- p2p/net/swarm/dial_test.go | 6 +++--- p2p/net/swarm/peers_test.go | 4 ++-- p2p/net/swarm/simul_test.go | 4 ++-- p2p/net/swarm/swarm.go | 20 ++++++++++---------- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 4 ++-- p2p/net/swarm/swarm_conn.go | 6 +++--- p2p/net/swarm/swarm_dial.go | 8 ++++---- p2p/net/swarm/swarm_listen.go | 6 +++--- p2p/net/swarm/swarm_net.go | 6 +++--- p2p/net/swarm/swarm_net_test.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 4 ++-- 16 files changed, 45 insertions(+), 45 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 00b8ed1c29..c74147527b 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,11 +3,11 @@ package addrutil import ( "fmt" - logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" + logging "gx/QmfZZB1aVXWA4kaR5R4e9NifERT366TTCSagkfhmAbYLsu/go-log" - manet "QmRCPT5WRph8aWXmaT2Rfn6ac98YRUUJnNURpD3hNAWp4f/go-multiaddr-net" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + manet "gx/QmNT7d1e4Xcp3KcsvxyzUHVtqrR43uypoxLLzdKj6YZga2/go-multiaddr-net" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 8a0bc9c35a..120195a122 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,8 +3,8 @@ package addrutil import ( "testing" - manet "QmRCPT5WRph8aWXmaT2Rfn6ac98YRUUJnNURpD3hNAWp4f/go-multiaddr-net" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + manet "gx/QmNT7d1e4Xcp3KcsvxyzUHVtqrR43uypoxLLzdKj6YZga2/go-multiaddr-net" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" ) func newMultiaddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index f915808b75..0df87fd293 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -13,9 +13,9 @@ import ( testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" - manet "QmRCPT5WRph8aWXmaT2Rfn6ac98YRUUJnNURpD3hNAWp4f/go-multiaddr-net" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + manet "gx/QmNT7d1e4Xcp3KcsvxyzUHVtqrR43uypoxLLzdKj6YZga2/go-multiaddr-net" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) func acceptAndHang(l net.Listener) { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 0068b37901..ef9c22ff60 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,8 +5,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index ec988bcd6c..408b39b244 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -9,8 +9,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" ci "github.com/ipfs/go-libp2p/testutil/ci" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 3a37d30b88..a6ddae3ccd 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -16,16 +16,16 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" - pst "QmPxuHs2NQjz16gnvndgkzHkm5PjtqbB5rwoSpLusBkQ7Q/go-stream-muxer" - psmss "QmPxuHs2NQjz16gnvndgkzHkm5PjtqbB5rwoSpLusBkQ7Q/go-stream-muxer/multistream" - "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess" - goprocessctx "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess/context" - ps "QmTgxFwS1nDK126fH5XPnLFcxcDFsxKbPPnCBwyRWNAjDX/go-peerstream" - mafilter "QmYhewVqJhkgEsm3AYVUbpT14q2P9V4Xb7np9JXKua6y7A/multiaddr-filter" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" - - logging "QmWRypnfEwrgH4k93KEHN5hng7VjKYkWmzDYRuTZeh2Mgh/go-log" + ps "gx/QmQMkNFMuh1UJbdaggASZpLtCroTnAYcWNenYL5hrzLJrT/go-peerstream" + pst "gx/QmRmT6MSnfhRDW1PTUGSd3z4fqXK48GUequQAZzeT4c5iC/go-stream-muxer" + psmss "gx/QmRmT6MSnfhRDW1PTUGSd3z4fqXK48GUequQAZzeT4c5iC/go-stream-muxer/multistream" + mafilter "gx/QmVCmuhgDFer5MW5737Z8GtBEGpUyEkFnLUv4ASDWnLZdC/multiaddr-filter" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + "gx/QmfDXyLfKNfja2XebomRZjZ2UZCa4BDyFoCymKtzNRVQ5b/goprocess" + goprocessctx "gx/QmfDXyLfKNfja2XebomRZjZ2UZCa4BDyFoCymKtzNRVQ5b/goprocess/context" + + logging "gx/QmfZZB1aVXWA4kaR5R4e9NifERT366TTCSagkfhmAbYLsu/go-log" ) var log = logging.Logger("swarm2") diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 6149eb66e3..81a77bb8e0 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -4,7 +4,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index b397a07f62..7383beeabe 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -8,8 +8,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index e07e82f41f..e7d1ec91dc 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,9 +8,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" - ps "QmTgxFwS1nDK126fH5XPnLFcxcDFsxKbPPnCBwyRWNAjDX/go-peerstream" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ps "gx/QmQMkNFMuh1UJbdaggASZpLtCroTnAYcWNenYL5hrzLJrT/go-peerstream" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) // a Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 11b3f20463..fa4fce1e47 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -8,14 +8,14 @@ import ( "sync" "time" - "QmRCPT5WRph8aWXmaT2Rfn6ac98YRUUJnNURpD3hNAWp4f/go-multiaddr-net" + lgbl "github.com/ipfs/go-libp2p/loggables" conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - lgbl "github.com/ipfs/go-libp2p/loggables" + "gx/QmNT7d1e4Xcp3KcsvxyzUHVtqrR43uypoxLLzdKj6YZga2/go-multiaddr-net" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index b986876825..bd1810b462 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -9,9 +9,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" transport "github.com/ipfs/go-libp2p/p2p/net/transport" - ps "QmTgxFwS1nDK126fH5XPnLFcxcDFsxKbPPnCBwyRWNAjDX/go-peerstream" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ps "gx/QmQMkNFMuh1UJbdaggASZpLtCroTnAYcWNenYL5hrzLJrT/go-peerstream" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) // Open listeners and reuse-dialers for the given addresses diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 36416fa394..d8724a49f9 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -8,9 +8,9 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - "QmSir6qPL1tjuxd8LkR8VZq6v625ExAUVs2eCLeqQuaPGU/goprocess" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + "gx/QmfDXyLfKNfja2XebomRZjZ2UZCa4BDyFoCymKtzNRVQ5b/goprocess" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 2db22fbb35..c40b89cc63 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/p2p/test/util" ) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 1180d641ff..9b59bb24d9 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" inet "github.com/ipfs/go-libp2p/p2p/net" peer "github.com/ipfs/go-libp2p/p2p/peer" diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index d52962ae83..f4ee314958 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,7 +3,7 @@ package swarm import ( inet "github.com/ipfs/go-libp2p/p2p/net" - ps "QmTgxFwS1nDK126fH5XPnLFcxcDFsxKbPPnCBwyRWNAjDX/go-peerstream" + ps "gx/QmQMkNFMuh1UJbdaggASZpLtCroTnAYcWNenYL5hrzLJrT/go-peerstream" ) // a Stream is a wrapper around a ps.Stream that exposes a way to get diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 670ea26592..c8a1ce548e 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -14,8 +14,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" - context "QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - ma "QmbWxL1aXQhBjc1XGjGF1f2KGBMCBYSuT2ThA8YXnXJK83/go-multiaddr" + ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" ) func EchoStreamHandler(stream inet.Stream) { From a478405a422fcfc24d8288d2232604208454e98d Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 19 Nov 2015 16:20:59 -0800 Subject: [PATCH 014/259] make new stream calls accept a context --- p2p/net/swarm/swarm.go | 5 +++-- p2p/net/swarm/swarm_net.go | 4 ++-- p2p/net/swarm/swarm_test.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index a6ddae3ccd..41c2ca95b0 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -192,16 +192,17 @@ func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) { } // NewStreamWithPeer creates a new stream on any available connection to p -func (s *Swarm) NewStreamWithPeer(p peer.ID) (*Stream, error) { +func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, error) { // if we have no connections, try connecting. if len(s.ConnectionsToPeer(p)) == 0 { log.Debug("Swarm: NewStreamWithPeer no connections. Attempting to connect...") - if _, err := s.Dial(s.Context(), p); err != nil { + if _, err := s.Dial(ctx, p); err != nil { return nil, err } } log.Debug("Swarm: NewStreamWithPeer...") + // TODO: think about passing a context down to NewStreamWithGroup st, err := s.swarm.NewStreamWithGroup(p) return wrapStream(st), err } diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index d8724a49f9..e5a6e0216a 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -132,9 +132,9 @@ func (n *Network) Connectedness(p peer.ID) inet.Connectedness { // NewStream returns a new stream to given peer p. // If there is no connection to p, attempts to create one. -func (n *Network) NewStream(p peer.ID) (inet.Stream, error) { +func (n *Network) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) { log.Debugf("[%s] network opening stream to peer [%s]", n.local, p) - s, err := n.Swarm().NewStreamWithPeer(p) + s, err := n.Swarm().NewStreamWithPeer(ctx, p) if err != nil { return nil, err } diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index c8a1ce548e..b6aa692cb0 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -129,7 +129,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { defer wg.Done() // first, one stream per peer (nice) - stream, err := s1.NewStreamWithPeer(p) + stream, err := s1.NewStreamWithPeer(ctx, p) if err != nil { errChan <- err return From 2d1b1db0e4f15d351dd3aa7dd64815c885d8588c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 6 Dec 2015 23:00:10 -0800 Subject: [PATCH 015/259] WIP --- p2p/net/swarm/addr/addr.go | 8 ++++---- p2p/net/swarm/addr/addr_test.go | 4 ++-- p2p/net/swarm/dial_test.go | 6 +++--- p2p/net/swarm/peers_test.go | 4 ++-- p2p/net/swarm/simul_test.go | 4 ++-- p2p/net/swarm/swarm.go | 20 ++++++++++---------- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 4 ++-- p2p/net/swarm/swarm_conn.go | 6 +++--- p2p/net/swarm/swarm_dial.go | 6 +++--- p2p/net/swarm/swarm_listen.go | 6 +++--- p2p/net/swarm/swarm_net.go | 6 +++--- p2p/net/swarm/swarm_net_test.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 4 ++-- 16 files changed, 44 insertions(+), 44 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index c74147527b..39f00dba1c 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,11 +3,11 @@ package addrutil import ( "fmt" - logging "gx/QmfZZB1aVXWA4kaR5R4e9NifERT366TTCSagkfhmAbYLsu/go-log" + logging "github.com/ipfs/go-log" - manet "gx/QmNT7d1e4Xcp3KcsvxyzUHVtqrR43uypoxLLzdKj6YZga2/go-multiaddr-net" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" + context "golang.org/x/net/context" ) var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 120195a122..49eec13511 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,8 +3,8 @@ package addrutil import ( "testing" - manet "gx/QmNT7d1e4Xcp3KcsvxyzUHVtqrR43uypoxLLzdKj6YZga2/go-multiaddr-net" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" ) func newMultiaddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 0df87fd293..57a4d7f161 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -13,9 +13,9 @@ import ( testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" - manet "gx/QmNT7d1e4Xcp3KcsvxyzUHVtqrR43uypoxLLzdKj6YZga2/go-multiaddr-net" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" + context "golang.org/x/net/context" ) func acceptAndHang(l net.Listener) { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index ef9c22ff60..b107fbfe83 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,8 +5,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 408b39b244..bd753e7660 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -9,8 +9,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" ci "github.com/ipfs/go-libp2p/testutil/ci" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 41c2ca95b0..c71ef359de 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -16,16 +16,16 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" - ps "gx/QmQMkNFMuh1UJbdaggASZpLtCroTnAYcWNenYL5hrzLJrT/go-peerstream" - pst "gx/QmRmT6MSnfhRDW1PTUGSd3z4fqXK48GUequQAZzeT4c5iC/go-stream-muxer" - psmss "gx/QmRmT6MSnfhRDW1PTUGSd3z4fqXK48GUequQAZzeT4c5iC/go-stream-muxer/multistream" - mafilter "gx/QmVCmuhgDFer5MW5737Z8GtBEGpUyEkFnLUv4ASDWnLZdC/multiaddr-filter" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - "gx/QmfDXyLfKNfja2XebomRZjZ2UZCa4BDyFoCymKtzNRVQ5b/goprocess" - goprocessctx "gx/QmfDXyLfKNfja2XebomRZjZ2UZCa4BDyFoCymKtzNRVQ5b/goprocess/context" - - logging "gx/QmfZZB1aVXWA4kaR5R4e9NifERT366TTCSagkfhmAbYLsu/go-log" + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + pst "github.com/jbenet/go-stream-muxer" + psmss "github.com/jbenet/go-stream-muxer/multistream" + "github.com/jbenet/goprocess" + goprocessctx "github.com/jbenet/goprocess/context" + mafilter "github.com/whyrusleeping/multiaddr-filter" + context "golang.org/x/net/context" + + logging "github.com/ipfs/go-log" ) var log = logging.Logger("swarm2") diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 81a77bb8e0..039683bfb9 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -4,7 +4,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 7383beeabe..7db46d3444 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -8,8 +8,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index e7d1ec91dc..840a9b465c 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,9 +8,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" - ps "gx/QmQMkNFMuh1UJbdaggASZpLtCroTnAYcWNenYL5hrzLJrT/go-peerstream" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + context "golang.org/x/net/context" ) // a Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index fa4fce1e47..0cb2858768 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -12,10 +12,10 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - "gx/QmNT7d1e4Xcp3KcsvxyzUHVtqrR43uypoxLLzdKj6YZga2/go-multiaddr-net" + "github.com/jbenet/go-multiaddr-net" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index bd1810b462..f184640658 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -9,9 +9,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" transport "github.com/ipfs/go-libp2p/p2p/net/transport" - ps "gx/QmQMkNFMuh1UJbdaggASZpLtCroTnAYcWNenYL5hrzLJrT/go-peerstream" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + context "golang.org/x/net/context" ) // Open listeners and reuse-dialers for the given addresses diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index e5a6e0216a..a8361fc5bb 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -8,9 +8,9 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" - "gx/QmfDXyLfKNfja2XebomRZjZ2UZCa4BDyFoCymKtzNRVQ5b/goprocess" + ma "github.com/jbenet/go-multiaddr" + "github.com/jbenet/goprocess" + context "golang.org/x/net/context" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index c40b89cc63..14c590b9d1 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -5,9 +5,9 @@ import ( "testing" "time" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/p2p/test/util" + context "golang.org/x/net/context" ) // TestConnectednessCorrect starts a few networks, connects a few diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 9b59bb24d9..0d976e81d0 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" inet "github.com/ipfs/go-libp2p/p2p/net" peer "github.com/ipfs/go-libp2p/p2p/peer" diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index f4ee314958..7965d27439 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,7 +3,7 @@ package swarm import ( inet "github.com/ipfs/go-libp2p/p2p/net" - ps "gx/QmQMkNFMuh1UJbdaggASZpLtCroTnAYcWNenYL5hrzLJrT/go-peerstream" + ps "github.com/jbenet/go-peerstream" ) // a Stream is a wrapper around a ps.Stream that exposes a way to get diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index b6aa692cb0..a1c35696dc 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -14,8 +14,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" - ma "gx/QmVUi2ncqnU48zsPgR1rQosDGwY3SSZ1Ndp33j33YjXdsj/go-multiaddr" - context "gx/QmacZi9WygGK7Me8mH53pypyscHzU386aUZXpr28GZgUct/context" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func EchoStreamHandler(stream inet.Stream) { From 3cdb2f50f7ac455ed13e646789f6ddb8f8ebe01a Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 4 Jan 2016 05:45:58 -0800 Subject: [PATCH 016/259] path rewrites --- p2p/net/swarm/addr/addr.go | 8 ++++---- p2p/net/swarm/addr/addr_test.go | 4 ++-- p2p/net/swarm/dial_test.go | 6 +++--- p2p/net/swarm/peers_test.go | 4 ++-- p2p/net/swarm/simul_test.go | 4 ++-- p2p/net/swarm/swarm.go | 20 ++++++++++---------- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 4 ++-- p2p/net/swarm/swarm_conn.go | 6 +++--- p2p/net/swarm/swarm_dial.go | 6 +++--- p2p/net/swarm/swarm_listen.go | 6 +++--- p2p/net/swarm/swarm_net.go | 6 +++--- p2p/net/swarm/swarm_net_test.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 4 ++-- 16 files changed, 44 insertions(+), 44 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 39f00dba1c..8711e8daec 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,11 +3,11 @@ package addrutil import ( "fmt" - logging "github.com/ipfs/go-log" + logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" - context "golang.org/x/net/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + manet "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 49eec13511..aa82097c3d 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,8 +3,8 @@ package addrutil import ( "testing" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + manet "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net" ) func newMultiaddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 57a4d7f161..72dd85860a 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -13,9 +13,9 @@ import ( testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" - context "golang.org/x/net/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + manet "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) func acceptAndHang(l net.Listener) { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index b107fbfe83..3451ec1f51 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,8 +5,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index bd753e7660..a838f3044d 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -9,8 +9,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" ci "github.com/ipfs/go-libp2p/testutil/ci" - ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c71ef359de..c422645c5c 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -16,16 +16,16 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "github.com/jbenet/go-multiaddr" - ps "github.com/jbenet/go-peerstream" - pst "github.com/jbenet/go-stream-muxer" - psmss "github.com/jbenet/go-stream-muxer/multistream" - "github.com/jbenet/goprocess" - goprocessctx "github.com/jbenet/goprocess/context" - mafilter "github.com/whyrusleeping/multiaddr-filter" - context "golang.org/x/net/context" - - logging "github.com/ipfs/go-log" + ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" + "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" + goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + pst "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer" + psmss "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer/multistream" + mafilter "gx/ipfs/QmW3Q7RQa8D1qCEEeyKCBV1drgFxvHBqAZ3zgCujEwKpHD/multiaddr-filter" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + + logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" ) var log = logging.Logger("swarm2") diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 039683bfb9..5b7aae5fb2 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -4,7 +4,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - ma "github.com/jbenet/go-multiaddr" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 7db46d3444..f48291bcd1 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -8,8 +8,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" - ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 840a9b465c..b8c5633f1e 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,9 +8,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "github.com/jbenet/go-multiaddr" - ps "github.com/jbenet/go-peerstream" - context "golang.org/x/net/context" + ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) // a Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 0cb2858768..e05a2a2f37 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -12,10 +12,10 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - "github.com/jbenet/go-multiaddr-net" + "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net" - ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index f184640658..4584250f7d 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -9,9 +9,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" transport "github.com/ipfs/go-libp2p/p2p/net/transport" - ma "github.com/jbenet/go-multiaddr" - ps "github.com/jbenet/go-peerstream" - context "golang.org/x/net/context" + ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) // Open listeners and reuse-dialers for the given addresses diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index a8361fc5bb..1d75814da1 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -8,9 +8,9 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - ma "github.com/jbenet/go-multiaddr" - "github.com/jbenet/goprocess" - context "golang.org/x/net/context" + "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 14c590b9d1..1ffb44812e 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -7,7 +7,7 @@ import ( inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/p2p/test/util" - context "golang.org/x/net/context" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) // TestConnectednessCorrect starts a few networks, connects a few diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 0d976e81d0..d6f06da7ce 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" inet "github.com/ipfs/go-libp2p/p2p/net" peer "github.com/ipfs/go-libp2p/p2p/peer" diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 7965d27439..3f9115fc6c 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,7 +3,7 @@ package swarm import ( inet "github.com/ipfs/go-libp2p/p2p/net" - ps "github.com/jbenet/go-peerstream" + ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" ) // a Stream is a wrapper around a ps.Stream that exposes a way to get diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index a1c35696dc..1166f52f1f 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -14,8 +14,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" - ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" + ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ) func EchoStreamHandler(stream inet.Stream) { From b9674575f9b130d076e1feda0ec5bf4d226d2870 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 5 Jan 2016 14:24:02 -0800 Subject: [PATCH 017/259] add utp to have feature parity with go-ipfs --- p2p/net/swarm/swarm.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c422645c5c..572933f9da 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -84,13 +84,16 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, } s := &Swarm{ - swarm: ps.NewSwarm(PSTransport), - local: local, - peers: peers, - ctx: ctx, - dialT: DialTimeout, - notifs: make(map[inet.Notifiee]ps.Notifiee), - transports: []transport.Transport{transport.NewTCPTransport()}, + swarm: ps.NewSwarm(PSTransport), + local: local, + peers: peers, + ctx: ctx, + dialT: DialTimeout, + notifs: make(map[inet.Notifiee]ps.Notifiee), + transports: []transport.Transport{ + transport.NewTCPTransport(), + transport.NewUtpTransport(), + }, bwc: bwc, fdRateLimit: make(chan struct{}, concurrentFdDials), Filters: filter.NewFilters(), From a8f9f6caca31f32d4d04b38be3e26cb3da465809 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 30 Jan 2016 17:12:33 -0800 Subject: [PATCH 018/259] update to utp code from master of go-ipfs --- p2p/net/swarm/addr/addr.go | 4 ++-- p2p/net/swarm/addr/addr_test.go | 8 +------- p2p/net/swarm/dial_test.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 6 +++--- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 8711e8daec..4fdda74d4a 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -18,8 +18,8 @@ var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") var SupportedTransportStrings = []string{ "/ip4/tcp", "/ip6/tcp", - // "/ip4/udp/utp", disabled because the lib is broken - // "/ip6/udp/utp", disabled because the lib is broken + "/ip4/udp/utp", + "/ip6/udp/utp", // "/ip4/udp/udt", disabled because the lib doesnt work on arm // "/ip6/udp/udt", disabled because the lib doesnt work on arm } diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index aa82097c3d..2b0532ceb3 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -20,7 +20,6 @@ func TestFilterAddrs(t *testing.T) { bad := []ma.Multiaddr{ newMultiaddr(t, "/ip4/1.2.3.4/udp/1234"), // unreliable newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/sctp/1234"), // not in manet - newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/utp"), // utp is broken newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/udt"), // udt is broken on arm newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), // link local newMultiaddr(t, "/ip6/fe80::100/tcp/1234"), // link local @@ -29,6 +28,7 @@ func TestFilterAddrs(t *testing.T) { good := []ma.Multiaddr{ newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), newMultiaddr(t, "/ip6/::1/tcp/1234"), + newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/utp"), } goodAndBad := append(good, bad...) @@ -39,18 +39,12 @@ func TestFilterAddrs(t *testing.T) { if AddrUsable(a, false) { t.Errorf("addr %s should be unusable", a) } - if AddrUsable(a, true) { - t.Errorf("addr %s should be unusable", a) - } } for _, a := range good { if !AddrUsable(a, false) { t.Errorf("addr %s should be usable", a) } - if !AddrUsable(a, true) { - t.Errorf("addr %s should be usable", a) - } } subtestAddrsEqual(t, FilterUsableAddrs(bad), []ma.Multiaddr{}) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 72dd85860a..127edffd52 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -151,7 +151,7 @@ func TestDialWait(t *testing.T) { func TestDialBackoff(t *testing.T) { // t.Skip("skipping for another test") if ci.IsRunning() { - t.Skip("travis and jenkins will never have fun with this test") + t.Skip("travis will never have fun with this test") } t.Parallel() diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index f48291bcd1..214916e47e 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -25,7 +25,6 @@ func TestFilterAddrs(t *testing.T) { bad := []ma.Multiaddr{ m("/ip4/1.2.3.4/udp/1234"), // unreliable m("/ip4/1.2.3.4/udp/1234/sctp/1234"), // not in manet - m("/ip4/1.2.3.4/udp/1234/utp"), // utp is broken m("/ip4/1.2.3.4/udp/1234/udt"), // udt is broken on arm m("/ip6/fe80::1/tcp/0"), // link local m("/ip6/fe80::100/tcp/1234"), // link local @@ -34,6 +33,7 @@ func TestFilterAddrs(t *testing.T) { good := []ma.Multiaddr{ m("/ip4/127.0.0.1/tcp/0"), m("/ip6/::1/tcp/0"), + m("/ip4/1.2.3.4/udp/1234/utp"), } goodAndBad := append(good, bad...) @@ -41,13 +41,13 @@ func TestFilterAddrs(t *testing.T) { // test filters for _, a := range bad { - if addrutil.AddrUsable(a, true) { + if addrutil.AddrUsable(a, false) { t.Errorf("addr %s should be unusable", a) } } for _, a := range good { - if !addrutil.AddrUsable(a, true) { + if !addrutil.AddrUsable(a, false) { t.Errorf("addr %s should be usable", a) } } From a7a48ae51108234952e077857cfe64b126525f10 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 13 Feb 2016 16:22:02 -0800 Subject: [PATCH 019/259] change listener errors to warnings --- p2p/net/swarm/swarm_listen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 4584250f7d..045ac9ccf8 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -126,7 +126,7 @@ func (s *Swarm) addConnListener(list conn.Listener) error { if !more { return } - log.Errorf("swarm listener accept error: %s", err) + log.Warningf("swarm listener accept error: %s", err) case <-ctx.Done(): return } From 55941459a299811d732c2482e02bffd49699a515 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 3 Mar 2016 13:03:24 -0800 Subject: [PATCH 020/259] update version of go-multiaddr --- p2p/net/swarm/addr/addr.go | 7 +++---- p2p/net/swarm/addr/addr_test.go | 4 ++-- p2p/net/swarm/dial_test.go | 4 ++-- p2p/net/swarm/peers_test.go | 3 +-- p2p/net/swarm/simul_test.go | 4 ++-- p2p/net/swarm/swarm.go | 5 ++--- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_dial.go | 5 ++--- p2p/net/swarm/swarm_listen.go | 5 ++--- p2p/net/swarm/swarm_net.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 5 ++--- p2p/net/swarm/swarm_test.go | 2 +- 14 files changed, 23 insertions(+), 29 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 4fdda74d4a..3cf1b7d149 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,11 +3,10 @@ package addrutil import ( "fmt" - logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" - - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" - manet "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net" + manet "gx/ipfs/QmQB7mNP3QE7b4zP2MQmsyJDqG5hzYE2CL8k1VyLWky2Ed/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 2b0532ceb3..af291fc3f7 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,8 +3,8 @@ package addrutil import ( "testing" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" - manet "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net" + manet "gx/ipfs/QmQB7mNP3QE7b4zP2MQmsyJDqG5hzYE2CL8k1VyLWky2Ed/go-multiaddr-net" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) func newMultiaddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 127edffd52..66bac550a8 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -13,9 +13,9 @@ import ( testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" - manet "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net" + manet "gx/ipfs/QmQB7mNP3QE7b4zP2MQmsyJDqG5hzYE2CL8k1VyLWky2Ed/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) func acceptAndHang(l net.Listener) { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 3451ec1f51..8b71f77f34 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,12 +5,11 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) func TestPeers(t *testing.T) { - // t.Skip("skipping for another test") ctx := context.Background() swarms := makeSwarms(ctx, t, 2) diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index a838f3044d..8c626b805d 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -9,12 +9,12 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" ci "github.com/ipfs/go-libp2p/testutil/ci" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) func TestSimultOpen(t *testing.T) { - // t.Skip("skipping for another test") + t.Parallel() ctx := context.Background() diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 572933f9da..8e4f32f2e1 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -19,13 +19,12 @@ import ( ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" pst "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer" psmss "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer/multistream" - mafilter "gx/ipfs/QmW3Q7RQa8D1qCEEeyKCBV1drgFxvHBqAZ3zgCujEwKpHD/multiaddr-filter" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" + mafilter "gx/ipfs/QmcR6dLYF8Eozaae3wGd5wjq76bofzmmbvQmtwobxvfhEt/multiaddr-filter" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) var log = logging.Logger("swarm2") diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 5b7aae5fb2..faab0a3fdb 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -4,7 +4,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 214916e47e..2e69e14e11 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -8,8 +8,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index b8c5633f1e..a07818d48a 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -9,8 +9,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) // a Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index e05a2a2f37..96cccd3c97 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -12,10 +12,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net" - - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" + "gx/ipfs/QmQB7mNP3QE7b4zP2MQmsyJDqG5hzYE2CL8k1VyLWky2Ed/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 045ac9ccf8..50550248f5 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -10,11 +10,10 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" -) + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" +) // Open listeners and reuse-dialers for the given addresses -// Open listeners and reuse-dialers for the given addresses func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { errs := make([]error, len(addrs)) var succeeded int diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 1d75814da1..863da8efec 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -9,8 +9,8 @@ import ( inet "github.com/ipfs/go-libp2p/p2p/net" "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index d6f06da7ce..9a8052e9ef 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,11 +4,10 @@ import ( "testing" "time" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - inet "github.com/ipfs/go-libp2p/p2p/net" peer "github.com/ipfs/go-libp2p/p2p/peer" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) func TestNotifications(t *testing.T) { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 1166f52f1f..121d1b0f35 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -14,8 +14,8 @@ import ( peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" - ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) func EchoStreamHandler(stream inet.Stream) { From 5f0753d75f2980c79b514505511295c91d72e658 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 4 Mar 2016 09:27:52 -0800 Subject: [PATCH 021/259] Add fallback dialer Previously we were unable to dial on a given transport if there were no listener addresses defined for that. This meant that most people couldnt use utp (as they had no utp listener transport to dial from). --- p2p/net/swarm/dial_test.go | 55 +++++++++++++++++++++++++++++++++++++ p2p/net/swarm/swarm_test.go | 25 ++++++++++++++--- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 66bac550a8..a6bc84e861 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -18,6 +18,61 @@ import ( ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) +func closeSwarms(swarms []*Swarm) { + for _, s := range swarms { + s.Close() + } +} + +func TestBasicDial(t *testing.T) { + t.Parallel() + ctx := context.Background() + + swarms := makeSwarms(ctx, t, 2) + defer closeSwarms(swarms) + s1 := swarms[0] + s2 := swarms[1] + + s1.peers.AddAddrs(s2.local, s2.ListenAddresses(), peer.PermanentAddrTTL) + + c, err := s1.Dial(ctx, s2.local) + if err != nil { + t.Fatal(err) + } + + s, err := c.NewStream() + if err != nil { + t.Fatal(err) + } + + s.Close() +} + +func TestDialWithNoListeners(t *testing.T) { + t.Parallel() + ctx := context.Background() + + s1 := makeDialOnlySwarm(ctx, t) + + swarms := makeSwarms(ctx, t, 1) + defer closeSwarms(swarms) + s2 := swarms[0] + + s1.peers.AddAddrs(s2.local, s2.ListenAddresses(), peer.PermanentAddrTTL) + + c, err := s1.Dial(ctx, s2.local) + if err != nil { + t.Fatal(err) + } + + s, err := c.NewStream() + if err != nil { + t.Fatal(err) + } + + s.Close() +} + func acceptAndHang(l net.Listener) { conns := make([]net.Conn, 0, 10) for { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 121d1b0f35..fe2ef2a207 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -24,31 +24,48 @@ func EchoStreamHandler(stream inet.Stream) { // pull out the ipfs conn c := stream.Conn() - log.Infof("%s ponging to %s", c.LocalPeer(), c.RemotePeer()) + log.Errorf("%s ponging to %s", c.LocalPeer(), c.RemotePeer()) buf := make([]byte, 4) for { if _, err := stream.Read(buf); err != nil { if err != io.EOF { - log.Info("ping receive error:", err) + log.Error("ping receive error:", err) } return } if !bytes.Equal(buf, []byte("ping")) { - log.Infof("ping receive error: ping != %s %v", buf, buf) + log.Errorf("ping receive error: ping != %s %v", buf, buf) return } if _, err := stream.Write([]byte("pong")); err != nil { - log.Info("pond send error:", err) + log.Error("pond send error:", err) return } } }() } +func makeDialOnlySwarm(ctx context.Context, t *testing.T) *Swarm { + id := testutil.RandIdentityOrFatal(t) + + peerstore := peer.NewPeerstore() + peerstore.AddPubKey(id.ID(), id.PublicKey()) + peerstore.AddPrivKey(id.ID(), id.PrivateKey()) + + swarm, err := NewSwarm(ctx, nil, id.ID(), peerstore, metrics.NewBandwidthCounter()) + if err != nil { + t.Fatal(err) + } + + swarm.SetStreamHandler(EchoStreamHandler) + + return swarm +} + func makeSwarms(ctx context.Context, t *testing.T, num int) []*Swarm { swarms := make([]*Swarm, 0, num) From 01d6273db10b17399e772652b497b885be4d9788 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 7 Mar 2016 22:43:13 -0800 Subject: [PATCH 022/259] switch to new version of go-stream-muxer --- p2p/net/swarm/swarm.go | 25 +++++++++++++++++++++---- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_listen.go | 6 +++--- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 2 +- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 8e4f32f2e1..0fd7a25e45 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -4,6 +4,7 @@ package swarm import ( "fmt" + "io/ioutil" "sync" "time" @@ -16,11 +17,13 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" - ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context" - pst "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer" - psmss "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer/multistream" + pst "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer" + psmss "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/multistream" + spdy "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/spdystream" + yamux "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/yamux" + ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" mafilter "gx/ipfs/QmcR6dLYF8Eozaae3wGd5wjq76bofzmmbvQmtwobxvfhEt/multiaddr-filter" @@ -32,7 +35,21 @@ var log = logging.Logger("swarm2") var PSTransport pst.Transport func init() { - PSTransport = psmss.NewTransport() + msstpt := psmss.NewBlankTransport() + + ymxtpt := &yamux.Transport{ + AcceptBacklog: 2048, + ConnectionWriteTimeout: time.Second * 10, + KeepAliveInterval: time.Second * 30, + EnableKeepAlive: true, + MaxStreamWindowSize: uint32(1024 * 256), + LogOutput: ioutil.Discard, + } + + msstpt.AddTransport("/yamux", ymxtpt) + msstpt.AddTransport("/spdystream", spdy.Transport) + + PSTransport = msstpt } // Swarm is a connection muxer, allowing connections to other peers to diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index a07818d48a..04e9a4034f 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -8,7 +8,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" peer "github.com/ipfs/go-libp2p/p2p/peer" - ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" + ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 50550248f5..a4d5e943d1 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -9,10 +9,10 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" transport "github.com/ipfs/go-libp2p/p2p/net/transport" - ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" + ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" -) // Open listeners and reuse-dialers for the given addresses +) // Open listeners and reuse-dialers for the given addresses func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { errs := make([]error, len(addrs)) @@ -152,7 +152,7 @@ func (s *Swarm) connHandler(c *ps.Conn) *Conn { if err != nil { log.Debug(err) log.Event(ctx, "newConnHandlerDisconnect", lgbl.NetConn(c.NetConn()), lgbl.Error(err)) - c.Close() // boom. close it. + c.Close() // boom. close it. return nil } diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 3f9115fc6c..3fbcc5d37c 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,7 +3,7 @@ package swarm import ( inet "github.com/ipfs/go-libp2p/p2p/net" - ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream" + ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" ) // a Stream is a wrapper around a ps.Stream that exposes a way to get diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index fe2ef2a207..c69c7db38f 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -24,7 +24,7 @@ func EchoStreamHandler(stream inet.Stream) { // pull out the ipfs conn c := stream.Conn() - log.Errorf("%s ponging to %s", c.LocalPeer(), c.RemotePeer()) + log.Infof("%s ponging to %s", c.LocalPeer(), c.RemotePeer()) buf := make([]byte, 4) From 7110cd7c80bc6ae2710f77a404b851b8757bcfe6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 8 Mar 2016 15:47:25 -0800 Subject: [PATCH 023/259] bump yamux backlog setting way up --- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_listen.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 0fd7a25e45..fd66f73027 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -38,7 +38,7 @@ func init() { msstpt := psmss.NewBlankTransport() ymxtpt := &yamux.Transport{ - AcceptBacklog: 2048, + AcceptBacklog: 8192, ConnectionWriteTimeout: time.Second * 10, KeepAliveInterval: time.Second * 30, EnableKeepAlive: true, diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index a4d5e943d1..b83df4551c 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -12,8 +12,9 @@ import ( ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" -) // Open listeners and reuse-dialers for the given addresses +) +// Open listeners and reuse-dialers for the given addresses func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { errs := make([]error, len(addrs)) var succeeded int @@ -152,7 +153,7 @@ func (s *Swarm) connHandler(c *ps.Conn) *Conn { if err != nil { log.Debug(err) log.Event(ctx, "newConnHandlerDisconnect", lgbl.NetConn(c.NetConn()), lgbl.Error(err)) - c.Close() // boom. close it. + c.Close() // boom. close it. return nil } From b4e7e8a85143b872c5078ecacbbc39c4b91655dc Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 29 Mar 2016 18:38:24 -0700 Subject: [PATCH 024/259] update utp lib --- p2p/net/swarm/addr/addr.go | 2 +- p2p/net/swarm/addr/addr_test.go | 2 +- p2p/net/swarm/dial_test.go | 2 +- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_dial.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 3cf1b7d149..14da595090 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,7 +3,7 @@ package addrutil import ( "fmt" - manet "gx/ipfs/QmQB7mNP3QE7b4zP2MQmsyJDqG5hzYE2CL8k1VyLWky2Ed/go-multiaddr-net" + manet "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index af291fc3f7..16599b4089 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,7 +3,7 @@ package addrutil import ( "testing" - manet "gx/ipfs/QmQB7mNP3QE7b4zP2MQmsyJDqG5hzYE2CL8k1VyLWky2Ed/go-multiaddr-net" + manet "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index a6bc84e861..6e987e24dd 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -13,7 +13,7 @@ import ( testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" - manet "gx/ipfs/QmQB7mNP3QE7b4zP2MQmsyJDqG5hzYE2CL8k1VyLWky2Ed/go-multiaddr-net" + manet "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index fd66f73027..543c01c214 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -17,6 +17,7 @@ import ( transport "github.com/ipfs/go-libp2p/p2p/net/transport" peer "github.com/ipfs/go-libp2p/p2p/peer" + mafilter "gx/ipfs/QmPwfFAHUmvWDucLHRS9Xz2Kb1TNX2cY4LJ7pQjg9kVcae/multiaddr-filter" "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context" pst "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer" @@ -26,7 +27,6 @@ import ( ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" - mafilter "gx/ipfs/QmcR6dLYF8Eozaae3wGd5wjq76bofzmmbvQmtwobxvfhEt/multiaddr-filter" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 96cccd3c97..283bef1afb 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -12,7 +12,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" peer "github.com/ipfs/go-libp2p/p2p/peer" - "gx/ipfs/QmQB7mNP3QE7b4zP2MQmsyJDqG5hzYE2CL8k1VyLWky2Ed/go-multiaddr-net" + "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) From a81c312df58891cfbcacf5a825a2ee8398055f59 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 6 Apr 2016 12:14:06 -0700 Subject: [PATCH 025/259] version multistream protocol tags --- p2p/net/swarm/swarm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 543c01c214..b8e041e811 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -46,8 +46,8 @@ func init() { LogOutput: ioutil.Discard, } - msstpt.AddTransport("/yamux", ymxtpt) - msstpt.AddTransport("/spdystream", spdy.Transport) + msstpt.AddTransport("/yamux/1.0.0", ymxtpt) + msstpt.AddTransport("/spdy/3.1.0", spdy.Transport) PSTransport = msstpt } From d4973ee01b624d92a93b3355d3db041662035863 Mon Sep 17 00:00:00 2001 From: Lars Gierth Date: Sun, 10 Apr 2016 22:58:16 -0700 Subject: [PATCH 026/259] Fix address filtering for /ip6, add tests License: MIT Signed-off-by: Lars Gierth --- p2p/net/swarm/swarm_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index c69c7db38f..6c81a9c14f 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -303,7 +303,7 @@ func TestAddrBlocking(t *testing.T) { swarms := makeSwarms(ctx, t, 2) swarms[0].SetConnHandler(func(conn *Conn) { - t.Fatal("no connections should happen!") + t.Fatalf("no connections should happen! -- %s", conn) }) _, block, err := net.ParseCIDR("127.0.0.1/8") From c218be3e56b1764e3648a9efdc933ba26259a1b9 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 11 Apr 2016 23:24:32 -0700 Subject: [PATCH 027/259] move some deps out as gx packages --- p2p/net/swarm/dial_test.go | 3 +-- p2p/net/swarm/peers_test.go | 3 +-- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm.go | 5 ++--- p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 4 ++-- p2p/net/swarm/swarm_dial.go | 4 ++-- p2p/net/swarm/swarm_listen.go | 5 ++--- p2p/net/swarm/swarm_net.go | 3 +-- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_test.go | 2 +- 11 files changed, 15 insertions(+), 20 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 6e987e24dd..d21af292d8 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -8,10 +8,9 @@ import ( "time" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - peer "github.com/ipfs/go-libp2p/p2p/peer" - testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" manet "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 8b71f77f34..9976d3bf4c 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -3,8 +3,7 @@ package swarm import ( "testing" - peer "github.com/ipfs/go-libp2p/p2p/peer" - + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 8c626b805d..b0256b8489 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p/p2p/peer" ci "github.com/ipfs/go-libp2p/testutil/ci" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index b8e041e811..5326bf4a7f 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -14,16 +14,15 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" filter "github.com/ipfs/go-libp2p/p2p/net/filter" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - transport "github.com/ipfs/go-libp2p/p2p/net/transport" - peer "github.com/ipfs/go-libp2p/p2p/peer" - mafilter "gx/ipfs/QmPwfFAHUmvWDucLHRS9Xz2Kb1TNX2cY4LJ7pQjg9kVcae/multiaddr-filter" "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context" + transport "gx/ipfs/QmR4HhZm1fL8epXz8661Ru8ks5Y1Uk2u51quAWaCTD1zHg/go-libp2p-transport" pst "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer" psmss "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/multistream" spdy "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/spdystream" yamux "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/yamux" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 2e69e14e11..9cef367c78 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -5,8 +5,8 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 04e9a4034f..3ff2b91fcc 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -3,13 +3,13 @@ package swarm import ( "fmt" - ic "github.com/ipfs/go-libp2p/p2p/crypto" inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" - peer "github.com/ipfs/go-libp2p/p2p/peer" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + ic "gx/ipfs/QmaP38GJApheTr84f8R89vsT7oJLQw1AeCz4HqrQgv2njB/go-libp2p-crypto" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 283bef1afb..b569f92fa4 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -8,10 +8,10 @@ import ( "sync" "time" - lgbl "github.com/ipfs/go-libp2p/loggables" conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - peer "github.com/ipfs/go-libp2p/p2p/peer" + lgbl "gx/ipfs/QmSyBhZt2upyQ3NJmTpab1pX6hesA59vcYTGmgoDorZZbw/go-libp2p-loggables" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index b83df4551c..cf0c3a3326 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -3,12 +3,11 @@ package swarm import ( "fmt" - lgbl "github.com/ipfs/go-libp2p/loggables" mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" - transport "github.com/ipfs/go-libp2p/p2p/net/transport" - + transport "gx/ipfs/QmR4HhZm1fL8epXz8661Ru8ks5Y1Uk2u51quAWaCTD1zHg/go-libp2p-transport" + lgbl "gx/ipfs/QmSyBhZt2upyQ3NJmTpab1pX6hesA59vcYTGmgoDorZZbw/go-libp2p-loggables" ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 863da8efec..d99e04e135 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -3,10 +3,9 @@ package swarm import ( "fmt" - peer "github.com/ipfs/go-libp2p/p2p/peer" - metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 9a8052e9ef..29292eedbd 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -5,7 +5,7 @@ import ( "time" inet "github.com/ipfs/go-libp2p/p2p/net" - peer "github.com/ipfs/go-libp2p/p2p/peer" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index c69c7db38f..f637203a30 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -11,8 +11,8 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - peer "github.com/ipfs/go-libp2p/p2p/peer" testutil "github.com/ipfs/go-libp2p/testutil" + peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" From 995f55cbe056c1448adfd2117e4da4a439c76691 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 27 Apr 2016 10:12:51 -0700 Subject: [PATCH 028/259] recursive dependency update of utp lib --- p2p/net/swarm/addr/addr.go | 2 +- p2p/net/swarm/addr/addr_test.go | 2 +- p2p/net/swarm/dial_test.go | 4 ++-- p2p/net/swarm/peers_test.go | 2 +- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm.go | 6 +++--- p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 4 ++-- p2p/net/swarm/swarm_dial.go | 6 +++--- p2p/net/swarm/swarm_listen.go | 4 ++-- p2p/net/swarm/swarm_net.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_test.go | 2 +- 13 files changed, 20 insertions(+), 20 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 14da595090..7cda66439c 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,7 +3,7 @@ package addrutil import ( "fmt" - manet "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" + manet "gx/ipfs/QmTrxSBY8Wqd5aBB4MeizeSzS5xFbK8dQBrYaMsiGnCBhb/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 16599b4089..8793bc0917 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,7 +3,7 @@ package addrutil import ( "testing" - manet "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" + manet "gx/ipfs/QmTrxSBY8Wqd5aBB4MeizeSzS5xFbK8dQBrYaMsiGnCBhb/go-multiaddr-net" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index d21af292d8..31efd1b467 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -10,9 +10,9 @@ import ( addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - manet "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" + manet "gx/ipfs/QmTrxSBY8Wqd5aBB4MeizeSzS5xFbK8dQBrYaMsiGnCBhb/go-multiaddr-net" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 9976d3bf4c..6429a86278 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -3,7 +3,7 @@ package swarm import ( "testing" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index b0256b8489..b0c5497743 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,7 +7,7 @@ import ( "time" ci "github.com/ipfs/go-libp2p/testutil/ci" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 5326bf4a7f..5f75b5494a 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -14,19 +14,19 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" filter "github.com/ipfs/go-libp2p/p2p/net/filter" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - mafilter "gx/ipfs/QmPwfFAHUmvWDucLHRS9Xz2Kb1TNX2cY4LJ7pQjg9kVcae/multiaddr-filter" "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context" - transport "gx/ipfs/QmR4HhZm1fL8epXz8661Ru8ks5Y1Uk2u51quAWaCTD1zHg/go-libp2p-transport" + transport "gx/ipfs/QmRHqYZs3Diy8YC3bW16zvs8VDDwS2ARKBuKwALxEMqibc/go-libp2p-transport" pst "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer" psmss "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/multistream" spdy "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/spdystream" yamux "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/yamux" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + mafilter "gx/ipfs/Qme8dipKnZAChkp5Kfgj2MYYyBbzjqqPXmxQx3g9v3MoxP/multiaddr-filter" ) var log = logging.Logger("swarm2") diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 9cef367c78..9822ff7d44 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -6,7 +6,7 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" testutil "github.com/ipfs/go-libp2p/testutil" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 3ff2b91fcc..d815f7dc0e 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -6,10 +6,10 @@ import ( inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" + ic "gx/ipfs/QmUEUu1CM8bxBJxc3ZLojAi8evhTr4byQogWstABet79oY/go-libp2p-crypto" ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ic "gx/ipfs/QmaP38GJApheTr84f8R89vsT7oJLQw1AeCz4HqrQgv2njB/go-libp2p-crypto" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index b569f92fa4..c069543d51 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -10,9 +10,9 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - lgbl "gx/ipfs/QmSyBhZt2upyQ3NJmTpab1pX6hesA59vcYTGmgoDorZZbw/go-libp2p-loggables" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" - "gx/ipfs/QmYVqhVfbK4BKvbW88Lhm26b3ud14sTBvcm1H7uWUx1Fkp/go-multiaddr-net" + "gx/ipfs/QmTrxSBY8Wqd5aBB4MeizeSzS5xFbK8dQBrYaMsiGnCBhb/go-multiaddr-net" + lgbl "gx/ipfs/QmYqiDbGUGqatbPFie11Py8cnyduoJYqcgKtjfhu1SQLh1/go-libp2p-loggables" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index cf0c3a3326..e17cda6e7f 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -6,8 +6,8 @@ import ( mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" - transport "gx/ipfs/QmR4HhZm1fL8epXz8661Ru8ks5Y1Uk2u51quAWaCTD1zHg/go-libp2p-transport" - lgbl "gx/ipfs/QmSyBhZt2upyQ3NJmTpab1pX6hesA59vcYTGmgoDorZZbw/go-libp2p-loggables" + transport "gx/ipfs/QmRHqYZs3Diy8YC3bW16zvs8VDDwS2ARKBuKwALxEMqibc/go-libp2p-transport" + lgbl "gx/ipfs/QmYqiDbGUGqatbPFie11Py8cnyduoJYqcgKtjfhu1SQLh1/go-libp2p-loggables" ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index d99e04e135..14a3a953d4 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -5,7 +5,7 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 29292eedbd..0230ac4ab7 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -5,7 +5,7 @@ import ( "time" inet "github.com/ipfs/go-libp2p/p2p/net" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 5d56d29fbd..c231e3b248 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -12,7 +12,7 @@ import ( metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/testutil" - peer "gx/ipfs/QmY1xNhBfF9xA1pmD8yejyQAyd77K68qNN6JPM1CN2eiRu/go-libp2p-peer" + peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" From b58b106ecd09285efbb77c47722e1eae9537be53 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 27 Apr 2016 12:45:16 -0700 Subject: [PATCH 029/259] rewrite all package paths to dvcs --- p2p/net/swarm/addr/addr.go | 8 ++++---- p2p/net/swarm/addr/addr_test.go | 4 ++-- p2p/net/swarm/dial_test.go | 8 ++++---- p2p/net/swarm/peers_test.go | 6 +++--- p2p/net/swarm/simul_test.go | 6 +++--- p2p/net/swarm/swarm.go | 26 +++++++++++++------------- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 6 +++--- p2p/net/swarm/swarm_conn.go | 10 +++++----- p2p/net/swarm/swarm_dial.go | 10 +++++----- p2p/net/swarm/swarm_listen.go | 10 +++++----- p2p/net/swarm/swarm_net.go | 8 ++++---- p2p/net/swarm/swarm_net_test.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 6 +++--- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 6 +++--- 16 files changed, 60 insertions(+), 60 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 7cda66439c..1e22bae261 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,10 +3,10 @@ package addrutil import ( "fmt" - manet "gx/ipfs/QmTrxSBY8Wqd5aBB4MeizeSzS5xFbK8dQBrYaMsiGnCBhb/go-multiaddr-net" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + logging "github.com/ipfs/go-log" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" + context "golang.org/x/net/context" ) var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 8793bc0917..17039967e3 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -3,8 +3,8 @@ package addrutil import ( "testing" - manet "gx/ipfs/QmTrxSBY8Wqd5aBB4MeizeSzS5xFbK8dQBrYaMsiGnCBhb/go-multiaddr-net" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" ) func newMultiaddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 31efd1b467..988f7f7f01 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -7,14 +7,14 @@ import ( "testing" "time" + peer "github.com/ipfs/go-libp2p-peer" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - manet "gx/ipfs/QmTrxSBY8Wqd5aBB4MeizeSzS5xFbK8dQBrYaMsiGnCBhb/go-multiaddr-net" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" + context "golang.org/x/net/context" ) func closeSwarms(swarms []*Swarm) { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 6429a86278..cacd88ece8 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -3,9 +3,9 @@ package swarm import ( "testing" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + peer "github.com/ipfs/go-libp2p-peer" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index b0c5497743..39ba6411b6 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -6,11 +6,11 @@ import ( "testing" "time" + peer "github.com/ipfs/go-libp2p-peer" ci "github.com/ipfs/go-libp2p/testutil/ci" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 5f75b5494a..c812f37565 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -8,25 +8,25 @@ import ( "sync" "time" + peer "github.com/ipfs/go-libp2p-peer" + transport "github.com/ipfs/go-libp2p-transport" metrics "github.com/ipfs/go-libp2p/p2p/metrics" mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" filter "github.com/ipfs/go-libp2p/p2p/net/filter" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" - goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context" - transport "gx/ipfs/QmRHqYZs3Diy8YC3bW16zvs8VDDwS2ARKBuKwALxEMqibc/go-libp2p-transport" - pst "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer" - psmss "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/multistream" - spdy "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/spdystream" - yamux "gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/yamux" - ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" - mafilter "gx/ipfs/Qme8dipKnZAChkp5Kfgj2MYYyBbzjqqPXmxQx3g9v3MoxP/multiaddr-filter" + logging "github.com/ipfs/go-log" + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + pst "github.com/jbenet/go-stream-muxer" + psmss "github.com/jbenet/go-stream-muxer/multistream" + spdy "github.com/jbenet/go-stream-muxer/spdystream" + yamux "github.com/jbenet/go-stream-muxer/yamux" + "github.com/jbenet/goprocess" + goprocessctx "github.com/jbenet/goprocess/context" + mafilter "github.com/whyrusleeping/multiaddr-filter" + context "golang.org/x/net/context" ) var log = logging.Logger("swarm2") diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index faab0a3fdb..039683bfb9 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -4,7 +4,7 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 9822ff7d44..42ee299e87 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -3,13 +3,13 @@ package swarm import ( "testing" + peer "github.com/ipfs/go-libp2p-peer" metrics "github.com/ipfs/go-libp2p/p2p/metrics" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" testutil "github.com/ipfs/go-libp2p/testutil" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index d815f7dc0e..96700ba835 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -6,11 +6,11 @@ import ( inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" - ic "gx/ipfs/QmUEUu1CM8bxBJxc3ZLojAi8evhTr4byQogWstABet79oY/go-libp2p-crypto" - ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ic "github.com/ipfs/go-libp2p-crypto" + peer "github.com/ipfs/go-libp2p-peer" + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + context "golang.org/x/net/context" ) // a Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index c069543d51..aa4a3514b4 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -8,13 +8,13 @@ import ( "sync" "time" + lgbl "github.com/ipfs/go-libp2p-loggables" + peer "github.com/ipfs/go-libp2p-peer" conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - "gx/ipfs/QmTrxSBY8Wqd5aBB4MeizeSzS5xFbK8dQBrYaMsiGnCBhb/go-multiaddr-net" - lgbl "gx/ipfs/QmYqiDbGUGqatbPFie11Py8cnyduoJYqcgKtjfhu1SQLh1/go-libp2p-loggables" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + "github.com/jbenet/go-multiaddr-net" + context "golang.org/x/net/context" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index e17cda6e7f..38457f16fe 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -3,14 +3,14 @@ package swarm import ( "fmt" + lgbl "github.com/ipfs/go-libp2p-loggables" + transport "github.com/ipfs/go-libp2p-transport" mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" inet "github.com/ipfs/go-libp2p/p2p/net" conn "github.com/ipfs/go-libp2p/p2p/net/conn" - transport "gx/ipfs/QmRHqYZs3Diy8YC3bW16zvs8VDDwS2ARKBuKwALxEMqibc/go-libp2p-transport" - lgbl "gx/ipfs/QmYqiDbGUGqatbPFie11Py8cnyduoJYqcgKtjfhu1SQLh1/go-libp2p-loggables" - ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + ps "github.com/jbenet/go-peerstream" + context "golang.org/x/net/context" ) // Open listeners and reuse-dialers for the given addresses diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 14a3a953d4..84cc6326eb 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -3,13 +3,13 @@ package swarm import ( "fmt" + peer "github.com/ipfs/go-libp2p-peer" metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + "github.com/jbenet/goprocess" + context "golang.org/x/net/context" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 1ffb44812e..14c590b9d1 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -7,7 +7,7 @@ import ( inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/p2p/test/util" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" + context "golang.org/x/net/context" ) // TestConnectednessCorrect starts a few networks, connects a few diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 0230ac4ab7..aa4fca2448 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,10 +4,10 @@ import ( "testing" "time" + peer "github.com/ipfs/go-libp2p-peer" inet "github.com/ipfs/go-libp2p/p2p/net" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func TestNotifications(t *testing.T) { diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 3fbcc5d37c..7965d27439 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,7 +3,7 @@ package swarm import ( inet "github.com/ipfs/go-libp2p/p2p/net" - ps "gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream" + ps "github.com/jbenet/go-peerstream" ) // a Stream is a wrapper around a ps.Stream that exposes a way to get diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index c231e3b248..6e9f121fcd 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -9,13 +9,13 @@ import ( "testing" "time" + peer "github.com/ipfs/go-libp2p-peer" metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/testutil" - peer "gx/ipfs/QmZwZjMVGss5rqYsJVGy18gNbkTJffFyq2x1uJ4e4p3ZAt/go-libp2p-peer" - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" - ma "gx/ipfs/QmcobAGsCjYt5DXoq9et9L8yR8er7o7Cu3DTvpaq12jYSz/go-multiaddr" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" ) func EchoStreamHandler(stream inet.Stream) { From 41807c04ac6264ee6d9bf8208618a32c78086959 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 5 May 2016 20:38:17 -0700 Subject: [PATCH 030/259] add env var for overriding muxer prefs --- p2p/net/swarm/swarm.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c812f37565..d31c03a63f 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -5,6 +5,8 @@ package swarm import ( "fmt" "io/ioutil" + "os" + "strings" "sync" "time" @@ -48,6 +50,11 @@ func init() { msstpt.AddTransport("/yamux/1.0.0", ymxtpt) msstpt.AddTransport("/spdy/3.1.0", spdy.Transport) + // allow overriding of muxer preferences + if prefs := os.Getenv("LIBP2P_MUX_PREFS"); prefs != "" { + msstpt.OrderPreference = strings.Fields(prefs) + } + PSTransport = msstpt } From 936b1571c3f622092870c0d12d4b560284694b16 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 13 May 2016 10:59:16 -0700 Subject: [PATCH 031/259] update muxers, includes yamux deadlock fix --- p2p/net/swarm/swarm.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index d31c03a63f..e8b53885cc 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -22,11 +22,11 @@ import ( ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" pst "github.com/jbenet/go-stream-muxer" - psmss "github.com/jbenet/go-stream-muxer/multistream" - spdy "github.com/jbenet/go-stream-muxer/spdystream" - yamux "github.com/jbenet/go-stream-muxer/yamux" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" + psmss "github.com/whyrusleeping/go-smux-multistream" + spdy "github.com/whyrusleeping/go-smux-spdystream" + yamux "github.com/whyrusleeping/go-smux-yamux" mafilter "github.com/whyrusleeping/multiaddr-filter" context "golang.org/x/net/context" ) @@ -39,11 +39,11 @@ func init() { msstpt := psmss.NewBlankTransport() ymxtpt := &yamux.Transport{ - AcceptBacklog: 8192, + AcceptBacklog: 1024, ConnectionWriteTimeout: time.Second * 10, KeepAliveInterval: time.Second * 30, EnableKeepAlive: true, - MaxStreamWindowSize: uint32(1024 * 256), + MaxStreamWindowSize: uint32(1024 * 512), LogOutput: ioutil.Discard, } From c6d47eea103ed4c0cb592f799f1ae52b88df5ef6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 15 May 2016 18:57:29 -0700 Subject: [PATCH 032/259] yamux patches to help mitigate hanging issue --- p2p/net/swarm/swarm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index e8b53885cc..74e43dde44 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -39,7 +39,7 @@ func init() { msstpt := psmss.NewBlankTransport() ymxtpt := &yamux.Transport{ - AcceptBacklog: 1024, + AcceptBacklog: 8192, ConnectionWriteTimeout: time.Second * 10, KeepAliveInterval: time.Second * 30, EnableKeepAlive: true, From eebf64993c15867806ea601d02b5bd45c4895332 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 14 Apr 2016 16:43:42 -0700 Subject: [PATCH 033/259] Refactor the swarm dialer --- p2p/net/swarm/addr/addr.go | 16 +- p2p/net/swarm/addr/filter.go | 27 ++++ p2p/net/swarm/dial_test.go | 36 ----- p2p/net/swarm/limiter.go | 136 +++++++++++++++++ p2p/net/swarm/limiter_test.go | 265 ++++++++++++++++++++++++++++++++++ p2p/net/swarm/swarm.go | 5 + p2p/net/swarm/swarm_dial.go | 253 +++++++++----------------------- p2p/net/swarm/swarm_test.go | 2 +- 8 files changed, 516 insertions(+), 224 deletions(-) create mode 100644 p2p/net/swarm/addr/filter.go create mode 100644 p2p/net/swarm/limiter.go create mode 100644 p2p/net/swarm/limiter_test.go diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 1e22bae261..8c0c94b2d6 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -42,10 +42,14 @@ func init() { // FilterAddrs is a filter that removes certain addresses, according to filter. // if filter returns true, the address is kept. -func FilterAddrs(a []ma.Multiaddr, filter func(ma.Multiaddr) bool) []ma.Multiaddr { +func FilterAddrs(a []ma.Multiaddr, filters ...func(ma.Multiaddr) bool) []ma.Multiaddr { b := make([]ma.Multiaddr, 0, len(a)) for _, addr := range a { - if filter(addr) { + good := true + for _, filter := range filters { + good = good && filter(addr) + } + if good { b = append(b, addr) } } @@ -56,9 +60,11 @@ func FilterAddrs(a []ma.Multiaddr, filter func(ma.Multiaddr) bool) []ma.Multiadd // from a list. the addresses removed are those known NOT // to work with our network. Namely, addresses with UTP. func FilterUsableAddrs(a []ma.Multiaddr) []ma.Multiaddr { - return FilterAddrs(a, func(m ma.Multiaddr) bool { - return AddrUsable(m, false) - }) + return FilterAddrs(a, AddrUsableFunc) +} + +func AddrUsableFunc(m ma.Multiaddr) bool { + return AddrUsable(m, false) } // AddrOverNonLocalIP returns whether the addr uses a non-local ip link diff --git a/p2p/net/swarm/addr/filter.go b/p2p/net/swarm/addr/filter.go new file mode 100644 index 0000000000..c67e1c624e --- /dev/null +++ b/p2p/net/swarm/addr/filter.go @@ -0,0 +1,27 @@ +package addrutil + +import ( + ma "github.com/jbenet/go-multiaddr" + mafmt "github.com/whyrusleeping/mafmt" +) + +func SubtractFilter(addrs ...ma.Multiaddr) func(ma.Multiaddr) bool { + addrmap := make(map[string]bool) + for _, a := range addrs { + addrmap[string(a.Bytes())] = true + } + + return func(a ma.Multiaddr) bool { + return !addrmap[string(a.Bytes())] + } +} + +func IsFDCostlyTransport(a ma.Multiaddr) bool { + return mafmt.TCP.Matches(a) +} + +func FilterNeg(f func(ma.Multiaddr) bool) func(ma.Multiaddr) bool { + return func(a ma.Multiaddr) bool { + return !f(a) + } +} diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 988f7f7f01..aea0f728cc 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -2,7 +2,6 @@ package swarm import ( "net" - "sort" "sync" "testing" "time" @@ -493,38 +492,3 @@ func TestDialBackoffClears(t *testing.T) { t.Log("correctly cleared backoff") } } - -func mkAddr(t *testing.T, s string) ma.Multiaddr { - a, err := ma.NewMultiaddr(s) - if err != nil { - t.Fatal(err) - } - - return a -} - -func TestAddressSorting(t *testing.T) { - u1 := mkAddr(t, "/ip4/152.12.23.53/udp/1234/utp") - u2l := mkAddr(t, "/ip4/127.0.0.1/udp/1234/utp") - local := mkAddr(t, "/ip4/127.0.0.1/tcp/1234") - norm := mkAddr(t, "/ip4/6.5.4.3/tcp/1234") - - l := AddrList{local, u1, u2l, norm} - sort.Sort(l) - - if !l[0].Equal(u2l) { - t.Fatal("expected utp local addr to be sorted first: ", l[0]) - } - - if !l[1].Equal(u1) { - t.Fatal("expected utp addr to be sorted second") - } - - if !l[2].Equal(local) { - t.Fatal("expected tcp localhost addr thid") - } - - if !l[3].Equal(norm) { - t.Fatal("expected normal addr last") - } -} diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go new file mode 100644 index 0000000000..fe8162d7f1 --- /dev/null +++ b/p2p/net/swarm/limiter.go @@ -0,0 +1,136 @@ +package swarm + +import ( + "sync" + + peer "github.com/ipfs/go-libp2p-peer" + ma "github.com/jbenet/go-multiaddr" + context "golang.org/x/net/context" + + conn "github.com/ipfs/go-libp2p/p2p/net/conn" + addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" +) + +type dialResult struct { + Conn conn.Conn + Err error +} + +type dialJob struct { + addr ma.Multiaddr + peer peer.ID + ctx context.Context + resp chan dialResult + success bool +} + +type dialLimiter struct { + rllock sync.Mutex + fdConsuming int + fdLimit int + waitingOnFd []*dialJob + + dialFunc func(context.Context, peer.ID, ma.Multiaddr) (conn.Conn, error) + + activePerPeer map[peer.ID]int + perPeerLimit int + waitingOnPeerLimit map[peer.ID][]*dialJob +} + +type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (conn.Conn, error) + +func newDialLimiter(df dialfunc) *dialLimiter { + return newDialLimiterWithParams(df, concurrentFdDials, defaultPerPeerRateLimit) +} + +func newDialLimiterWithParams(df dialfunc, fdl, ppl int) *dialLimiter { + return &dialLimiter{ + fdLimit: fdl, + perPeerLimit: ppl, + waitingOnPeerLimit: make(map[peer.ID][]*dialJob), + activePerPeer: make(map[peer.ID]int), + dialFunc: df, + } +} + +func (dl *dialLimiter) finishedDial(dj *dialJob) { + dl.rllock.Lock() + defer dl.rllock.Unlock() + + // release tokens in reverse order than we take them + dl.activePerPeer[dj.peer]-- + if dl.activePerPeer[dj.peer] == 0 { + delete(dl.activePerPeer, dj.peer) + } + + waitlist := dl.waitingOnPeerLimit[dj.peer] + if !dj.success && len(waitlist) > 0 { + next := waitlist[0] + if len(waitlist) == 1 { + delete(dl.waitingOnPeerLimit, dj.peer) + } else { + dl.waitingOnPeerLimit[dj.peer] = waitlist[1:] + } + dl.activePerPeer[dj.peer]++ // just kidding, we still want this token + + // can kick this off right here, dials in this list already + // have the other tokens needed + go dl.executeDial(next) + } + + if addrutil.IsFDCostlyTransport(dj.addr) { + dl.fdConsuming-- + if len(dl.waitingOnFd) > 0 { + next := dl.waitingOnFd[0] + dl.waitingOnFd = dl.waitingOnFd[1:] + dl.fdConsuming++ + + // now, attempt to take the 'per peer limit' token + dl.schedulePerPeerDial(next) + } + } +} + +// AddDialJob tries to take the needed tokens for starting the given dial job. +// If it acquires all needed tokens, it immediately starts the dial, otherwise +// it will put it on the waitlist for the requested token. +func (dl *dialLimiter) AddDialJob(dj *dialJob) { + dl.rllock.Lock() + defer dl.rllock.Unlock() + + if addrutil.IsFDCostlyTransport(dj.addr) { + if dl.fdConsuming >= dl.fdLimit { + dl.waitingOnFd = append(dl.waitingOnFd, dj) + return + } + + // take token + dl.fdConsuming++ + } + + dl.schedulePerPeerDial(dj) +} + +// executeDial calls the dialFunc, and reports the result through the response +// channel when finished. Once the response is sent it also releases all tokens +// it held during the dial. +func (dl *dialLimiter) executeDial(j *dialJob) { + defer dl.finishedDial(j) + con, err := dl.dialFunc(j.ctx, j.peer, j.addr) + select { + case j.resp <- dialResult{Conn: con, Err: err}: + case <-j.ctx.Done(): + } +} + +func (dl *dialLimiter) schedulePerPeerDial(j *dialJob) { + if dl.activePerPeer[j.peer] >= dl.perPeerLimit { + wlist := dl.waitingOnPeerLimit[j.peer] + dl.waitingOnPeerLimit[j.peer] = append(wlist, j) + return + } + + // take second needed token and start dial! + dl.activePerPeer[j.peer]++ + go dl.executeDial(j) +} diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go new file mode 100644 index 0000000000..b5bbde847e --- /dev/null +++ b/p2p/net/swarm/limiter_test.go @@ -0,0 +1,265 @@ +package swarm + +import ( + "fmt" + "strconv" + "testing" + "time" + + peer "github.com/ipfs/go-libp2p-peer" + ma "github.com/jbenet/go-multiaddr" + mafmt "github.com/whyrusleeping/mafmt" + context "golang.org/x/net/context" + + conn "github.com/ipfs/go-libp2p/p2p/net/conn" +) + +func mustAddr(t *testing.T, s string) ma.Multiaddr { + a, err := ma.NewMultiaddr(s) + if err != nil { + t.Fatal(err) + } + return a +} + +func addrWithPort(t *testing.T, p int) ma.Multiaddr { + return mustAddr(t, fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p)) +} + +// in these tests I use addresses with tcp ports over a certain number to +// signify 'good' addresses that will succeed, and addresses below that number +// will fail. This lets us more easily test these different scenarios. +func tcpPortOver(a ma.Multiaddr, n int) bool { + port, err := a.ValueForProtocol(ma.P_TCP) + if err != nil { + panic(err) + } + + pnum, err := strconv.Atoi(port) + if err != nil { + panic(err) + } + + return pnum > n +} + +func tryDialAddrs(ctx context.Context, l *dialLimiter, p peer.ID, addrs []ma.Multiaddr, res chan dialResult) { + for _, a := range addrs { + l.AddDialJob(&dialJob{ + ctx: ctx, + peer: p, + addr: a, + resp: res, + }) + } +} + +func hangDialFunc(hang chan struct{}) dialfunc { + return func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { + if mafmt.UTP.Matches(a) { + return conn.Conn(nil), nil + } + + if tcpPortOver(a, 10) { + return conn.Conn(nil), nil + } else { + <-hang + return nil, fmt.Errorf("test bad dial") + } + } +} + +func TestLimiterBasicDials(t *testing.T) { + hang := make(chan struct{}) + defer close(hang) + + l := newDialLimiterWithParams(hangDialFunc(hang), concurrentFdDials, 4) + + bads := []ma.Multiaddr{ + addrWithPort(t, 1), + addrWithPort(t, 2), + addrWithPort(t, 3), + addrWithPort(t, 4), + } + + good := addrWithPort(t, 20) + + resch := make(chan dialResult) + pid := peer.ID("testpeer") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tryDialAddrs(ctx, l, pid, bads, resch) + + l.AddDialJob(&dialJob{ + ctx: ctx, + peer: pid, + addr: good, + resp: resch, + }) + + select { + case <-resch: + t.Fatal("no dials should have completed!") + case <-time.After(time.Millisecond * 100): + } + + // complete a single hung dial + hang <- struct{}{} + + select { + case r := <-resch: + if r.Err == nil { + t.Fatal("should have gotten failed dial result") + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for dial completion") + } + + select { + case r := <-resch: + if r.Err != nil { + t.Fatal("expected second result to be success!") + } + case <-time.After(time.Second): + } +} + +func TestFDLimiting(t *testing.T) { + hang := make(chan struct{}) + defer close(hang) + l := newDialLimiterWithParams(hangDialFunc(hang), 16, 5) + + bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} + pids := []peer.ID{"testpeer1", "testpeer2", "testpeer3", "testpeer4"} + good_tcp := addrWithPort(t, 20) + + ctx := context.Background() + resch := make(chan dialResult) + + // take all fd limit tokens with hang dials + for _, pid := range pids { + tryDialAddrs(ctx, l, pid, bads, resch) + } + + // these dials should work normally, but will hang because we have taken + // up all the fd limiting + for _, pid := range pids { + l.AddDialJob(&dialJob{ + ctx: ctx, + peer: pid, + addr: good_tcp, + resp: resch, + }) + } + + select { + case <-resch: + t.Fatal("no dials should have completed!") + case <-time.After(time.Millisecond * 100): + } + + pid5 := peer.ID("testpeer5") + utpaddr := mustAddr(t, "/ip4/127.0.0.1/udp/7777/utp") + + l.AddDialJob(&dialJob{ctx: ctx, peer: pid5, addr: utpaddr, resp: resch}) + + select { + case res := <-resch: + if res.Err != nil { + t.Fatal("should have gotten successful response") + } + case <-time.After(time.Second * 5): + t.Fatal("timeout waiting for utp addr success") + } +} + +func TestTokenRedistribution(t *testing.T) { + hangchs := make(map[peer.ID]chan struct{}) + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { + if tcpPortOver(a, 10) { + return (conn.Conn)(nil), nil + } else { + <-hangchs[p] + return nil, fmt.Errorf("test bad dial") + } + } + l := newDialLimiterWithParams(df, 8, 4) + + bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} + pids := []peer.ID{"testpeer1", "testpeer2"} + + ctx := context.Background() + resch := make(chan dialResult) + + // take all fd limit tokens with hang dials + for _, pid := range pids { + hangchs[pid] = make(chan struct{}) + tryDialAddrs(ctx, l, pid, bads, resch) + } + + good := mustAddr(t, "/ip4/127.0.0.1/tcp/1001") + + // add a good dial job for peer 1 + l.AddDialJob(&dialJob{ + ctx: ctx, + peer: pids[1], + addr: good, + resp: resch, + }) + + select { + case <-resch: + t.Fatal("no dials should have completed!") + case <-time.After(time.Millisecond * 100): + } + + // unblock one dial for peer 0 + hangchs[pids[0]] <- struct{}{} + + select { + case res := <-resch: + if res.Err == nil { + t.Fatal("should have only been a failure here") + } + case <-time.After(time.Millisecond * 100): + t.Fatal("expected a dial failure here") + } + + select { + case <-resch: + t.Fatal("no more dials should have completed!") + case <-time.After(time.Millisecond * 100): + } + + // add a bad dial job to peer 0 to fill their rate limiter + // and test that more dials for this peer won't interfere with peer 1's successful dial incoming + l.AddDialJob(&dialJob{ + ctx: ctx, + peer: pids[0], + addr: addrWithPort(t, 7), + resp: resch, + }) + + hangchs[pids[1]] <- struct{}{} + + // now one failed dial from peer 1 should get through and fail + // which will in turn unblock the successful dial on peer 1 + select { + case res := <-resch: + if res.Err == nil { + t.Fatal("should have only been a failure here") + } + case <-time.After(time.Millisecond * 100): + t.Fatal("expected a dial failure here") + } + + select { + case res := <-resch: + if res.Err != nil { + t.Fatal("should have succeeded!") + } + case <-time.After(time.Millisecond * 100): + t.Fatal("should have gotten successful dial") + } +} diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 74e43dde44..54dd903056 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -90,6 +90,8 @@ type Swarm struct { proc goprocess.Process ctx context.Context bwc metrics.Reporter + + limiter *dialLimiter } // NewSwarm constructs a Swarm, with a Chan. @@ -122,6 +124,8 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, dialer: conn.NewDialer(local, peers.PrivKey(local), wrap), } + s.limiter = newDialLimiter(s.dialAddr) + // configure Swarm s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown) s.SetConnHandler(nil) // make sure to setup our own conn handler. @@ -155,6 +159,7 @@ func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { } listenAddrs = filtered } + return listenAddrs, nil } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index aa4a3514b4..c1b3a73922 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -1,10 +1,8 @@ package swarm import ( - "bytes" "errors" "fmt" - "sort" "sync" "time" @@ -13,7 +11,6 @@ import ( conn "github.com/ipfs/go-libp2p/p2p/net/conn" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" ma "github.com/jbenet/go-multiaddr" - "github.com/jbenet/go-multiaddr-net" context "golang.org/x/net/context" ) @@ -42,6 +39,9 @@ const dialAttempts = 1 // number of concurrent outbound dials over transports that consume file descriptors const concurrentFdDials = 160 +// number of concurrent outbound dials to make per peer +const defaultPerPeerRateLimit = 8 + // DialTimeout is the amount of time each dial attempt has. We can think about making // this larger down the road, or putting more granular timeouts (i.e. within each // subcomponent of Dial) @@ -319,32 +319,34 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { log.Debug("Dial not given PrivateKey, so WILL NOT SECURE conn.") } - // get remote peer addrs - remoteAddrs := s.peers.Addrs(p) - // make sure we can use the addresses. - remoteAddrs = addrutil.FilterUsableAddrs(remoteAddrs) - // drop out any addrs that would just dial ourselves. use ListenAddresses - // as that is a more authoritative view than localAddrs. ila, _ := s.InterfaceListenAddresses() - remoteAddrs = addrutil.Subtract(remoteAddrs, ila) - remoteAddrs = addrutil.Subtract(remoteAddrs, s.peers.Addrs(s.local)) - - log.Debugf("%s swarm dialing %s -- local:%s remote:%s", s.local, p, s.ListenAddresses(), remoteAddrs) - if len(remoteAddrs) == 0 { - err := errors.New("peer has no addresses") - logdial["error"] = err - return nil, err - } - - remoteAddrs = s.filterAddrs(remoteAddrs) - if len(remoteAddrs) == 0 { - err := errors.New("all adresses for peer have been filtered out") - logdial["error"] = err - return nil, err + subtract_filter := addrutil.SubtractFilter(append(ila, s.peers.Addrs(s.local)...)...) + + // get live channel of addresses for peer, filtered by the given filters + /* + remoteAddrChan := s.peers.AddrsChan(ctx, p, + addrutil.AddrUsableFilter, + subtract_filter, + s.Filters.AddrBlocked) + */ + + ////// TEMP UNTIL PEERSTORE GETS UPGRADED + // Ref: https://github.com/ipfs/go-libp2p-peer/pull/1 + paddrs := s.peers.Addrs(p) + good_addrs := addrutil.FilterAddrs(paddrs, + addrutil.AddrUsableFunc, + subtract_filter, + addrutil.FilterNeg(s.Filters.AddrBlocked), + ) + remoteAddrChan := make(chan ma.Multiaddr, len(good_addrs)) + for _, a := range good_addrs { + remoteAddrChan <- a } + close(remoteAddrChan) + ///////// // try to get a connection to any addr - connC, err := s.dialAddrs(ctx, p, remoteAddrs) + connC, err := s.dialAddrs(ctx, p, remoteAddrChan) if err != nil { logdial["error"] = err return nil, err @@ -364,98 +366,64 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return swarmC, nil } -func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs []ma.Multiaddr) (conn.Conn, error) { - - // sort addresses so preferred addresses are dialed sooner - sort.Sort(AddrList(remoteAddrs)) - - // try to connect to one of the peer's known addresses. - // we dial concurrently to each of the addresses, which: - // * makes the process faster overall - // * attempts to get the fastest connection available. - // * mitigates the waste of trying bad addresses +func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (conn.Conn, error) { log.Debugf("%s swarm dialing %s %s", s.local, p, remoteAddrs) ctx, cancel := context.WithCancel(ctx) defer cancel() // cancel work when we exit func - conns := make(chan conn.Conn) - errs := make(chan error, len(remoteAddrs)) + // use a single response type instead of errs and conns, reduces complexity *a ton* + respch := make(chan dialResult) - // dialSingleAddr is used in the rate-limited async thing below. - dialSingleAddr := func(addr ma.Multiaddr) { - // rebind chans in scope so we can nil them out easily - connsout := conns - errsout := errs + defaultDialFail := fmt.Errorf("failed to dial %s (default failure)", p) + exitErr := defaultDialFail - connC, err := s.dialAddr(ctx, p, addr) - if err != nil { - connsout = nil - } else if connC == nil { - // NOTE: this really should never happen - log.Errorf("failed to dial %s %s and got no error!", p, addr) - err = fmt.Errorf("failed to dial %s %s", p, addr) - connsout = nil - } else { - errsout = nil - } - - // check parent still wants our results + var active int + for { select { - case <-ctx.Done(): - if connC != nil { - connC.Close() + case addr, ok := <-remoteAddrs: + if !ok { + remoteAddrs = nil + if active == 0 { + return nil, exitErr + } + continue } - case errsout <- err: - case connsout <- connC: - } - } - // this whole thing is in a goroutine so we can use foundConn - // to end early. - go func() { - limiter := make(chan struct{}, 8) - for _, addr := range remoteAddrs { - // returns whatever ratelimiting is acceptable for workerAddr. - // may not rate limit at all. - rl := s.addrDialRateLimit(addr) - select { - case <-ctx.Done(): // our context was cancelled - return - case rl <- struct{}{}: - // take the token, move on + // limitedDial will start a dial to the given peer when + // it is able, respecting the various different types of rate + // limiting that occur without using extra goroutines per addr + s.limitedDial(ctx, p, addr, respch) + active++ + case <-ctx.Done(): + if exitErr == defaultDialFail { + exitErr = ctx.Err() } - - select { - case <-ctx.Done(): // our context was cancelled - return - case limiter <- struct{}{}: - // take the token, move on + return nil, exitErr + case resp := <-respch: + active-- + if resp.Err != nil { + log.Error("got error on dial: ", resp.Err) + // Errors are normal, lots of dials will fail + exitErr = resp.Err + + if remoteAddrs == nil && active == 0 { + return nil, exitErr + } + } else if resp.Conn != nil { + return resp.Conn, nil } - - go func(rlc <-chan struct{}, a ma.Multiaddr) { - dialSingleAddr(a) - <-limiter - <-rlc - }(rl, addr) - } - }() - - // wair for the results. - exitErr := fmt.Errorf("failed to dial %s", p) - for range remoteAddrs { - select { - case exitErr = <-errs: // - log.Debug("dial error: ", exitErr) - case connC := <-conns: - // take the first + return asap - return connC, nil - case <-ctx.Done(): - // break out and return error - break } } - return nil, exitErr +} + +func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp chan dialResult) { + s.limiter.AddDialJob(&dialJob{ + addr: a, + peer: p, + resp: resp, + ctx: ctx, + }) } func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (conn.Conn, error) { @@ -485,16 +453,6 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (con return connC, nil } -func (s *Swarm) filterAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { - var out []ma.Multiaddr - for _, a := range addrs { - if !s.Filters.AddrBlocked(a) { - out = append(out, a) - } - } - return out -} - // dialConnSetup is the setup logic for a connection from the dial side. it // needs to add the Conn to the StreamSwarm, then run newConnSetup func dialConnSetup(ctx context.Context, s *Swarm, connC conn.Conn) (*Conn, error) { @@ -514,72 +472,3 @@ func dialConnSetup(ctx context.Context, s *Swarm, connC conn.Conn) (*Conn, error return swarmC, err } - -// addrDialRateLimit returns a ratelimiting channel for dialing transport -// addrs like a. for example, tcp is fd-ratelimited. utp is not ratelimited. -func (s *Swarm) addrDialRateLimit(a ma.Multiaddr) chan struct{} { - if isFDCostlyTransport(a) { - return s.fdRateLimit - } - - // do not rate limit it at all - return make(chan struct{}, 1) -} - -func isFDCostlyTransport(a ma.Multiaddr) bool { - return isTCPMultiaddr(a) -} - -func isTCPMultiaddr(a ma.Multiaddr) bool { - p := a.Protocols() - return len(p) == 2 && (p[0].Name == "ip4" || p[0].Name == "ip6") && p[1].Name == "tcp" -} - -type AddrList []ma.Multiaddr - -func (al AddrList) Len() int { - return len(al) -} - -func (al AddrList) Swap(i, j int) { - al[i], al[j] = al[j], al[i] -} - -func (al AddrList) Less(i, j int) bool { - a := al[i] - b := al[j] - - // dial localhost addresses next, they should fail immediately - lba := manet.IsIPLoopback(a) - lbb := manet.IsIPLoopback(b) - if lba { - if !lbb { - return true - } - } - - // dial utp and similar 'non-fd-consuming' addresses first - fda := isFDCostlyTransport(a) - fdb := isFDCostlyTransport(b) - if !fda { - if fdb { - return true - } - - // if neither consume fd's, assume equal ordering - return false - } - - // if 'b' doesnt take a file descriptor - if !fdb { - return false - } - - // if 'b' is loopback and both take file descriptors - if lbb { - return false - } - - // for the rest, just sort by bytes - return bytes.Compare(a.Bytes(), b.Bytes()) > 0 -} diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 6e9f121fcd..f5e454956c 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -303,7 +303,7 @@ func TestAddrBlocking(t *testing.T) { swarms := makeSwarms(ctx, t, 2) swarms[0].SetConnHandler(func(conn *Conn) { - t.Fatalf("no connections should happen! -- %s", conn) + t.Errorf("no connections should happen! -- %s", conn) }) _, block, err := net.ParseCIDR("127.0.0.1/8") From 19e1b7f9623a08993d39ae71f778b9ed28b755f1 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 6 May 2016 12:39:08 -0700 Subject: [PATCH 034/259] don't execute cancelled jobs --- p2p/net/swarm/limiter.go | 13 +++++++ p2p/net/swarm/limiter_test.go | 64 +++++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index fe8162d7f1..7835fe5729 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -24,6 +24,15 @@ type dialJob struct { success bool } +func (dj *dialJob) cancelled() bool { + select { + case <-dj.ctx.Done(): + return true + default: + return false + } +} + type dialLimiter struct { rllock sync.Mutex fdConsuming int @@ -116,6 +125,10 @@ func (dl *dialLimiter) AddDialJob(dj *dialJob) { // it held during the dial. func (dl *dialLimiter) executeDial(j *dialJob) { defer dl.finishedDial(j) + if j.cancelled() { + return + } + con, err := dl.dialFunc(j.ctx, j.peer, j.addr) select { case j.resp <- dialResult{Conn: con, Err: err}: diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index b5bbde847e..28733c5ab5 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -2,6 +2,7 @@ package swarm import ( "fmt" + "math/rand" "strconv" "testing" "time" @@ -75,13 +76,7 @@ func TestLimiterBasicDials(t *testing.T) { l := newDialLimiterWithParams(hangDialFunc(hang), concurrentFdDials, 4) - bads := []ma.Multiaddr{ - addrWithPort(t, 1), - addrWithPort(t, 2), - addrWithPort(t, 3), - addrWithPort(t, 4), - } - + bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} good := addrWithPort(t, 20) resch := make(chan dialResult) @@ -162,6 +157,7 @@ func TestFDLimiting(t *testing.T) { pid5 := peer.ID("testpeer5") utpaddr := mustAddr(t, "/ip4/127.0.0.1/udp/7777/utp") + // This should complete immediately since utp addresses arent blocked by fd rate limiting l.AddDialJob(&dialJob{ctx: ctx, peer: pid5, addr: utpaddr, resp: resch}) select { @@ -263,3 +259,57 @@ func TestTokenRedistribution(t *testing.T) { t.Fatal("should have gotten successful dial") } } + +func TestStressLimiter(t *testing.T) { + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { + if tcpPortOver(a, 1000) { + return conn.Conn(nil), nil + } else { + time.Sleep(time.Millisecond * time.Duration(5+rand.Intn(100))) + return nil, fmt.Errorf("test bad dial") + } + } + + l := newDialLimiterWithParams(df, 20, 5) + + var bads []ma.Multiaddr + for i := 0; i < 100; i++ { + bads = append(bads, addrWithPort(t, i)) + } + + addresses := append(bads, addrWithPort(t, 2000)) + success := make(chan struct{}) + + for i := 0; i < 20; i++ { + go func(id peer.ID) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resp := make(chan dialResult) + time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond) + for _, i := range rand.Perm(len(addresses)) { + l.AddDialJob(&dialJob{ + addr: addresses[i], + ctx: ctx, + peer: id, + resp: resp, + }) + } + + for res := range resp { + if res.Err == nil { + success <- struct{}{} + return + } + } + }(peer.ID(fmt.Sprintf("testpeer%d", i))) + } + + for i := 0; i < 20; i++ { + select { + case <-success: + case <-time.After(time.Second * 5): + t.Fatal("expected a success within five seconds") + } + } +} From 475ca0cb712609716400b44a88758a941d4d2deb Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 13 May 2016 09:28:52 -0700 Subject: [PATCH 035/259] refactor locking order structure --- p2p/net/swarm/limiter.go | 59 +++++++++++++++++++++-------------- p2p/net/swarm/limiter_test.go | 4 +++ 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 7835fe5729..94ce05bb86 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -66,6 +66,20 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { dl.rllock.Lock() defer dl.rllock.Unlock() + if addrutil.IsFDCostlyTransport(dj.addr) { + dl.fdConsuming-- + if len(dl.waitingOnFd) > 0 { + next := dl.waitingOnFd[0] + dl.waitingOnFd = dl.waitingOnFd[1:] + if len(dl.waitingOnFd) == 0 { + dl.waitingOnFd = nil // clear out memory + } + dl.fdConsuming++ + + go dl.executeDial(next) + } + } + // release tokens in reverse order than we take them dl.activePerPeer[dj.peer]-- if dl.activePerPeer[dj.peer] == 0 { @@ -87,17 +101,6 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { go dl.executeDial(next) } - if addrutil.IsFDCostlyTransport(dj.addr) { - dl.fdConsuming-- - if len(dl.waitingOnFd) > 0 { - next := dl.waitingOnFd[0] - dl.waitingOnFd = dl.waitingOnFd[1:] - dl.fdConsuming++ - - // now, attempt to take the 'per peer limit' token - dl.schedulePerPeerDial(next) - } - } } // AddDialJob tries to take the needed tokens for starting the given dial job. @@ -107,6 +110,13 @@ func (dl *dialLimiter) AddDialJob(dj *dialJob) { dl.rllock.Lock() defer dl.rllock.Unlock() + if dl.activePerPeer[dj.peer] >= dl.perPeerLimit { + wlist := dl.waitingOnPeerLimit[dj.peer] + dl.waitingOnPeerLimit[dj.peer] = append(wlist, dj) + return + } + dl.activePerPeer[dj.peer]++ + if addrutil.IsFDCostlyTransport(dj.addr) { if dl.fdConsuming >= dl.fdLimit { dl.waitingOnFd = append(dl.waitingOnFd, dj) @@ -117,7 +127,20 @@ func (dl *dialLimiter) AddDialJob(dj *dialJob) { dl.fdConsuming++ } - dl.schedulePerPeerDial(dj) + // take second needed token and start dial! + go dl.executeDial(dj) +} + +func (dl *dialLimiter) schedulePerPeerDial(j *dialJob) { + if dl.activePerPeer[j.peer] >= dl.perPeerLimit { + wlist := dl.waitingOnPeerLimit[j.peer] + dl.waitingOnPeerLimit[j.peer] = append(wlist, j) + return + } + + // take second needed token and start dial! + dl.activePerPeer[j.peer]++ + go dl.executeDial(j) } // executeDial calls the dialFunc, and reports the result through the response @@ -135,15 +158,3 @@ func (dl *dialLimiter) executeDial(j *dialJob) { case <-j.ctx.Done(): } } - -func (dl *dialLimiter) schedulePerPeerDial(j *dialJob) { - if dl.activePerPeer[j.peer] >= dl.perPeerLimit { - wlist := dl.waitingOnPeerLimit[j.peer] - dl.waitingOnPeerLimit[j.peer] = append(wlist, j) - return - } - - // take second needed token and start dial! - dl.activePerPeer[j.peer]++ - go dl.executeDial(j) -} diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 28733c5ab5..fb1be191ee 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -3,6 +3,7 @@ package swarm import ( "fmt" "math/rand" + "runtime" "strconv" "testing" "time" @@ -262,6 +263,7 @@ func TestTokenRedistribution(t *testing.T) { func TestStressLimiter(t *testing.T) { df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { + fmt.Println("dial for peer: ", string(p)) if tcpPortOver(a, 1000) { return conn.Conn(nil), nil } else { @@ -305,6 +307,8 @@ func TestStressLimiter(t *testing.T) { }(peer.ID(fmt.Sprintf("testpeer%d", i))) } + time.Sleep(time.Millisecond * 1000) + fmt.Println("NUM GOROS: ", runtime.NumGoroutine()) for i := 0; i < 20; i++ { select { case <-success: From 37883b42dfe638c029926b1a9c887f5948123878 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 14 May 2016 19:32:28 -0700 Subject: [PATCH 036/259] address CR feedback --- p2p/net/swarm/addr/addr.go | 4 ++-- p2p/net/swarm/addr/filter.go | 4 ++++ p2p/net/swarm/swarm_dial.go | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 8c0c94b2d6..d9ba87216a 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -40,8 +40,8 @@ func init() { SupportedTransportProtocols = transports } -// FilterAddrs is a filter that removes certain addresses, according to filter. -// if filter returns true, the address is kept. +// FilterAddrs is a filter that removes certain addresses, according the given filters. +// if all filters return true, the address is kept. func FilterAddrs(a []ma.Multiaddr, filters ...func(ma.Multiaddr) bool) []ma.Multiaddr { b := make([]ma.Multiaddr, 0, len(a)) for _, addr := range a { diff --git a/p2p/net/swarm/addr/filter.go b/p2p/net/swarm/addr/filter.go index c67e1c624e..d87ba816a7 100644 --- a/p2p/net/swarm/addr/filter.go +++ b/p2p/net/swarm/addr/filter.go @@ -5,6 +5,7 @@ import ( mafmt "github.com/whyrusleeping/mafmt" ) +// SubtractFilter returns a filter func that filters all of the given addresses func SubtractFilter(addrs ...ma.Multiaddr) func(ma.Multiaddr) bool { addrmap := make(map[string]bool) for _, a := range addrs { @@ -16,10 +17,13 @@ func SubtractFilter(addrs ...ma.Multiaddr) func(ma.Multiaddr) bool { } } +// IsFDCostlyTransport returns true for transports that require a new file +// descriptor per connection created func IsFDCostlyTransport(a ma.Multiaddr) bool { return mafmt.TCP.Matches(a) } +// FilterNeg returns a negated version of the passed in filter func FilterNeg(f func(ma.Multiaddr) bool) func(ma.Multiaddr) bool { return func(a ma.Multiaddr) bool { return !f(a) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index c1b3a73922..33b087efc3 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -390,9 +390,6 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. continue } - // limitedDial will start a dial to the given peer when - // it is able, respecting the various different types of rate - // limiting that occur without using extra goroutines per addr s.limitedDial(ctx, p, addr, respch) active++ case <-ctx.Done(): @@ -403,7 +400,7 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. case resp := <-respch: active-- if resp.Err != nil { - log.Error("got error on dial: ", resp.Err) + log.Info("got error on dial: ", resp.Err) // Errors are normal, lots of dials will fail exitErr = resp.Err @@ -417,6 +414,9 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. } } +// limitedDial will start a dial to the given peer when +// it is able, respecting the various different types of rate +// limiting that occur without using extra goroutines per addr func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp chan dialResult) { s.limiter.AddDialJob(&dialJob{ addr: a, From 4aea49efe54458febbb3ae8350a94bfe34241b0d Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 14 May 2016 22:51:25 -0700 Subject: [PATCH 037/259] test cleanup --- p2p/net/swarm/limiter_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index fb1be191ee..28733c5ab5 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -3,7 +3,6 @@ package swarm import ( "fmt" "math/rand" - "runtime" "strconv" "testing" "time" @@ -263,7 +262,6 @@ func TestTokenRedistribution(t *testing.T) { func TestStressLimiter(t *testing.T) { df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { - fmt.Println("dial for peer: ", string(p)) if tcpPortOver(a, 1000) { return conn.Conn(nil), nil } else { @@ -307,8 +305,6 @@ func TestStressLimiter(t *testing.T) { }(peer.ID(fmt.Sprintf("testpeer%d", i))) } - time.Sleep(time.Millisecond * 1000) - fmt.Println("NUM GOROS: ", runtime.NumGoroutine()) for i := 0; i < 20; i++ { select { case <-success: From 4e786eb9472b6be10c866c85fc2516b44694a754 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 31 May 2016 16:24:11 -0700 Subject: [PATCH 038/259] split peerstore from peer package --- p2p/net/swarm/dial_test.go | 19 ++++++++++--------- p2p/net/swarm/peers_test.go | 3 ++- p2p/net/swarm/simul_test.go | 5 +++-- p2p/net/swarm/swarm.go | 5 +++-- p2p/net/swarm/swarm_addr_test.go | 6 +++--- p2p/net/swarm/swarm_net.go | 5 +++-- p2p/net/swarm/swarm_test.go | 13 +++++++------ 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 988f7f7f01..833d1087ce 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -7,11 +7,12 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p-peer" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" testutil "github.com/ipfs/go-libp2p/testutil" ci "github.com/ipfs/go-libp2p/testutil/ci" + peer "github.com/ipfs/go-libp2p-peer" + pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" manet "github.com/jbenet/go-multiaddr-net" context "golang.org/x/net/context" @@ -32,7 +33,7 @@ func TestBasicDial(t *testing.T) { s1 := swarms[0] s2 := swarms[1] - s1.peers.AddAddrs(s2.local, s2.ListenAddresses(), peer.PermanentAddrTTL) + s1.peers.AddAddrs(s2.local, s2.ListenAddresses(), pstore.PermanentAddrTTL) c, err := s1.Dial(ctx, s2.local) if err != nil { @@ -57,7 +58,7 @@ func TestDialWithNoListeners(t *testing.T) { defer closeSwarms(swarms) s2 := swarms[0] - s1.peers.AddAddrs(s2.local, s2.ListenAddresses(), peer.PermanentAddrTTL) + s1.peers.AddAddrs(s2.local, s2.ListenAddresses(), pstore.PermanentAddrTTL) c, err := s1.Dial(ctx, s2.local) if err != nil { @@ -101,7 +102,7 @@ func TestSimultDials(t *testing.T) { connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr) - s.peers.AddAddr(dst, addr, peer.TempAddrTTL) + s.peers.AddAddr(dst, addr, pstore.TempAddrTTL) if _, err := s.Dial(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } @@ -178,7 +179,7 @@ func TestDialWait(t *testing.T) { s2p, s2addr, s2l := newSilentPeer(t) go acceptAndHang(s2l) defer s2l.Close() - s1.peers.AddAddr(s2p, s2addr, peer.PermanentAddrTTL) + s1.peers.AddAddr(s2p, s2addr, pstore.PermanentAddrTTL) before := time.Now() if c, err := s1.Dial(ctx, s2p); err == nil { @@ -224,13 +225,13 @@ func TestDialBackoff(t *testing.T) { if err != nil { t.Fatal(err) } - s1.peers.AddAddrs(s2.local, s2addrs, peer.PermanentAddrTTL) + s1.peers.AddAddrs(s2.local, s2addrs, pstore.PermanentAddrTTL) // dial to a non-existent peer. s3p, s3addr, s3l := newSilentPeer(t) go acceptAndHang(s3l) defer s3l.Close() - s1.peers.AddAddr(s3p, s3addr, peer.PermanentAddrTTL) + s1.peers.AddAddr(s3p, s3addr, pstore.PermanentAddrTTL) // in this test we will: // 1) dial 10x to each node. @@ -442,7 +443,7 @@ func TestDialBackoffClears(t *testing.T) { defer s2l.Close() // phase 1 -- dial to non-operational addresses - s1.peers.AddAddr(s2.local, s2bad, peer.PermanentAddrTTL) + s1.peers.AddAddr(s2.local, s2bad, pstore.PermanentAddrTTL) before := time.Now() if c, err := s1.Dial(ctx, s2.local); err == nil { @@ -472,7 +473,7 @@ func TestDialBackoffClears(t *testing.T) { if err != nil { t.Fatal(err) } - s1.peers.AddAddrs(s2.local, ifaceAddrs1, peer.PermanentAddrTTL) + s1.peers.AddAddrs(s2.local, ifaceAddrs1, pstore.PermanentAddrTTL) if _, err := s1.Dial(ctx, s2.local); err == nil { t.Fatal("should have failed to dial backed off peer") diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index cacd88ece8..b63e655b11 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -4,6 +4,7 @@ import ( "testing" peer "github.com/ipfs/go-libp2p-peer" + pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" context "golang.org/x/net/context" ) @@ -17,7 +18,7 @@ func TestPeers(t *testing.T) { connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. - s.peers.AddAddr(dst, addr, peer.PermanentAddrTTL) + s.peers.AddAddr(dst, addr, pstore.PermanentAddrTTL) // t.Logf("connections from %s", s.LocalPeer()) // for _, c := range s.ConnectionsToPeer(dst) { // t.Logf("connection from %s to %s: %v", s.LocalPeer(), dst, c) diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 39ba6411b6..e0165262e3 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -6,9 +6,10 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p-peer" ci "github.com/ipfs/go-libp2p/testutil/ci" + peer "github.com/ipfs/go-libp2p-peer" + pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" context "golang.org/x/net/context" ) @@ -26,7 +27,7 @@ func TestSimultOpen(t *testing.T) { connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr) - s.peers.AddAddr(dst, addr, peer.PermanentAddrTTL) + s.peers.AddAddr(dst, addr, pstore.PermanentAddrTTL) if _, err := s.Dial(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 74e43dde44..69591d0caa 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -11,6 +11,7 @@ import ( "time" peer "github.com/ipfs/go-libp2p-peer" + pstore "github.com/ipfs/go-libp2p-peerstore" transport "github.com/ipfs/go-libp2p-transport" metrics "github.com/ipfs/go-libp2p/p2p/metrics" mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" @@ -67,7 +68,7 @@ func init() { type Swarm struct { swarm *ps.Swarm local peer.ID - peers peer.Peerstore + peers pstore.Peerstore connh ConnHandler dsync dialsync @@ -94,7 +95,7 @@ type Swarm struct { // NewSwarm constructs a Swarm, with a Chan. func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, - local peer.ID, peers peer.Peerstore, bwc metrics.Reporter) (*Swarm, error) { + local peer.ID, peers pstore.Peerstore, bwc metrics.Reporter) (*Swarm, error) { listenAddrs, err := filterAddrs(listenAddrs) if err != nil { diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 42ee299e87..0f9419bf34 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -3,11 +3,11 @@ package swarm import ( "testing" - peer "github.com/ipfs/go-libp2p-peer" metrics "github.com/ipfs/go-libp2p/p2p/metrics" addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" testutil "github.com/ipfs/go-libp2p/testutil" + pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" context "golang.org/x/net/context" ) @@ -63,7 +63,7 @@ func TestFilterAddrs(t *testing.T) { t.Fatal(err) } - ps := peer.NewPeerstore() + ps := pstore.NewPeerstore() ctx := context.Background() if _, err := NewNetwork(ctx, bad, id, ps, metrics.NewBandwidthCounter()); err == nil { @@ -111,7 +111,7 @@ func TestDialBadAddrs(t *testing.T) { test := func(a ma.Multiaddr) { p := testutil.RandPeerIDFatal(t) - s.peers.AddAddr(p, a, peer.PermanentAddrTTL) + s.peers.AddAddr(p, a, pstore.PermanentAddrTTL) if _, err := s.Dial(ctx, p); err == nil { t.Error("swarm should not dial: %s", m) } diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 84cc6326eb..f13da04083 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -4,6 +4,7 @@ import ( "fmt" peer "github.com/ipfs/go-libp2p-peer" + pstore "github.com/ipfs/go-libp2p-peerstore" metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" @@ -19,7 +20,7 @@ type Network Swarm // NewNetwork constructs a new network and starts listening on given addresses. func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, - peers peer.Peerstore, bwc metrics.Reporter) (*Network, error) { + peers pstore.Peerstore, bwc metrics.Reporter) (*Network, error) { s, err := NewSwarm(ctx, listen, local, peers, bwc) if err != nil { @@ -63,7 +64,7 @@ func (n *Network) Peers() []peer.ID { } // Peers returns the Peerstore, which tracks known peers -func (n *Network) Peerstore() peer.Peerstore { +func (n *Network) Peerstore() pstore.Peerstore { return n.Swarm().peers } diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 6e9f121fcd..9160c4c6ae 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -9,11 +9,12 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p-peer" metrics "github.com/ipfs/go-libp2p/p2p/metrics" inet "github.com/ipfs/go-libp2p/p2p/net" testutil "github.com/ipfs/go-libp2p/testutil" + peer "github.com/ipfs/go-libp2p-peer" + pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" context "golang.org/x/net/context" ) @@ -52,7 +53,7 @@ func EchoStreamHandler(stream inet.Stream) { func makeDialOnlySwarm(ctx context.Context, t *testing.T) *Swarm { id := testutil.RandIdentityOrFatal(t) - peerstore := peer.NewPeerstore() + peerstore := pstore.NewPeerstore() peerstore.AddPubKey(id.ID(), id.PublicKey()) peerstore.AddPrivKey(id.ID(), id.PrivateKey()) @@ -72,7 +73,7 @@ func makeSwarms(ctx context.Context, t *testing.T, num int) []*Swarm { for i := 0; i < num; i++ { localnp := testutil.RandPeerNetParamsOrFatal(t) - peerstore := peer.NewPeerstore() + peerstore := pstore.NewPeerstore() peerstore.AddPubKey(localnp.ID, localnp.PubKey) peerstore.AddPrivKey(localnp.ID, localnp.PrivKey) @@ -94,7 +95,7 @@ func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { var wg sync.WaitGroup connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. - s.peers.AddAddr(dst, addr, peer.PermanentAddrTTL) + s.peers.AddAddr(dst, addr, pstore.PermanentAddrTTL) if _, err := s.Dial(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } @@ -313,13 +314,13 @@ func TestAddrBlocking(t *testing.T) { swarms[1].Filters.AddDialFilter(block) - swarms[1].peers.AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], peer.PermanentAddrTTL) + swarms[1].peers.AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], pstore.PermanentAddrTTL) _, err = swarms[1].Dial(ctx, swarms[0].LocalPeer()) if err == nil { t.Fatal("dial should have failed") } - swarms[0].peers.AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], peer.PermanentAddrTTL) + swarms[0].peers.AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], pstore.PermanentAddrTTL) _, err = swarms[0].Dial(ctx, swarms[1].LocalPeer()) if err == nil { t.Fatal("dial should have failed") From f374c546885999dd01d8e34d0b43922f1071f5d6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 1 Jun 2016 10:02:26 -0700 Subject: [PATCH 039/259] improve comment on temp code --- p2p/net/swarm/swarm_dial.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 33b087efc3..1100ca544d 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -330,8 +330,14 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { s.Filters.AddrBlocked) */ - ////// TEMP UNTIL PEERSTORE GETS UPGRADED - // Ref: https://github.com/ipfs/go-libp2p-peer/pull/1 + ////// + /* + This code is temporary, the peerstore can currently provide + a channel as an interface for receiving addresses, but more thought + needs to be put into the execution. For now, this allows us to use + the improved rate limiter, while maintaining the outward behaviour + that we previously had (halting a dial when we run out of addrs) + */ paddrs := s.peers.Addrs(p) good_addrs := addrutil.FilterAddrs(paddrs, addrutil.AddrUsableFunc, From 7b26e0b818ce3a2bb855f76309556afa9ce2b63c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 3 Jul 2016 13:28:06 -0700 Subject: [PATCH 040/259] expose backoff manager in swarm --- p2p/net/swarm/swarm.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 100a9826d8..15e1ec5e1b 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -152,6 +152,7 @@ func (s *Swarm) AddAddrFilter(f string) error { s.Filters.AddDialFilter(m) return nil } + func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { if len(listenAddrs) > 0 { filtered := addrutil.FilterUsableAddrs(listenAddrs) @@ -285,6 +286,10 @@ func (s *Swarm) LocalPeer() peer.ID { return s.local } +func (s *Swarm) Backoff() *dialbackoff { + return &s.backf +} + // notifyAll sends a signal to all Notifiees func (s *Swarm) notifyAll(notify func(inet.Notifiee)) { s.notifmu.RLock() From 133ace853665ec95b2f40a2522de7aad44e453f4 Mon Sep 17 00:00:00 2001 From: John Steidley Date: Sat, 23 Jul 2016 13:24:45 -0700 Subject: [PATCH 041/259] gosimple --- p2p/net/swarm/dial_test.go | 4 ++-- p2p/net/swarm/swarm_net.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 515deb100f..031f3b3a5e 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -187,7 +187,7 @@ func TestDialWait(t *testing.T) { } else { t.Log("correctly got error:", err) } - duration := time.Now().Sub(before) + duration := time.Since(before) dt := s1.dialT if duration < dt*dialAttempts { @@ -451,7 +451,7 @@ func TestDialBackoffClears(t *testing.T) { } else { t.Log("correctly got error:", err) } - duration := time.Now().Sub(before) + duration := time.Since(before) dt := s1.dialT if duration < dt*dialAttempts { diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index f13da04083..92d2a801eb 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -124,7 +124,7 @@ func (n *Network) InterfaceListenAddresses() ([]ma.Multiaddr, error) { // For now only returns Connected || NotConnected. Expand into more later. func (n *Network) Connectedness(p peer.ID) inet.Connectedness { c := n.Swarm().ConnectionsToPeer(p) - if c != nil && len(c) > 0 { + if len(c) > 0 { return inet.Connected } return inet.NotConnected From f223e85a366e42106cdf8123980453a30a4d3600 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Aug 2016 09:48:34 -0700 Subject: [PATCH 042/259] update multistream deps and fix code to work with new changes --- p2p/net/swarm/swarm.go | 4 ++-- p2p/net/swarm/swarm_notif_test.go | 10 ++++++++-- p2p/net/swarm/swarm_stream.go | 27 ++++++++++++++++++++------- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 15e1ec5e1b..1fcf0f1c14 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -340,9 +340,9 @@ func (n *ps2netNotifee) Disconnected(c *ps.Conn) { } func (n *ps2netNotifee) OpenedStream(s *ps.Stream) { - n.not.OpenedStream(n.net, inet.Stream((*Stream)(s))) + n.not.OpenedStream(n.net, &Stream{stream: s}) } func (n *ps2netNotifee) ClosedStream(s *ps.Stream) { - n.not.ClosedStream(n.net, inet.Stream((*Stream)(s))) + n.not.ClosedStream(n.net, &Stream{stream: s}) } diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index aa4fca2448..baced67e61 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -10,6 +10,12 @@ import ( context "golang.org/x/net/context" ) +func streamsSame(a, b inet.Stream) bool { + sa := a.(*Stream) + sb := b.(*Stream) + return sa.Stream() == sb.Stream() +} + func TestNotifications(t *testing.T) { ctx := context.Background() swarms := makeSwarms(ctx, t, 5) @@ -98,7 +104,7 @@ func TestNotifications(t *testing.T) { case <-time.After(timeout): t.Fatal("timeout") } - if s != s2 { + if !streamsSame(s, s2) { t.Fatal("got incorrect stream", s.Conn(), s2.Conn()) } @@ -108,7 +114,7 @@ func TestNotifications(t *testing.T) { case <-time.After(timeout): t.Fatal("timeout") } - if s != s2 { + if !streamsSame(s, s2) { t.Fatal("got incorrect stream", s.Conn(), s2.Conn()) } } diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 7965d27439..16aa6e07c9 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -8,11 +8,14 @@ import ( // a Stream is a wrapper around a ps.Stream that exposes a way to get // our Conn and Swarm (instead of just the ps.Conn and ps.Swarm) -type Stream ps.Stream +type Stream struct { + stream *ps.Stream + protocol string +} // Stream returns the underlying peerstream.Stream func (s *Stream) Stream() *ps.Stream { - return (*ps.Stream)(s) + return s.stream } // Conn returns the Conn associated with this Stream, as an inet.Conn @@ -22,27 +25,37 @@ func (s *Stream) Conn() inet.Conn { // SwarmConn returns the Conn associated with this Stream, as a *Conn func (s *Stream) SwarmConn() *Conn { - return (*Conn)(s.Stream().Conn()) + return (*Conn)(s.stream.Conn()) } // Read reads bytes from a stream. func (s *Stream) Read(p []byte) (n int, err error) { - return s.Stream().Read(p) + return s.stream.Read(p) } // Write writes bytes to a stream, flushing for each call. func (s *Stream) Write(p []byte) (n int, err error) { - return s.Stream().Write(p) + return s.stream.Write(p) } // Close closes the stream, indicating this side is finished // with the stream. func (s *Stream) Close() error { - return s.Stream().Close() + return s.stream.Close() +} + +func (s *Stream) Protocol() string { + return s.protocol +} + +func (s *Stream) SetProtocol(p string) { + s.protocol = p } func wrapStream(pss *ps.Stream) *Stream { - return (*Stream)(pss) + return &Stream{ + stream: pss, + } } func wrapStreams(st []*ps.Stream) []*Stream { From a2a399b0ee18614276e5b75fe564df3bbd600bbd Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Aug 2016 09:59:14 -0700 Subject: [PATCH 043/259] update local import paths to reflect org change --- p2p/net/swarm/addr/addr.go | 2 +- p2p/net/swarm/dial_test.go | 6 +++--- p2p/net/swarm/limiter.go | 4 ++-- p2p/net/swarm/limiter_test.go | 2 +- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm.go | 12 ++++++------ p2p/net/swarm/swarm_addr.go | 4 ++-- p2p/net/swarm/swarm_addr_test.go | 6 +++--- p2p/net/swarm/swarm_conn.go | 4 ++-- p2p/net/swarm/swarm_dial.go | 4 ++-- p2p/net/swarm/swarm_listen.go | 6 +++--- p2p/net/swarm/swarm_net.go | 4 ++-- p2p/net/swarm/swarm_net_test.go | 4 ++-- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 6 +++--- 16 files changed, 35 insertions(+), 35 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index d9ba87216a..7759625f11 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -9,7 +9,7 @@ import ( context "golang.org/x/net/context" ) -var log = logging.Logger("github.com/ipfs/go-libp2p/p2p/net/swarm/addr") +var log = logging.Logger("github.com/libp2p/go-libp2p/p2p/net/swarm/addr") // SupportedTransportStrings is the list of supported transports for the swarm. // These are strings of encapsulated multiaddr protocols. E.g.: diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 031f3b3a5e..cfdb996c28 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - testutil "github.com/ipfs/go-libp2p/testutil" - ci "github.com/ipfs/go-libp2p/testutil/ci" + addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" + testutil "github.com/libp2p/go-libp2p/testutil" + ci "github.com/libp2p/go-libp2p/testutil/ci" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 94ce05bb86..19fc84c329 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -7,8 +7,8 @@ import ( ma "github.com/jbenet/go-multiaddr" context "golang.org/x/net/context" - conn "github.com/ipfs/go-libp2p/p2p/net/conn" - addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" + conn "github.com/libp2p/go-libp2p/p2p/net/conn" + addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" ) type dialResult struct { diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 28733c5ab5..6ee6349a7b 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -12,7 +12,7 @@ import ( mafmt "github.com/whyrusleeping/mafmt" context "golang.org/x/net/context" - conn "github.com/ipfs/go-libp2p/p2p/net/conn" + conn "github.com/libp2p/go-libp2p/p2p/net/conn" ) func mustAddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index e0165262e3..c6ee6cf985 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - ci "github.com/ipfs/go-libp2p/testutil/ci" + ci "github.com/libp2p/go-libp2p/testutil/ci" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 1fcf0f1c14..a86edb372f 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -13,18 +13,18 @@ import ( peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" transport "github.com/ipfs/go-libp2p-transport" - metrics "github.com/ipfs/go-libp2p/p2p/metrics" - mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" - inet "github.com/ipfs/go-libp2p/p2p/net" - conn "github.com/ipfs/go-libp2p/p2p/net/conn" - filter "github.com/ipfs/go-libp2p/p2p/net/filter" - addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" logging "github.com/ipfs/go-log" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" pst "github.com/jbenet/go-stream-muxer" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" + metrics "github.com/libp2p/go-libp2p/p2p/metrics" + mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" + inet "github.com/libp2p/go-libp2p/p2p/net" + conn "github.com/libp2p/go-libp2p/p2p/net/conn" + filter "github.com/libp2p/go-libp2p/p2p/net/filter" + addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" psmss "github.com/whyrusleeping/go-smux-multistream" spdy "github.com/whyrusleeping/go-smux-spdystream" yamux "github.com/whyrusleeping/go-smux-yamux" diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 039683bfb9..85d4aafe85 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -1,8 +1,8 @@ package swarm import ( - conn "github.com/ipfs/go-libp2p/p2p/net/conn" - addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" + conn "github.com/libp2p/go-libp2p/p2p/net/conn" + addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" ma "github.com/jbenet/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 0f9419bf34..37eacbfd71 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -3,9 +3,9 @@ package swarm import ( "testing" - metrics "github.com/ipfs/go-libp2p/p2p/metrics" - addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" - testutil "github.com/ipfs/go-libp2p/testutil" + metrics "github.com/libp2p/go-libp2p/p2p/metrics" + addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" + testutil "github.com/libp2p/go-libp2p/testutil" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 96700ba835..36932784c9 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -3,8 +3,8 @@ package swarm import ( "fmt" - inet "github.com/ipfs/go-libp2p/p2p/net" - conn "github.com/ipfs/go-libp2p/p2p/net/conn" + inet "github.com/libp2p/go-libp2p/p2p/net" + conn "github.com/libp2p/go-libp2p/p2p/net/conn" ic "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 1100ca544d..2f4d29fa51 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -8,9 +8,9 @@ import ( lgbl "github.com/ipfs/go-libp2p-loggables" peer "github.com/ipfs/go-libp2p-peer" - conn "github.com/ipfs/go-libp2p/p2p/net/conn" - addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr" ma "github.com/jbenet/go-multiaddr" + conn "github.com/libp2p/go-libp2p/p2p/net/conn" + addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 38457f16fe..64fdb83bd0 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -5,11 +5,11 @@ import ( lgbl "github.com/ipfs/go-libp2p-loggables" transport "github.com/ipfs/go-libp2p-transport" - mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn" - inet "github.com/ipfs/go-libp2p/p2p/net" - conn "github.com/ipfs/go-libp2p/p2p/net/conn" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" + mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" + inet "github.com/libp2p/go-libp2p/p2p/net" + conn "github.com/libp2p/go-libp2p/p2p/net/conn" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 92d2a801eb..8624dedba1 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -5,8 +5,8 @@ import ( peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" - metrics "github.com/ipfs/go-libp2p/p2p/metrics" - inet "github.com/ipfs/go-libp2p/p2p/net" + metrics "github.com/libp2p/go-libp2p/p2p/metrics" + inet "github.com/libp2p/go-libp2p/p2p/net" ma "github.com/jbenet/go-multiaddr" "github.com/jbenet/goprocess" diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 14c590b9d1..fddc8523fa 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - inet "github.com/ipfs/go-libp2p/p2p/net" - testutil "github.com/ipfs/go-libp2p/p2p/test/util" + inet "github.com/libp2p/go-libp2p/p2p/net" + testutil "github.com/libp2p/go-libp2p/p2p/test/util" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index baced67e61..06777469f5 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -5,8 +5,8 @@ import ( "time" peer "github.com/ipfs/go-libp2p-peer" - inet "github.com/ipfs/go-libp2p/p2p/net" ma "github.com/jbenet/go-multiaddr" + inet "github.com/libp2p/go-libp2p/p2p/net" context "golang.org/x/net/context" ) diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 16aa6e07c9..b305bf917e 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -1,7 +1,7 @@ package swarm import ( - inet "github.com/ipfs/go-libp2p/p2p/net" + inet "github.com/libp2p/go-libp2p/p2p/net" ps "github.com/jbenet/go-peerstream" ) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index a19688d90b..c11377ed5e 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -9,9 +9,9 @@ import ( "testing" "time" - metrics "github.com/ipfs/go-libp2p/p2p/metrics" - inet "github.com/ipfs/go-libp2p/p2p/net" - testutil "github.com/ipfs/go-libp2p/testutil" + metrics "github.com/libp2p/go-libp2p/p2p/metrics" + inet "github.com/libp2p/go-libp2p/p2p/net" + testutil "github.com/libp2p/go-libp2p/testutil" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" From 2ca1eb131e40fb67233609157f1c17cee730b4a5 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 4 Aug 2016 05:47:26 -0700 Subject: [PATCH 044/259] lint: fixed a bunch of issues reported by gometalinter --- p2p/net/swarm/limiter_test.go | 22 +++++++++++----------- p2p/net/swarm/swarm.go | 8 +++++++- p2p/net/swarm/swarm_addr.go | 12 ------------ p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 3 ++- p2p/net/swarm/swarm_dial.go | 24 +++++++++++++++--------- p2p/net/swarm/swarm_net.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 2 +- 8 files changed, 39 insertions(+), 38 deletions(-) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 6ee6349a7b..f5fc18746f 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -63,10 +63,10 @@ func hangDialFunc(hang chan struct{}) dialfunc { if tcpPortOver(a, 10) { return conn.Conn(nil), nil - } else { - <-hang - return nil, fmt.Errorf("test bad dial") } + + <-hang + return nil, fmt.Errorf("test bad dial") } } @@ -127,7 +127,7 @@ func TestFDLimiting(t *testing.T) { bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} pids := []peer.ID{"testpeer1", "testpeer2", "testpeer3", "testpeer4"} - good_tcp := addrWithPort(t, 20) + goodTCP := addrWithPort(t, 20) ctx := context.Background() resch := make(chan dialResult) @@ -143,7 +143,7 @@ func TestFDLimiting(t *testing.T) { l.AddDialJob(&dialJob{ ctx: ctx, peer: pid, - addr: good_tcp, + addr: goodTCP, resp: resch, }) } @@ -175,10 +175,10 @@ func TestTokenRedistribution(t *testing.T) { df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { if tcpPortOver(a, 10) { return (conn.Conn)(nil), nil - } else { - <-hangchs[p] - return nil, fmt.Errorf("test bad dial") } + + <-hangchs[p] + return nil, fmt.Errorf("test bad dial") } l := newDialLimiterWithParams(df, 8, 4) @@ -264,10 +264,10 @@ func TestStressLimiter(t *testing.T) { df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { if tcpPortOver(a, 1000) { return conn.Conn(nil), nil - } else { - time.Sleep(time.Millisecond * time.Duration(5+rand.Intn(100))) - return nil, fmt.Errorf("test bad dial") } + + time.Sleep(time.Millisecond * time.Duration(5+rand.Intn(100))) + return nil, fmt.Errorf("test bad dial") } l := newDialLimiterWithParams(df, 20, 5) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index a86edb372f..704beb761e 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -1,4 +1,4 @@ -// package swarm implements a connection muxer with a pair of channels +// Package swarm implements a connection muxer with a pair of channels // to synchronize all network communication. package swarm @@ -34,6 +34,8 @@ import ( var log = logging.Logger("swarm2") +// PSTransport is the default peerstream transport that will be used by +// any libp2p swarms. var PSTransport pst.Transport func init() { @@ -143,6 +145,8 @@ func (s *Swarm) teardown() error { return s.swarm.Close() } +// AddAddrFilter adds a multiaddr filter to the set of filters the swarm will +// use to determine which addresses not to dial to. func (s *Swarm) AddAddrFilter(f string) error { m, err := mafilter.NewMask(f) if err != nil { @@ -165,6 +169,7 @@ func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { return listenAddrs, nil } +// Listen sets up listeners for all of the given addresses func (s *Swarm) Listen(addrs ...ma.Multiaddr) error { addrs, err := filterAddrs(addrs) if err != nil { @@ -286,6 +291,7 @@ func (s *Swarm) LocalPeer() peer.ID { return s.local } +// Backoff returns the dialbackoff object for this swarm. func (s *Swarm) Backoff() *dialbackoff { return &s.backf } diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 85d4aafe85..39fbe02afc 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -25,15 +25,3 @@ func (s *Swarm) ListenAddresses() []ma.Multiaddr { func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { return addrutil.ResolveUnspecifiedAddresses(s.ListenAddresses(), nil) } - -// checkNATWarning checks if our observed addresses differ. if so, -// informs the user that certain things might not work yet -func checkNATWarning(s *Swarm, observed ma.Multiaddr, expected ma.Multiaddr) { - listen, err := s.InterfaceListenAddresses() - if err != nil { - log.Debugf("Error retrieving swarm.InterfaceListenAddresses: %s", err) - return - } - - addrutil.CheckNATWarning(observed, expected, listen) -} diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 37eacbfd71..772413b286 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -113,7 +113,7 @@ func TestDialBadAddrs(t *testing.T) { p := testutil.RandPeerIDFatal(t) s.peers.AddAddr(p, a, pstore.PermanentAddrTTL) if _, err := s.Dial(ctx, p); err == nil { - t.Error("swarm should not dial: %s", m) + t.Errorf("swarm should not dial: %s", p) } } diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 36932784c9..762ba6e1d3 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -13,7 +13,7 @@ import ( context "golang.org/x/net/context" ) -// a Conn is a simple wrapper around a ps.Conn that also exposes +// Conn is a simple wrapper around a ps.Conn that also exposes // some of the methods from the underlying conn.Conn. // There's **five** "layers" to each connection: // * 0. the net.Conn - underlying net.Conn (TCP/UDP/UTP/etc) @@ -87,6 +87,7 @@ func (c *Conn) NewStream() (inet.Stream, error) { return inet.Stream(s), err } +// Close closes the underlying stream connection func (c *Conn) Close() error { return c.StreamConn().Close() } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 2f4d29fa51..6774721247 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -26,9 +26,15 @@ import ( // retry dialAttempt x var ( + // ErrDialBackoff is returned by the backoff code when a given peer has + // been dialed too frequently ErrDialBackoff = errors.New("dial backoff") - ErrDialFailed = errors.New("dial attempt failed") - ErrDialToSelf = errors.New("dial to self attempted") + + // ErrDialFailed is returned when connecting to a peer has ultimately failed + ErrDialFailed = errors.New("dial attempt failed") + + // ErrDialToSelf is returned if we attempt to dial our own peer + ErrDialToSelf = errors.New("dial to self attempted") ) // dialAttempts governs how many times a goroutine will try to dial a given peer. @@ -45,7 +51,7 @@ const defaultPerPeerRateLimit = 8 // DialTimeout is the amount of time each dial attempt has. We can think about making // this larger down the road, or putting more granular timeouts (i.e. within each // subcomponent of Dial) -var DialTimeout time.Duration = time.Second * 10 +var DialTimeout = time.Second * 10 // dialsync is a small object that helps manage ongoing dials. // this way, if we receive many simultaneous dial requests, one @@ -320,13 +326,13 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { } ila, _ := s.InterfaceListenAddresses() - subtract_filter := addrutil.SubtractFilter(append(ila, s.peers.Addrs(s.local)...)...) + subtractFilter := addrutil.SubtractFilter(append(ila, s.peers.Addrs(s.local)...)...) // get live channel of addresses for peer, filtered by the given filters /* remoteAddrChan := s.peers.AddrsChan(ctx, p, addrutil.AddrUsableFilter, - subtract_filter, + subtractFilter, s.Filters.AddrBlocked) */ @@ -339,13 +345,13 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { that we previously had (halting a dial when we run out of addrs) */ paddrs := s.peers.Addrs(p) - good_addrs := addrutil.FilterAddrs(paddrs, + goodAddrs := addrutil.FilterAddrs(paddrs, addrutil.AddrUsableFunc, - subtract_filter, + subtractFilter, addrutil.FilterNeg(s.Filters.AddrBlocked), ) - remoteAddrChan := make(chan ma.Multiaddr, len(good_addrs)) - for _, a := range good_addrs { + remoteAddrChan := make(chan ma.Multiaddr, len(goodAddrs)) + for _, a := range goodAddrs { remoteAddrChan <- a } close(remoteAddrChan) diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 8624dedba1..48defb11ad 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -63,7 +63,7 @@ func (n *Network) Peers() []peer.ID { return n.Swarm().Peers() } -// Peers returns the Peerstore, which tracks known peers +// Peerstore returns the Peerstore, which tracks known peers func (n *Network) Peerstore() pstore.Peerstore { return n.Swarm().peers } @@ -142,7 +142,7 @@ func (n *Network) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) return inet.Stream(s), nil } -// SetHandler sets the protocol handler on the Network's Muxer. +// SetStreamHandler sets the protocol handler on the Network's Muxer. // This operation is threadsafe. func (n *Network) SetStreamHandler(h inet.StreamHandler) { n.Swarm().SetStreamHandler(h) diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index b305bf917e..86719819a5 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -6,7 +6,7 @@ import ( ps "github.com/jbenet/go-peerstream" ) -// a Stream is a wrapper around a ps.Stream that exposes a way to get +// Stream is a wrapper around a ps.Stream that exposes a way to get // our Conn and Swarm (instead of just the ps.Conn and ps.Swarm) type Stream struct { stream *ps.Stream From 3471cda23c3fd24e9e9de2c1197b20cce11a2402 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 16 Aug 2016 15:04:24 -0700 Subject: [PATCH 045/259] p2p/net/swarm: improve code coverage --- p2p/net/swarm/limiter.go | 12 ---- p2p/net/swarm/swarm_addr.go | 12 ---- p2p/net/swarm/swarm_net_test.go | 104 +++++++++++++++++++++++++++++--- 3 files changed, 96 insertions(+), 32 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 19fc84c329..4bbfb11862 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -131,18 +131,6 @@ func (dl *dialLimiter) AddDialJob(dj *dialJob) { go dl.executeDial(dj) } -func (dl *dialLimiter) schedulePerPeerDial(j *dialJob) { - if dl.activePerPeer[j.peer] >= dl.perPeerLimit { - wlist := dl.waitingOnPeerLimit[j.peer] - dl.waitingOnPeerLimit[j.peer] = append(wlist, j) - return - } - - // take second needed token and start dial! - dl.activePerPeer[j.peer]++ - go dl.executeDial(j) -} - // executeDial calls the dialFunc, and reports the result through the response // channel when finished. Once the response is sent it also releases all tokens // it held during the dial. diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 85d4aafe85..39fbe02afc 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -25,15 +25,3 @@ func (s *Swarm) ListenAddresses() []ma.Multiaddr { func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { return addrutil.ResolveUnspecifiedAddresses(s.ListenAddresses(), nil) } - -// checkNATWarning checks if our observed addresses differ. if so, -// informs the user that certain things might not work yet -func checkNATWarning(s *Swarm, observed ma.Multiaddr, expected ma.Multiaddr) { - listen, err := s.InterfaceListenAddresses() - if err != nil { - log.Debugf("Error retrieving swarm.InterfaceListenAddresses: %s", err) - return - } - - addrutil.CheckNATWarning(observed, expected, listen) -} diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index fddc8523fa..1294e899ba 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -35,9 +35,10 @@ func TestConnectednessCorrect(t *testing.T) { dial(nets[1], nets[2]) dial(nets[3], nets[2]) - // there's something wrong with dial, i think. it's not finishing - // completely. there must be some async stuff. - <-time.After(100 * time.Millisecond) + // The notifications for new connections get sent out asynchronously. + // There is the potential for a race condition here, so we sleep to ensure + // that they have been received. + time.Sleep(time.Millisecond * 100) // test those connected show up correctly @@ -51,20 +52,44 @@ func TestConnectednessCorrect(t *testing.T) { expectConnectedness(t, nets[0], nets[2], inet.NotConnected) expectConnectedness(t, nets[1], nets[3], inet.NotConnected) + if len(nets[0].Peers()) != 2 { + t.Fatal("expected net 0 to have two peers") + } + + if len(nets[2].Conns()) != 2 { + t.Fatal("expected net 2 to have two conns") + } + + if len(nets[1].ConnsToPeer(nets[3].LocalPeer())) != 0 { + t.Fatal("net 1 should have no connections to net 3") + } + + if err := nets[2].ClosePeer(nets[1].LocalPeer()); err != nil { + t.Fatal(err) + } + + expectConnectedness(t, nets[2], nets[1], inet.NotConnected) + for _, n := range nets { n.Close() } + + for _, n := range nets { + <-n.Process().Closed() + } } func expectConnectedness(t *testing.T, a, b inet.Network, expected inet.Connectedness) { - es := "%s is connected to %s, but Connectedness incorrect. %s %s" - if a.Connectedness(b.LocalPeer()) != expected { - t.Errorf(es, a, b, printConns(a), printConns(b)) + es := "%s is connected to %s, but Connectedness incorrect. %s %s %s" + atob := a.Connectedness(b.LocalPeer()) + btoa := b.Connectedness(a.LocalPeer()) + if atob != expected { + t.Errorf(es, a, b, printConns(a), printConns(b), atob) } // test symmetric case - if b.Connectedness(a.LocalPeer()) != expected { - t.Errorf(es, b, a, printConns(b), printConns(a)) + if btoa != expected { + t.Errorf(es, b, a, printConns(b), printConns(a), btoa) } } @@ -75,3 +100,66 @@ func printConns(n inet.Network) string { } return s } + +func TestNetworkOpenStream(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + nets := make([]inet.Network, 4) + for i := 0; i < 4; i++ { + nets[i] = testutil.GenSwarmNetwork(t, ctx) + } + + dial := func(a, b inet.Network) { + testutil.DivulgeAddresses(b, a) + if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { + t.Fatalf("Failed to dial: %s", err) + } + } + + dial(nets[0], nets[1]) + dial(nets[0], nets[3]) + dial(nets[1], nets[2]) + + done := make(chan bool) + nets[1].SetStreamHandler(func(s inet.Stream) { + defer close(done) + defer s.Close() + + buf := make([]byte, 10) + _, err := s.Read(buf) + if err != nil { + t.Error(err) + return + } + if string(buf) != "hello ipfs" { + t.Error("got wrong message") + } + }) + + s, err := nets[0].NewStream(ctx, nets[1].LocalPeer()) + if err != nil { + t.Fatal(err) + } + + _, err = s.Write([]byte("hello ipfs")) + if err != nil { + t.Fatal(err) + } + + err = s.Close() + if err != nil { + t.Fatal(err) + } + + select { + case <-done: + case <-time.After(time.Millisecond * 100): + t.Fatal("timed out waiting on stream") + } + + _, err = nets[1].NewStream(ctx, nets[3].LocalPeer()) + if err == nil { + t.Fatal("expected stream open 1->3 to fail") + } +} From 873e29641af5dcda8a75f80c7de48fc5d5be5351 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 17 Aug 2016 18:40:08 -0700 Subject: [PATCH 046/259] swarm: make stream.Protocol() return type protocol.ID --- p2p/net/swarm/swarm_stream.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 86719819a5..dc365e8c09 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -2,6 +2,7 @@ package swarm import ( inet "github.com/libp2p/go-libp2p/p2p/net" + protocol "github.com/libp2p/go-libp2p/p2p/protocol" ps "github.com/jbenet/go-peerstream" ) @@ -10,7 +11,7 @@ import ( // our Conn and Swarm (instead of just the ps.Conn and ps.Swarm) type Stream struct { stream *ps.Stream - protocol string + protocol protocol.ID } // Stream returns the underlying peerstream.Stream @@ -44,11 +45,11 @@ func (s *Stream) Close() error { return s.stream.Close() } -func (s *Stream) Protocol() string { +func (s *Stream) Protocol() protocol.ID { return s.protocol } -func (s *Stream) SetProtocol(p string) { +func (s *Stream) SetProtocol(p protocol.ID) { s.protocol = p } From 275e18319f81683d7bc207da0e145da5cae3bc36 Mon Sep 17 00:00:00 2001 From: Jeromy Johnson Date: Thu, 18 Aug 2016 11:19:20 -0700 Subject: [PATCH 047/259] sleep before final connectedness check --- p2p/net/swarm/swarm_net_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 1294e899ba..4d564e3901 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -68,6 +68,8 @@ func TestConnectednessCorrect(t *testing.T) { t.Fatal(err) } + time.Sleep(time.Millisecond * 50) + expectConnectedness(t, nets[2], nets[1], inet.NotConnected) for _, n := range nets { From b619447584cafbce0304db59e24136c376568fb8 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 16 May 2016 15:27:58 -0700 Subject: [PATCH 048/259] add in experimental websocket support --- p2p/net/swarm/addr/addr.go | 4 ++++ p2p/net/swarm/addr/addr_test.go | 1 + p2p/net/swarm/swarm.go | 2 ++ 3 files changed, 7 insertions(+) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 7759625f11..08ca90ac46 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -7,6 +7,8 @@ import ( ma "github.com/jbenet/go-multiaddr" manet "github.com/jbenet/go-multiaddr-net" context "golang.org/x/net/context" + + _ "github.com/whyrusleeping/ws-transport" ) var log = logging.Logger("github.com/libp2p/go-libp2p/p2p/net/swarm/addr") @@ -19,6 +21,8 @@ var SupportedTransportStrings = []string{ "/ip6/tcp", "/ip4/udp/utp", "/ip6/udp/utp", + "/ip4/tcp/ws", + "/ip6/tcp/ws", // "/ip4/udp/udt", disabled because the lib doesnt work on arm // "/ip6/udp/udt", disabled because the lib doesnt work on arm } diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go index 17039967e3..213b7b9f1a 100644 --- a/p2p/net/swarm/addr/addr_test.go +++ b/p2p/net/swarm/addr/addr_test.go @@ -29,6 +29,7 @@ func TestFilterAddrs(t *testing.T) { newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), newMultiaddr(t, "/ip6/::1/tcp/1234"), newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/utp"), + newMultiaddr(t, "/ip4/1.2.3.4/tcp/1234/ws"), } goodAndBad := append(good, bad...) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 704beb761e..c342f411c4 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -29,6 +29,7 @@ import ( spdy "github.com/whyrusleeping/go-smux-spdystream" yamux "github.com/whyrusleeping/go-smux-yamux" mafilter "github.com/whyrusleeping/multiaddr-filter" + ws "github.com/whyrusleeping/ws-transport" context "golang.org/x/net/context" ) @@ -120,6 +121,7 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, transports: []transport.Transport{ transport.NewTCPTransport(), transport.NewUtpTransport(), + new(ws.WebsocketTransport), }, bwc: bwc, fdRateLimit: make(chan struct{}, concurrentFdDials), From afe0aa7f2b4f40da7770ab9f219730450f634e03 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 24 Aug 2016 13:03:30 -0700 Subject: [PATCH 049/259] add note to test failure --- p2p/net/swarm/swarm_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index c11377ed5e..db8a464423 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -294,7 +294,7 @@ func TestConnHandler(t *testing.T) { select { case <-gotconn: - t.Fatalf("should have connected to %d swarms", expect) + t.Fatalf("should have connected to %d swarms, got an extra.", expect) default: } } From a9132bea7b15e7735915bf23c4c86d34d5a0501e Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 3 Sep 2016 09:38:00 -0700 Subject: [PATCH 050/259] swarm: refactor to make modular construction easier --- p2p/net/swarm/swarm.go | 39 ++++++++++++++++++++---- p2p/net/swarm/swarm_listen.go | 56 +++++++++++++++++++---------------- 2 files changed, 64 insertions(+), 31 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c342f411c4..05402f5052 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -10,6 +10,14 @@ import ( "sync" "time" + metrics "github.com/libp2p/go-libp2p/p2p/metrics" + mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" + inet "github.com/libp2p/go-libp2p/p2p/net" + conn "github.com/libp2p/go-libp2p/p2p/net/conn" + filter "github.com/libp2p/go-libp2p/p2p/net/filter" + addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" + + ci "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" transport "github.com/ipfs/go-libp2p-transport" @@ -19,12 +27,6 @@ import ( pst "github.com/jbenet/go-stream-muxer" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" - metrics "github.com/libp2p/go-libp2p/p2p/metrics" - mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" - inet "github.com/libp2p/go-libp2p/p2p/net" - conn "github.com/libp2p/go-libp2p/p2p/net/conn" - filter "github.com/libp2p/go-libp2p/p2p/net/filter" - addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" psmss "github.com/whyrusleeping/go-smux-multistream" spdy "github.com/whyrusleeping/go-smux-spdystream" yamux "github.com/whyrusleeping/go-smux-yamux" @@ -143,6 +145,31 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, return s, nil } +func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey) *Swarm { + s := &Swarm{ + swarm: ps.NewSwarm(PSTransport), + local: id, + peers: pstore.NewPeerstore(), + ctx: ctx, + dialT: DialTimeout, + notifs: make(map[inet.Notifiee]ps.Notifiee), + fdRateLimit: make(chan struct{}, concurrentFdDials), + Filters: filter.NewFilters(), + dialer: conn.NewDialer(id, privkey, nil), + } + + // configure Swarm + s.limiter = newDialLimiter(s.dialAddr) + s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown) + s.SetConnHandler(nil) // make sure to setup our own conn handler. + + return s +} + +func (s *Swarm) AddTransport(t transport.Transport) { + s.transports = append(s.transports, t) +} + func (s *Swarm) teardown() error { return s.swarm.Close() } diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 64fdb83bd0..4eb61d5a41 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -13,37 +13,42 @@ import ( context "golang.org/x/net/context" ) +func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { + tpt := s.transportForAddr(a) + if tpt == nil { + return fmt.Errorf("no transport for address: %s", a) + } + + d, err := tpt.Dialer(a, transport.TimeoutOpt(DialTimeout), transport.ReusePorts) + if err != nil { + return err + } + + s.dialer.AddDialer(d) + + list, err := tpt.Listen(a) + if err != nil { + return err + } + + err = s.addListener(list) + if err != nil { + return err + } + + return nil +} + // Open listeners and reuse-dialers for the given addresses func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { errs := make([]error, len(addrs)) var succeeded int for i, a := range addrs { - tpt := s.transportForAddr(a) - if tpt == nil { - errs[i] = fmt.Errorf("no transport for address: %s", a) - continue - } - - d, err := tpt.Dialer(a, transport.TimeoutOpt(DialTimeout), transport.ReusePorts) - if err != nil { - errs[i] = err - continue - } - - s.dialer.AddDialer(d) - - list, err := tpt.Listen(a) - if err != nil { + if err := s.AddListenAddr(a); err != nil { errs[i] = err - continue + } else { + succeeded++ } - - err = s.addListener(list) - if err != nil { - errs[i] = err - continue - } - succeeded++ } for i, e := range errs { @@ -51,6 +56,7 @@ func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { log.Warning("listen on %s failed: %s", addrs[i], errs[i]) } } + if succeeded == 0 && len(addrs) > 0 { return fmt.Errorf("failed to listen on any addresses: %s", errs) } @@ -83,7 +89,7 @@ func (s *Swarm) addListener(tptlist transport.Listener) error { list.SetAddrFilters(s.Filters) - if cw, ok := list.(conn.ListenerConnWrapper); ok { + if cw, ok := list.(conn.ListenerConnWrapper); ok && s.bwc != nil { cw.SetConnWrapper(func(c transport.Conn) transport.Conn { return mconn.WrapConn(s.bwc, c) }) From 423a82fa7aa8568260401cdb3544db5ecad00501 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 4 Sep 2016 08:13:36 -0700 Subject: [PATCH 051/259] swarm: pass in stream muxer on construct --- p2p/net/swarm/swarm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 05402f5052..a3787ced0e 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -145,9 +145,9 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, return s, nil } -func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey) *Swarm { +func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey, pstpt pst.Transport) *Swarm { s := &Swarm{ - swarm: ps.NewSwarm(PSTransport), + swarm: ps.NewSwarm(pstpt), local: id, peers: pstore.NewPeerstore(), ctx: ctx, From 6856aa870c8c16b2a83fc34aae063d1fea9d0087 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 4 Sep 2016 10:50:07 -0700 Subject: [PATCH 052/259] don't wrap conns with metrics if bwc is nil --- p2p/net/swarm/swarm.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index a3787ced0e..05eecec59a 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -109,8 +109,11 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, return nil, err } - wrap := func(c transport.Conn) transport.Conn { - return mconn.WrapConn(bwc, c) + var wrap func(c transport.Conn) transport.Conn + if bwc != nil { + wrap = func(c transport.Conn) transport.Conn { + return mconn.WrapConn(bwc, c) + } } s := &Swarm{ From a7b803d838df91a528fab55c8e698da467377995 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 5 Sep 2016 13:25:24 -0700 Subject: [PATCH 053/259] swarm: add optimized method for checking connectedness to peer --- p2p/net/swarm/swarm.go | 9 +++++++++ p2p/net/swarm/swarm_net.go | 3 +-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 05eecec59a..abab8e8a43 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -286,6 +286,15 @@ func (s *Swarm) ConnectionsToPeer(p peer.ID) []*Conn { return wrapConns(ps.ConnsWithGroup(p, s.swarm.Conns())) } +func (s *Swarm) HaveConnsToPeer(p peer.ID) bool { + for _, c := range s.swarm.Conns() { + if c.InGroup(p) { + return true + } + } + return false +} + // Connections returns a slice of all connections. func (s *Swarm) Connections() []*Conn { return wrapConns(s.swarm.Conns()) diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 48defb11ad..abf2f21ce0 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -123,8 +123,7 @@ func (n *Network) InterfaceListenAddresses() ([]ma.Multiaddr, error) { // Connectedness returns a state signaling connection capabilities // For now only returns Connected || NotConnected. Expand into more later. func (n *Network) Connectedness(p peer.ID) inet.Connectedness { - c := n.Swarm().ConnectionsToPeer(p) - if len(c) > 0 { + if n.Swarm().HaveConnsToPeer(p) { return inet.Connected } return inet.NotConnected From ef69b8b035a3193179ff2509e3fd70211c848b43 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 5 Sep 2016 14:48:28 -0700 Subject: [PATCH 054/259] swarm: externalize secio handshake errors --- p2p/net/swarm/swarm_dial.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 6774721247..6f82204347 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -450,7 +450,8 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (con remotep := connC.RemotePeer() if remotep != p { connC.Close() - return nil, fmt.Errorf("misdial to %s through %s (got %s)", p, addr, remotep) + _, err := connC.Read(nil) // should return any potential errors (ex: from secio) + return nil, fmt.Errorf("misdial to %s through %s (got %s): %s", p, addr, remotep, err) } // if the connection is to ourselves... From b97c307993d6389a8860825112dc027bcd1cfdbf Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 7 Sep 2016 18:48:22 -0700 Subject: [PATCH 055/259] swarm: perform backoff check before taking dialsync lock --- p2p/net/swarm/swarm_dial.go | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 6f82204347..eaf8d1f63a 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -255,16 +255,16 @@ func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) return conn, nil } + // if this peer has been backed off, lets get out of here + if s.backf.Backoff(p) { + log.Event(ctx, "swarmDialBackoff", logdial) + return nil, ErrDialBackoff + } + // check if there's an ongoing dial to this peer if ok, wait := s.dsync.Lock(p); ok { defer s.dsync.Unlock(p) - // if this peer has been backed off, lets get out of here - if s.backf.Backoff(p) { - log.Event(ctx, "swarmDialBackoff", logdial) - return nil, ErrDialBackoff - } - // ok, we have been charged to dial! let's do it. // if it succeeds, dial will add the conn to the swarm itself. defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done() @@ -285,13 +285,6 @@ func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) } else { // we did not dial. we must wait for someone else to dial. - - // check whether we should backoff first... - if s.backf.Backoff(p) { - log.Event(ctx, "swarmDialBackoff", logdial) - return nil, ErrDialBackoff - } - defer log.EventBegin(ctx, "swarmDialWait", logdial).Done() select { case <-wait: // wait for that other dial to finish. From da731da3f5f4fdbd4ca29a1aa4078665528e5b80 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 7 Sep 2016 18:48:55 -0700 Subject: [PATCH 056/259] swarm: implement new DialSync manager code --- p2p/net/swarm/dial_sync.go | 92 +++++++++++++++ p2p/net/swarm/dial_sync_test.go | 203 ++++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 p2p/net/swarm/dial_sync.go create mode 100644 p2p/net/swarm/dial_sync_test.go diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go new file mode 100644 index 0000000000..a63c047ac8 --- /dev/null +++ b/p2p/net/swarm/dial_sync.go @@ -0,0 +1,92 @@ +package swarm + +import ( + "context" + "sync" + + peer "github.com/ipfs/go-libp2p-peer" +) + +type DialFunc func(context.Context, peer.ID) (*Conn, error) + +func NewDialSync(dfn DialFunc) *DialSync { + return &DialSync{ + dials: make(map[peer.ID]*activeDial), + dialFunc: dfn, + } +} + +type DialSync struct { + dials map[peer.ID]*activeDial + dialsLk sync.Mutex + dialFunc DialFunc +} + +type activeDial struct { + id peer.ID + refCnt int + refCntLk sync.Mutex + cancel func() + + err error + conn *Conn + waitch chan struct{} + + ds *DialSync +} + +func (dr *activeDial) wait(ctx context.Context) (*Conn, error) { + defer dr.decref() + select { + case <-dr.waitch: + return dr.conn, dr.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (ad *activeDial) incref() { + ad.refCntLk.Lock() + defer ad.refCntLk.Unlock() + ad.refCnt++ +} + +func (ad *activeDial) decref() { + ad.refCntLk.Lock() + defer ad.refCntLk.Unlock() + ad.refCnt-- + if ad.refCnt <= 0 { + ad.cancel() + ad.ds.dialsLk.Lock() + delete(ad.ds.dials, ad.id) + ad.ds.dialsLk.Unlock() + } +} + +func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { + ds.dialsLk.Lock() + + actd, ok := ds.dials[p] + if !ok { + ctx, cancel := context.WithCancel(context.Background()) + actd = &activeDial{ + id: p, + cancel: cancel, + waitch: make(chan struct{}), + ds: ds, + } + ds.dials[p] = actd + + go func(ctx context.Context, p peer.ID, ad *activeDial) { + ad.conn, ad.err = ds.dialFunc(ctx, p) + close(ad.waitch) + ad.cancel() + ad.waitch = nil // to ensure nobody tries reusing this + }(ctx, p, actd) + } + + actd.incref() + ds.dialsLk.Unlock() + + return actd.wait(ctx) +} diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go new file mode 100644 index 0000000000..61a69ce386 --- /dev/null +++ b/p2p/net/swarm/dial_sync_test.go @@ -0,0 +1,203 @@ +package swarm + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + peer "github.com/ipfs/go-libp2p-peer" +) + +func getMockDialFunc() (DialFunc, func(), context.Context, <-chan struct{}) { + dfcalls := make(chan struct{}, 512) // buffer it large enough that we won't care + dialctx, cancel := context.WithCancel(context.Background()) + ch := make(chan struct{}) + f := func(ctx context.Context, p peer.ID) (*Conn, error) { + dfcalls <- struct{}{} + defer cancel() + select { + case <-ch: + return new(Conn), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + o := new(sync.Once) + + return f, func() { o.Do(func() { close(ch) }) }, dialctx, dfcalls +} + +func TestBasicDialSync(t *testing.T) { + df, done, _, callsch := getMockDialFunc() + + dsync := NewDialSync(df) + + p := peer.ID("testpeer") + + ctx := context.Background() + + finished := make(chan struct{}) + go func() { + _, err := dsync.DialLock(ctx, p) + if err != nil { + t.Error(err) + } + finished <- struct{}{} + }() + + go func() { + _, err := dsync.DialLock(ctx, p) + if err != nil { + t.Error(err) + } + finished <- struct{}{} + }() + + // short sleep just to make sure we've moved around in the scheduler + time.Sleep(time.Millisecond * 20) + done() + + <-finished + <-finished + + if len(callsch) > 1 { + t.Fatal("should only have called dial func once!") + } +} + +func TestDialSyncCancel(t *testing.T) { + df, done, _, dcall := getMockDialFunc() + + dsync := NewDialSync(df) + + p := peer.ID("testpeer") + + ctx1, cancel1 := context.WithCancel(context.Background()) + + finished := make(chan struct{}) + go func() { + _, err := dsync.DialLock(ctx1, p) + if err != ctx1.Err() { + t.Error("should have gotten context error") + } + finished <- struct{}{} + }() + + // make sure the above makes it through the wait code first + select { + case <-dcall: + case <-time.After(time.Second): + t.Fatal("timed out waiting for dial to start") + } + + // Add a second dialwait in so two actors are waiting on the same dial + go func() { + _, err := dsync.DialLock(context.Background(), p) + if err != nil { + t.Error(err) + } + finished <- struct{}{} + }() + + time.Sleep(time.Millisecond * 20) + + // cancel the first dialwait, it should not affect the second at all + cancel1() + select { + case <-finished: + case <-time.After(time.Second): + t.Fatal("timed out waiting for wait to exit") + } + + // short sleep just to make sure we've moved around in the scheduler + time.Sleep(time.Millisecond * 20) + done() + + <-finished +} + +func TestDialSyncAllCancel(t *testing.T) { + df, done, dctx, _ := getMockDialFunc() + + dsync := NewDialSync(df) + + p := peer.ID("testpeer") + + ctx1, cancel1 := context.WithCancel(context.Background()) + + finished := make(chan struct{}) + go func() { + _, err := dsync.DialLock(ctx1, p) + if err != ctx1.Err() { + t.Error("should have gotten context error") + } + finished <- struct{}{} + }() + + // Add a second dialwait in so two actors are waiting on the same dial + go func() { + _, err := dsync.DialLock(ctx1, p) + if err != ctx1.Err() { + t.Error("should have gotten context error") + } + finished <- struct{}{} + }() + + cancel1() + for i := 0; i < 2; i++ { + select { + case <-finished: + case <-time.After(time.Second): + t.Fatal("timed out waiting for wait to exit") + } + } + + // the dial should have exited now + select { + case <-dctx.Done(): + case <-time.After(time.Second): + t.Fatal("timed out waiting for dial to return") + } + + // should be able to successfully dial that peer again + done() + _, err := dsync.DialLock(context.Background(), p) + if err != nil { + t.Fatal(err) + } +} + +func TestFailFirst(t *testing.T) { + var count int + f := func(ctx context.Context, p peer.ID) (*Conn, error) { + if count > 0 { + return new(Conn), nil + } + count++ + return nil, fmt.Errorf("gophers ate the modem") + } + + ds := NewDialSync(f) + + p := peer.ID("testing") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + _, err := ds.DialLock(ctx, p) + if err == nil { + t.Fatal("expected gophers to have eaten the modem") + } + + c, err := ds.DialLock(ctx, p) + if err != nil { + t.Fatal(err) + } + + if c == nil { + t.Fatal("should have gotten a 'real' conn back") + } +} From ad0576ff0feae24bf0a3ca9ab18fcfd0d0d61800 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 8 Sep 2016 09:59:28 -0700 Subject: [PATCH 057/259] swarm: integrate new dialsync code into swarm --- p2p/net/swarm/dial_test.go | 1 - p2p/net/swarm/limiter.go | 2 +- p2p/net/swarm/limiter_test.go | 2 +- p2p/net/swarm/swarm.go | 5 +- p2p/net/swarm/swarm_dial.go | 135 ++++++---------------------------- 5 files changed, 29 insertions(+), 116 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index cfdb996c28..aeeb6ae42d 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -415,7 +415,6 @@ func TestDialBackoff(t *testing.T) { if !s1.backf.Backoff(s3p) { t.Error("s3 should be on backoff") } - } } diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 4bbfb11862..c34954a527 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -1,11 +1,11 @@ package swarm import ( + "context" "sync" peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" conn "github.com/libp2p/go-libp2p/p2p/net/conn" addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index f5fc18746f..93761a5048 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -1,6 +1,7 @@ package swarm import ( + "context" "fmt" "math/rand" "strconv" @@ -10,7 +11,6 @@ import ( peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" mafmt "github.com/whyrusleeping/mafmt" - context "golang.org/x/net/context" conn "github.com/libp2p/go-libp2p/p2p/net/conn" ) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index abab8e8a43..e36f763fb9 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -3,6 +3,7 @@ package swarm import ( + "context" "fmt" "io/ioutil" "os" @@ -32,7 +33,6 @@ import ( yamux "github.com/whyrusleeping/go-smux-yamux" mafilter "github.com/whyrusleeping/multiaddr-filter" ws "github.com/whyrusleeping/ws-transport" - context "golang.org/x/net/context" ) var log = logging.Logger("swarm2") @@ -76,7 +76,7 @@ type Swarm struct { peers pstore.Peerstore connh ConnHandler - dsync dialsync + dsync *DialSync backf dialbackoff dialT time.Duration // mainly for tests @@ -134,6 +134,7 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, dialer: conn.NewDialer(local, peers.PrivKey(local), wrap), } + s.dsync = NewDialSync(s.doDial) s.limiter = newDialLimiter(s.dialAddr) // configure Swarm diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index eaf8d1f63a..573651778b 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -1,6 +1,7 @@ package swarm import ( + "context" "errors" "fmt" "sync" @@ -11,7 +12,6 @@ import ( ma "github.com/jbenet/go-multiaddr" conn "github.com/libp2p/go-libp2p/p2p/net/conn" addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" - context "golang.org/x/net/context" ) // Diagram of dial sync: @@ -53,78 +53,6 @@ const defaultPerPeerRateLimit = 8 // subcomponent of Dial) var DialTimeout = time.Second * 10 -// dialsync is a small object that helps manage ongoing dials. -// this way, if we receive many simultaneous dial requests, one -// can do its thing, while the rest wait. -// -// this interface is so would-be dialers can just: -// -// for { -// c := findConnectionToPeer(peer) -// if c != nil { -// return c -// } -// -// // ok, no connections. should we dial? -// if ok, wait := dialsync.Lock(peer); !ok { -// <-wait // can optionally wait -// continue -// } -// defer dialsync.Unlock(peer) -// -// c := actuallyDial(peer) -// return c -// } -// -type dialsync struct { - // ongoing is a map of tickets for the current peers being dialed. - // this way, we dont kick off N dials simultaneously. - ongoing map[peer.ID]chan struct{} - lock sync.Mutex -} - -// Lock governs the beginning of a dial attempt. -// If there are no ongoing dials, it returns true, and the client is now -// scheduled to dial. Every other goroutine that calls startDial -- with -//the same dst -- will block until client is done. The client MUST call -// ds.Unlock(p) when it is done, to unblock the other callers. -// The client is not reponsible for achieving a successful dial, only for -// reporting the end of the attempt (calling ds.Unlock(p)). -// -// see the example below `dialsync` -func (ds *dialsync) Lock(dst peer.ID) (bool, chan struct{}) { - ds.lock.Lock() - if ds.ongoing == nil { // init if not ready - ds.ongoing = make(map[peer.ID]chan struct{}) - } - wait, found := ds.ongoing[dst] - if !found { - ds.ongoing[dst] = make(chan struct{}) - } - ds.lock.Unlock() - - if found { - return false, wait - } - - // ok! you're signed up to dial! - return true, nil -} - -// Unlock releases waiters to a dial attempt. see Lock. -// if Unlock(p) is called without calling Lock(p) first, Unlock panics. -func (ds *dialsync) Unlock(dst peer.ID) { - ds.lock.Lock() - wait, found := ds.ongoing[dst] - if !found { - panic("called dialDone with no ongoing dials to peer: " + dst.Pretty()) - } - - delete(ds.ongoing, dst) // remove ongoing dial - close(wait) // release everyone else - ds.lock.Unlock() -} - // dialbackoff is a struct used to avoid over-dialing the same, dead peers. // Whenever we totally time out on a peer (all three attempts), we add them // to dialbackoff. Then, whenevers goroutines would _wait_ (dialsync), they @@ -246,8 +174,7 @@ func (s *Swarm) bestConnectionToPeer(p peer.ID) *Conn { // gatedDialAttempt is an attempt to dial a node. It is gated by the swarm's // dial synchronization systems: dialsync and dialbackoff. func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) { - var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) - defer log.EventBegin(ctx, "swarmDialAttemptSync", logdial).Done() + defer log.EventBegin(ctx, "swarmDialAttemptSync", p).Done() // check if we already have an open connection first conn := s.bestConnectionToPeer(p) @@ -257,48 +184,34 @@ func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) // if this peer has been backed off, lets get out of here if s.backf.Backoff(p) { - log.Event(ctx, "swarmDialBackoff", logdial) + log.Event(ctx, "swarmDialBackoff", p) return nil, ErrDialBackoff } - // check if there's an ongoing dial to this peer - if ok, wait := s.dsync.Lock(p); ok { - defer s.dsync.Unlock(p) - - // ok, we have been charged to dial! let's do it. - // if it succeeds, dial will add the conn to the swarm itself. - defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done() - ctxT, cancel := context.WithTimeout(ctx, s.dialT) - conn, err := s.dial(ctxT, p) - cancel() - log.Debugf("dial end %s", conn) - if err != nil { - log.Event(ctx, "swarmDialBackoffAdd", logdial) - s.backf.AddBackoff(p) // let others know to backoff - - // ok, we failed. try again. (if loop is done, our error is output) - return nil, fmt.Errorf("dial attempt failed: %s", err) - } - log.Event(ctx, "swarmDialBackoffClear", logdial) - s.backf.Clear(p) // okay, no longer need to backoff - return conn, nil + return s.dsync.DialLock(ctx, p) +} - } else { - // we did not dial. we must wait for someone else to dial. - defer log.EventBegin(ctx, "swarmDialWait", logdial).Done() - select { - case <-wait: // wait for that other dial to finish. +// doDial is an ugly shim method to retain all the logging and backoff logic +// of the old dialsync code +func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { + var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + // ok, we have been charged to dial! let's do it. + // if it succeeds, dial will add the conn to the swarm itself. + defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done() + ctxT, cancel := context.WithTimeout(ctx, s.dialT) + conn, err := s.dial(ctxT, p) + cancel() + log.Debugf("dial end %s", conn) + if err != nil { + log.Event(ctx, "swarmDialBackoffAdd", logdial) + s.backf.AddBackoff(p) // let others know to backoff - // see if it worked, OR we got an incoming dial in the meantime... - conn := s.bestConnectionToPeer(p) - if conn != nil { - return conn, nil - } - return nil, ErrDialFailed - case <-ctx.Done(): // or we may have to bail... - return nil, ctx.Err() - } + // ok, we failed. try again. (if loop is done, our error is output) + return nil, fmt.Errorf("dial attempt failed: %s", err) } + log.Event(ctx, "swarmDialBackoffClear", logdial) + s.backf.Clear(p) // okay, no longer need to backoff + return conn, nil } // dial is the actual swarm's dial logic, gated by Dial. From 7f414b2b1426f7ec48de59f3bc4020a6c67776b9 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 18 Aug 2016 15:45:18 -0700 Subject: [PATCH 058/259] swarm: add deadline for connection setup --- p2p/net/swarm/swarm_dial.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 573651778b..6d1abd3342 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -372,10 +372,21 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (con return connC, nil } +var ConnSetupTimeout = time.Minute * 5 + // dialConnSetup is the setup logic for a connection from the dial side. it // needs to add the Conn to the StreamSwarm, then run newConnSetup func dialConnSetup(ctx context.Context, s *Swarm, connC conn.Conn) (*Conn, error) { + deadline, ok := ctx.Deadline() + if !ok { + deadline = time.Now().Add(ConnSetupTimeout) + } + + if err := connC.SetDeadline(deadline); err != nil { + return nil, err + } + psC, err := s.swarm.AddConn(connC) if err != nil { // connC is closed by caller if we fail. @@ -389,5 +400,10 @@ func dialConnSetup(ctx context.Context, s *Swarm, connC conn.Conn) (*Conn, error return nil, err } + if err := connC.SetDeadline(time.Time{}); err != nil { + log.Error("failed to reset connection deadline after setup: ", err) + return nil, err + } + return swarmC, err } From af5031ca5e01cd7b20c3d898a3576ea3dc62c8bc Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 25 Sep 2016 04:51:27 -0700 Subject: [PATCH 059/259] split tcp transports out of the base package --- p2p/net/swarm/swarm.go | 6 +++--- p2p/net/swarm/swarm_listen.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index e36f763fb9..4eebf626be 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -21,13 +21,14 @@ import ( ci "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" - transport "github.com/ipfs/go-libp2p-transport" logging "github.com/ipfs/go-log" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" pst "github.com/jbenet/go-stream-muxer" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" + transport "github.com/libp2p/go-libp2p-transport" + tcpt "github.com/libp2p/go-tcp-transport" psmss "github.com/whyrusleeping/go-smux-multistream" spdy "github.com/whyrusleeping/go-smux-spdystream" yamux "github.com/whyrusleeping/go-smux-yamux" @@ -124,8 +125,7 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, dialT: DialTimeout, notifs: make(map[inet.Notifiee]ps.Notifiee), transports: []transport.Transport{ - transport.NewTCPTransport(), - transport.NewUtpTransport(), + tcpt.NewTCPTransport(), new(ws.WebsocketTransport), }, bwc: bwc, diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 4eb61d5a41..92ccd833af 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -4,9 +4,9 @@ import ( "fmt" lgbl "github.com/ipfs/go-libp2p-loggables" - transport "github.com/ipfs/go-libp2p-transport" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" + transport "github.com/libp2p/go-libp2p-transport" mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" inet "github.com/libp2p/go-libp2p/p2p/net" conn "github.com/libp2p/go-libp2p/p2p/net/conn" From 17194ccda6021fa8cd0621335324f14a79d5385f Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 3 Oct 2016 11:00:00 -0700 Subject: [PATCH 060/259] switch to using stdlib context package --- p2p/net/swarm/addr/addr.go | 2 +- p2p/net/swarm/dial_test.go | 2 +- p2p/net/swarm/peers_test.go | 2 +- p2p/net/swarm/simul_test.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_net.go | 2 +- p2p/net/swarm/swarm_net_test.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_test.go | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go index 08ca90ac46..4a14c43778 100644 --- a/p2p/net/swarm/addr/addr.go +++ b/p2p/net/swarm/addr/addr.go @@ -3,10 +3,10 @@ package addrutil import ( "fmt" + "context" logging "github.com/ipfs/go-log" ma "github.com/jbenet/go-multiaddr" manet "github.com/jbenet/go-multiaddr-net" - context "golang.org/x/net/context" _ "github.com/whyrusleeping/ws-transport" ) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index aeeb6ae42d..ab65b8c59f 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -10,11 +10,11 @@ import ( testutil "github.com/libp2p/go-libp2p/testutil" ci "github.com/libp2p/go-libp2p/testutil/ci" + "context" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" manet "github.com/jbenet/go-multiaddr-net" - context "golang.org/x/net/context" ) func closeSwarms(swarms []*Swarm) { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index b63e655b11..5eadf4e671 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -3,10 +3,10 @@ package swarm import ( "testing" + "context" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index c6ee6cf985..bc9cf3ae0f 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -8,10 +8,10 @@ import ( ci "github.com/libp2p/go-libp2p/testutil/ci" + "context" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 772413b286..e221f7bc87 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -7,9 +7,9 @@ import ( addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" testutil "github.com/libp2p/go-libp2p/testutil" + "context" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 762ba6e1d3..40f5bdf0ab 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -6,11 +6,11 @@ import ( inet "github.com/libp2p/go-libp2p/p2p/net" conn "github.com/libp2p/go-libp2p/p2p/net/conn" + "context" ic "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" - context "golang.org/x/net/context" ) // Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 92ccd833af..e1b9a876fb 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -3,6 +3,7 @@ package swarm import ( "fmt" + "context" lgbl "github.com/ipfs/go-libp2p-loggables" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" @@ -10,7 +11,6 @@ import ( mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" inet "github.com/libp2p/go-libp2p/p2p/net" conn "github.com/libp2p/go-libp2p/p2p/net/conn" - context "golang.org/x/net/context" ) func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index abf2f21ce0..d9e83d5fdc 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -8,9 +8,9 @@ import ( metrics "github.com/libp2p/go-libp2p/p2p/metrics" inet "github.com/libp2p/go-libp2p/p2p/net" + "context" ma "github.com/jbenet/go-multiaddr" "github.com/jbenet/goprocess" - context "golang.org/x/net/context" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 4d564e3901..205b841a19 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -7,7 +7,7 @@ import ( inet "github.com/libp2p/go-libp2p/p2p/net" testutil "github.com/libp2p/go-libp2p/p2p/test/util" - context "golang.org/x/net/context" + "context" ) // TestConnectednessCorrect starts a few networks, connects a few diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 06777469f5..2bc663e0a5 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -4,10 +4,10 @@ import ( "testing" "time" + "context" peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" inet "github.com/libp2p/go-libp2p/p2p/net" - context "golang.org/x/net/context" ) func streamsSame(a, b inet.Stream) bool { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index db8a464423..b7520dc757 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -13,10 +13,10 @@ import ( inet "github.com/libp2p/go-libp2p/p2p/net" testutil "github.com/libp2p/go-libp2p/testutil" + "context" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" - context "golang.org/x/net/context" ) func EchoStreamHandler(stream inet.Stream) { From c03a78ff28e77862a44faa836b3c8db2644111b9 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 3 Oct 2016 21:35:40 -0700 Subject: [PATCH 061/259] extract protocol and maddr-filter --- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_stream.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 4eebf626be..19ccf13eef 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -15,7 +15,6 @@ import ( mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" inet "github.com/libp2p/go-libp2p/p2p/net" conn "github.com/libp2p/go-libp2p/p2p/net/conn" - filter "github.com/libp2p/go-libp2p/p2p/net/filter" addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" ci "github.com/ipfs/go-libp2p-crypto" @@ -28,6 +27,7 @@ import ( "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" transport "github.com/libp2p/go-libp2p-transport" + filter "github.com/libp2p/go-maddr-filter" tcpt "github.com/libp2p/go-tcp-transport" psmss "github.com/whyrusleeping/go-smux-multistream" spdy "github.com/whyrusleeping/go-smux-spdystream" diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index dc365e8c09..51c846001a 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -1,8 +1,8 @@ package swarm import ( + protocol "github.com/libp2p/go-libp2p-protocol" inet "github.com/libp2p/go-libp2p/p2p/net" - protocol "github.com/libp2p/go-libp2p/p2p/protocol" ps "github.com/jbenet/go-peerstream" ) From 7f10dbda8be07f737fab768d4e627c42cfe069a6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 4 Oct 2016 12:33:25 -0700 Subject: [PATCH 062/259] extract conn, addr-util, and testutil --- p2p/net/swarm/addr/addr.go | 284 ------------------------------- p2p/net/swarm/addr/addr_test.go | 227 ------------------------ p2p/net/swarm/addr/filter.go | 31 ---- p2p/net/swarm/dial_test.go | 9 +- p2p/net/swarm/limiter.go | 5 +- p2p/net/swarm/limiter_test.go | 3 +- p2p/net/swarm/simul_test.go | 5 +- p2p/net/swarm/swarm.go | 4 +- p2p/net/swarm/swarm_addr.go | 5 +- p2p/net/swarm/swarm_addr_test.go | 6 +- p2p/net/swarm/swarm_conn.go | 4 +- p2p/net/swarm/swarm_dial.go | 4 +- p2p/net/swarm/swarm_listen.go | 7 +- p2p/net/swarm/swarm_test.go | 4 +- 14 files changed, 26 insertions(+), 572 deletions(-) delete mode 100644 p2p/net/swarm/addr/addr.go delete mode 100644 p2p/net/swarm/addr/addr_test.go delete mode 100644 p2p/net/swarm/addr/filter.go diff --git a/p2p/net/swarm/addr/addr.go b/p2p/net/swarm/addr/addr.go deleted file mode 100644 index 4a14c43778..0000000000 --- a/p2p/net/swarm/addr/addr.go +++ /dev/null @@ -1,284 +0,0 @@ -package addrutil - -import ( - "fmt" - - "context" - logging "github.com/ipfs/go-log" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" - - _ "github.com/whyrusleeping/ws-transport" -) - -var log = logging.Logger("github.com/libp2p/go-libp2p/p2p/net/swarm/addr") - -// SupportedTransportStrings is the list of supported transports for the swarm. -// These are strings of encapsulated multiaddr protocols. E.g.: -// /ip4/tcp -var SupportedTransportStrings = []string{ - "/ip4/tcp", - "/ip6/tcp", - "/ip4/udp/utp", - "/ip6/udp/utp", - "/ip4/tcp/ws", - "/ip6/tcp/ws", - // "/ip4/udp/udt", disabled because the lib doesnt work on arm - // "/ip6/udp/udt", disabled because the lib doesnt work on arm -} - -// SupportedTransportProtocols is the list of supported transports for the swarm. -// These are []ma.Protocol lists. Populated at runtime from SupportedTransportStrings -var SupportedTransportProtocols = [][]ma.Protocol{} - -func init() { - // initialize SupportedTransportProtocols - transports := make([][]ma.Protocol, len(SupportedTransportStrings)) - for _, s := range SupportedTransportStrings { - t, err := ma.ProtocolsWithString(s) - if err != nil { - panic(err) // important to fix this in the codebase - } - transports = append(transports, t) - } - SupportedTransportProtocols = transports -} - -// FilterAddrs is a filter that removes certain addresses, according the given filters. -// if all filters return true, the address is kept. -func FilterAddrs(a []ma.Multiaddr, filters ...func(ma.Multiaddr) bool) []ma.Multiaddr { - b := make([]ma.Multiaddr, 0, len(a)) - for _, addr := range a { - good := true - for _, filter := range filters { - good = good && filter(addr) - } - if good { - b = append(b, addr) - } - } - return b -} - -// FilterUsableAddrs removes certain addresses -// from a list. the addresses removed are those known NOT -// to work with our network. Namely, addresses with UTP. -func FilterUsableAddrs(a []ma.Multiaddr) []ma.Multiaddr { - return FilterAddrs(a, AddrUsableFunc) -} - -func AddrUsableFunc(m ma.Multiaddr) bool { - return AddrUsable(m, false) -} - -// AddrOverNonLocalIP returns whether the addr uses a non-local ip link -func AddrOverNonLocalIP(a ma.Multiaddr) bool { - split := ma.Split(a) - if len(split) < 1 { - return false - } - if manet.IsIP6LinkLocal(split[0]) { - return false - } - return true -} - -// AddrUsable returns whether our network can use this addr. -// We only use the transports in SupportedTransportStrings, -// and we do not link local addresses. Loopback is ok -// as we need to be able to connect to multiple ipfs nodes -// in the same machine. -func AddrUsable(a ma.Multiaddr, partial bool) bool { - if a == nil { - return false - } - - if !AddrOverNonLocalIP(a) { - return false - } - - // test the address protocol list is in SupportedTransportProtocols - matches := func(supported, test []ma.Protocol) bool { - if len(test) > len(supported) { - return false - } - - // when partial, it's ok if test < supported. - if !partial && len(supported) != len(test) { - return false - } - - for i := range test { - if supported[i].Code != test[i].Code { - return false - } - } - return true - } - - transport := a.Protocols() - for _, supported := range SupportedTransportProtocols { - if matches(supported, transport) { - return true - } - } - - return false -} - -// ResolveUnspecifiedAddress expands an unspecified ip addresses (/ip4/0.0.0.0, /ip6/::) to -// use the known local interfaces. If ifaceAddr is nil, we request interface addresses -// from the network stack. (this is so you can provide a cached value if resolving many addrs) -func ResolveUnspecifiedAddress(resolve ma.Multiaddr, ifaceAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { - // split address into its components - split := ma.Split(resolve) - - // if first component (ip) is not unspecified, use it as is. - if !manet.IsIPUnspecified(split[0]) { - return []ma.Multiaddr{resolve}, nil - } - - out := make([]ma.Multiaddr, 0, len(ifaceAddrs)) - for _, ia := range ifaceAddrs { - // must match the first protocol to be resolve. - if ia.Protocols()[0].Code != resolve.Protocols()[0].Code { - continue - } - - split[0] = ia - joined := ma.Join(split...) - out = append(out, joined) - log.Debug("adding resolved addr:", resolve, joined, out) - } - if len(out) < 1 { - return nil, fmt.Errorf("failed to resolve: %s", resolve) - } - return out, nil -} - -// ResolveUnspecifiedAddresses expands unspecified ip addresses (/ip4/0.0.0.0, /ip6/::) to -// use the known local interfaces. -func ResolveUnspecifiedAddresses(unspecAddrs, ifaceAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { - - // todo optimize: only fetch these if we have a "any" addr. - if len(ifaceAddrs) < 1 { - var err error - ifaceAddrs, err = InterfaceAddresses() - if err != nil { - return nil, err - } - // log.Debug("InterfaceAddresses:", ifaceAddrs) - } - - var outputAddrs []ma.Multiaddr - for _, a := range unspecAddrs { - // unspecified? - resolved, err := ResolveUnspecifiedAddress(a, ifaceAddrs) - if err != nil { - continue // optimistic. if we cant resolve anything, we'll know at the bottom. - } - // log.Debug("resolved:", a, resolved) - outputAddrs = append(outputAddrs, resolved...) - } - - if len(outputAddrs) < 1 { - return nil, fmt.Errorf("failed to specify addrs: %s", unspecAddrs) - } - - log.Event(context.TODO(), "interfaceListenAddresses", func() logging.Loggable { - var addrs []string - for _, addr := range outputAddrs { - addrs = append(addrs, addr.String()) - } - return logging.Metadata{"addresses": addrs} - }()) - - log.Debug("ResolveUnspecifiedAddresses:", unspecAddrs, ifaceAddrs, outputAddrs) - return outputAddrs, nil -} - -// InterfaceAddresses returns a list of addresses associated with local machine -// Note: we do not return link local addresses. IP loopback is ok, because we -// may be connecting to other nodes in the same machine. -func InterfaceAddresses() ([]ma.Multiaddr, error) { - maddrs, err := manet.InterfaceMultiaddrs() - if err != nil { - return nil, err - } - log.Debug("InterfaceAddresses: from manet:", maddrs) - - var out []ma.Multiaddr - for _, a := range maddrs { - if !AddrUsable(a, true) { // partial - // log.Debug("InterfaceAddresses: skipping unusable:", a) - continue - } - - out = append(out, a) - } - - log.Debug("InterfaceAddresses: usable:", out) - return out, nil -} - -// AddrInList returns whether or not an address is part of a list. -// this is useful to check if NAT is happening (or other bugs?) -func AddrInList(addr ma.Multiaddr, list []ma.Multiaddr) bool { - for _, addr2 := range list { - if addr.Equal(addr2) { - return true - } - } - return false -} - -// AddrIsShareableOnWAN returns whether the given address should be shareable on the -// wide area network (wide internet). -func AddrIsShareableOnWAN(addr ma.Multiaddr) bool { - s := ma.Split(addr) - if len(s) < 1 { - return false - } - a := s[0] - if manet.IsIPLoopback(a) || manet.IsIP6LinkLocal(a) || manet.IsIPUnspecified(a) { - return false - } - return manet.IsThinWaist(a) -} - -// WANShareableAddrs filters addresses based on whether they're shareable on WAN -func WANShareableAddrs(inp []ma.Multiaddr) []ma.Multiaddr { - return FilterAddrs(inp, AddrIsShareableOnWAN) -} - -// Subtract filters out all addrs in b from a -func Subtract(a, b []ma.Multiaddr) []ma.Multiaddr { - return FilterAddrs(a, func(m ma.Multiaddr) bool { - for _, bb := range b { - if m.Equal(bb) { - return false - } - } - return true - }) -} - -// CheckNATWarning checks if our observed addresses differ. if so, -// informs the user that certain things might not work yet -func CheckNATWarning(observed, expected ma.Multiaddr, listen []ma.Multiaddr) { - if observed.Equal(expected) { - return - } - - if !AddrInList(observed, listen) { // probably a nat - log.Warningf(natWarning, observed, listen) - } -} - -const natWarning = `Remote peer observed our address to be: %s -The local addresses are: %s -Thus, connection is going through NAT, and other connections may fail. - -IPFS NAT traversal is still under development. Please bug us on github or irc to fix this. -Baby steps: http://jbenet.static.s3.amazonaws.com/271dfcf/baby-steps.gif -` diff --git a/p2p/net/swarm/addr/addr_test.go b/p2p/net/swarm/addr/addr_test.go deleted file mode 100644 index 213b7b9f1a..0000000000 --- a/p2p/net/swarm/addr/addr_test.go +++ /dev/null @@ -1,227 +0,0 @@ -package addrutil - -import ( - "testing" - - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" -) - -func newMultiaddr(t *testing.T, s string) ma.Multiaddr { - maddr, err := ma.NewMultiaddr(s) - if err != nil { - t.Fatal(err) - } - return maddr -} - -func TestFilterAddrs(t *testing.T) { - - bad := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/1.2.3.4/udp/1234"), // unreliable - newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/sctp/1234"), // not in manet - newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/udt"), // udt is broken on arm - newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), // link local - newMultiaddr(t, "/ip6/fe80::100/tcp/1234"), // link local - } - - good := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), - newMultiaddr(t, "/ip6/::1/tcp/1234"), - newMultiaddr(t, "/ip4/1.2.3.4/udp/1234/utp"), - newMultiaddr(t, "/ip4/1.2.3.4/tcp/1234/ws"), - } - - goodAndBad := append(good, bad...) - - // test filters - - for _, a := range bad { - if AddrUsable(a, false) { - t.Errorf("addr %s should be unusable", a) - } - } - - for _, a := range good { - if !AddrUsable(a, false) { - t.Errorf("addr %s should be usable", a) - } - } - - subtestAddrsEqual(t, FilterUsableAddrs(bad), []ma.Multiaddr{}) - subtestAddrsEqual(t, FilterUsableAddrs(good), good) - subtestAddrsEqual(t, FilterUsableAddrs(goodAndBad), good) -} - -func subtestAddrsEqual(t *testing.T, a, b []ma.Multiaddr) { - if len(a) != len(b) { - t.Error(t) - } - - in := func(addr ma.Multiaddr, l []ma.Multiaddr) bool { - for _, addr2 := range l { - if addr.Equal(addr2) { - return true - } - } - return false - } - - for _, aa := range a { - if !in(aa, b) { - t.Errorf("%s not in %s", aa, b) - } - } -} - -func TestInterfaceAddrs(t *testing.T) { - addrs, err := InterfaceAddresses() - if err != nil { - t.Fatal(err) - } - - if len(addrs) < 1 { - t.Error("no addresses") - } - - for _, a := range addrs { - if manet.IsIP6LinkLocal(a) { - t.Error("should not return ip link local addresses", a) - } - } - - if len(addrs) < 1 { - t.Error("no good interface addrs") - } -} - -func TestResolvingAddrs(t *testing.T) { - - unspec := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/0.0.0.0/tcp/1234"), - newMultiaddr(t, "/ip4/1.2.3.4/tcp/1234"), - newMultiaddr(t, "/ip6/::/tcp/1234"), - newMultiaddr(t, "/ip6/::100/tcp/1234"), - } - - iface := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/127.0.0.1"), - newMultiaddr(t, "/ip4/10.20.30.40"), - newMultiaddr(t, "/ip6/::1"), - newMultiaddr(t, "/ip6/::f"), - } - - spec := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), - newMultiaddr(t, "/ip4/10.20.30.40/tcp/1234"), - newMultiaddr(t, "/ip4/1.2.3.4/tcp/1234"), - newMultiaddr(t, "/ip6/::1/tcp/1234"), - newMultiaddr(t, "/ip6/::f/tcp/1234"), - newMultiaddr(t, "/ip6/::100/tcp/1234"), - } - - actual, err := ResolveUnspecifiedAddresses(unspec, iface) - if err != nil { - t.Fatal(err) - } - - for i, a := range actual { - if !a.Equal(spec[i]) { - t.Error(a, " != ", spec[i]) - } - } - - ip4u := []ma.Multiaddr{newMultiaddr(t, "/ip4/0.0.0.0")} - ip4i := []ma.Multiaddr{newMultiaddr(t, "/ip4/1.2.3.4")} - - ip6u := []ma.Multiaddr{newMultiaddr(t, "/ip6/::")} - ip6i := []ma.Multiaddr{newMultiaddr(t, "/ip6/::1")} - - if _, err := ResolveUnspecifiedAddress(ip4u[0], ip6i); err == nil { - t.Fatal("should have failed") - } - if _, err := ResolveUnspecifiedAddress(ip6u[0], ip4i); err == nil { - t.Fatal("should have failed") - } - - if _, err := ResolveUnspecifiedAddresses(ip6u, ip4i); err == nil { - t.Fatal("should have failed") - } - if _, err := ResolveUnspecifiedAddresses(ip4u, ip6i); err == nil { - t.Fatal("should have failed") - } - -} - -func TestWANShareable(t *testing.T) { - - wanok := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/1.2.3.4/tcp/1234"), - newMultiaddr(t, "/ip6/abcd::1/tcp/1234"), - } - - wanbad := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), - newMultiaddr(t, "/ip4/0.0.0.0/tcp/1234"), - newMultiaddr(t, "/ip6/::1/tcp/1234"), - newMultiaddr(t, "/ip6/::/tcp/1234"), - newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), - newMultiaddr(t, "/ip6/fe80::/tcp/1234"), - } - - for _, a := range wanok { - if !AddrIsShareableOnWAN(a) { - t.Error("should be true", a) - } - } - - for _, a := range wanbad { - if AddrIsShareableOnWAN(a) { - t.Error("should be false", a) - } - } - - wanok2 := WANShareableAddrs(wanok) - if len(wanok) != len(wanok2) { - t.Error("should be the same") - } - - wanbad2 := WANShareableAddrs(wanbad) - if len(wanbad2) != 0 { - t.Error("should be zero") - } -} - -func TestSubtract(t *testing.T) { - - a := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), - newMultiaddr(t, "/ip4/0.0.0.0/tcp/1234"), - newMultiaddr(t, "/ip6/::1/tcp/1234"), - newMultiaddr(t, "/ip6/::/tcp/1234"), - newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), - newMultiaddr(t, "/ip6/fe80::/tcp/1234"), - } - - b := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/127.0.0.1/tcp/1234"), - newMultiaddr(t, "/ip6/::1/tcp/1234"), - newMultiaddr(t, "/ip6/fe80::1/tcp/1234"), - } - - c1 := []ma.Multiaddr{ - newMultiaddr(t, "/ip4/0.0.0.0/tcp/1234"), - newMultiaddr(t, "/ip6/::/tcp/1234"), - newMultiaddr(t, "/ip6/fe80::/tcp/1234"), - } - - c2 := Subtract(a, b) - if len(c1) != len(c2) { - t.Error("should be the same") - } - for i, ca := range c1 { - if !c2[i].Equal(ca) { - t.Error("should be the same", ca, c2[i]) - } - } -} diff --git a/p2p/net/swarm/addr/filter.go b/p2p/net/swarm/addr/filter.go deleted file mode 100644 index d87ba816a7..0000000000 --- a/p2p/net/swarm/addr/filter.go +++ /dev/null @@ -1,31 +0,0 @@ -package addrutil - -import ( - ma "github.com/jbenet/go-multiaddr" - mafmt "github.com/whyrusleeping/mafmt" -) - -// SubtractFilter returns a filter func that filters all of the given addresses -func SubtractFilter(addrs ...ma.Multiaddr) func(ma.Multiaddr) bool { - addrmap := make(map[string]bool) - for _, a := range addrs { - addrmap[string(a.Bytes())] = true - } - - return func(a ma.Multiaddr) bool { - return !addrmap[string(a.Bytes())] - } -} - -// IsFDCostlyTransport returns true for transports that require a new file -// descriptor per connection created -func IsFDCostlyTransport(a ma.Multiaddr) bool { - return mafmt.TCP.Matches(a) -} - -// FilterNeg returns a negated version of the passed in filter -func FilterNeg(f func(ma.Multiaddr) bool) func(ma.Multiaddr) bool { - return func(a ma.Multiaddr) bool { - return !f(a) - } -} diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index ab65b8c59f..8e7b48fc2b 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -1,20 +1,19 @@ package swarm import ( + "context" "net" "sync" "testing" "time" - addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" - testutil "github.com/libp2p/go-libp2p/testutil" - ci "github.com/libp2p/go-libp2p/testutil/ci" - - "context" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" manet "github.com/jbenet/go-multiaddr-net" + addrutil "github.com/libp2p/go-addr-util" + testutil "github.com/libp2p/go-testutil" + ci "github.com/libp2p/go-testutil/ci" ) func closeSwarms(swarms []*Swarm) { diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index c34954a527..5c8d235b8e 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -6,9 +6,8 @@ import ( peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" - - conn "github.com/libp2p/go-libp2p/p2p/net/conn" - addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" + addrutil "github.com/libp2p/go-addr-util" + conn "github.com/libp2p/go-libp2p-conn" ) type dialResult struct { diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 93761a5048..7332c0e06a 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -10,9 +10,8 @@ import ( peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" + conn "github.com/libp2p/go-libp2p-conn" mafmt "github.com/whyrusleeping/mafmt" - - conn "github.com/libp2p/go-libp2p/p2p/net/conn" ) func mustAddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index bc9cf3ae0f..3518f139e6 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -1,17 +1,16 @@ package swarm import ( + "context" "runtime" "sync" "testing" "time" - ci "github.com/libp2p/go-libp2p/testutil/ci" - - "context" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" + ci "github.com/libp2p/go-testutil/ci" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 19ccf13eef..9107d7160a 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -14,8 +14,6 @@ import ( metrics "github.com/libp2p/go-libp2p/p2p/metrics" mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" inet "github.com/libp2p/go-libp2p/p2p/net" - conn "github.com/libp2p/go-libp2p/p2p/net/conn" - addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" ci "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" @@ -26,6 +24,8 @@ import ( pst "github.com/jbenet/go-stream-muxer" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" + addrutil "github.com/libp2p/go-addr-util" + conn "github.com/libp2p/go-libp2p-conn" transport "github.com/libp2p/go-libp2p-transport" filter "github.com/libp2p/go-maddr-filter" tcpt "github.com/libp2p/go-tcp-transport" diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 39fbe02afc..87b0829da1 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -1,10 +1,9 @@ package swarm import ( - conn "github.com/libp2p/go-libp2p/p2p/net/conn" - addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" - ma "github.com/jbenet/go-multiaddr" + addrutil "github.com/libp2p/go-addr-util" + conn "github.com/libp2p/go-libp2p-conn" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index e221f7bc87..bd48d4f601 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -1,15 +1,15 @@ package swarm import ( + "context" "testing" metrics "github.com/libp2p/go-libp2p/p2p/metrics" - addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" - testutil "github.com/libp2p/go-libp2p/testutil" - "context" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" + addrutil "github.com/libp2p/go-addr-util" + testutil "github.com/libp2p/go-testutil" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 40f5bdf0ab..30ef800edb 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -1,16 +1,16 @@ package swarm import ( + "context" "fmt" inet "github.com/libp2p/go-libp2p/p2p/net" - conn "github.com/libp2p/go-libp2p/p2p/net/conn" - "context" ic "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" + conn "github.com/libp2p/go-libp2p-conn" ) // Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 6d1abd3342..29694df73b 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -10,8 +10,8 @@ import ( lgbl "github.com/ipfs/go-libp2p-loggables" peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" - conn "github.com/libp2p/go-libp2p/p2p/net/conn" - addrutil "github.com/libp2p/go-libp2p/p2p/net/swarm/addr" + addrutil "github.com/libp2p/go-addr-util" + conn "github.com/libp2p/go-libp2p-conn" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index e1b9a876fb..702ef54b7e 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -1,16 +1,17 @@ package swarm import ( + "context" "fmt" - "context" + inet "github.com/libp2p/go-libp2p/p2p/net" + lgbl "github.com/ipfs/go-libp2p-loggables" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" + conn "github.com/libp2p/go-libp2p-conn" transport "github.com/libp2p/go-libp2p-transport" mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" - inet "github.com/libp2p/go-libp2p/p2p/net" - conn "github.com/libp2p/go-libp2p/p2p/net/conn" ) func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index b7520dc757..e7099812db 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -2,6 +2,7 @@ package swarm import ( "bytes" + "context" "fmt" "io" "net" @@ -11,12 +12,11 @@ import ( metrics "github.com/libp2p/go-libp2p/p2p/metrics" inet "github.com/libp2p/go-libp2p/p2p/net" - testutil "github.com/libp2p/go-libp2p/testutil" - "context" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" + testutil "github.com/libp2p/go-testutil" ) func EchoStreamHandler(stream inet.Stream) { From dfae5c7dd6e6c6aff67f1ffc3002236e6b48ed1f Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 4 Oct 2016 13:50:34 -0700 Subject: [PATCH 063/259] extract net interface and metrics --- p2p/net/swarm/swarm.go | 7 +++---- p2p/net/swarm/swarm_addr_test.go | 3 +-- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_listen.go | 5 ++--- p2p/net/swarm/swarm_net.go | 7 +++---- p2p/net/swarm/swarm_net_test.go | 4 ++-- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_stream.go | 2 +- p2p/net/swarm/swarm_test.go | 5 ++--- 9 files changed, 16 insertions(+), 21 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 9107d7160a..77c7daff04 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -11,10 +11,6 @@ import ( "sync" "time" - metrics "github.com/libp2p/go-libp2p/p2p/metrics" - mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" - inet "github.com/libp2p/go-libp2p/p2p/net" - ci "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" @@ -26,6 +22,9 @@ import ( goprocessctx "github.com/jbenet/goprocess/context" addrutil "github.com/libp2p/go-addr-util" conn "github.com/libp2p/go-libp2p-conn" + metrics "github.com/libp2p/go-libp2p-metrics" + mconn "github.com/libp2p/go-libp2p-metrics/conn" + inet "github.com/libp2p/go-libp2p-net" transport "github.com/libp2p/go-libp2p-transport" filter "github.com/libp2p/go-maddr-filter" tcpt "github.com/libp2p/go-tcp-transport" diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index bd48d4f601..5d2225405a 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -4,11 +4,10 @@ import ( "context" "testing" - metrics "github.com/libp2p/go-libp2p/p2p/metrics" - pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" addrutil "github.com/libp2p/go-addr-util" + metrics "github.com/libp2p/go-libp2p-metrics" testutil "github.com/libp2p/go-testutil" ) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 30ef800edb..42fc8905da 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - inet "github.com/libp2p/go-libp2p/p2p/net" + inet "github.com/libp2p/go-libp2p-net" ic "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 702ef54b7e..0c9d859735 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -4,14 +4,13 @@ import ( "context" "fmt" - inet "github.com/libp2p/go-libp2p/p2p/net" - lgbl "github.com/ipfs/go-libp2p-loggables" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" conn "github.com/libp2p/go-libp2p-conn" + mconn "github.com/libp2p/go-libp2p-metrics/conn" + inet "github.com/libp2p/go-libp2p-net" transport "github.com/libp2p/go-libp2p-transport" - mconn "github.com/libp2p/go-libp2p/p2p/metrics/conn" ) func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index d9e83d5fdc..86755cf8be 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -1,16 +1,15 @@ package swarm import ( + "context" "fmt" peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" - metrics "github.com/libp2p/go-libp2p/p2p/metrics" - inet "github.com/libp2p/go-libp2p/p2p/net" - - "context" ma "github.com/jbenet/go-multiaddr" "github.com/jbenet/goprocess" + metrics "github.com/libp2p/go-libp2p-metrics" + inet "github.com/libp2p/go-libp2p-net" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 205b841a19..ebf3bb82a2 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -5,9 +5,9 @@ import ( "testing" "time" - inet "github.com/libp2p/go-libp2p/p2p/net" - testutil "github.com/libp2p/go-libp2p/p2p/test/util" "context" + inet "github.com/libp2p/go-libp2p-net" + testutil "github.com/libp2p/go-libp2p/p2p/test/util" ) // TestConnectednessCorrect starts a few networks, connects a few diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 2bc663e0a5..9e914b8218 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -7,7 +7,7 @@ import ( "context" peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" - inet "github.com/libp2p/go-libp2p/p2p/net" + inet "github.com/libp2p/go-libp2p-net" ) func streamsSame(a, b inet.Stream) bool { diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 51c846001a..ef3ab3c8fd 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -1,8 +1,8 @@ package swarm import ( + inet "github.com/libp2p/go-libp2p-net" protocol "github.com/libp2p/go-libp2p-protocol" - inet "github.com/libp2p/go-libp2p/p2p/net" ps "github.com/jbenet/go-peerstream" ) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index e7099812db..f0ed560cf2 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -10,12 +10,11 @@ import ( "testing" "time" - metrics "github.com/libp2p/go-libp2p/p2p/metrics" - inet "github.com/libp2p/go-libp2p/p2p/net" - peer "github.com/ipfs/go-libp2p-peer" pstore "github.com/ipfs/go-libp2p-peerstore" ma "github.com/jbenet/go-multiaddr" + metrics "github.com/libp2p/go-libp2p-metrics" + inet "github.com/libp2p/go-libp2p-net" testutil "github.com/libp2p/go-testutil" ) From 26422f282b81406b2ddbf985bb17a1642cee3af0 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 4 Oct 2016 15:39:24 -0700 Subject: [PATCH 064/259] switching to separated conn and interface-conn packages --- p2p/net/swarm/limiter.go | 8 ++++---- p2p/net/swarm/limiter_test.go | 16 ++++++++-------- p2p/net/swarm/swarm_addr.go | 4 ++-- p2p/net/swarm/swarm_conn.go | 11 +++++------ p2p/net/swarm/swarm_dial.go | 8 ++++---- p2p/net/swarm/swarm_listen.go | 3 ++- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 5c8d235b8e..bf4bcf0f5c 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -7,11 +7,11 @@ import ( peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" addrutil "github.com/libp2p/go-addr-util" - conn "github.com/libp2p/go-libp2p-conn" + iconn "github.com/libp2p/go-libp2p-interface-conn" ) type dialResult struct { - Conn conn.Conn + Conn iconn.Conn Err error } @@ -38,14 +38,14 @@ type dialLimiter struct { fdLimit int waitingOnFd []*dialJob - dialFunc func(context.Context, peer.ID, ma.Multiaddr) (conn.Conn, error) + dialFunc func(context.Context, peer.ID, ma.Multiaddr) (iconn.Conn, error) activePerPeer map[peer.ID]int perPeerLimit int waitingOnPeerLimit map[peer.ID][]*dialJob } -type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (conn.Conn, error) +type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (iconn.Conn, error) func newDialLimiter(df dialfunc) *dialLimiter { return newDialLimiterWithParams(df, concurrentFdDials, defaultPerPeerRateLimit) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 7332c0e06a..31ac8435df 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -10,7 +10,7 @@ import ( peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" - conn "github.com/libp2p/go-libp2p-conn" + iconn "github.com/libp2p/go-libp2p-interface-conn" mafmt "github.com/whyrusleeping/mafmt" ) @@ -55,13 +55,13 @@ func tryDialAddrs(ctx context.Context, l *dialLimiter, p peer.ID, addrs []ma.Mul } func hangDialFunc(hang chan struct{}) dialfunc { - return func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { + return func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { if mafmt.UTP.Matches(a) { - return conn.Conn(nil), nil + return iconn.Conn(nil), nil } if tcpPortOver(a, 10) { - return conn.Conn(nil), nil + return iconn.Conn(nil), nil } <-hang @@ -171,9 +171,9 @@ func TestFDLimiting(t *testing.T) { func TestTokenRedistribution(t *testing.T) { hangchs := make(map[peer.ID]chan struct{}) - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { if tcpPortOver(a, 10) { - return (conn.Conn)(nil), nil + return (iconn.Conn)(nil), nil } <-hangchs[p] @@ -260,9 +260,9 @@ func TestTokenRedistribution(t *testing.T) { } func TestStressLimiter(t *testing.T) { - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (conn.Conn, error) { + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { if tcpPortOver(a, 1000) { - return conn.Conn(nil), nil + return iconn.Conn(nil), nil } time.Sleep(time.Millisecond * time.Duration(5+rand.Intn(100))) diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 87b0829da1..0ab2c7ea1c 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -3,7 +3,7 @@ package swarm import ( ma "github.com/jbenet/go-multiaddr" addrutil "github.com/libp2p/go-addr-util" - conn "github.com/libp2p/go-libp2p-conn" + iconn "github.com/libp2p/go-libp2p-interface-conn" ) // ListenAddresses returns a list of addresses at which this swarm listens. @@ -11,7 +11,7 @@ func (s *Swarm) ListenAddresses() []ma.Multiaddr { listeners := s.swarm.Listeners() addrs := make([]ma.Multiaddr, 0, len(listeners)) for _, l := range listeners { - if l2, ok := l.NetListener().(conn.Listener); ok { + if l2, ok := l.NetListener().(iconn.Listener); ok { addrs = append(addrs, l2.Multiaddr()) } } diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 42fc8905da..8128a3e6ab 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -4,13 +4,12 @@ import ( "context" "fmt" - inet "github.com/libp2p/go-libp2p-net" - ic "github.com/ipfs/go-libp2p-crypto" peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" - conn "github.com/libp2p/go-libp2p-conn" + iconn "github.com/libp2p/go-libp2p-interface-conn" + inet "github.com/libp2p/go-libp2p-net" ) // Conn is a simple wrapper around a ps.Conn that also exposes @@ -33,12 +32,12 @@ func (c *Conn) StreamConn() *ps.Conn { return (*ps.Conn)(c) } -func (c *Conn) RawConn() conn.Conn { +func (c *Conn) RawConn() iconn.Conn { // righly panic if these things aren't true. it is an expected // invariant that these Conns are all of the typewe expect: // ps.Conn wrapping a conn.Conn // if we get something else it is programmer error. - return (*ps.Conn)(c).NetConn().(conn.Conn) + return (*ps.Conn)(c).NetConn().(iconn.Conn) } func (c *Conn) String() string { @@ -94,7 +93,7 @@ func (c *Conn) Close() error { func wrapConn(psc *ps.Conn) (*Conn, error) { // grab the underlying connection. - if _, ok := psc.NetConn().(conn.Conn); !ok { + if _, ok := psc.NetConn().(iconn.Conn); !ok { // this should never happen. if we see it ocurring it means that we added // a Listener to the ps.Swarm that is NOT one of our net/conn.Listener. return nil, fmt.Errorf("swarm connHandler: invalid conn (not a conn.Conn): %s", psc) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 29694df73b..bfe9f0382e 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -11,7 +11,7 @@ import ( peer "github.com/ipfs/go-libp2p-peer" ma "github.com/jbenet/go-multiaddr" addrutil "github.com/libp2p/go-addr-util" - conn "github.com/libp2p/go-libp2p-conn" + iconn "github.com/libp2p/go-libp2p-interface-conn" ) // Diagram of dial sync: @@ -284,7 +284,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return swarmC, nil } -func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (conn.Conn, error) { +func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (iconn.Conn, error) { log.Debugf("%s swarm dialing %s %s", s.local, p, remoteAddrs) ctx, cancel := context.WithCancel(ctx) @@ -344,7 +344,7 @@ func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp }) } -func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (conn.Conn, error) { +func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (iconn.Conn, error) { log.Debugf("%s swarm dialing %s %s", s.local, p, addr) connC, err := s.dialer.Dial(ctx, addr, p) @@ -376,7 +376,7 @@ var ConnSetupTimeout = time.Minute * 5 // dialConnSetup is the setup logic for a connection from the dial side. it // needs to add the Conn to the StreamSwarm, then run newConnSetup -func dialConnSetup(ctx context.Context, s *Swarm, connC conn.Conn) (*Conn, error) { +func dialConnSetup(ctx context.Context, s *Swarm, connC iconn.Conn) (*Conn, error) { deadline, ok := ctx.Deadline() if !ok { diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 0c9d859735..756084abc7 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -8,6 +8,7 @@ import ( ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" conn "github.com/libp2p/go-libp2p-conn" + iconn "github.com/libp2p/go-libp2p-interface-conn" mconn "github.com/libp2p/go-libp2p-metrics/conn" inet "github.com/libp2p/go-libp2p-net" transport "github.com/libp2p/go-libp2p-transport" @@ -98,7 +99,7 @@ func (s *Swarm) addListener(tptlist transport.Listener) error { return s.addConnListener(list) } -func (s *Swarm) addConnListener(list conn.Listener) error { +func (s *Swarm) addConnListener(list iconn.Listener) error { // AddListener to the peerstream Listener. this will begin accepting connections // and streams! sl, err := s.swarm.AddListener(list) From 05b9ca603912fee3020476d425142732d8b10d64 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 4 Oct 2016 21:18:07 -0700 Subject: [PATCH 065/259] update deps --- p2p/net/swarm/dial_sync.go | 2 +- p2p/net/swarm/dial_sync_test.go | 2 +- p2p/net/swarm/dial_test.go | 8 ++++---- p2p/net/swarm/limiter.go | 4 ++-- p2p/net/swarm/limiter_test.go | 4 ++-- p2p/net/swarm/peers_test.go | 6 +++--- p2p/net/swarm/simul_test.go | 6 +++--- p2p/net/swarm/swarm.go | 8 ++++---- p2p/net/swarm/swarm_addr.go | 2 +- p2p/net/swarm/swarm_addr_test.go | 4 ++-- p2p/net/swarm/swarm_conn.go | 6 +++--- p2p/net/swarm/swarm_dial.go | 6 +++--- p2p/net/swarm/swarm_listen.go | 4 ++-- p2p/net/swarm/swarm_net.go | 6 +++--- p2p/net/swarm/swarm_notif_test.go | 4 ++-- p2p/net/swarm/swarm_test.go | 6 +++--- 16 files changed, 39 insertions(+), 39 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index a63c047ac8..75d3f0fbee 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -4,7 +4,7 @@ import ( "context" "sync" - peer "github.com/ipfs/go-libp2p-peer" + peer "github.com/libp2p/go-libp2p-peer" ) type DialFunc func(context.Context, peer.ID) (*Conn, error) diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 61a69ce386..ca81a9c872 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p-peer" + peer "github.com/libp2p/go-libp2p-peer" ) func getMockDialFunc() (DialFunc, func(), context.Context, <-chan struct{}) { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 8e7b48fc2b..dabb0bdb00 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -7,13 +7,13 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p-peer" - pstore "github.com/ipfs/go-libp2p-peerstore" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" addrutil "github.com/libp2p/go-addr-util" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" testutil "github.com/libp2p/go-testutil" ci "github.com/libp2p/go-testutil/ci" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" ) func closeSwarms(swarms []*Swarm) { diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index bf4bcf0f5c..5e96908e41 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -4,10 +4,10 @@ import ( "context" "sync" - peer "github.com/ipfs/go-libp2p-peer" - ma "github.com/jbenet/go-multiaddr" addrutil "github.com/libp2p/go-addr-util" iconn "github.com/libp2p/go-libp2p-interface-conn" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" ) type dialResult struct { diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 31ac8435df..646019d265 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p-peer" - ma "github.com/jbenet/go-multiaddr" iconn "github.com/libp2p/go-libp2p-interface-conn" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" mafmt "github.com/whyrusleeping/mafmt" ) diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 5eadf4e671..3259264ae8 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -4,9 +4,9 @@ import ( "testing" "context" - peer "github.com/ipfs/go-libp2p-peer" - pstore "github.com/ipfs/go-libp2p-peerstore" - ma "github.com/jbenet/go-multiaddr" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + ma "github.com/multiformats/go-multiaddr" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 3518f139e6..8935773296 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p-peer" - pstore "github.com/ipfs/go-libp2p-peerstore" - ma "github.com/jbenet/go-multiaddr" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" ci "github.com/libp2p/go-testutil/ci" + ma "github.com/multiformats/go-multiaddr" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 77c7daff04..bb179bbc69 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -11,23 +11,23 @@ import ( "sync" "time" - ci "github.com/ipfs/go-libp2p-crypto" - peer "github.com/ipfs/go-libp2p-peer" - pstore "github.com/ipfs/go-libp2p-peerstore" logging "github.com/ipfs/go-log" - ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" pst "github.com/jbenet/go-stream-muxer" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" addrutil "github.com/libp2p/go-addr-util" conn "github.com/libp2p/go-libp2p-conn" + ci "github.com/libp2p/go-libp2p-crypto" metrics "github.com/libp2p/go-libp2p-metrics" mconn "github.com/libp2p/go-libp2p-metrics/conn" inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" transport "github.com/libp2p/go-libp2p-transport" filter "github.com/libp2p/go-maddr-filter" tcpt "github.com/libp2p/go-tcp-transport" + ma "github.com/multiformats/go-multiaddr" psmss "github.com/whyrusleeping/go-smux-multistream" spdy "github.com/whyrusleeping/go-smux-spdystream" yamux "github.com/whyrusleeping/go-smux-yamux" diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 0ab2c7ea1c..78a3f5351a 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -1,9 +1,9 @@ package swarm import ( - ma "github.com/jbenet/go-multiaddr" addrutil "github.com/libp2p/go-addr-util" iconn "github.com/libp2p/go-libp2p-interface-conn" + ma "github.com/multiformats/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 5d2225405a..011b706d81 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -4,11 +4,11 @@ import ( "context" "testing" - pstore "github.com/ipfs/go-libp2p-peerstore" - ma "github.com/jbenet/go-multiaddr" addrutil "github.com/libp2p/go-addr-util" metrics "github.com/libp2p/go-libp2p-metrics" + pstore "github.com/libp2p/go-libp2p-peerstore" testutil "github.com/libp2p/go-testutil" + ma "github.com/multiformats/go-multiaddr" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 8128a3e6ab..72609f0f32 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -4,12 +4,12 @@ import ( "context" "fmt" - ic "github.com/ipfs/go-libp2p-crypto" - peer "github.com/ipfs/go-libp2p-peer" - ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" + ic "github.com/libp2p/go-libp2p-crypto" iconn "github.com/libp2p/go-libp2p-interface-conn" inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" ) // Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index bfe9f0382e..0f15301698 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -7,11 +7,11 @@ import ( "sync" "time" - lgbl "github.com/ipfs/go-libp2p-loggables" - peer "github.com/ipfs/go-libp2p-peer" - ma "github.com/jbenet/go-multiaddr" addrutil "github.com/libp2p/go-addr-util" iconn "github.com/libp2p/go-libp2p-interface-conn" + lgbl "github.com/libp2p/go-libp2p-loggables" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 756084abc7..d76d1e640e 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -4,14 +4,14 @@ import ( "context" "fmt" - lgbl "github.com/ipfs/go-libp2p-loggables" - ma "github.com/jbenet/go-multiaddr" ps "github.com/jbenet/go-peerstream" conn "github.com/libp2p/go-libp2p-conn" iconn "github.com/libp2p/go-libp2p-interface-conn" + lgbl "github.com/libp2p/go-libp2p-loggables" mconn "github.com/libp2p/go-libp2p-metrics/conn" inet "github.com/libp2p/go-libp2p-net" transport "github.com/libp2p/go-libp2p-transport" + ma "github.com/multiformats/go-multiaddr" ) func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 86755cf8be..90356a0fc7 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -4,12 +4,12 @@ import ( "context" "fmt" - peer "github.com/ipfs/go-libp2p-peer" - pstore "github.com/ipfs/go-libp2p-peerstore" - ma "github.com/jbenet/go-multiaddr" "github.com/jbenet/goprocess" metrics "github.com/libp2p/go-libp2p-metrics" inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + ma "github.com/multiformats/go-multiaddr" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 9e914b8218..d9e9df62a5 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -5,9 +5,9 @@ import ( "time" "context" - peer "github.com/ipfs/go-libp2p-peer" - ma "github.com/jbenet/go-multiaddr" inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" ) func streamsSame(a, b inet.Stream) bool { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index f0ed560cf2..9deb918f3c 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -10,12 +10,12 @@ import ( "testing" "time" - peer "github.com/ipfs/go-libp2p-peer" - pstore "github.com/ipfs/go-libp2p-peerstore" - ma "github.com/jbenet/go-multiaddr" metrics "github.com/libp2p/go-libp2p-metrics" inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" testutil "github.com/libp2p/go-testutil" + ma "github.com/multiformats/go-multiaddr" ) func EchoStreamHandler(stream inet.Stream) { From ba533150b4fd7df41e7718e3f374af727623b924 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 5 Oct 2016 10:05:05 -0700 Subject: [PATCH 066/259] extract from go-libp2p --- p2p/net/swarm/swarm_net_test.go | 35 ++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index ebf3bb82a2..ff13934088 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -1,15 +1,36 @@ -package swarm_test +package swarm import ( + "context" "fmt" "testing" "time" - "context" inet "github.com/libp2p/go-libp2p-net" - testutil "github.com/libp2p/go-libp2p/p2p/test/util" + pstore "github.com/libp2p/go-libp2p-peerstore" + tu "github.com/libp2p/go-testutil" + ma "github.com/multiformats/go-multiaddr" ) +func GenSwarmNetwork(t *testing.T, ctx context.Context) *Network { + p := tu.RandPeerNetParamsOrFatal(t) + ps := pstore.NewPeerstore() + ps.AddPubKey(p.ID, p.PubKey) + ps.AddPrivKey(p.ID, p.PrivKey) + n, err := NewNetwork(ctx, []ma.Multiaddr{p.Addr}, p.ID, ps, nil) + if err != nil { + t.Fatal(err) + } + ps.AddAddrs(p.ID, n.ListenAddresses(), pstore.PermanentAddrTTL) + return n +} + +func DivulgeAddresses(a, b inet.Network) { + id := a.LocalPeer() + addrs := a.Peerstore().Addrs(id) + b.Peerstore().AddAddrs(id, addrs, pstore.PermanentAddrTTL) +} + // TestConnectednessCorrect starts a few networks, connects a few // and tests Connectedness value is correct. func TestConnectednessCorrect(t *testing.T) { @@ -18,13 +39,13 @@ func TestConnectednessCorrect(t *testing.T) { nets := make([]inet.Network, 4) for i := 0; i < 4; i++ { - nets[i] = testutil.GenSwarmNetwork(t, ctx) + nets[i] = GenSwarmNetwork(t, ctx) } // connect 0-1, 0-2, 0-3, 1-2, 2-3 dial := func(a, b inet.Network) { - testutil.DivulgeAddresses(b, a) + DivulgeAddresses(b, a) if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { t.Fatalf("Failed to dial: %s", err) } @@ -109,11 +130,11 @@ func TestNetworkOpenStream(t *testing.T) { nets := make([]inet.Network, 4) for i := 0; i < 4; i++ { - nets[i] = testutil.GenSwarmNetwork(t, ctx) + nets[i] = GenSwarmNetwork(t, ctx) } dial := func(a, b inet.Network) { - testutil.DivulgeAddresses(b, a) + DivulgeAddresses(b, a) if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { t.Fatalf("Failed to dial: %s", err) } From 1fc8868191454ee03a626380f7e2b1b2f4283eb6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 18 Oct 2016 17:10:22 -0700 Subject: [PATCH 067/259] fix race conditions in dial_sync and limiter tests --- p2p/net/swarm/dial_sync.go | 1 - p2p/net/swarm/limiter_test.go | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 75d3f0fbee..48b77899f2 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -81,7 +81,6 @@ func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { ad.conn, ad.err = ds.dialFunc(ctx, p) close(ad.waitch) ad.cancel() - ad.waitch = nil // to ensure nobody tries reusing this }(ctx, p, actd) } diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 646019d265..cbb6afdbc6 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -5,6 +5,7 @@ import ( "fmt" "math/rand" "strconv" + "sync" "testing" "time" @@ -170,13 +171,17 @@ func TestFDLimiting(t *testing.T) { } func TestTokenRedistribution(t *testing.T) { + var lk sync.Mutex hangchs := make(map[peer.ID]chan struct{}) df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { if tcpPortOver(a, 10) { return (iconn.Conn)(nil), nil } - <-hangchs[p] + lk.Lock() + ch := hangchs[p] + lk.Unlock() + <-ch return nil, fmt.Errorf("test bad dial") } l := newDialLimiterWithParams(df, 8, 4) @@ -190,6 +195,9 @@ func TestTokenRedistribution(t *testing.T) { // take all fd limit tokens with hang dials for _, pid := range pids { hangchs[pid] = make(chan struct{}) + } + + for _, pid := range pids { tryDialAddrs(ctx, l, pid, bads, resch) } From c88fe55f11a4e3b6baa4b94927154a496f52d200 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 24 Oct 2016 17:42:27 -0700 Subject: [PATCH 068/259] expose errors in log messages --- p2p/net/swarm/swarm_dial.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 0f15301698..d32d156dff 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -266,7 +266,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { // try to get a connection to any addr connC, err := s.dialAddrs(ctx, p, remoteAddrChan) if err != nil { - logdial["error"] = err + logdial["error"] = err.Error() return nil, err } logdial["netconn"] = lgbl.NetConn(connC) @@ -275,7 +275,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { defer log.EventBegin(ctx, "swarmDialDoSetup", logdial, lgbl.NetConn(connC)).Done() swarmC, err := dialConnSetup(ctx, s, connC) if err != nil { - logdial["error"] = err + logdial["error"] = err.Error() connC.Close() // close the connection. didn't work out :( return nil, err } From 3b5d44bb21262b5dd85b717f03e1442b2e444df4 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 28 Oct 2016 14:06:31 -0700 Subject: [PATCH 069/259] make sure to always take locks in correct order --- p2p/net/swarm/dial_sync.go | 42 +++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 48b77899f2..8ef5183755 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -36,6 +36,7 @@ type activeDial struct { } func (dr *activeDial) wait(ctx context.Context) (*Conn, error) { + dr.incref() defer dr.decref() select { case <-dr.waitch: @@ -53,22 +54,38 @@ func (ad *activeDial) incref() { func (ad *activeDial) decref() { ad.refCntLk.Lock() - defer ad.refCntLk.Unlock() ad.refCnt-- - if ad.refCnt <= 0 { - ad.cancel() + maybeZero := (ad.refCnt <= 0) + ad.refCntLk.Unlock() + + // make sure to always take locks in correct order. + if maybeZero { ad.ds.dialsLk.Lock() - delete(ad.ds.dials, ad.id) + ad.refCntLk.Lock() + // check again after lock swap drop to make sure nobody else called incref + // in between locks + if ad.refCnt <= 0 { + ad.cancel() + delete(ad.ds.dials, ad.id) + } ad.ds.dialsLk.Unlock() + ad.refCntLk.Unlock() } } -func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { +func (ad *activeDial) start(ctx context.Context) { + ad.conn, ad.err = ad.ds.dialFunc(ctx, ad.id) + close(ad.waitch) + ad.cancel() +} + +func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { ds.dialsLk.Lock() + defer ds.dialsLk.Unlock() actd, ok := ds.dials[p] if !ok { - ctx, cancel := context.WithCancel(context.Background()) + adctx, cancel := context.WithCancel(context.Background()) actd = &activeDial{ id: p, cancel: cancel, @@ -77,15 +94,12 @@ func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { } ds.dials[p] = actd - go func(ctx context.Context, p peer.ID, ad *activeDial) { - ad.conn, ad.err = ds.dialFunc(ctx, p) - close(ad.waitch) - ad.cancel() - }(ctx, p, actd) + go actd.start(adctx) } - actd.incref() - ds.dialsLk.Unlock() + return actd +} - return actd.wait(ctx) +func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { + return ds.getActiveDial(p).wait(ctx) } From 9bd9701f20c20c96119e23de7267042e0f61f031 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 28 Oct 2016 14:14:44 -0700 Subject: [PATCH 070/259] add a test to stress the lock order race condition --- p2p/net/swarm/dial_sync_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index ca81a9c872..0d70e226af 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -201,3 +201,27 @@ func TestFailFirst(t *testing.T) { t.Fatal("should have gotten a 'real' conn back") } } + +func TestStressActiveDial(t *testing.T) { + ds := NewDialSync(func(ctx context.Context, p peer.ID) (*Conn, error) { + return nil, nil + }) + + wg := sync.WaitGroup{} + + pid := peer.ID("foo") + + makeDials := func() { + for i := 0; i < 10000; i++ { + ds.DialLock(context.Background(), pid) + } + wg.Done() + } + + for i := 0; i < 100; i++ { + wg.Add(1) + go makeDials() + } + + wg.Wait() +} From 0cd1fc7907ed83cb22ee3afbf86e48e9af6c49f9 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 28 Oct 2016 16:43:20 -0700 Subject: [PATCH 071/259] nitpick: lock release order --- p2p/net/swarm/dial_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 8ef5183755..734901fc15 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -68,8 +68,8 @@ func (ad *activeDial) decref() { ad.cancel() delete(ad.ds.dials, ad.id) } - ad.ds.dialsLk.Unlock() ad.refCntLk.Unlock() + ad.ds.dialsLk.Unlock() } } From 034bcab5e6761da4a5204d433b92d9883de22a3b Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 28 Oct 2016 17:09:45 -0700 Subject: [PATCH 072/259] increase refcount before dropping dialsLk initially --- p2p/net/swarm/dial_sync.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 734901fc15..69739e54ec 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -36,7 +36,6 @@ type activeDial struct { } func (dr *activeDial) wait(ctx context.Context) (*Conn, error) { - dr.incref() defer dr.decref() select { case <-dr.waitch: @@ -97,6 +96,9 @@ func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { go actd.start(adctx) } + // increase ref count before dropping dialsLk + actd.incref() + return actd } From 4edf3cfa4c6dc9e4f3effb748d797471493ef132 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 2 Nov 2016 23:35:03 -0700 Subject: [PATCH 073/259] move protocol methods down into peerstream --- p2p/net/swarm/swarm.go | 13 ++++--------- p2p/net/swarm/swarm_conn.go | 6 +++++- p2p/net/swarm/swarm_stream.go | 33 ++++++++------------------------- 3 files changed, 17 insertions(+), 35 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index bb179bbc69..ec156d4e4b 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -256,7 +256,7 @@ func (s *Swarm) SetConnHandler(handler ConnHandler) { // See peerstream. func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) { s.swarm.SetStreamHandler(func(s *ps.Stream) { - handler(wrapStream(s)) + handler((*Stream)(s)) }) } @@ -273,12 +273,7 @@ func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, erro // TODO: think about passing a context down to NewStreamWithGroup st, err := s.swarm.NewStreamWithGroup(p) - return wrapStream(st), err -} - -// StreamsWithPeer returns all the live Streams to p -func (s *Swarm) StreamsWithPeer(p peer.ID) []*Stream { - return wrapStreams(ps.StreamsWithGroup(p, s.swarm.Streams())) + return (*Stream)(st), err } // ConnectionsToPeer returns all the live connections to p @@ -387,9 +382,9 @@ func (n *ps2netNotifee) Disconnected(c *ps.Conn) { } func (n *ps2netNotifee) OpenedStream(s *ps.Stream) { - n.not.OpenedStream(n.net, &Stream{stream: s}) + n.not.OpenedStream(n.net, (*Stream)(s)) } func (n *ps2netNotifee) ClosedStream(s *ps.Stream) { - n.not.ClosedStream(n.net, &Stream{stream: s}) + n.not.ClosedStream(n.net, (*Stream)(s)) } diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 72609f0f32..bc5b69e4ad 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -77,7 +77,7 @@ func (c *Conn) RemotePublicKey() ic.PubKey { // NewSwarmStream returns a new Stream from this connection func (c *Conn) NewSwarmStream() (*Stream, error) { s, err := c.StreamConn().NewStream() - return wrapStream(s), err + return (*Stream)(s), err } // NewStream returns a new Stream from this connection @@ -91,6 +91,10 @@ func (c *Conn) Close() error { return c.StreamConn().Close() } +func (c *Conn) GetStreams() ([]inet.Stream, error) { + return nil, fmt.Errorf("GetStreams() not yet implemented") +} + func wrapConn(psc *ps.Conn) (*Conn, error) { // grab the underlying connection. if _, ok := psc.NetConn().(iconn.Conn); !ok { diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index ef3ab3c8fd..e3b12563bb 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -9,14 +9,11 @@ import ( // Stream is a wrapper around a ps.Stream that exposes a way to get // our Conn and Swarm (instead of just the ps.Conn and ps.Swarm) -type Stream struct { - stream *ps.Stream - protocol protocol.ID -} +type Stream ps.Stream // Stream returns the underlying peerstream.Stream func (s *Stream) Stream() *ps.Stream { - return s.stream + return (*ps.Stream)(s) } // Conn returns the Conn associated with this Stream, as an inet.Conn @@ -26,43 +23,29 @@ func (s *Stream) Conn() inet.Conn { // SwarmConn returns the Conn associated with this Stream, as a *Conn func (s *Stream) SwarmConn() *Conn { - return (*Conn)(s.stream.Conn()) + return (*Conn)(s.Stream().Conn()) } // Read reads bytes from a stream. func (s *Stream) Read(p []byte) (n int, err error) { - return s.stream.Read(p) + return s.Stream().Read(p) } // Write writes bytes to a stream, flushing for each call. func (s *Stream) Write(p []byte) (n int, err error) { - return s.stream.Write(p) + return s.Stream().Write(p) } // Close closes the stream, indicating this side is finished // with the stream. func (s *Stream) Close() error { - return s.stream.Close() + return s.Stream().Close() } func (s *Stream) Protocol() protocol.ID { - return s.protocol + return (*ps.Stream)(s).Protocol() } func (s *Stream) SetProtocol(p protocol.ID) { - s.protocol = p -} - -func wrapStream(pss *ps.Stream) *Stream { - return &Stream{ - stream: pss, - } -} - -func wrapStreams(st []*ps.Stream) []*Stream { - out := make([]*Stream, len(st)) - for i, s := range st { - out[i] = wrapStream(s) - } - return out + (*ps.Stream)(s).SetProtocol(p) } From 0ac5cdf74458835bbcc18d9bfb703666f912bd1c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 3 Nov 2016 13:15:48 -0700 Subject: [PATCH 074/259] bubble up peerstream modifications --- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_conn.go | 2 +- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_stream.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index ec156d4e4b..3a59517c34 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -12,7 +12,6 @@ import ( "time" logging "github.com/ipfs/go-log" - ps "github.com/jbenet/go-peerstream" pst "github.com/jbenet/go-stream-muxer" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" @@ -26,6 +25,7 @@ import ( pstore "github.com/libp2p/go-libp2p-peerstore" transport "github.com/libp2p/go-libp2p-transport" filter "github.com/libp2p/go-maddr-filter" + ps "github.com/libp2p/go-peerstream" tcpt "github.com/libp2p/go-tcp-transport" ma "github.com/multiformats/go-multiaddr" psmss "github.com/whyrusleeping/go-smux-multistream" diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index bc5b69e4ad..4b13be54f1 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -4,11 +4,11 @@ import ( "context" "fmt" - ps "github.com/jbenet/go-peerstream" ic "github.com/libp2p/go-libp2p-crypto" iconn "github.com/libp2p/go-libp2p-interface-conn" inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" + ps "github.com/libp2p/go-peerstream" ma "github.com/multiformats/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index d76d1e640e..571a305e3a 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -4,13 +4,13 @@ import ( "context" "fmt" - ps "github.com/jbenet/go-peerstream" conn "github.com/libp2p/go-libp2p-conn" iconn "github.com/libp2p/go-libp2p-interface-conn" lgbl "github.com/libp2p/go-libp2p-loggables" mconn "github.com/libp2p/go-libp2p-metrics/conn" inet "github.com/libp2p/go-libp2p-net" transport "github.com/libp2p/go-libp2p-transport" + ps "github.com/libp2p/go-peerstream" ma "github.com/multiformats/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index e3b12563bb..771038621f 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -4,7 +4,7 @@ import ( inet "github.com/libp2p/go-libp2p-net" protocol "github.com/libp2p/go-libp2p-protocol" - ps "github.com/jbenet/go-peerstream" + ps "github.com/libp2p/go-peerstream" ) // Stream is a wrapper around a ps.Stream that exposes a way to get From 23bb096d3ebaf22a7490507d73b76824a0c570c8 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 3 Nov 2016 14:01:18 -0700 Subject: [PATCH 075/259] implement GetStreams --- p2p/net/swarm/swarm_conn.go | 8 +++++++- p2p/net/swarm/swarm_net_test.go | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 4b13be54f1..0b0c626cea 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -92,7 +92,13 @@ func (c *Conn) Close() error { } func (c *Conn) GetStreams() ([]inet.Stream, error) { - return nil, fmt.Errorf("GetStreams() not yet implemented") + ss := c.StreamConn().Streams() + out := make([]inet.Stream, len(ss)) + + for i, s := range ss { + out[i] = (*Stream)(s) + } + return out, nil } func wrapConn(psc *ps.Conn) (*Conn, error) { diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index ff13934088..95bc268307 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -165,6 +165,15 @@ func TestNetworkOpenStream(t *testing.T) { t.Fatal(err) } + streams, err := nets[0].ConnsToPeer(nets[1].LocalPeer())[0].GetStreams() + if err != nil { + t.Fatal(err) + } + + if len(streams) != 1 { + t.Fatal("should only have one stream there") + } + _, err = s.Write([]byte("hello ipfs")) if err != nil { t.Fatal(err) From 02a743fd6f2faa3852b9e7bcdf2d65698360721e Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 8 Nov 2016 10:37:06 -0800 Subject: [PATCH 076/259] add deadline methods to stream --- p2p/net/swarm/swarm_stream.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 771038621f..adadca78ec 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -1,9 +1,10 @@ package swarm import ( + "time" + inet "github.com/libp2p/go-libp2p-net" protocol "github.com/libp2p/go-libp2p-protocol" - ps "github.com/libp2p/go-peerstream" ) @@ -49,3 +50,15 @@ func (s *Stream) Protocol() protocol.ID { func (s *Stream) SetProtocol(p protocol.ID) { (*ps.Stream)(s).SetProtocol(p) } + +func (s *Stream) SetDeadline(t time.Time) error { + return s.Stream().SetDeadline(t) +} + +func (s *Stream) SetReadDeadline(t time.Time) error { + return s.Stream().SetReadDeadline(t) +} + +func (s *Stream) SetWriteDeadline(t time.Time) error { + return s.Stream().SetWriteDeadline(t) +} From 82fb7d1a18c7b55f955d33066ca2a759ea378005 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 9 Nov 2016 12:04:55 +0100 Subject: [PATCH 077/259] Add Private Network protector to swarm --- p2p/net/swarm/swarm.go | 12 ++++++++---- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_net.go | 5 +++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 3a59517c34..fe03a8e111 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -18,6 +18,7 @@ import ( addrutil "github.com/libp2p/go-addr-util" conn "github.com/libp2p/go-libp2p-conn" ci "github.com/libp2p/go-libp2p-crypto" + ipnet "github.com/libp2p/go-libp2p-interface-pnet" metrics "github.com/libp2p/go-libp2p-metrics" mconn "github.com/libp2p/go-libp2p-metrics/conn" inet "github.com/libp2p/go-libp2p-net" @@ -98,11 +99,13 @@ type Swarm struct { bwc metrics.Reporter limiter *dialLimiter + + protec ipnet.Protector } // NewSwarm constructs a Swarm, with a Chan. -func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, - local peer.ID, peers pstore.Peerstore, bwc metrics.Reporter) (*Swarm, error) { +func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID, + peers pstore.Peerstore, protec ipnet.Protector, bwc metrics.Reporter) (*Swarm, error) { listenAddrs, err := filterAddrs(listenAddrs) if err != nil { @@ -130,7 +133,8 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, bwc: bwc, fdRateLimit: make(chan struct{}, concurrentFdDials), Filters: filter.NewFilters(), - dialer: conn.NewDialer(local, peers.PrivKey(local), wrap), + dialer: conn.NewDialer(local, peers.PrivKey(local), protec, wrap), + protec: protec, } s.dsync = NewDialSync(s.doDial) @@ -158,7 +162,7 @@ func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey, pstpt ps notifs: make(map[inet.Notifiee]ps.Notifiee), fdRateLimit: make(chan struct{}, concurrentFdDials), Filters: filter.NewFilters(), - dialer: conn.NewDialer(id, privkey, nil), + dialer: conn.NewDialer(id, privkey, nil, nil), } // configure Swarm diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 571a305e3a..edb075c18e 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -83,7 +83,7 @@ func (s *Swarm) addListener(tptlist transport.Listener) error { log.Warning("Listener not given PrivateKey, so WILL NOT SECURE conns.") } - list, err := conn.WrapTransportListener(s.Context(), tptlist, s.local, sk) + list, err := conn.WrapTransportListener(s.Context(), tptlist, s.local, sk, s.protec) if err != nil { return err } diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 90356a0fc7..d522f87262 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/jbenet/goprocess" + ipnet "github.com/libp2p/go-libp2p-interface-pnet" metrics "github.com/libp2p/go-libp2p-metrics" inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" @@ -19,9 +20,9 @@ type Network Swarm // NewNetwork constructs a new network and starts listening on given addresses. func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, - peers pstore.Peerstore, bwc metrics.Reporter) (*Network, error) { + peers pstore.Peerstore, protec ipnet.Protector, bwc metrics.Reporter) (*Network, error) { - s, err := NewSwarm(ctx, listen, local, peers, bwc) + s, err := NewSwarm(ctx, listen, local, peers, protec, bwc) if err != nil { return nil, err } From b4dc58aa3be1c8fcad51cd53f2ebab67a13d8ef3 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 10 Nov 2016 16:00:57 -0800 Subject: [PATCH 078/259] log addr with failures --- p2p/net/swarm/limiter.go | 3 ++- p2p/net/swarm/swarm_dial.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 5e96908e41..67c6af4ddf 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -12,6 +12,7 @@ import ( type dialResult struct { Conn iconn.Conn + Addr ma.Multiaddr Err error } @@ -141,7 +142,7 @@ func (dl *dialLimiter) executeDial(j *dialJob) { con, err := dl.dialFunc(j.ctx, j.peer, j.addr) select { - case j.resp <- dialResult{Conn: con, Err: err}: + case j.resp <- dialResult{Conn: con, Addr: j.addr, Err: err}: case <-j.ctx.Done(): } } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index d32d156dff..c7cfe91110 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -318,7 +318,7 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. case resp := <-respch: active-- if resp.Err != nil { - log.Info("got error on dial: ", resp.Err) + log.Info("got error on dial to %s: ", resp.Addr, resp.Err) // Errors are normal, lots of dials will fail exitErr = resp.Err From 8dcdc4f994f4cab63f91a3e4dfd07a32cab66798 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 11 Nov 2016 21:34:55 +0100 Subject: [PATCH 079/259] Update for changed libp2p-conn --- p2p/net/swarm/swarm.go | 12 +++++++++--- p2p/net/swarm/swarm_addr_test.go | 4 ++-- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_net.go | 2 +- p2p/net/swarm/swarm_net_test.go | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index fe03a8e111..b1878ff0e2 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -103,8 +103,13 @@ type Swarm struct { protec ipnet.Protector } -// NewSwarm constructs a Swarm, with a Chan. func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID, + peers pstore.Peerstore, bwc metrics.Reporter) (*Swarm, error) { + return NewSwarmWithProtector(ctx, listenAddrs, local, peers, nil, bwc) +} + +// NewSwarm constructs a Swarm, with a Chan. +func NewSwarmWithProtector(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID, peers pstore.Peerstore, protec ipnet.Protector, bwc metrics.Reporter) (*Swarm, error) { listenAddrs, err := filterAddrs(listenAddrs) @@ -133,9 +138,10 @@ func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID, bwc: bwc, fdRateLimit: make(chan struct{}, concurrentFdDials), Filters: filter.NewFilters(), - dialer: conn.NewDialer(local, peers.PrivKey(local), protec, wrap), + dialer: conn.NewDialer(local, peers.PrivKey(local), wrap), protec: protec, } + s.dialer.Protector = protec s.dsync = NewDialSync(s.doDial) s.limiter = newDialLimiter(s.dialAddr) @@ -162,7 +168,7 @@ func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey, pstpt ps notifs: make(map[inet.Notifiee]ps.Notifiee), fdRateLimit: make(chan struct{}, concurrentFdDials), Filters: filter.NewFilters(), - dialer: conn.NewDialer(id, privkey, nil, nil), + dialer: conn.NewDialer(id, privkey, nil), } // configure Swarm diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 011b706d81..ab0ad029dd 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -65,11 +65,11 @@ func TestFilterAddrs(t *testing.T) { ps := pstore.NewPeerstore() ctx := context.Background() - if _, err := NewNetwork(ctx, bad, id, ps, metrics.NewBandwidthCounter()); err == nil { + if _, err := NewNetwork(ctx, bad, id, ps, nil, metrics.NewBandwidthCounter()); err == nil { t.Fatal("should have failed to create swarm") } - if _, err := NewNetwork(ctx, goodAndBad, id, ps, metrics.NewBandwidthCounter()); err != nil { + if _, err := NewNetwork(ctx, goodAndBad, id, ps, nil, metrics.NewBandwidthCounter()); err != nil { t.Fatal("should have succeeded in creating swarm", err) } } diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index edb075c18e..93099fe398 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -83,7 +83,7 @@ func (s *Swarm) addListener(tptlist transport.Listener) error { log.Warning("Listener not given PrivateKey, so WILL NOT SECURE conns.") } - list, err := conn.WrapTransportListener(s.Context(), tptlist, s.local, sk, s.protec) + list, err := conn.WrapTransportListenerWithProtector(s.Context(), tptlist, s.local, sk, s.protec) if err != nil { return err } diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index d522f87262..f6fa8c5796 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -22,7 +22,7 @@ type Network Swarm func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, peers pstore.Peerstore, protec ipnet.Protector, bwc metrics.Reporter) (*Network, error) { - s, err := NewSwarm(ctx, listen, local, peers, protec, bwc) + s, err := NewSwarmWithProtector(ctx, listen, local, peers, protec, bwc) if err != nil { return nil, err } diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 95bc268307..7dcdbba99f 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -17,7 +17,7 @@ func GenSwarmNetwork(t *testing.T, ctx context.Context) *Network { ps := pstore.NewPeerstore() ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) - n, err := NewNetwork(ctx, []ma.Multiaddr{p.Addr}, p.ID, ps, nil) + n, err := NewNetwork(ctx, []ma.Multiaddr{p.Addr}, p.ID, ps, nil, nil) if err != nil { t.Fatal(err) } From 94cbca5f521d33a6adc51f3805ff8060e9431d12 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 14 Nov 2016 19:39:18 +0100 Subject: [PATCH 080/259] Revert to normal NewNetwork create NewNetworkWithProtector --- p2p/net/swarm/swarm_addr_test.go | 4 ++-- p2p/net/swarm/swarm_net.go | 8 +++++++- p2p/net/swarm/swarm_net_test.go | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index ab0ad029dd..011b706d81 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -65,11 +65,11 @@ func TestFilterAddrs(t *testing.T) { ps := pstore.NewPeerstore() ctx := context.Background() - if _, err := NewNetwork(ctx, bad, id, ps, nil, metrics.NewBandwidthCounter()); err == nil { + if _, err := NewNetwork(ctx, bad, id, ps, metrics.NewBandwidthCounter()); err == nil { t.Fatal("should have failed to create swarm") } - if _, err := NewNetwork(ctx, goodAndBad, id, ps, nil, metrics.NewBandwidthCounter()); err != nil { + if _, err := NewNetwork(ctx, goodAndBad, id, ps, metrics.NewBandwidthCounter()); err != nil { t.Fatal("should have succeeded in creating swarm", err) } } diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index f6fa8c5796..b8f66d958a 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -18,8 +18,14 @@ import ( // to implement inet.Network. type Network Swarm -// NewNetwork constructs a new network and starts listening on given addresses. func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, + peers pstore.Peerstore, bwc metrics.Reporter) (*Network, error) { + + return NewNetworkWithProtector(ctx, listen, local, peers, nil, bwc) +} + +// NewNetwork constructs a new network and starts listening on given addresses. +func NewNetworkWithProtector(ctx context.Context, listen []ma.Multiaddr, local peer.ID, peers pstore.Peerstore, protec ipnet.Protector, bwc metrics.Reporter) (*Network, error) { s, err := NewSwarmWithProtector(ctx, listen, local, peers, protec, bwc) diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 7dcdbba99f..95bc268307 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -17,7 +17,7 @@ func GenSwarmNetwork(t *testing.T, ctx context.Context) *Network { ps := pstore.NewPeerstore() ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) - n, err := NewNetwork(ctx, []ma.Multiaddr{p.Addr}, p.ID, ps, nil, nil) + n, err := NewNetwork(ctx, []ma.Multiaddr{p.Addr}, p.ID, ps, nil) if err != nil { t.Fatal(err) } From c1991809dc7ed76f1fb8cd35cd5d31bb2a7667a8 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 22 Nov 2016 19:20:21 -0800 Subject: [PATCH 081/259] expose transport parameter to swarm creation --- p2p/net/swarm/swarm.go | 6 +++--- p2p/net/swarm/swarm_net.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index b1878ff0e2..6eba8f819b 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -105,12 +105,12 @@ type Swarm struct { func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID, peers pstore.Peerstore, bwc metrics.Reporter) (*Swarm, error) { - return NewSwarmWithProtector(ctx, listenAddrs, local, peers, nil, bwc) + return NewSwarmWithProtector(ctx, listenAddrs, local, peers, nil, PSTransport, bwc) } // NewSwarm constructs a Swarm, with a Chan. func NewSwarmWithProtector(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID, - peers pstore.Peerstore, protec ipnet.Protector, bwc metrics.Reporter) (*Swarm, error) { + peers pstore.Peerstore, protec ipnet.Protector, tpt pst.Transport, bwc metrics.Reporter) (*Swarm, error) { listenAddrs, err := filterAddrs(listenAddrs) if err != nil { @@ -125,7 +125,7 @@ func NewSwarmWithProtector(ctx context.Context, listenAddrs []ma.Multiaddr, loca } s := &Swarm{ - swarm: ps.NewSwarm(PSTransport), + swarm: ps.NewSwarm(tpt), local: local, peers: peers, ctx: ctx, diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index b8f66d958a..eb11ecb7da 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -28,7 +28,7 @@ func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, func NewNetworkWithProtector(ctx context.Context, listen []ma.Multiaddr, local peer.ID, peers pstore.Peerstore, protec ipnet.Protector, bwc metrics.Reporter) (*Network, error) { - s, err := NewSwarmWithProtector(ctx, listen, local, peers, protec, bwc) + s, err := NewSwarmWithProtector(ctx, listen, local, peers, protec, PSTransport, bwc) if err != nil { return nil, err } From addabcef30381b96e4893838e599c0aa48bff551 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 24 Jan 2017 10:20:46 -0800 Subject: [PATCH 082/259] fix log message --- p2p/net/swarm/swarm_listen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 93099fe398..b04c47f11a 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -54,7 +54,7 @@ func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { for i, e := range errs { if e != nil { - log.Warning("listen on %s failed: %s", addrs[i], errs[i]) + log.Warningf("listen on %s failed: %s", addrs[i], errs[i]) } } From 5c58b97791cff762d9d821c13ea2223b42984aee Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 3 Feb 2017 11:02:21 -0800 Subject: [PATCH 083/259] gx publish 1.6.8 --- p2p/net/swarm/dial_sync.go | 2 +- p2p/net/swarm/dial_sync_test.go | 2 +- p2p/net/swarm/dial_test.go | 14 +++++----- p2p/net/swarm/limiter.go | 8 +++--- p2p/net/swarm/limiter_test.go | 8 +++--- p2p/net/swarm/peers_test.go | 6 ++-- p2p/net/swarm/simul_test.go | 8 +++--- p2p/net/swarm/swarm.go | 46 +++++++++++++++---------------- p2p/net/swarm/swarm_addr.go | 6 ++-- p2p/net/swarm/swarm_addr_test.go | 10 +++---- p2p/net/swarm/swarm_conn.go | 12 ++++---- p2p/net/swarm/swarm_dial.go | 10 +++---- p2p/net/swarm/swarm_listen.go | 16 +++++------ p2p/net/swarm/swarm_net.go | 14 +++++----- p2p/net/swarm/swarm_net_test.go | 8 +++--- p2p/net/swarm/swarm_notif_test.go | 6 ++-- p2p/net/swarm/swarm_stream.go | 6 ++-- p2p/net/swarm/swarm_test.go | 12 ++++---- 18 files changed, 97 insertions(+), 97 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 69739e54ec..d22acb69f8 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -4,7 +4,7 @@ import ( "context" "sync" - peer "github.com/libp2p/go-libp2p-peer" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" ) type DialFunc func(context.Context, peer.ID) (*Conn, error) diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 0d70e226af..bb189d2f2f 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - peer "github.com/libp2p/go-libp2p-peer" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" ) func getMockDialFunc() (DialFunc, func(), context.Context, <-chan struct{}) { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index dabb0bdb00..84df140263 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -7,13 +7,13 @@ import ( "testing" "time" - addrutil "github.com/libp2p/go-addr-util" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - testutil "github.com/libp2p/go-testutil" - ci "github.com/libp2p/go-testutil/ci" - ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + manet "gx/ipfs/QmVCNGTyD4EkvNYaAp253uMQ9Rjsjy2oGMvcdJJUoVRfja/go-multiaddr-net" + addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + testutil "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil" + ci "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil/ci" + pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" ) func closeSwarms(swarms []*Swarm) { diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 67c6af4ddf..db29b874d3 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -4,10 +4,10 @@ import ( "context" "sync" - addrutil "github.com/libp2p/go-addr-util" - iconn "github.com/libp2p/go-libp2p-interface-conn" - peer "github.com/libp2p/go-libp2p-peer" - ma "github.com/multiformats/go-multiaddr" + iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" ) type dialResult struct { diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index cbb6afdbc6..948e7f1513 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -9,10 +9,10 @@ import ( "testing" "time" - iconn "github.com/libp2p/go-libp2p-interface-conn" - peer "github.com/libp2p/go-libp2p-peer" - ma "github.com/multiformats/go-multiaddr" - mafmt "github.com/whyrusleeping/mafmt" + iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + mafmt "gx/ipfs/QmYjJnSTfXWhYL2cV1xFphPqjqowJqH7ZKLA1As8QrPHbn/mafmt" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" ) func mustAddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 3259264ae8..28d6d14cdf 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -4,9 +4,9 @@ import ( "testing" "context" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - ma "github.com/multiformats/go-multiaddr" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 8935773296..c6d1195b5d 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - ci "github.com/libp2p/go-testutil/ci" - ma "github.com/multiformats/go-multiaddr" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + ci "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil/ci" + pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 6eba8f819b..90ec067372 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -11,29 +11,29 @@ import ( "sync" "time" - logging "github.com/ipfs/go-log" - pst "github.com/jbenet/go-stream-muxer" - "github.com/jbenet/goprocess" - goprocessctx "github.com/jbenet/goprocess/context" - addrutil "github.com/libp2p/go-addr-util" - conn "github.com/libp2p/go-libp2p-conn" - ci "github.com/libp2p/go-libp2p-crypto" - ipnet "github.com/libp2p/go-libp2p-interface-pnet" - metrics "github.com/libp2p/go-libp2p-metrics" - mconn "github.com/libp2p/go-libp2p-metrics/conn" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - transport "github.com/libp2p/go-libp2p-transport" - filter "github.com/libp2p/go-maddr-filter" - ps "github.com/libp2p/go-peerstream" - tcpt "github.com/libp2p/go-tcp-transport" - ma "github.com/multiformats/go-multiaddr" - psmss "github.com/whyrusleeping/go-smux-multistream" - spdy "github.com/whyrusleeping/go-smux-spdystream" - yamux "github.com/whyrusleeping/go-smux-yamux" - mafilter "github.com/whyrusleeping/multiaddr-filter" - ws "github.com/whyrusleeping/ws-transport" + metrics "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics" + mconn "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics/conn" + transport "gx/ipfs/QmR4YrM1DdvYmox249o8aday8oiotkcsT1Qzv42ujPEQDb/go-libp2p-transport" + psmss "gx/ipfs/QmRVYfZ7tWNHPBzWiG6KWGzvT2hcGems8srihsQE29x1U5/go-smux-multistream" + "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" + goprocessctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context" + mafilter "gx/ipfs/QmSMZwvs3n4GBikZ7hKzT17c3bk65FmyZo2JqtJ16swqCv/multiaddr-filter" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + ps "gx/ipfs/QmSxPTJ2Jb3nekq3AGMgDyXZR3dhPrSPVSggT3FqJWw4Sa/go-peerstream" + inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" + ci "gx/ipfs/QmTuX6VtWTbWgPwd5PMXHyp411RKsW5nBqLKVVRfJMNneb/go-libp2p-crypto" + filter "gx/ipfs/QmUuCcuet4eneEzto6PxvoC1MrHcxszy7MALrWSKH9inLy/go-maddr-filter" + ipnet "gx/ipfs/QmWNwP8oYzPUK6LqYEH1hoTMuVmQUUbhKQgHeT1AENGkEm/go-libp2p-interface-pnet" + spdy "gx/ipfs/QmWUNsat6Jb19nC5CiJCDXepTkxjdxi3eZqeoB6mrmmaGu/go-smux-spdystream" + tcpt "gx/ipfs/QmXJfmGL3GycHcuqft7vvhf6Xhzad6nuTiEnoEWwAv3zcd/go-tcp-transport" + addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + yamux "gx/ipfs/Qmbn7RYyWzBVXiUp9jZ1dA4VADHy9DtS7iZLwfhEUQvm3U/go-smux-yamux" + ws "gx/ipfs/QmeHbitozRtLYuPrtpcAoJFJixZzTctPtkSsBQGSQuz8pG/ws-transport" + pst "gx/ipfs/QmeZBgYBHvxMukGK5ojg28BCNLB9SeXqT7XXg6o7r2GbJy/go-stream-muxer" + pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" + conn "gx/ipfs/Qmf8Br5gq5XgMrSA6Y6dzY3kaBWwJu6aqXJxxuxCCKsbyD/go-libp2p-conn" ) var log = logging.Logger("swarm2") diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 78a3f5351a..518138f874 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -1,9 +1,9 @@ package swarm import ( - addrutil "github.com/libp2p/go-addr-util" - iconn "github.com/libp2p/go-libp2p-interface-conn" - ma "github.com/multiformats/go-multiaddr" + iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 011b706d81..e9db3697df 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -4,11 +4,11 @@ import ( "context" "testing" - addrutil "github.com/libp2p/go-addr-util" - metrics "github.com/libp2p/go-libp2p-metrics" - pstore "github.com/libp2p/go-libp2p-peerstore" - testutil "github.com/libp2p/go-testutil" - ma "github.com/multiformats/go-multiaddr" + metrics "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" + testutil "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil" + pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 0b0c626cea..f40dbdb5be 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -4,12 +4,12 @@ import ( "context" "fmt" - ic "github.com/libp2p/go-libp2p-crypto" - iconn "github.com/libp2p/go-libp2p-interface-conn" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - ps "github.com/libp2p/go-peerstream" - ma "github.com/multiformats/go-multiaddr" + iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + ps "gx/ipfs/QmSxPTJ2Jb3nekq3AGMgDyXZR3dhPrSPVSggT3FqJWw4Sa/go-peerstream" + inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" + ic "gx/ipfs/QmTuX6VtWTbWgPwd5PMXHyp411RKsW5nBqLKVVRfJMNneb/go-libp2p-crypto" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" ) // Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index c7cfe91110..84da324a43 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -7,11 +7,11 @@ import ( "sync" "time" - addrutil "github.com/libp2p/go-addr-util" - iconn "github.com/libp2p/go-libp2p-interface-conn" - lgbl "github.com/libp2p/go-libp2p-loggables" - peer "github.com/libp2p/go-libp2p-peer" - ma "github.com/multiformats/go-multiaddr" + iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + lgbl "gx/ipfs/Qma1v2AjS7d2ceYgye4wUVBFFSdB6UPp7jgL8kNtxvFzWu/go-libp2p-loggables" + addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index b04c47f11a..128668bf54 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -4,14 +4,14 @@ import ( "context" "fmt" - conn "github.com/libp2p/go-libp2p-conn" - iconn "github.com/libp2p/go-libp2p-interface-conn" - lgbl "github.com/libp2p/go-libp2p-loggables" - mconn "github.com/libp2p/go-libp2p-metrics/conn" - inet "github.com/libp2p/go-libp2p-net" - transport "github.com/libp2p/go-libp2p-transport" - ps "github.com/libp2p/go-peerstream" - ma "github.com/multiformats/go-multiaddr" + iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" + mconn "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics/conn" + transport "gx/ipfs/QmR4YrM1DdvYmox249o8aday8oiotkcsT1Qzv42ujPEQDb/go-libp2p-transport" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + ps "gx/ipfs/QmSxPTJ2Jb3nekq3AGMgDyXZR3dhPrSPVSggT3FqJWw4Sa/go-peerstream" + inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" + lgbl "gx/ipfs/Qma1v2AjS7d2ceYgye4wUVBFFSdB6UPp7jgL8kNtxvFzWu/go-libp2p-loggables" + conn "gx/ipfs/Qmf8Br5gq5XgMrSA6Y6dzY3kaBWwJu6aqXJxxuxCCKsbyD/go-libp2p-conn" ) func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index eb11ecb7da..2b3ba020f6 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -4,13 +4,13 @@ import ( "context" "fmt" - "github.com/jbenet/goprocess" - ipnet "github.com/libp2p/go-libp2p-interface-pnet" - metrics "github.com/libp2p/go-libp2p-metrics" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - ma "github.com/multiformats/go-multiaddr" + metrics "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics" + "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" + ipnet "gx/ipfs/QmWNwP8oYzPUK6LqYEH1hoTMuVmQUUbhKQgHeT1AENGkEm/go-libp2p-interface-pnet" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 95bc268307..9130c8a082 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -6,10 +6,10 @@ import ( "testing" "time" - inet "github.com/libp2p/go-libp2p-net" - pstore "github.com/libp2p/go-libp2p-peerstore" - tu "github.com/libp2p/go-testutil" - ma "github.com/multiformats/go-multiaddr" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" + tu "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil" + pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" ) func GenSwarmNetwork(t *testing.T, ctx context.Context) *Network { diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index d9e9df62a5..18eef7cd61 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -5,9 +5,9 @@ import ( "time" "context" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - ma "github.com/multiformats/go-multiaddr" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" ) func streamsSame(a, b inet.Stream) bool { diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index adadca78ec..cf59b169cd 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,9 +3,9 @@ package swarm import ( "time" - inet "github.com/libp2p/go-libp2p-net" - protocol "github.com/libp2p/go-libp2p-protocol" - ps "github.com/libp2p/go-peerstream" + ps "gx/ipfs/QmSxPTJ2Jb3nekq3AGMgDyXZR3dhPrSPVSggT3FqJWw4Sa/go-peerstream" + inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) // Stream is a wrapper around a ps.Stream that exposes a way to get diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 9deb918f3c..0b5ab4bdba 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -10,12 +10,12 @@ import ( "testing" "time" - metrics "github.com/libp2p/go-libp2p-metrics" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - testutil "github.com/libp2p/go-testutil" - ma "github.com/multiformats/go-multiaddr" + metrics "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics" + ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" + inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" + peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + testutil "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil" + pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" ) func EchoStreamHandler(stream inet.Stream) { From 9661f5e7ab49772f9229e607990f823bb917f330 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 3 Feb 2017 13:05:16 -0800 Subject: [PATCH 084/259] gx publish 1.6.9 --- p2p/net/swarm/dial_sync.go | 2 +- p2p/net/swarm/dial_sync_test.go | 2 +- p2p/net/swarm/dial_test.go | 14 +++++----- p2p/net/swarm/limiter.go | 8 +++--- p2p/net/swarm/limiter_test.go | 8 +++--- p2p/net/swarm/peers_test.go | 6 ++-- p2p/net/swarm/simul_test.go | 8 +++--- p2p/net/swarm/swarm.go | 46 +++++++++++++++---------------- p2p/net/swarm/swarm_addr.go | 6 ++-- p2p/net/swarm/swarm_addr_test.go | 10 +++---- p2p/net/swarm/swarm_conn.go | 12 ++++---- p2p/net/swarm/swarm_dial.go | 10 +++---- p2p/net/swarm/swarm_listen.go | 16 +++++------ p2p/net/swarm/swarm_net.go | 14 +++++----- p2p/net/swarm/swarm_net_test.go | 8 +++--- p2p/net/swarm/swarm_notif_test.go | 6 ++-- p2p/net/swarm/swarm_stream.go | 6 ++-- p2p/net/swarm/swarm_test.go | 12 ++++---- 18 files changed, 97 insertions(+), 97 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index d22acb69f8..69739e54ec 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -4,7 +4,7 @@ import ( "context" "sync" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + peer "github.com/libp2p/go-libp2p-peer" ) type DialFunc func(context.Context, peer.ID) (*Conn, error) diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index bb189d2f2f..0d70e226af 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + peer "github.com/libp2p/go-libp2p-peer" ) func getMockDialFunc() (DialFunc, func(), context.Context, <-chan struct{}) { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 84df140263..dabb0bdb00 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -7,13 +7,13 @@ import ( "testing" "time" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - manet "gx/ipfs/QmVCNGTyD4EkvNYaAp253uMQ9Rjsjy2oGMvcdJJUoVRfja/go-multiaddr-net" - addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" - testutil "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil" - ci "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil/ci" - pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" + addrutil "github.com/libp2p/go-addr-util" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + testutil "github.com/libp2p/go-testutil" + ci "github.com/libp2p/go-testutil/ci" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" ) func closeSwarms(swarms []*Swarm) { diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index db29b874d3..67c6af4ddf 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -4,10 +4,10 @@ import ( "context" "sync" - iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + addrutil "github.com/libp2p/go-addr-util" + iconn "github.com/libp2p/go-libp2p-interface-conn" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" ) type dialResult struct { diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 948e7f1513..cbb6afdbc6 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -9,10 +9,10 @@ import ( "testing" "time" - iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - mafmt "gx/ipfs/QmYjJnSTfXWhYL2cV1xFphPqjqowJqH7ZKLA1As8QrPHbn/mafmt" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + iconn "github.com/libp2p/go-libp2p-interface-conn" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" + mafmt "github.com/whyrusleeping/mafmt" ) func mustAddr(t *testing.T, s string) ma.Multiaddr { diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 28d6d14cdf..3259264ae8 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -4,9 +4,9 @@ import ( "testing" "context" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" - pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + ma "github.com/multiformats/go-multiaddr" ) func TestPeers(t *testing.T) { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index c6d1195b5d..8935773296 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" - ci "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil/ci" - pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + ci "github.com/libp2p/go-testutil/ci" + ma "github.com/multiformats/go-multiaddr" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 90ec067372..6eba8f819b 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -11,29 +11,29 @@ import ( "sync" "time" - metrics "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics" - mconn "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics/conn" - transport "gx/ipfs/QmR4YrM1DdvYmox249o8aday8oiotkcsT1Qzv42ujPEQDb/go-libp2p-transport" - psmss "gx/ipfs/QmRVYfZ7tWNHPBzWiG6KWGzvT2hcGems8srihsQE29x1U5/go-smux-multistream" - "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" - goprocessctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context" - mafilter "gx/ipfs/QmSMZwvs3n4GBikZ7hKzT17c3bk65FmyZo2JqtJ16swqCv/multiaddr-filter" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" - ps "gx/ipfs/QmSxPTJ2Jb3nekq3AGMgDyXZR3dhPrSPVSggT3FqJWw4Sa/go-peerstream" - inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" - ci "gx/ipfs/QmTuX6VtWTbWgPwd5PMXHyp411RKsW5nBqLKVVRfJMNneb/go-libp2p-crypto" - filter "gx/ipfs/QmUuCcuet4eneEzto6PxvoC1MrHcxszy7MALrWSKH9inLy/go-maddr-filter" - ipnet "gx/ipfs/QmWNwP8oYzPUK6LqYEH1hoTMuVmQUUbhKQgHeT1AENGkEm/go-libp2p-interface-pnet" - spdy "gx/ipfs/QmWUNsat6Jb19nC5CiJCDXepTkxjdxi3eZqeoB6mrmmaGu/go-smux-spdystream" - tcpt "gx/ipfs/QmXJfmGL3GycHcuqft7vvhf6Xhzad6nuTiEnoEWwAv3zcd/go-tcp-transport" - addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" - yamux "gx/ipfs/Qmbn7RYyWzBVXiUp9jZ1dA4VADHy9DtS7iZLwfhEUQvm3U/go-smux-yamux" - ws "gx/ipfs/QmeHbitozRtLYuPrtpcAoJFJixZzTctPtkSsBQGSQuz8pG/ws-transport" - pst "gx/ipfs/QmeZBgYBHvxMukGK5ojg28BCNLB9SeXqT7XXg6o7r2GbJy/go-stream-muxer" - pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" - conn "gx/ipfs/Qmf8Br5gq5XgMrSA6Y6dzY3kaBWwJu6aqXJxxuxCCKsbyD/go-libp2p-conn" + logging "github.com/ipfs/go-log" + pst "github.com/jbenet/go-stream-muxer" + "github.com/jbenet/goprocess" + goprocessctx "github.com/jbenet/goprocess/context" + addrutil "github.com/libp2p/go-addr-util" + conn "github.com/libp2p/go-libp2p-conn" + ci "github.com/libp2p/go-libp2p-crypto" + ipnet "github.com/libp2p/go-libp2p-interface-pnet" + metrics "github.com/libp2p/go-libp2p-metrics" + mconn "github.com/libp2p/go-libp2p-metrics/conn" + inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + transport "github.com/libp2p/go-libp2p-transport" + filter "github.com/libp2p/go-maddr-filter" + ps "github.com/libp2p/go-peerstream" + tcpt "github.com/libp2p/go-tcp-transport" + ma "github.com/multiformats/go-multiaddr" + psmss "github.com/whyrusleeping/go-smux-multistream" + spdy "github.com/whyrusleeping/go-smux-spdystream" + yamux "github.com/whyrusleeping/go-smux-yamux" + mafilter "github.com/whyrusleeping/multiaddr-filter" + ws "github.com/whyrusleeping/ws-transport" ) var log = logging.Logger("swarm2") diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 518138f874..78a3f5351a 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -1,9 +1,9 @@ package swarm import ( - iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" + addrutil "github.com/libp2p/go-addr-util" + iconn "github.com/libp2p/go-libp2p-interface-conn" + ma "github.com/multiformats/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index e9db3697df..011b706d81 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -4,11 +4,11 @@ import ( "context" "testing" - metrics "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" - testutil "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil" - pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" + addrutil "github.com/libp2p/go-addr-util" + metrics "github.com/libp2p/go-libp2p-metrics" + pstore "github.com/libp2p/go-libp2p-peerstore" + testutil "github.com/libp2p/go-testutil" + ma "github.com/multiformats/go-multiaddr" ) func TestFilterAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index f40dbdb5be..0b0c626cea 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -4,12 +4,12 @@ import ( "context" "fmt" - iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - ps "gx/ipfs/QmSxPTJ2Jb3nekq3AGMgDyXZR3dhPrSPVSggT3FqJWw4Sa/go-peerstream" - inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" - ic "gx/ipfs/QmTuX6VtWTbWgPwd5PMXHyp411RKsW5nBqLKVVRfJMNneb/go-libp2p-crypto" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + ic "github.com/libp2p/go-libp2p-crypto" + iconn "github.com/libp2p/go-libp2p-interface-conn" + inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + ps "github.com/libp2p/go-peerstream" + ma "github.com/multiformats/go-multiaddr" ) // Conn is a simple wrapper around a ps.Conn that also exposes diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 84da324a43..c7cfe91110 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -7,11 +7,11 @@ import ( "sync" "time" - iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - lgbl "gx/ipfs/Qma1v2AjS7d2ceYgye4wUVBFFSdB6UPp7jgL8kNtxvFzWu/go-libp2p-loggables" - addrutil "gx/ipfs/Qmb86KiitngVm5AGQ18viwCBwwcTvULAtz3X74YWKAevnc/go-addr-util" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + addrutil "github.com/libp2p/go-addr-util" + iconn "github.com/libp2p/go-libp2p-interface-conn" + lgbl "github.com/libp2p/go-libp2p-loggables" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 128668bf54..b04c47f11a 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -4,14 +4,14 @@ import ( "context" "fmt" - iconn "gx/ipfs/QmNmKcQiJB5cUTXXXS1o44s4PdeYevmH8aHhQJCGYjwBj3/go-libp2p-interface-conn" - mconn "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics/conn" - transport "gx/ipfs/QmR4YrM1DdvYmox249o8aday8oiotkcsT1Qzv42ujPEQDb/go-libp2p-transport" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - ps "gx/ipfs/QmSxPTJ2Jb3nekq3AGMgDyXZR3dhPrSPVSggT3FqJWw4Sa/go-peerstream" - inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" - lgbl "gx/ipfs/Qma1v2AjS7d2ceYgye4wUVBFFSdB6UPp7jgL8kNtxvFzWu/go-libp2p-loggables" - conn "gx/ipfs/Qmf8Br5gq5XgMrSA6Y6dzY3kaBWwJu6aqXJxxuxCCKsbyD/go-libp2p-conn" + conn "github.com/libp2p/go-libp2p-conn" + iconn "github.com/libp2p/go-libp2p-interface-conn" + lgbl "github.com/libp2p/go-libp2p-loggables" + mconn "github.com/libp2p/go-libp2p-metrics/conn" + inet "github.com/libp2p/go-libp2p-net" + transport "github.com/libp2p/go-libp2p-transport" + ps "github.com/libp2p/go-peerstream" + ma "github.com/multiformats/go-multiaddr" ) func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go index 2b3ba020f6..eb11ecb7da 100644 --- a/p2p/net/swarm/swarm_net.go +++ b/p2p/net/swarm/swarm_net.go @@ -4,13 +4,13 @@ import ( "context" "fmt" - metrics "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics" - "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" - ipnet "gx/ipfs/QmWNwP8oYzPUK6LqYEH1hoTMuVmQUUbhKQgHeT1AENGkEm/go-libp2p-interface-pnet" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" - pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" + "github.com/jbenet/goprocess" + ipnet "github.com/libp2p/go-libp2p-interface-pnet" + metrics "github.com/libp2p/go-libp2p-metrics" + inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + ma "github.com/multiformats/go-multiaddr" ) // Network implements the inet.Network interface. diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 9130c8a082..95bc268307 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -6,10 +6,10 @@ import ( "testing" "time" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" - tu "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil" - pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" + inet "github.com/libp2p/go-libp2p-net" + pstore "github.com/libp2p/go-libp2p-peerstore" + tu "github.com/libp2p/go-testutil" + ma "github.com/multiformats/go-multiaddr" ) func GenSwarmNetwork(t *testing.T, ctx context.Context) *Network { diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 18eef7cd61..d9e9df62a5 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -5,9 +5,9 @@ import ( "time" "context" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" + inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" ) func streamsSame(a, b inet.Stream) bool { diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index cf59b169cd..adadca78ec 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -3,9 +3,9 @@ package swarm import ( "time" - ps "gx/ipfs/QmSxPTJ2Jb3nekq3AGMgDyXZR3dhPrSPVSggT3FqJWw4Sa/go-peerstream" - inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" - protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + inet "github.com/libp2p/go-libp2p-net" + protocol "github.com/libp2p/go-libp2p-protocol" + ps "github.com/libp2p/go-peerstream" ) // Stream is a wrapper around a ps.Stream that exposes a way to get diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 0b5ab4bdba..9deb918f3c 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -10,12 +10,12 @@ import ( "testing" "time" - metrics "gx/ipfs/QmPfeoLzBPsiwbHLwev8ecgjd1RPCYfsAHhdFA8avEnXQm/go-libp2p-metrics" - ma "gx/ipfs/QmSWLfmj5frN9xVLMMN846dMDriy5wN5jeghUm7aTW3DAG/go-multiaddr" - inet "gx/ipfs/QmTWdBq4ZaLGbXcMiNasSR5ywomkgQiJDqQknEU16z3pZR/go-libp2p-net" - peer "gx/ipfs/QmbKtZxyDqUJp7Ad8tGr5nrLqoi9nfgqFxcNbmLJbfaHPe/go-libp2p-peer" - testutil "gx/ipfs/QmbN6CcaGuVSWwhYq5nDQs98hWHgu5PTBgVjnGV28YeGRk/go-testutil" - pstore "gx/ipfs/QmekwEJM81TqzKzupMvK68hVNfBNvBengWkpc8nAw2bjBf/go-libp2p-peerstore" + metrics "github.com/libp2p/go-libp2p-metrics" + inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + testutil "github.com/libp2p/go-testutil" + ma "github.com/multiformats/go-multiaddr" ) func EchoStreamHandler(stream inet.Stream) { From 3a37e5880db42748ca54e04ce9623592e20feac5 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 17 Mar 2017 12:50:12 +0700 Subject: [PATCH 085/259] fix logging of dial errors --- p2p/net/swarm/swarm_dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index c7cfe91110..627cce45ad 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -318,7 +318,7 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. case resp := <-respch: active-- if resp.Err != nil { - log.Info("got error on dial to %s: ", resp.Addr, resp.Err) + log.Infof("got error on dial to %s: %s", resp.Addr, resp.Err) // Errors are normal, lots of dials will fail exitErr = resp.Err From 3e754c4b0abd6c6ca80095fae4be51b1b9e4dff1 Mon Sep 17 00:00:00 2001 From: Lars Gierth Date: Tue, 6 Jun 2017 02:42:05 +0200 Subject: [PATCH 086/259] Fix go-ws-transport package name --- p2p/net/swarm/swarm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 6eba8f819b..a282f7429b 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -28,12 +28,12 @@ import ( filter "github.com/libp2p/go-maddr-filter" ps "github.com/libp2p/go-peerstream" tcpt "github.com/libp2p/go-tcp-transport" + ws "github.com/libp2p/go-ws-transport" ma "github.com/multiformats/go-multiaddr" psmss "github.com/whyrusleeping/go-smux-multistream" spdy "github.com/whyrusleeping/go-smux-spdystream" yamux "github.com/whyrusleeping/go-smux-yamux" mafilter "github.com/whyrusleeping/multiaddr-filter" - ws "github.com/whyrusleeping/ws-transport" ) var log = logging.Logger("swarm2") From 4333b4cf3029e2931cfa3944ee4443ef32fd9148 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 20 Jul 2017 01:24:31 -0700 Subject: [PATCH 087/259] update deps * Also, make notifications async to prevent them from deadlocking. --- p2p/net/swarm/swarm_notif_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index d9e9df62a5..8ebcccb973 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -17,8 +17,10 @@ func streamsSame(a, b inet.Stream) bool { } func TestNotifications(t *testing.T) { + const swarmSize = 5 + ctx := context.Background() - swarms := makeSwarms(ctx, t, 5) + swarms := makeSwarms(ctx, t, swarmSize) defer func() { for _, s := range swarms { s.Close() @@ -30,7 +32,7 @@ func TestNotifications(t *testing.T) { // signup notifs notifiees := make([]*netNotifiee, len(swarms)) for i, swarm := range swarms { - n := newNetNotifiee() + n := newNetNotifiee(swarmSize) swarm.Notify(n) notifiees[i] = n } @@ -184,14 +186,14 @@ type netNotifiee struct { closedStream chan inet.Stream } -func newNetNotifiee() *netNotifiee { +func newNetNotifiee(buffer int) *netNotifiee { return &netNotifiee{ - listen: make(chan ma.Multiaddr), - listenClose: make(chan ma.Multiaddr), - connected: make(chan inet.Conn), - disconnected: make(chan inet.Conn), - openedStream: make(chan inet.Stream), - closedStream: make(chan inet.Stream), + listen: make(chan ma.Multiaddr, buffer), + listenClose: make(chan ma.Multiaddr, buffer), + connected: make(chan inet.Conn, buffer), + disconnected: make(chan inet.Conn, buffer), + openedStream: make(chan inet.Stream, buffer), + closedStream: make(chan inet.Stream, buffer), } } From 07e3a6db8d09e654c1f32d489e4ff665b87b3cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 30 Jul 2017 23:25:59 +0200 Subject: [PATCH 088/259] Fix dialLimiter.fdConsuming counting --- p2p/net/swarm/limiter.go | 19 +++++++--- p2p/net/swarm/limiter_test.go | 66 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 67c6af4ddf..f2513d8e10 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -52,10 +52,10 @@ func newDialLimiter(df dialfunc) *dialLimiter { return newDialLimiterWithParams(df, concurrentFdDials, defaultPerPeerRateLimit) } -func newDialLimiterWithParams(df dialfunc, fdl, ppl int) *dialLimiter { +func newDialLimiterWithParams(df dialfunc, fdLimit, perPeerLimit int) *dialLimiter { return &dialLimiter{ - fdLimit: fdl, - perPeerLimit: ppl, + fdLimit: fdLimit, + perPeerLimit: perPeerLimit, waitingOnPeerLimit: make(map[peer.ID][]*dialJob), activePerPeer: make(map[peer.ID]int), dialFunc: df, @@ -68,6 +68,7 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { if addrutil.IsFDCostlyTransport(dj.addr) { dl.fdConsuming-- + if len(dl.waitingOnFd) > 0 { next := dl.waitingOnFd[0] dl.waitingOnFd = dl.waitingOnFd[1:] @@ -89,6 +90,7 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { waitlist := dl.waitingOnPeerLimit[dj.peer] if !dj.success && len(waitlist) > 0 { next := waitlist[0] + if len(waitlist) == 1 { delete(dl.waitingOnPeerLimit, dj.peer) } else { @@ -96,11 +98,20 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { } dl.activePerPeer[dj.peer]++ // just kidding, we still want this token + if addrutil.IsFDCostlyTransport(next.addr) { + if dl.fdConsuming >= dl.fdLimit { + dl.waitingOnFd = append(dl.waitingOnFd, next) + return + } + + // take token + dl.fdConsuming++ + } + // can kick this off right here, dials in this list already // have the other tokens needed go dl.executeDial(next) } - } // AddDialJob tries to take the needed tokens for starting the given dial job. diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index cbb6afdbc6..3111a5d107 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -320,3 +320,69 @@ func TestStressLimiter(t *testing.T) { } } } + +func TestFDLimitUnderflow(t *testing.T) { + dials := 0 + + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { + dials++ + + timeout := make(chan bool, 1) + go func() { + time.Sleep(time.Second * 5) + timeout <- true + }() + + select { + case <-ctx.Done(): + case <-timeout: + } + + return nil, fmt.Errorf("df timed out") + } + + l := newDialLimiterWithParams(df, 20, 3) + + var addrs []ma.Multiaddr + for i := 0; i <= 1000; i++ { + addrs = append(addrs, addrWithPort(t, i)) + } + + for i := 0; i < 1000; i++ { + go func(id peer.ID, i int) { + ctx, cancel := context.WithCancel(context.Background()) + + resp := make(chan dialResult) + l.AddDialJob(&dialJob{ + addr: addrs[i], + ctx: ctx, + peer: id, + resp: resp, + }) + + //cancel first 60 after 1s, next 60 after 2s + if i > 60 { + time.Sleep(time.Second * 1) + } + if i < 120 { + time.Sleep(time.Second * 1) + cancel() + return + } + defer cancel() + + for res := range resp { + if res.Err != nil { + return + } + t.Fatal("got dial res, shouldn't") + } + }(peer.ID(fmt.Sprintf("testpeer%d", i % 20)), i) + } + + time.Sleep(time.Second * 3) + + if l.fdConsuming < 0 { + t.Fatalf("l.fdConsuming < 0") + } +} From bb8f8d0696b43d3dbb02ec80b93b50a27bdd08d8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 31 Jul 2017 13:40:11 -0700 Subject: [PATCH 089/259] go fmt --- p2p/net/swarm/limiter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 3111a5d107..6c6eeb68fa 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -377,7 +377,7 @@ func TestFDLimitUnderflow(t *testing.T) { } t.Fatal("got dial res, shouldn't") } - }(peer.ID(fmt.Sprintf("testpeer%d", i % 20)), i) + }(peer.ID(fmt.Sprintf("testpeer%d", i%20)), i) } time.Sleep(time.Second * 3) From 755830ea064e0af3b89deb7caaf0eae182dd23b1 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Aug 2017 18:17:44 -0700 Subject: [PATCH 090/259] test: connect each peer precisely once. Otherwise, some tests will get confused by duplicate (temporary, we close duplicates quickly) connections. fixes libp2p/go-libp2p#100 --- p2p/net/swarm/swarm_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 9deb918f3c..e14de3faad 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -102,12 +102,10 @@ func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { } log.Info("Connecting swarms simultaneously.") - for _, s1 := range swarms { - for _, s2 := range swarms { - if s2.local != s1.local { // don't connect to self. - wg.Add(1) - connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) // try the first. - } + for i, s1 := range swarms { + for _, s2 := range swarms[i+1:] { + wg.Add(1) + connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) // try the first. } } wg.Wait() From 707639d5429588add14d555c28f2083cc6beaab4 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 4 Sep 2017 22:53:46 -0700 Subject: [PATCH 091/259] gx publish 1.7.6 --- p2p/net/swarm/swarm_dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 627cce45ad..d866c57b99 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -285,7 +285,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { } func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (iconn.Conn, error) { - log.Debugf("%s swarm dialing %s %s", s.local, p, remoteAddrs) + log.Debugf("%s swarm dialing %s", s.local, p) ctx, cancel := context.WithCancel(ctx) defer cancel() // cancel work when we exit func From 10de0c316cbefb08d1f80ff8ee0b9dac214eeead Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 12 Sep 2017 17:38:02 -0700 Subject: [PATCH 092/259] Update stream muxer. * Add in the reset method and use it instead of Close. Close now only closes one side of the connection. --- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_notif_test.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 5 +++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index a282f7429b..8794048b5c 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -12,7 +12,6 @@ import ( "time" logging "github.com/ipfs/go-log" - pst "github.com/jbenet/go-stream-muxer" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" addrutil "github.com/libp2p/go-addr-util" @@ -27,6 +26,7 @@ import ( transport "github.com/libp2p/go-libp2p-transport" filter "github.com/libp2p/go-maddr-filter" ps "github.com/libp2p/go-peerstream" + pst "github.com/libp2p/go-stream-muxer" tcpt "github.com/libp2p/go-tcp-transport" ws "github.com/libp2p/go-ws-transport" ma "github.com/multiformats/go-multiaddr" diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index b04c47f11a..98d3549a4d 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -145,7 +145,7 @@ func (s *Swarm) addConnListener(list iconn.Listener) error { // connHandler is called by the StreamSwarm whenever a new connection is added // here we configure it slightly. Note that this is sequential, so if anything // will take a while do it in a goroutine. -// See https://godoc.org/github.com/jbenet/go-peerstream for more information +// See https://godoc.org/github.com/libp2p/go-peerstream for more information func (s *Swarm) connHandler(c *ps.Conn) *Conn { ctx := context.Background() // this context is for running the handshake, which -- when receiveing connections diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 8ebcccb973..8a574aa070 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -125,7 +125,7 @@ func TestNotifications(t *testing.T) { for _, s := range swarms { s.SetStreamHandler(func(s inet.Stream) { streams <- s - s.Close() + s.Reset() }) } @@ -139,7 +139,7 @@ func TestNotifications(t *testing.T) { t.Error(err) } else { st1.Write([]byte("hello")) - st1.Close() + st1.Reset() testOCStream(notifiees[i], st1) st2 := <-streams testOCStream(n2, st2) diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index adadca78ec..7e83dde1d8 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -43,6 +43,11 @@ func (s *Stream) Close() error { return s.Stream().Close() } +// Reset resets the stream, closing both ends. +func (s *Stream) Reset() error { + return s.Stream().Reset() +} + func (s *Stream) Protocol() protocol.ID { return (*ps.Stream)(s).Protocol() } From 5be8da502992ad261bfe975fa001105314c0da7c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 20 Sep 2017 15:04:07 -0700 Subject: [PATCH 093/259] use optimized 'HaveConnToPeer' checks --- p2p/net/swarm/swarm.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 8794048b5c..dac2624d96 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -273,7 +273,7 @@ func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) { // NewStreamWithPeer creates a new stream on any available connection to p func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, error) { // if we have no connections, try connecting. - if len(s.ConnectionsToPeer(p)) == 0 { + if !s.HaveConnsToPeer(p) { log.Debug("Swarm: NewStreamWithPeer no connections. Attempting to connect...") if _, err := s.Dial(ctx, p); err != nil { return nil, err @@ -288,16 +288,11 @@ func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, erro // ConnectionsToPeer returns all the live connections to p func (s *Swarm) ConnectionsToPeer(p peer.ID) []*Conn { - return wrapConns(ps.ConnsWithGroup(p, s.swarm.Conns())) + return wrapConns(s.swarm.ConnsWithGroup(p)) } func (s *Swarm) HaveConnsToPeer(p peer.ID) bool { - for _, c := range s.swarm.Conns() { - if c.InGroup(p) { - return true - } - } - return false + return len(s.swarm.ConnsWithGroup(p)) > 0 } // Connections returns a slice of all connections. From 43f337097797bdde2963f9af84cc2952f2ef12d7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 18 Oct 2017 13:42:31 -0700 Subject: [PATCH 094/259] don't backoff dialing when the context is canceled fixes libp2p/go-libp2p-kad-dht#96 --- p2p/net/swarm/swarm_dial.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index d866c57b99..3b0b676cd5 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -203,8 +203,10 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { cancel() log.Debugf("dial end %s", conn) if err != nil { - log.Event(ctx, "swarmDialBackoffAdd", logdial) - s.backf.AddBackoff(p) // let others know to backoff + if err != context.Canceled { + log.Event(ctx, "swarmDialBackoffAdd", logdial) + s.backf.AddBackoff(p) // let others know to backoff + } // ok, we failed. try again. (if loop is done, our error is output) return nil, fmt.Errorf("dial attempt failed: %s", err) From 4381ea23a48c256a77e5d0080a1a4cb92c75103f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 9 Nov 2017 16:02:02 -0800 Subject: [PATCH 095/259] don't pass the dial timeout option It doesn't exist anymore. This is now handled entirely though contexts. --- p2p/net/swarm/swarm.go | 4 ++-- p2p/net/swarm/swarm_dial.go | 5 ----- p2p/net/swarm/swarm_listen.go | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index dac2624d96..eabd6d67f6 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -129,7 +129,7 @@ func NewSwarmWithProtector(ctx context.Context, listenAddrs []ma.Multiaddr, loca local: local, peers: peers, ctx: ctx, - dialT: DialTimeout, + dialT: conn.DialTimeout, notifs: make(map[inet.Notifiee]ps.Notifiee), transports: []transport.Transport{ tcpt.NewTCPTransport(), @@ -164,7 +164,7 @@ func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey, pstpt ps local: id, peers: pstore.NewPeerstore(), ctx: ctx, - dialT: DialTimeout, + dialT: conn.DialTimeout, notifs: make(map[inet.Notifiee]ps.Notifiee), fdRateLimit: make(chan struct{}, concurrentFdDials), Filters: filter.NewFilters(), diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 3b0b676cd5..1cc03580fd 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -48,11 +48,6 @@ const concurrentFdDials = 160 // number of concurrent outbound dials to make per peer const defaultPerPeerRateLimit = 8 -// DialTimeout is the amount of time each dial attempt has. We can think about making -// this larger down the road, or putting more granular timeouts (i.e. within each -// subcomponent of Dial) -var DialTimeout = time.Second * 10 - // dialbackoff is a struct used to avoid over-dialing the same, dead peers. // Whenever we totally time out on a peer (all three attempts), we add them // to dialbackoff. Then, whenevers goroutines would _wait_ (dialsync), they diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 98d3549a4d..1e4a92efc9 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -20,7 +20,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { return fmt.Errorf("no transport for address: %s", a) } - d, err := tpt.Dialer(a, transport.TimeoutOpt(DialTimeout), transport.ReusePorts) + d, err := tpt.Dialer(a, transport.ReusePorts) if err != nil { return err } From 140bad3585f20b59b19413d8777b149fc963a532 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 20 Nov 2017 17:52:09 -0800 Subject: [PATCH 096/259] nil out references to finished dials We need a better solution (this is a memory hotspot when mass dialing) but that will take some thought and effort. This is a simple fix that should reduce memory usage a bit. --- p2p/net/swarm/limiter.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index f2513d8e10..f6cf6dda26 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -71,6 +71,7 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { if len(dl.waitingOnFd) > 0 { next := dl.waitingOnFd[0] + dl.waitingOnFd[0] = nil // clear out memory dl.waitingOnFd = dl.waitingOnFd[1:] if len(dl.waitingOnFd) == 0 { dl.waitingOnFd = nil // clear out memory @@ -94,6 +95,7 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { if len(waitlist) == 1 { delete(dl.waitingOnPeerLimit, dj.peer) } else { + waitlist[0] = nil // clear out memory dl.waitingOnPeerLimit[dj.peer] = waitlist[1:] } dl.activePerPeer[dj.peer]++ // just kidding, we still want this token From 00ded6820ef6b410a88266227b56af69fcde9494 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 5 Dec 2017 16:20:51 -0800 Subject: [PATCH 097/259] preallocate duplicate checking map in Peers We call this method in `FindProvidersAsync`. Really, we shouldn't even be doing that (but this is "good enough" for now). --- p2p/net/swarm/swarm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index eabd6d67f6..e7d6b9a7f4 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -313,7 +313,7 @@ func (s *Swarm) CloseConnection(p peer.ID) error { func (s *Swarm) Peers() []peer.ID { conns := s.Connections() - seen := make(map[peer.ID]struct{}) + seen := make(map[peer.ID]struct{}, len(conns)) peers := make([]peer.ID, 0, len(conns)) for _, c := range conns { p := c.RemotePeer() From 6138f692cd631105856b962b56db525e09763723 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 2 Jan 2018 21:09:00 -0800 Subject: [PATCH 098/259] clear out extra dial jobs after dial finishes --- p2p/net/swarm/limiter.go | 9 +++++++++ p2p/net/swarm/swarm_dial.go | 2 ++ 2 files changed, 11 insertions(+) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index f6cf6dda26..c3d8951894 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -144,6 +144,15 @@ func (dl *dialLimiter) AddDialJob(dj *dialJob) { go dl.executeDial(dj) } +func (dl *dialLimiter) clearAllPeerDials(p peer.ID) { + dl.rllock.Lock() + defer dl.rllock.Unlock() + delete(dl.waitingOnPeerLimit, p) + // NB: the waitingOnFd list doesnt need to be cleaned out here, we will + // remove them as we encounter them because they are 'cancelled' at this + // point +} + // executeDial calls the dialFunc, and reports the result through the response // channel when finished. Once the response is sent it also releases all tokens // it held during the dial. diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 1cc03580fd..1531bb2ed9 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -293,6 +293,8 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. defaultDialFail := fmt.Errorf("failed to dial %s (default failure)", p) exitErr := defaultDialFail + defer s.limiter.clearAllPeerDials(p) + var active int for { select { From 22b78018de8df8e4d84b0530d3f72b41fca44af2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 9 Jan 2018 11:14:54 -0800 Subject: [PATCH 099/259] cleanup connection setup We already setup the connection from within the new connection handler. No need to do it *again* on dial. --- p2p/net/swarm/swarm_conn.go | 29 ----------------------------- p2p/net/swarm/swarm_dial.go | 22 ++++++++++------------ p2p/net/swarm/swarm_listen.go | 25 +++++++++++++------------ 3 files changed, 23 insertions(+), 53 deletions(-) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 0b0c626cea..c143062cee 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -1,7 +1,6 @@ package swarm import ( - "context" "fmt" ic "github.com/libp2p/go-libp2p-crypto" @@ -121,31 +120,3 @@ func wrapConns(conns1 []*ps.Conn) []*Conn { } return conns2 } - -// newConnSetup does the swarm's "setup" for a connection. returns the underlying -// conn.Conn this method is used by both swarm.Dial and ps.Swarm connHandler -func (s *Swarm) newConnSetup(ctx context.Context, psConn *ps.Conn) (*Conn, error) { - - // wrap with a Conn - sc, err := wrapConn(psConn) - if err != nil { - return nil, err - } - - // if we have a public key, make sure we add it to our peerstore! - // This is an important detail. Otherwise we must fetch the public - // key from the DHT or some other system. - if pk := sc.RemotePublicKey(); pk != nil { - s.peers.AddPubKey(sc.RemotePeer(), pk) - } - - // ok great! we can use it. add it to our group. - - // set the RemotePeer as a group on the conn. this lets us group - // connections in the StreamSwarm by peer, and get a streams from - // any available connection in the group (better multiconn): - // swarm.StreamSwarm().NewStreamWithGroup(remotePeer) - psConn.AddGroup(sc.RemotePeer()) - - return sc, nil -} diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 1531bb2ed9..d977d3752b 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -189,14 +189,16 @@ func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) // doDial is an ugly shim method to retain all the logging and backoff logic // of the old dialsync code func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { - var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + ctx, cancel := context.WithTimeout(ctx, s.dialT) + defer cancel() + + logdial := lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + // ok, we have been charged to dial! let's do it. // if it succeeds, dial will add the conn to the swarm itself. defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done() - ctxT, cancel := context.WithTimeout(ctx, s.dialT) - conn, err := s.dial(ctxT, p) - cancel() - log.Debugf("dial end %s", conn) + + conn, err := s.dial(ctx, p) if err != nil { if err != context.Canceled { log.Event(ctx, "swarmDialBackoffAdd", logdial) @@ -206,8 +208,6 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { // ok, we failed. try again. (if loop is done, our error is output) return nil, fmt.Errorf("dial attempt failed: %s", err) } - log.Event(ctx, "swarmDialBackoffClear", logdial) - s.backf.Clear(p) // okay, no longer need to backoff return conn, nil } @@ -373,10 +373,8 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (ico var ConnSetupTimeout = time.Minute * 5 -// dialConnSetup is the setup logic for a connection from the dial side. it -// needs to add the Conn to the StreamSwarm, then run newConnSetup +// dialConnSetup is the setup logic for a connection from the dial side. func dialConnSetup(ctx context.Context, s *Swarm, connC iconn.Conn) (*Conn, error) { - deadline, ok := ctx.Deadline() if !ok { deadline = time.Now().Add(ConnSetupTimeout) @@ -386,14 +384,14 @@ func dialConnSetup(ctx context.Context, s *Swarm, connC iconn.Conn) (*Conn, erro return nil, err } + // Add conn to ps swarm. Setup will be done by the connection handler. psC, err := s.swarm.AddConn(connC) if err != nil { // connC is closed by caller if we fail. return nil, fmt.Errorf("failed to add conn to ps.Swarm: %s", err) } - // ok try to setup the new connection. (newConnSetup will add to group) - swarmC, err := s.newConnSetup(ctx, psC) + swarmC, err := wrapConn(psC) if err != nil { psC.Close() // we need to make sure psC is Closed. return nil, err diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 1e4a92efc9..148d9a8881 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -147,23 +147,24 @@ func (s *Swarm) addConnListener(list iconn.Listener) error { // will take a while do it in a goroutine. // See https://godoc.org/github.com/libp2p/go-peerstream for more information func (s *Swarm) connHandler(c *ps.Conn) *Conn { - ctx := context.Background() - // this context is for running the handshake, which -- when receiveing connections - // -- we have no bound on beyond what the transport protocol bounds it at. - // note that setup + the handshake are bounded by underlying io. - // (i.e. if TCP or UDP disconnects (or the swarm closes), we're done. - // Q: why not have a shorter handshake? think about an HTTP server on really slow conns. - // as long as the conn is live (TCP says its online), it tries its best. we follow suit.) - - sc, err := s.newConnSetup(ctx, c) + sc, err := wrapConn(c) if err != nil { - log.Debug(err) - log.Event(ctx, "newConnHandlerDisconnect", lgbl.NetConn(c.NetConn()), lgbl.Error(err)) + log.Event(s.Context(), "newConnHandlerDisconnect", lgbl.NetConn(c.NetConn()), lgbl.Error(err)) c.Close() // boom. close it. return nil } - // if a peer dials us, remove from dial backoff. + // Add the public key. + if pk := sc.RemotePublicKey(); pk != nil { + s.peers.AddPubKey(sc.RemotePeer(), pk) + } + + // Add the group + c.AddGroup(sc.RemotePeer()) + + // clear backoff on successful connection. + logdial := lgbl.Dial("swarm", sc.LocalPeer(), sc.RemotePeer(), nil, nil) + log.Event(s.Context(), "swarmDialBackoffClear", logdial) s.backf.Clear(sc.RemotePeer()) return sc From 7d45aa84ea7b79aadb66e1c457e806bcedd610a1 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 8 Mar 2018 20:51:56 -0800 Subject: [PATCH 100/259] don't leak connections when canceling dials --- p2p/net/swarm/limiter.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index c3d8951894..836edbc9f5 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -166,5 +166,8 @@ func (dl *dialLimiter) executeDial(j *dialJob) { select { case j.resp <- dialResult{Conn: con, Addr: j.addr, Err: err}: case <-j.ctx.Done(): + if err == nil { + con.Close() + } } } From 4e39954ac98603db3510b257e5d2d4287ea0b5a3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 19 Jan 2018 16:28:45 -0800 Subject: [PATCH 101/259] refactor for new transports Removes: * go-libp2p-conn logic (moved to transports) * go-libp2p-peerstream (moved here) Changes: * New transport interface. * Explicit over implicit (the implicit automagic now all lives in go-libp2p): * No more default transports, muxers, etc. * No more fallback dialer. Transports are now *required*. * Stream opening: * Connection picking logic (instead of just picking the first). * Tries harder to open a stream of some connections happen to be closed. * Stream closing: * No longer treats half-closed streams as fully closed. Users *must* read the an EOF or reset the stream for it to be garbage collected. * No more polling for dead connections. * Multiplexers are now *non-optional*. Really, they haven't been optional for a while but we still pretended that they were. * No more Network type alias. It added a bunch of code and didn't really provide anything but an alternative set of methods that do the same thing. * Notifications: * New guarantee: connection open notifications will complete before connection close notifications begin. * Given that, notifications are now delivered in parallel. No more notification backlogs blocking connection closing/opening. --- p2p/net/swarm/dial_sync.go | 23 +- p2p/net/swarm/dial_sync_test.go | 4 +- p2p/net/swarm/dial_test.go | 125 +++--- p2p/net/swarm/limiter.go | 10 +- p2p/net/swarm/limiter_test.go | 30 +- p2p/net/swarm/peers_test.go | 23 +- p2p/net/swarm/simul_test.go | 17 +- p2p/net/swarm/swarm.go | 610 +++++++++++++++++------------- p2p/net/swarm/swarm_addr.go | 12 +- p2p/net/swarm/swarm_addr_test.go | 92 +---- p2p/net/swarm/swarm_conn.go | 229 +++++++---- p2p/net/swarm/swarm_dial.go | 211 ++++++----- p2p/net/swarm/swarm_listen.go | 159 ++------ p2p/net/swarm/swarm_net.go | 177 --------- p2p/net/swarm/swarm_net_test.go | 32 +- p2p/net/swarm/swarm_notif_test.go | 26 +- p2p/net/swarm/swarm_stream.go | 146 +++++-- p2p/net/swarm/swarm_test.go | 75 ++-- p2p/net/swarm/swarm_transport.go | 87 +++++ p2p/net/swarm/testing/testing.go | 97 +++++ 20 files changed, 1151 insertions(+), 1034 deletions(-) delete mode 100644 p2p/net/swarm/swarm_net.go create mode 100644 p2p/net/swarm/swarm_transport.go create mode 100644 p2p/net/swarm/testing/testing.go diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 69739e54ec..82256eca48 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -7,8 +7,10 @@ import ( peer "github.com/libp2p/go-libp2p-peer" ) +// DialFunc is the type of function expected by DialSync. type DialFunc func(context.Context, peer.ID) (*Conn, error) +// NewDialSync constructs a new DialSync func NewDialSync(dfn DialFunc) *DialSync { return &DialSync{ dials: make(map[peer.ID]*activeDial), @@ -16,6 +18,8 @@ func NewDialSync(dfn DialFunc) *DialSync { } } +// DialSync is a dial synchronization helper that ensures that at most one dial +// to any given peer is active at any given time. type DialSync struct { dials map[peer.ID]*activeDial dialsLk sync.Mutex @@ -35,11 +39,11 @@ type activeDial struct { ds *DialSync } -func (dr *activeDial) wait(ctx context.Context) (*Conn, error) { - defer dr.decref() +func (ad *activeDial) wait(ctx context.Context) (*Conn, error) { + defer ad.decref() select { - case <-dr.waitch: - return dr.conn, dr.err + case <-ad.waitch: + return ad.conn, ad.err case <-ctx.Done(): return nil, ctx.Err() } @@ -102,6 +106,17 @@ func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { return actd } +// DialLock initiates a dial to the given peer if there are none in progress +// then waits for the dial to that peer to complete. func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { return ds.getActiveDial(p).wait(ctx) } + +// CancelDial cancels all in-progress dials to the given peer. +func (ds *DialSync) CancelDial(p peer.ID) { + ds.dialsLk.Lock() + defer ds.dialsLk.Unlock() + if ad, ok := ds.dials[p]; ok { + ad.cancel() + } +} diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 0d70e226af..53c3fc25b4 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -1,4 +1,4 @@ -package swarm +package swarm_test import ( "context" @@ -7,6 +7,8 @@ import ( "testing" "time" + . "github.com/libp2p/go-libp2p-swarm" + peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index dabb0bdb00..a7eebcea1f 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -1,4 +1,4 @@ -package swarm +package swarm_test import ( "context" @@ -10,19 +10,26 @@ import ( addrutil "github.com/libp2p/go-addr-util" peer "github.com/libp2p/go-libp2p-peer" pstore "github.com/libp2p/go-libp2p-peerstore" + swarmt "github.com/libp2p/go-libp2p-swarm/testing" testutil "github.com/libp2p/go-testutil" ci "github.com/libp2p/go-testutil/ci" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" + + . "github.com/libp2p/go-libp2p-swarm" ) +func init() { + DialTimeout = time.Second +} + func closeSwarms(swarms []*Swarm) { for _, s := range swarms { s.Close() } } -func TestBasicDial(t *testing.T) { +func TestBasicDialPeer(t *testing.T) { t.Parallel() ctx := context.Background() @@ -31,9 +38,9 @@ func TestBasicDial(t *testing.T) { s1 := swarms[0] s2 := swarms[1] - s1.peers.AddAddrs(s2.local, s2.ListenAddresses(), pstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), pstore.PermanentAddrTTL) - c, err := s1.Dial(ctx, s2.local) + c, err := s1.DialPeer(ctx, s2.LocalPeer()) if err != nil { t.Fatal(err) } @@ -56,9 +63,9 @@ func TestDialWithNoListeners(t *testing.T) { defer closeSwarms(swarms) s2 := swarms[0] - s1.peers.AddAddrs(s2.local, s2.ListenAddresses(), pstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), pstore.PermanentAddrTTL) - c, err := s1.Dial(ctx, s2.local) + c, err := s1.DialPeer(ctx, s2.LocalPeer()) if err != nil { t.Fatal(err) } @@ -92,16 +99,16 @@ func TestSimultDials(t *testing.T) { t.Parallel() ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(ctx, t, 2, swarmt.OptDisableReuseport) // connect everyone { var wg sync.WaitGroup connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // copy for other peer - log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr) - s.peers.AddAddr(dst, addr, pstore.TempAddrTTL) - if _, err := s.Dial(ctx, dst); err != nil { + log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) + s.Peerstore().AddAddr(dst, addr, pstore.TempAddrTTL) + if _, err := s.DialPeer(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } wg.Done() @@ -119,18 +126,18 @@ func TestSimultDials(t *testing.T) { log.Info("Connecting swarms simultaneously.") for i := 0; i < 10; i++ { // connect 10x for each. wg.Add(2) - go connect(swarms[0], swarms[1].local, ifaceAddrs1[0]) - go connect(swarms[1], swarms[0].local, ifaceAddrs0[0]) + go connect(swarms[0], swarms[1].LocalPeer(), ifaceAddrs1[0]) + go connect(swarms[1], swarms[0].LocalPeer(), ifaceAddrs0[0]) } wg.Wait() } // should still just have 1, at most 2 connections :) - c01l := len(swarms[0].ConnectionsToPeer(swarms[1].local)) + c01l := len(swarms[0].ConnsToPeer(swarms[1].LocalPeer())) if c01l > 2 { t.Error("0->1 has", c01l) } - c10l := len(swarms[1].ConnectionsToPeer(swarms[0].local)) + c10l := len(swarms[1].ConnsToPeer(swarms[0].LocalPeer())) if c10l > 2 { t.Error("1->0 has", c10l) } @@ -168,19 +175,14 @@ func TestDialWait(t *testing.T) { s1 := swarms[0] defer s1.Close() - s1.dialT = time.Millisecond * 300 // lower timeout for tests. - if ci.IsRunning() { - s1.dialT = time.Second - } - // dial to a non-existent peer. s2p, s2addr, s2l := newSilentPeer(t) go acceptAndHang(s2l) defer s2l.Close() - s1.peers.AddAddr(s2p, s2addr, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddr(s2p, s2addr, pstore.PermanentAddrTTL) before := time.Now() - if c, err := s1.Dial(ctx, s2p); err == nil { + if c, err := s1.DialPeer(ctx, s2p); err == nil { defer c.Close() t.Fatal("error swarm dialing to unknown peer worked...", err) } else { @@ -188,15 +190,14 @@ func TestDialWait(t *testing.T) { } duration := time.Since(before) - dt := s1.dialT - if duration < dt*dialAttempts { - t.Error("< DialTimeout * dialAttempts not being respected", duration, dt*dialAttempts) + if duration < DialTimeout*DialAttempts { + t.Error("< DialTimeout * DialAttempts not being respected", duration, DialTimeout*DialAttempts) } - if duration > 2*dt*dialAttempts { - t.Error("> 2*DialTimeout * dialAttempts not being respected", duration, 2*dt*dialAttempts) + if duration > 2*DialTimeout*DialAttempts { + t.Error("> 2*DialTimeout * DialAttempts not being respected", duration, 2*DialTimeout*DialAttempts) } - if !s1.backf.Backoff(s2p) { + if !s1.Backoff().Backoff(s2p) { t.Error("s2 should now be on backoff") } } @@ -216,20 +217,17 @@ func TestDialBackoff(t *testing.T) { defer s1.Close() defer s2.Close() - s1.dialT = time.Second // lower timeout for tests. - s2.dialT = time.Second // lower timeout for tests. - s2addrs, err := s2.InterfaceListenAddresses() if err != nil { t.Fatal(err) } - s1.peers.AddAddrs(s2.local, s2addrs, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2addrs, pstore.PermanentAddrTTL) // dial to a non-existent peer. s3p, s3addr, s3l := newSilentPeer(t) go acceptAndHang(s3l) defer s3l.Close() - s1.peers.AddAddr(s3p, s3addr, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddr(s3p, s3addr, pstore.PermanentAddrTTL) // in this test we will: // 1) dial 10x to each node. @@ -246,7 +244,7 @@ func TestDialBackoff(t *testing.T) { ch := make(chan bool) for i := 0; i < times; i++ { go func() { - if _, err := s1.Dial(ctx, dst); err != nil { + if _, err := s1.DialPeer(ctx, dst); err != nil { t.Error("error dialing", dst, err) ch <- false } else { @@ -261,7 +259,7 @@ func TestDialBackoff(t *testing.T) { ch := make(chan bool) for i := 0; i < times; i++ { go func() { - if c, err := s1.Dial(ctx, dst); err != nil { + if c, err := s1.DialPeer(ctx, dst); err != nil { ch <- false } else { t.Error("succeeded in dialing", dst) @@ -276,13 +274,12 @@ func TestDialBackoff(t *testing.T) { { // 1) dial 10x to each node. N := 10 - s2done := dialOnlineNode(s2.local, N) + s2done := dialOnlineNode(s2.LocalPeer(), N) s3done := dialOfflineNode(s3p, N) // when all dials should be done by: - dialTimeout1x := time.After(s1.dialT) - // dialTimeout1Ax := time.After(s1.dialT * 2) // dialAttempts) - dialTimeout10Ax := time.After(s1.dialT * 2 * 10) // dialAttempts * 10) + dialTimeout1x := time.After(DialTimeout) + dialTimeout10Ax := time.After(DialTimeout * 2 * 10) // DialAttempts * 10) // 2) all dials should hang select { @@ -337,22 +334,22 @@ func TestDialBackoff(t *testing.T) { } // check backoff state - if s1.backf.Backoff(s2.local) { + if s1.Backoff().Backoff(s2.LocalPeer()) { t.Error("s2 should not be on backoff") } - if !s1.backf.Backoff(s3p) { + if !s1.Backoff().Backoff(s3p) { t.Error("s3 should be on backoff") } // 5) disconnect entirely - for _, c := range s1.Connections() { + for _, c := range s1.Conns() { c.Close() } - for i := 0; i < 100 && len(s1.Connections()) > 0; i++ { + for i := 0; i < 100 && len(s1.Conns()) > 0; i++ { <-time.After(time.Millisecond) } - if len(s1.Connections()) > 0 { + if len(s1.Conns()) > 0 { t.Fatal("s1 conns must exit") } } @@ -360,13 +357,12 @@ func TestDialBackoff(t *testing.T) { { // 6) dial 10x to each node again N := 10 - s2done := dialOnlineNode(s2.local, N) + s2done := dialOnlineNode(s2.LocalPeer(), N) s3done := dialOfflineNode(s3p, N) // when all dials should be done by: - dialTimeout1x := time.After(s1.dialT) - // dialTimeout1Ax := time.After(s1.dialT * 2) // dialAttempts) - dialTimeout10Ax := time.After(s1.dialT * 2 * 10) // dialAttempts * 10) + dialTimeout1x := time.After(DialTimeout) + dialTimeout10Ax := time.After(DialTimeout * 2 * 10) // DialAttempts * 10) // 7) s3 dials should all return immediately (except 1) for i := 0; i < N-1; i++ { @@ -408,10 +404,10 @@ func TestDialBackoff(t *testing.T) { } // check backoff state (the same) - if s1.backf.Backoff(s2.local) { + if s1.Backoff().Backoff(s2.LocalPeer()) { t.Error("s2 should not be on backoff") } - if !s1.backf.Backoff(s3p) { + if !s1.Backoff().Backoff(s3p) { t.Error("s3 should be on backoff") } } @@ -427,12 +423,6 @@ func TestDialBackoffClears(t *testing.T) { s2 := swarms[1] defer s1.Close() defer s2.Close() - s1.dialT = time.Millisecond * 300 // lower timeout for tests. - s2.dialT = time.Millisecond * 300 // lower timeout for tests. - if ci.IsRunning() { - s1.dialT = 2 * time.Second - s2.dialT = 2 * time.Second - } // use another address first, that accept and hang on conns _, s2bad, s2l := newSilentPeer(t) @@ -440,10 +430,10 @@ func TestDialBackoffClears(t *testing.T) { defer s2l.Close() // phase 1 -- dial to non-operational addresses - s1.peers.AddAddr(s2.local, s2bad, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddr(s2.LocalPeer(), s2bad, pstore.PermanentAddrTTL) before := time.Now() - if c, err := s1.Dial(ctx, s2.local); err == nil { + if c, err := s1.DialPeer(ctx, s2.LocalPeer()); err == nil { t.Fatal("dialing to broken addr worked...", err) defer c.Close() } else { @@ -451,15 +441,14 @@ func TestDialBackoffClears(t *testing.T) { } duration := time.Since(before) - dt := s1.dialT - if duration < dt*dialAttempts { - t.Error("< DialTimeout * dialAttempts not being respected", duration, dt*dialAttempts) + if duration < DialTimeout*DialAttempts { + t.Error("< DialTimeout * DialAttempts not being respected", duration, DialTimeout*DialAttempts) } - if duration > 2*dt*dialAttempts { - t.Error("> 2*DialTimeout * dialAttempts not being respected", duration, 2*dt*dialAttempts) + if duration > 2*DialTimeout*DialAttempts { + t.Error("> 2*DialTimeout * DialAttempts not being respected", duration, 2*DialTimeout*DialAttempts) } - if !s1.backf.Backoff(s2.local) { + if !s1.Backoff().Backoff(s2.LocalPeer()) { t.Error("s2 should now be on backoff") } else { t.Log("correctly added to backoff") @@ -470,22 +459,22 @@ func TestDialBackoffClears(t *testing.T) { if err != nil { t.Fatal(err) } - s1.peers.AddAddrs(s2.local, ifaceAddrs1, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), ifaceAddrs1, pstore.PermanentAddrTTL) - if _, err := s1.Dial(ctx, s2.local); err == nil { + if _, err := s1.DialPeer(ctx, s2.LocalPeer()); err == nil { t.Fatal("should have failed to dial backed off peer") } - time.Sleep(baseBackoffTime) + time.Sleep(BackoffBase) - if c, err := s1.Dial(ctx, s2.local); err != nil { + if c, err := s1.DialPeer(ctx, s2.LocalPeer()); err != nil { t.Fatal(err) } else { c.Close() t.Log("correctly connected") } - if s1.backf.Backoff(s2.local) { + if s1.Backoff().Backoff(s2.LocalPeer()) { t.Error("s2 should no longer be on backoff") } else { t.Log("correctly cleared backoff") diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 836edbc9f5..3d3437b95f 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -5,13 +5,13 @@ import ( "sync" addrutil "github.com/libp2p/go-addr-util" - iconn "github.com/libp2p/go-libp2p-interface-conn" peer "github.com/libp2p/go-libp2p-peer" + transport "github.com/libp2p/go-libp2p-transport" ma "github.com/multiformats/go-multiaddr" ) type dialResult struct { - Conn iconn.Conn + Conn transport.Conn Addr ma.Multiaddr Err error } @@ -39,17 +39,17 @@ type dialLimiter struct { fdLimit int waitingOnFd []*dialJob - dialFunc func(context.Context, peer.ID, ma.Multiaddr) (iconn.Conn, error) + dialFunc func(context.Context, peer.ID, ma.Multiaddr) (transport.Conn, error) activePerPeer map[peer.ID]int perPeerLimit int waitingOnPeerLimit map[peer.ID][]*dialJob } -type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (iconn.Conn, error) +type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (transport.Conn, error) func newDialLimiter(df dialfunc) *dialLimiter { - return newDialLimiterWithParams(df, concurrentFdDials, defaultPerPeerRateLimit) + return newDialLimiterWithParams(df, ConcurrentFdDials, DefaultPerPeerRateLimit) } func newDialLimiterWithParams(df dialfunc, fdLimit, perPeerLimit int) *dialLimiter { diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 6c6eeb68fa..6ae3935caf 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - iconn "github.com/libp2p/go-libp2p-interface-conn" peer "github.com/libp2p/go-libp2p-peer" + transport "github.com/libp2p/go-libp2p-transport" ma "github.com/multiformats/go-multiaddr" mafmt "github.com/whyrusleeping/mafmt" ) @@ -56,13 +56,13 @@ func tryDialAddrs(ctx context.Context, l *dialLimiter, p peer.ID, addrs []ma.Mul } func hangDialFunc(hang chan struct{}) dialfunc { - return func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { + return func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.Conn, error) { if mafmt.UTP.Matches(a) { - return iconn.Conn(nil), nil + return transport.Conn(nil), nil } if tcpPortOver(a, 10) { - return iconn.Conn(nil), nil + return transport.Conn(nil), nil } <-hang @@ -74,7 +74,7 @@ func TestLimiterBasicDials(t *testing.T) { hang := make(chan struct{}) defer close(hang) - l := newDialLimiterWithParams(hangDialFunc(hang), concurrentFdDials, 4) + l := newDialLimiterWithParams(hangDialFunc(hang), ConcurrentFdDials, 4) bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} good := addrWithPort(t, 20) @@ -173,9 +173,9 @@ func TestFDLimiting(t *testing.T) { func TestTokenRedistribution(t *testing.T) { var lk sync.Mutex hangchs := make(map[peer.ID]chan struct{}) - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.Conn, error) { if tcpPortOver(a, 10) { - return (iconn.Conn)(nil), nil + return (transport.Conn)(nil), nil } lk.Lock() @@ -268,9 +268,9 @@ func TestTokenRedistribution(t *testing.T) { } func TestStressLimiter(t *testing.T) { - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.Conn, error) { if tcpPortOver(a, 1000) { - return iconn.Conn(nil), nil + return transport.Conn(nil), nil } time.Sleep(time.Millisecond * time.Duration(5+rand.Intn(100))) @@ -322,11 +322,7 @@ func TestStressLimiter(t *testing.T) { } func TestFDLimitUnderflow(t *testing.T) { - dials := 0 - - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (iconn.Conn, error) { - dials++ - + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.Conn, error) { timeout := make(chan bool, 1) go func() { time.Sleep(time.Second * 5) @@ -382,7 +378,11 @@ func TestFDLimitUnderflow(t *testing.T) { time.Sleep(time.Second * 3) - if l.fdConsuming < 0 { + l.rllock.Lock() + fdConsuming := l.fdConsuming + l.rllock.Unlock() + + if fdConsuming < 0 { t.Fatalf("l.fdConsuming < 0") } } diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 3259264ae8..76cc00b022 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -1,12 +1,15 @@ -package swarm +package swarm_test import ( "testing" "context" + inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" pstore "github.com/libp2p/go-libp2p-peerstore" ma "github.com/multiformats/go-multiaddr" + + . "github.com/libp2p/go-libp2p-swarm" ) func TestPeers(t *testing.T) { @@ -18,13 +21,13 @@ func TestPeers(t *testing.T) { connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. - s.peers.AddAddr(dst, addr, pstore.PermanentAddrTTL) + s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL) // t.Logf("connections from %s", s.LocalPeer()) - // for _, c := range s.ConnectionsToPeer(dst) { + // for _, c := range s.ConnsToPeer(dst) { // t.Logf("connection from %s to %s: %v", s.LocalPeer(), dst, c) // } // t.Logf("") - if _, err := s.Dial(ctx, dst); err != nil { + if _, err := s.DialPeer(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } // t.Log(s.swarm.Dump()) @@ -32,10 +35,10 @@ func TestPeers(t *testing.T) { s1GotConn := make(chan struct{}, 0) s2GotConn := make(chan struct{}, 0) - s1.SetConnHandler(func(c *Conn) { + s1.SetConnHandler(func(c inet.Conn) { s1GotConn <- struct{}{} }) - s2.SetConnHandler(func(c *Conn) { + s2.SetConnHandler(func(c inet.Conn) { s2GotConn <- struct{}{} }) @@ -49,7 +52,7 @@ func TestPeers(t *testing.T) { } for _, s := range swarms { - log.Infof("%s swarm routing table: %s", s.local, s.Peers()) + log.Infof("%s swarm routing table: %s", s.LocalPeer(), s.Peers()) } test := func(s *Swarm) { @@ -57,12 +60,10 @@ func TestPeers(t *testing.T) { actual := len(s.Peers()) if actual != expect { t.Errorf("%s has %d peers, not %d: %v", s.LocalPeer(), actual, expect, s.Peers()) - t.Log(s.swarm.Dump()) } - actual = len(s.Connections()) + actual = len(s.Conns()) if actual != expect { - t.Errorf("%s has %d conns, not %d: %v", s.LocalPeer(), actual, expect, s.Connections()) - t.Log(s.swarm.Dump()) + t.Errorf("%s has %d conns, not %d: %v", s.LocalPeer(), actual, expect, s.Conns()) } } diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 8935773296..22b0dbf330 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -1,4 +1,4 @@ -package swarm +package swarm_test import ( "context" @@ -11,6 +11,9 @@ import ( pstore "github.com/libp2p/go-libp2p-peerstore" ci "github.com/libp2p/go-testutil/ci" ma "github.com/multiformats/go-multiaddr" + + . "github.com/libp2p/go-libp2p-swarm" + swarmt "github.com/libp2p/go-libp2p-swarm/testing" ) func TestSimultOpen(t *testing.T) { @@ -18,16 +21,16 @@ func TestSimultOpen(t *testing.T) { t.Parallel() ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(ctx, t, 2, swarmt.OptDisableReuseport) // connect everyone { var wg sync.WaitGroup connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // copy for other peer - log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr) - s.peers.AddAddr(dst, addr, pstore.PermanentAddrTTL) - if _, err := s.Dial(ctx, dst); err != nil { + log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) + s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL) + if _, err := s.DialPeer(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } wg.Done() @@ -35,8 +38,8 @@ func TestSimultOpen(t *testing.T) { log.Info("Connecting swarms simultaneously.") wg.Add(2) - go connect(swarms[0], swarms[1].local, swarms[1].ListenAddresses()[0]) - go connect(swarms[1], swarms[0].local, swarms[0].ListenAddresses()[0]) + go connect(swarms[0], swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0]) + go connect(swarms[1], swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0]) wg.Wait() } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index e7d6b9a7f4..dbf81613f9 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -1,190 +1,150 @@ -// Package swarm implements a connection muxer with a pair of channels -// to synchronize all network communication. package swarm import ( "context" + "errors" "fmt" - "io/ioutil" - "os" "strings" "sync" + "sync/atomic" "time" logging "github.com/ipfs/go-log" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" - addrutil "github.com/libp2p/go-addr-util" - conn "github.com/libp2p/go-libp2p-conn" - ci "github.com/libp2p/go-libp2p-crypto" - ipnet "github.com/libp2p/go-libp2p-interface-pnet" metrics "github.com/libp2p/go-libp2p-metrics" - mconn "github.com/libp2p/go-libp2p-metrics/conn" inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" pstore "github.com/libp2p/go-libp2p-peerstore" transport "github.com/libp2p/go-libp2p-transport" filter "github.com/libp2p/go-maddr-filter" - ps "github.com/libp2p/go-peerstream" - pst "github.com/libp2p/go-stream-muxer" - tcpt "github.com/libp2p/go-tcp-transport" - ws "github.com/libp2p/go-ws-transport" - ma "github.com/multiformats/go-multiaddr" - psmss "github.com/whyrusleeping/go-smux-multistream" - spdy "github.com/whyrusleeping/go-smux-spdystream" - yamux "github.com/whyrusleeping/go-smux-yamux" mafilter "github.com/whyrusleeping/multiaddr-filter" ) -var log = logging.Logger("swarm2") - -// PSTransport is the default peerstream transport that will be used by -// any libp2p swarms. -var PSTransport pst.Transport - -func init() { - msstpt := psmss.NewBlankTransport() - - ymxtpt := &yamux.Transport{ - AcceptBacklog: 8192, - ConnectionWriteTimeout: time.Second * 10, - KeepAliveInterval: time.Second * 30, - EnableKeepAlive: true, - MaxStreamWindowSize: uint32(1024 * 512), - LogOutput: ioutil.Discard, - } +// DialTimeout is the maximum duration a Dial is allowed to take. +// This includes the time between dialing the raw network connection, +// protocol selection as well the handshake, if applicable. +var DialTimeout = 60 * time.Second - msstpt.AddTransport("/yamux/1.0.0", ymxtpt) - msstpt.AddTransport("/spdy/3.1.0", spdy.Transport) +var log = logging.Logger("swarm2") - // allow overriding of muxer preferences - if prefs := os.Getenv("LIBP2P_MUX_PREFS"); prefs != "" { - msstpt.OrderPreference = strings.Fields(prefs) - } +// ErrSwarmClosed is returned when one attempts to operate on a closed swarm. +var ErrSwarmClosed = errors.New("swarm closed") - PSTransport = msstpt -} +// ErrAddrFiltered is returned when trying to register a connection to a +// filtered address. You shouldn't see this error unless some underlying +// transport is misbehaving. +var ErrAddrFiltered = errors.New("address filtered") // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the // destination or source Peer. -// -// Uses peerstream.Swarm type Swarm struct { - swarm *ps.Swarm + // Close refcount. This allows us to fully wait for the swarm to be torn + // down before continuing. + refs sync.WaitGroup + local peer.ID peers pstore.Peerstore - connh ConnHandler - dsync *DialSync - backf dialbackoff - dialT time.Duration // mainly for tests + conns struct { + sync.RWMutex + m map[peer.ID][]*Conn + } - dialer *conn.Dialer + listeners struct { + sync.RWMutex + m map[transport.Listener]struct{} + } - notifmu sync.RWMutex - notifs map[inet.Notifiee]ps.Notifiee + notifs struct { + sync.RWMutex + m map[inet.Notifiee]struct{} + } - transports []transport.Transport + transports struct { + sync.RWMutex + m map[int]transport.Transport + } - // filters for addresses that shouldnt be dialed - Filters *filter.Filters + // new connection and stream handlers + connh atomic.Value + streamh atomic.Value - // file descriptor rate limited - fdRateLimit chan struct{} + // dialing helpers + dsync *DialSync + backf DialBackoff + limiter *dialLimiter + + // filters for addresses that shouldnt be dialed (or accepted) + Filters *filter.Filters proc goprocess.Process ctx context.Context bwc metrics.Reporter - - limiter *dialLimiter - - protec ipnet.Protector -} - -func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID, - peers pstore.Peerstore, bwc metrics.Reporter) (*Swarm, error) { - return NewSwarmWithProtector(ctx, listenAddrs, local, peers, nil, PSTransport, bwc) } -// NewSwarm constructs a Swarm, with a Chan. -func NewSwarmWithProtector(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID, - peers pstore.Peerstore, protec ipnet.Protector, tpt pst.Transport, bwc metrics.Reporter) (*Swarm, error) { - - listenAddrs, err := filterAddrs(listenAddrs) - if err != nil { - return nil, err - } - - var wrap func(c transport.Conn) transport.Conn - if bwc != nil { - wrap = func(c transport.Conn) transport.Conn { - return mconn.WrapConn(bwc, c) - } - } - +// NewSwarm constructs a Swarm +func NewSwarm(ctx context.Context, local peer.ID, peers pstore.Peerstore, bwc metrics.Reporter) *Swarm { s := &Swarm{ - swarm: ps.NewSwarm(tpt), - local: local, - peers: peers, - ctx: ctx, - dialT: conn.DialTimeout, - notifs: make(map[inet.Notifiee]ps.Notifiee), - transports: []transport.Transport{ - tcpt.NewTCPTransport(), - new(ws.WebsocketTransport), - }, - bwc: bwc, - fdRateLimit: make(chan struct{}, concurrentFdDials), - Filters: filter.NewFilters(), - dialer: conn.NewDialer(local, peers.PrivKey(local), wrap), - protec: protec, + local: local, + peers: peers, + bwc: bwc, + Filters: filter.NewFilters(), } - s.dialer.Protector = protec + + s.conns.m = make(map[peer.ID][]*Conn) + s.listeners.m = make(map[transport.Listener]struct{}) + s.transports.m = make(map[int]transport.Transport) + s.notifs.m = make(map[inet.Notifiee]struct{}) s.dsync = NewDialSync(s.doDial) s.limiter = newDialLimiter(s.dialAddr) - - // configure Swarm s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown) - s.SetConnHandler(nil) // make sure to setup our own conn handler. - - err = s.setupInterfaces(listenAddrs) - if err != nil { - return nil, err - } + s.ctx = goprocessctx.OnClosingContext(s.proc) - return s, nil + return s } -func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey, pstpt pst.Transport) *Swarm { - s := &Swarm{ - swarm: ps.NewSwarm(pstpt), - local: id, - peers: pstore.NewPeerstore(), - ctx: ctx, - dialT: conn.DialTimeout, - notifs: make(map[inet.Notifiee]ps.Notifiee), - fdRateLimit: make(chan struct{}, concurrentFdDials), - Filters: filter.NewFilters(), - dialer: conn.NewDialer(id, privkey, nil), +func (s *Swarm) teardown() error { + // Prevents new connections and/or listeners from being added to the swarm. + + s.listeners.Lock() + listeners := s.listeners.m + s.listeners.m = nil + s.listeners.Unlock() + + s.conns.Lock() + conns := s.conns.m + s.conns.m = nil + s.conns.Unlock() + + // Lots of go routines but we might as well do this in parallel. We want + // to shut down as fast as possible. + + for l := range listeners { + go func(l transport.Listener) { + if err := l.Close(); err != nil { + log.Errorf("error when shutting down listener: %s", err) + } + }(l) } - // configure Swarm - s.limiter = newDialLimiter(s.dialAddr) - s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown) - s.SetConnHandler(nil) // make sure to setup our own conn handler. + for _, cs := range conns { + for _, c := range cs { + go func(c *Conn) { + if err := c.Close(); err != nil { + log.Errorf("error when shutting down connection: %s", err) + } + }(c) + } + } - return s -} + // Wait for everything to finish. + s.refs.Wait() -func (s *Swarm) AddTransport(t transport.Transport) { - s.transports = append(s.transports, t) -} - -func (s *Swarm) teardown() error { - return s.swarm.Close() + return nil } // AddAddrFilter adds a multiaddr filter to the set of filters the swarm will @@ -199,31 +159,81 @@ func (s *Swarm) AddAddrFilter(f string) error { return nil } -func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) { - if len(listenAddrs) > 0 { - filtered := addrutil.FilterUsableAddrs(listenAddrs) - if len(filtered) < 1 { - return nil, fmt.Errorf("swarm cannot use any addr in: %s", listenAddrs) - } - listenAddrs = filtered +// Process returns the Process of the swarm +func (s *Swarm) Process() goprocess.Process { + return s.proc +} + +func (s *Swarm) addConn(tc transport.Conn) (*Conn, error) { + // The underlying transport (or the dialer) *should* filter it's own + // connections but we should double check anyways. + raddr := tc.RemoteMultiaddr() + if s.Filters.AddrBlocked(raddr) { + tc.Close() + return nil, ErrAddrFiltered } - return listenAddrs, nil -} + p := tc.RemotePeer() -// Listen sets up listeners for all of the given addresses -func (s *Swarm) Listen(addrs ...ma.Multiaddr) error { - addrs, err := filterAddrs(addrs) - if err != nil { - return err + // Add the public key. + if pk := tc.RemotePublicKey(); pk != nil { + s.peers.AddPubKey(p, pk) + } + + // Clear any backoffs + s.backf.Clear(p) + + // Finally, add the peer. + s.conns.Lock() + // Check if we're still online + if s.conns.m == nil { + s.conns.Unlock() + tc.Close() + return nil, ErrSwarmClosed + } + + // Wrap and register the connection. + c := &Conn{ + conn: tc, + swarm: s, } + c.streams.m = make(map[*Stream]struct{}) + s.conns.m[p] = append(s.conns.m[p], c) + + // Add two swarm refs: + // * One will be decremented after the close notifications fire in Conn.doClose + // * The other will be decremented when Conn.start exits. + s.refs.Add(2) + + // Take the notification lock before releasing the conns lock to block + // Disconnect notifications until after the Connect notifications done. + c.notifyLk.Lock() + s.conns.Unlock() + + // We have a connection now. Cancel all other in-progress dials. + // This should be fast, no reason to wait till later. + s.dsync.CancelDial(p) + + s.notifyAll(func(f inet.Notifiee) { + f.Connected(s, c) + }) + c.notifyLk.Unlock() - return s.setupInterfaces(addrs) + c.start() + + // TODO: Get rid of this. We use it for identify but that happen much + // earlier (really, inside the transport and, if not then, during the + // notifications). + if h := s.ConnHandler(); h != nil { + go h(c) + } + + return c, nil } -// Process returns the Process of the swarm -func (s *Swarm) Process() goprocess.Process { - return s.proc +// Peerstore returns this swarms internal Peerstore. +func (s *Swarm) Peerstore() pstore.Peerstore { + return s.peers } // Context returns the context of the swarm @@ -236,94 +246,180 @@ func (s *Swarm) Close() error { return s.proc.Close() } -// StreamSwarm returns the underlying peerstream.Swarm -func (s *Swarm) StreamSwarm() *ps.Swarm { - return s.swarm -} +// TODO: We probably don't need the conn handlers. // SetConnHandler assigns the handler for new connections. -// See peerstream. You will rarely use this. See SetStreamHandler -func (s *Swarm) SetConnHandler(handler ConnHandler) { - - // handler is nil if user wants to clear the old handler. - if handler == nil { - s.swarm.SetConnHandler(func(psconn *ps.Conn) { - s.connHandler(psconn) - }) - return - } +// You will rarely use this. See SetStreamHandler +func (s *Swarm) SetConnHandler(handler inet.ConnHandler) { + s.connh.Store(handler) +} - s.swarm.SetConnHandler(func(psconn *ps.Conn) { - // sc is nil if closed in our handler. - if sc := s.connHandler(psconn); sc != nil { - // call the user's handler. in a goroutine for sync safety. - go handler(sc) - } - }) +// ConnHandler gets the handler for new connections. +func (s *Swarm) ConnHandler() inet.ConnHandler { + handler, _ := s.connh.Load().(inet.ConnHandler) + return handler } // SetStreamHandler assigns the handler for new streams. -// See peerstream. func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) { - s.swarm.SetStreamHandler(func(s *ps.Stream) { - handler((*Stream)(s)) - }) -} - -// NewStreamWithPeer creates a new stream on any available connection to p -func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, error) { - // if we have no connections, try connecting. - if !s.HaveConnsToPeer(p) { - log.Debug("Swarm: NewStreamWithPeer no connections. Attempting to connect...") - if _, err := s.Dial(ctx, p); err != nil { + s.streamh.Store(handler) +} + +// StreamHandler gets the handler for new streams. +func (s *Swarm) StreamHandler() inet.StreamHandler { + handler, _ := s.streamh.Load().(inet.StreamHandler) + return handler +} + +// NewStream creates a new stream on any available connection to peer, dialing +// if necessary. +func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) { + log.Debugf("[%s] opening stream to peer [%s]", s.local, p) + + // Algorithm: + // 1. Find the best connection, otherwise, dial. + // 2. Try opening a stream. + // 3. If the underlying connection is, in fact, closed, close the outer + // connection and try again. We do this in case we have a closed + // connection but don't notice it until we actually try to open a + // stream. + // + // Note: We only dial once. + // + // TODO: Try all connections even if we get an error opening a stream on + // a non-closed connection. + dials := 0 + for { + c := s.bestConnToPeer(p) + if c == nil { + if dials >= DialAttempts { + return nil, errors.New("max dial attempts exceeded") + } + dials++ + + var err error + c, err = s.dialPeer(ctx, p) + if err != nil { + return nil, err + } + } + s, err := c.NewStream() + if err != nil { + if c.conn.IsClosed() { + continue + } return nil, err } + return s, nil } - log.Debug("Swarm: NewStreamWithPeer...") - - // TODO: think about passing a context down to NewStreamWithGroup - st, err := s.swarm.NewStreamWithGroup(p) - return (*Stream)(st), err } -// ConnectionsToPeer returns all the live connections to p -func (s *Swarm) ConnectionsToPeer(p peer.ID) []*Conn { - return wrapConns(s.swarm.ConnsWithGroup(p)) -} +// ConnsToPeer returns all the live connections to peer. +func (s *Swarm) ConnsToPeer(p peer.ID) []inet.Conn { + // TODO: Consider sorting the connection list best to worst. Currently, + // it's sorted oldest to newest. + s.conns.RLock() + defer s.conns.RUnlock() + conns := s.conns.m[p] + output := make([]inet.Conn, len(conns)) + for i, c := range conns { + output[i] = c + } + return output +} + +// bestConnToPeer returns the best connection to peer. +func (s *Swarm) bestConnToPeer(p peer.ID) *Conn { + // Selects the best connection we have to the peer. + // TODO: Prefer some transports over others. Currently, we just select + // the newest non-closed connection with the most streams. + s.conns.RLock() + defer s.conns.RUnlock() + + var best *Conn + bestLen := 0 + for _, c := range s.conns.m[p] { + if c.conn.IsClosed() { + // We *will* garbage collect this soon anyways. + continue + } + c.streams.Lock() + cLen := len(c.streams.m) + c.streams.Unlock() -func (s *Swarm) HaveConnsToPeer(p peer.ID) bool { - return len(s.swarm.ConnsWithGroup(p)) > 0 -} + if cLen >= bestLen { + best = c + bestLen = cLen + } -// Connections returns a slice of all connections. -func (s *Swarm) Connections() []*Conn { - return wrapConns(s.swarm.Conns()) + } + return best } -// CloseConnection removes a given peer from swarm + closes the connection -func (s *Swarm) CloseConnection(p peer.ID) error { - conns := s.swarm.ConnsWithGroup(p) // boom. - for _, c := range conns { - c.Close() +// Connectedness returns our "connectedness" state with the given peer. +// +// To check if we have an open connection, use `s.Connectedness(p) == +// inet.Connected`. +func (s *Swarm) Connectedness(p peer.ID) inet.Connectedness { + if s.bestConnToPeer(p) != nil { + return inet.Connected } - return nil + return inet.NotConnected } -// Peers returns a copy of the set of peers swarm is connected to. -func (s *Swarm) Peers() []peer.ID { - conns := s.Connections() +// Conns returns a slice of all connections. +func (s *Swarm) Conns() []inet.Conn { + s.conns.RLock() + defer s.conns.RUnlock() - seen := make(map[peer.ID]struct{}, len(conns)) - peers := make([]peer.ID, 0, len(conns)) - for _, c := range conns { - p := c.RemotePeer() - if _, found := seen[p]; found { - continue + conns := make([]inet.Conn, 0, len(s.conns.m)) + for _, cs := range s.conns.m { + for _, c := range cs { + conns = append(conns, c) + } + } + return conns +} + +// ClosePeer closes all connections to the given peer. +func (s *Swarm) ClosePeer(p peer.ID) error { + conns := s.ConnsToPeer(p) + switch len(conns) { + case 0: + return nil + case 1: + return conns[0].Close() + default: + errCh := make(chan error) + for _, c := range conns { + go func(c inet.Conn) { + errCh <- c.Close() + }(c) } - seen[p] = struct{}{} + var errs []string + for _ = range conns { + err := <-errCh + if err != nil { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("when disconnecting from peer %s: %s", p, strings.Join(errs, ", ")) + } + return nil + } +} + +// Peers returns a copy of the set of peers swarm is connected to. +func (s *Swarm) Peers() []peer.ID { + s.conns.RLock() + defer s.conns.RUnlock() + peers := make([]peer.ID, 0, len(s.conns.m)) + for p := range s.conns.m { peers = append(peers, p) } + return peers } @@ -332,64 +428,70 @@ func (s *Swarm) LocalPeer() peer.ID { return s.local } -// Backoff returns the dialbackoff object for this swarm. -func (s *Swarm) Backoff() *dialbackoff { +// Backoff returns the DialBackoff object for this swarm. +func (s *Swarm) Backoff() *DialBackoff { return &s.backf } // notifyAll sends a signal to all Notifiees func (s *Swarm) notifyAll(notify func(inet.Notifiee)) { - s.notifmu.RLock() - for f := range s.notifs { - go notify(f) + var wg sync.WaitGroup + + s.notifs.RLock() + wg.Add(len(s.notifs.m)) + for f := range s.notifs.m { + go func(f inet.Notifiee) { + defer wg.Done() + notify(f) + }(f) } - s.notifmu.RUnlock() + + wg.Wait() + s.notifs.RUnlock() } // Notify signs up Notifiee to receive signals when events happen func (s *Swarm) Notify(f inet.Notifiee) { - // wrap with our notifiee, to translate function calls - n := &ps2netNotifee{net: (*Network)(s), not: f} - - s.notifmu.Lock() - s.notifs[f] = n - s.notifmu.Unlock() - - // register for notifications in the peer swarm. - s.swarm.Notify(n) + s.notifs.Lock() + s.notifs.m[f] = struct{}{} + s.notifs.Unlock() } // StopNotify unregisters Notifiee fromr receiving signals func (s *Swarm) StopNotify(f inet.Notifiee) { - s.notifmu.Lock() - n, found := s.notifs[f] - if found { - delete(s.notifs, f) - } - s.notifmu.Unlock() - - if found { - s.swarm.StopNotify(n) + s.notifs.Lock() + delete(s.notifs.m, f) + s.notifs.Unlock() +} + +func (s *Swarm) removeConn(c *Conn) { + p := c.RemotePeer() + + s.conns.Lock() + defer s.conns.Unlock() + cs := s.conns.m[p] + for i, ci := range cs { + if ci == c { + if len(cs) == 1 { + delete(s.conns.m, p) + } else { + // NOTE: We're intentionally preserving order. + // This way, connections to a peer are always + // sorted oldest to newest. + copy(cs[i:], cs[i+1:]) + cs[len(cs)-1] = nil + s.conns.m[p] = cs[:len(cs)-1] + } + return + } } } -type ps2netNotifee struct { - net *Network - not inet.Notifiee +// String returns a string representation of Network. +func (s *Swarm) String() string { + return fmt.Sprintf("", s.LocalPeer()) } -func (n *ps2netNotifee) Connected(c *ps.Conn) { - n.not.Connected(n.net, inet.Conn((*Conn)(c))) -} - -func (n *ps2netNotifee) Disconnected(c *ps.Conn) { - n.not.Disconnected(n.net, inet.Conn((*Conn)(c))) -} - -func (n *ps2netNotifee) OpenedStream(s *ps.Stream) { - n.not.OpenedStream(n.net, (*Stream)(s)) -} - -func (n *ps2netNotifee) ClosedStream(s *ps.Stream) { - n.not.ClosedStream(n.net, (*Stream)(s)) -} +// Swarm is a Network. +var _ inet.Network = (*Swarm)(nil) +var _ transport.Network = (*Swarm)(nil) diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 78a3f5351a..c86d58bb4d 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -2,18 +2,16 @@ package swarm import ( addrutil "github.com/libp2p/go-addr-util" - iconn "github.com/libp2p/go-libp2p-interface-conn" ma "github.com/multiformats/go-multiaddr" ) // ListenAddresses returns a list of addresses at which this swarm listens. func (s *Swarm) ListenAddresses() []ma.Multiaddr { - listeners := s.swarm.Listeners() - addrs := make([]ma.Multiaddr, 0, len(listeners)) - for _, l := range listeners { - if l2, ok := l.NetListener().(iconn.Listener); ok { - addrs = append(addrs, l2.Multiaddr()) - } + s.listeners.RLock() + defer s.listeners.RUnlock() + addrs := make([]ma.Multiaddr, 0, len(s.listeners.m)) + for l := range s.listeners.m { + addrs = append(addrs, l.Multiaddr()) } return addrs } diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 011b706d81..bafcfb6697 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -1,100 +1,14 @@ -package swarm +package swarm_test import ( "context" "testing" - addrutil "github.com/libp2p/go-addr-util" - metrics "github.com/libp2p/go-libp2p-metrics" pstore "github.com/libp2p/go-libp2p-peerstore" testutil "github.com/libp2p/go-testutil" ma "github.com/multiformats/go-multiaddr" ) -func TestFilterAddrs(t *testing.T) { - - m := func(s string) ma.Multiaddr { - maddr, err := ma.NewMultiaddr(s) - if err != nil { - t.Fatal(err) - } - return maddr - } - - bad := []ma.Multiaddr{ - m("/ip4/1.2.3.4/udp/1234"), // unreliable - m("/ip4/1.2.3.4/udp/1234/sctp/1234"), // not in manet - m("/ip4/1.2.3.4/udp/1234/udt"), // udt is broken on arm - m("/ip6/fe80::1/tcp/0"), // link local - m("/ip6/fe80::100/tcp/1234"), // link local - } - - good := []ma.Multiaddr{ - m("/ip4/127.0.0.1/tcp/0"), - m("/ip6/::1/tcp/0"), - m("/ip4/1.2.3.4/udp/1234/utp"), - } - - goodAndBad := append(good, bad...) - - // test filters - - for _, a := range bad { - if addrutil.AddrUsable(a, false) { - t.Errorf("addr %s should be unusable", a) - } - } - - for _, a := range good { - if !addrutil.AddrUsable(a, false) { - t.Errorf("addr %s should be usable", a) - } - } - - subtestAddrsEqual(t, addrutil.FilterUsableAddrs(bad), []ma.Multiaddr{}) - subtestAddrsEqual(t, addrutil.FilterUsableAddrs(good), good) - subtestAddrsEqual(t, addrutil.FilterUsableAddrs(goodAndBad), good) - - // now test it with swarm - - id, err := testutil.RandPeerID() - if err != nil { - t.Fatal(err) - } - - ps := pstore.NewPeerstore() - ctx := context.Background() - - if _, err := NewNetwork(ctx, bad, id, ps, metrics.NewBandwidthCounter()); err == nil { - t.Fatal("should have failed to create swarm") - } - - if _, err := NewNetwork(ctx, goodAndBad, id, ps, metrics.NewBandwidthCounter()); err != nil { - t.Fatal("should have succeeded in creating swarm", err) - } -} - -func subtestAddrsEqual(t *testing.T, a, b []ma.Multiaddr) { - if len(a) != len(b) { - t.Error(t) - } - - in := func(addr ma.Multiaddr, l []ma.Multiaddr) bool { - for _, addr2 := range l { - if addr.Equal(addr2) { - return true - } - } - return false - } - - for _, aa := range a { - if !in(aa, b) { - t.Errorf("%s not in %s", aa, b) - } - } -} - func TestDialBadAddrs(t *testing.T) { m := func(s string) ma.Multiaddr { @@ -110,8 +24,8 @@ func TestDialBadAddrs(t *testing.T) { test := func(a ma.Multiaddr) { p := testutil.RandPeerIDFatal(t) - s.peers.AddAddr(p, a, pstore.PermanentAddrTTL) - if _, err := s.Dial(ctx, p); err == nil { + s.Peerstore().AddAddr(p, a, pstore.PermanentAddrTTL) + if _, err := s.DialPeer(ctx, p); err == nil { t.Errorf("swarm should not dial: %s", p) } } diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index c143062cee..5b2420cd5f 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -1,122 +1,213 @@ package swarm import ( + "errors" "fmt" + "sync" ic "github.com/libp2p/go-libp2p-crypto" - iconn "github.com/libp2p/go-libp2p-interface-conn" inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" - ps "github.com/libp2p/go-peerstream" + transport "github.com/libp2p/go-libp2p-transport" + smux "github.com/libp2p/go-stream-muxer" ma "github.com/multiformats/go-multiaddr" ) -// Conn is a simple wrapper around a ps.Conn that also exposes -// some of the methods from the underlying conn.Conn. -// There's **five** "layers" to each connection: -// * 0. the net.Conn - underlying net.Conn (TCP/UDP/UTP/etc) -// * 1. the manet.Conn - provides multiaddr friendly Conn -// * 2. the conn.Conn - provides Peer friendly Conn (inc Secure channel) -// * 3. the peerstream.Conn - provides peerstream / spdysptream happiness -// * 4. the Conn - abstracts everyting out, exposing only key parts of underlying layers -// (I know, this is kinda crazy. it's more historical than a good design. though the -// layers do build up pieces of functionality. and they're all just io.RW :) ) -type Conn ps.Conn - -// ConnHandler is called when new conns are opened from remote peers. -// See peerstream.ConnHandler -type ConnHandler func(*Conn) - -func (c *Conn) StreamConn() *ps.Conn { - return (*ps.Conn)(c) +// TODO: Put this elsewhere. + +// ErrConnClosed is returned when operating on a closed connection. +var ErrConnClosed = errors.New("connection closed") + +// Conn is the connection type used by swarm. In general, you won't use this +// type directly. +type Conn struct { + conn transport.Conn + swarm *Swarm + + closeOnce sync.Once + err error + + notifyLk sync.Mutex + + streams struct { + sync.Mutex + m map[*Stream]struct{} + } +} + +// Close closes this connection. +// +// Note: This method won't wait for the close notifications to finish as that +// would create a deadlock when called from an open notification (because all +// open notifications must finish before we can fire off the close +// notifications). +func (c *Conn) Close() error { + c.closeOnce.Do(c.doClose) + return c.err +} + +func (c *Conn) doClose() { + c.swarm.removeConn(c) + + // Prevent new streams from opening. + c.streams.Lock() + streams := c.streams.m + c.streams.m = nil + c.streams.Unlock() + + c.err = c.conn.Close() + + // This is just for cleaning up state. The connection has already been closed. + // We *could* optimize this but it really isn't worth it. + for s := range streams { + s.Reset() + } + + // do this in a goroutine to avoid deadlocking if we call close in an open notification. + go func() { + // prevents us from issuing close notifications before finishing the open notifications + c.notifyLk.Lock() + defer c.notifyLk.Unlock() + + c.swarm.notifyAll(func(f inet.Notifiee) { + f.Disconnected(c.swarm, c) + }) + c.swarm.refs.Done() // taken in Swarm.addConn + }() } -func (c *Conn) RawConn() iconn.Conn { - // righly panic if these things aren't true. it is an expected - // invariant that these Conns are all of the typewe expect: - // ps.Conn wrapping a conn.Conn - // if we get something else it is programmer error. - return (*ps.Conn)(c).NetConn().(iconn.Conn) +func (c *Conn) removeStream(s *Stream) { + c.streams.Lock() + delete(c.streams.m, s) + c.streams.Unlock() +} + +// listens for new streams. +// +// The caller must take a swarm ref before calling. This function decrements the +// swarm ref count. +func (c *Conn) start() { + go func() { + defer c.swarm.refs.Done() + defer c.Close() + + for { + ts, err := c.conn.AcceptStream() + if err != nil { + return + } + c.swarm.refs.Add(1) + go func() { + s, err := c.addStream(ts) + + // Don't defer this. We don't want to block + // swarm shutdown on the connection handler. + c.swarm.refs.Done() + + // We only get an error here when the swarm is closed or closing. + if err != nil { + return + } + + if h := c.swarm.StreamHandler(); h != nil { + h(s) + } + }() + } + }() } func (c *Conn) String() string { - return fmt.Sprintf("", c.RawConn()) + return fmt.Sprintf( + " %s (%s)>", + c.conn.Transport(), + c.conn.LocalMultiaddr(), + c.conn.LocalPeer().Pretty(), + c.conn.RemoteMultiaddr(), + c.conn.RemotePeer().Pretty(), + ) } // LocalMultiaddr is the Multiaddr on this side func (c *Conn) LocalMultiaddr() ma.Multiaddr { - return c.RawConn().LocalMultiaddr() + return c.conn.LocalMultiaddr() } // LocalPeer is the Peer on our side of the connection func (c *Conn) LocalPeer() peer.ID { - return c.RawConn().LocalPeer() + return c.conn.LocalPeer() } // RemoteMultiaddr is the Multiaddr on the remote side func (c *Conn) RemoteMultiaddr() ma.Multiaddr { - return c.RawConn().RemoteMultiaddr() + return c.conn.RemoteMultiaddr() } // RemotePeer is the Peer on the remote side func (c *Conn) RemotePeer() peer.ID { - return c.RawConn().RemotePeer() + return c.conn.RemotePeer() } // LocalPrivateKey is the public key of the peer on this side func (c *Conn) LocalPrivateKey() ic.PrivKey { - return c.RawConn().LocalPrivateKey() + return c.conn.LocalPrivateKey() } // RemotePublicKey is the public key of the peer on the remote side func (c *Conn) RemotePublicKey() ic.PubKey { - return c.RawConn().RemotePublicKey() -} - -// NewSwarmStream returns a new Stream from this connection -func (c *Conn) NewSwarmStream() (*Stream, error) { - s, err := c.StreamConn().NewStream() - return (*Stream)(s), err + return c.conn.RemotePublicKey() } // NewStream returns a new Stream from this connection func (c *Conn) NewStream() (inet.Stream, error) { - s, err := c.NewSwarmStream() - return inet.Stream(s), err -} - -// Close closes the underlying stream connection -func (c *Conn) Close() error { - return c.StreamConn().Close() + ts, err := c.conn.OpenStream() + if err != nil { + return nil, err + } + return c.addStream(ts) } -func (c *Conn) GetStreams() ([]inet.Stream, error) { - ss := c.StreamConn().Streams() - out := make([]inet.Stream, len(ss)) - - for i, s := range ss { - out[i] = (*Stream)(s) +func (c *Conn) addStream(ts smux.Stream) (*Stream, error) { + c.streams.Lock() + // Are we still online? + if c.streams.m == nil { + c.streams.Unlock() + ts.Reset() + return nil, ErrConnClosed } - return out, nil -} -func wrapConn(psc *ps.Conn) (*Conn, error) { - // grab the underlying connection. - if _, ok := psc.NetConn().(iconn.Conn); !ok { - // this should never happen. if we see it ocurring it means that we added - // a Listener to the ps.Swarm that is NOT one of our net/conn.Listener. - return nil, fmt.Errorf("swarm connHandler: invalid conn (not a conn.Conn): %s", psc) + // Wrap and register the stream. + s := &Stream{ + stream: ts, + conn: c, } - return (*Conn)(psc), nil + c.streams.m[s] = struct{}{} + + // Released once the stream disconnect notifications have finished + // firing (in Swarm.remove). + c.swarm.refs.Add(1) + + // Take the notification lock before releasing the streams lock to block + // StreamClose notifications until after the StreamOpen notifications + // done. + s.notifyLk.Lock() + c.streams.Unlock() + + c.swarm.notifyAll(func(f inet.Notifiee) { + f.OpenedStream(c.swarm, s) + }) + s.notifyLk.Unlock() + + return s, nil } -// wrapConns returns a *Conn for all these ps.Conns -func wrapConns(conns1 []*ps.Conn) []*Conn { - conns2 := make([]*Conn, len(conns1)) - for i, c1 := range conns1 { - if c2, err := wrapConn(c1); err == nil { - conns2[i] = c2 - } +// GetStreams returns the streams associated with this connection. +func (c *Conn) GetStreams() []inet.Stream { + c.streams.Lock() + defer c.streams.Unlock() + streams := make([]inet.Stream, 0, len(c.streams.m)) + for s := range c.streams.m { + streams = append(streams, s) } - return conns2 + return streams } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index d977d3752b..7125bc3cb2 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -7,10 +7,12 @@ import ( "sync" "time" + logging "github.com/ipfs/go-log" addrutil "github.com/libp2p/go-addr-util" - iconn "github.com/libp2p/go-libp2p-interface-conn" lgbl "github.com/libp2p/go-libp2p-loggables" + inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" + transport "github.com/libp2p/go-libp2p-transport" ma "github.com/multiformats/go-multiaddr" ) @@ -35,18 +37,24 @@ var ( // ErrDialToSelf is returned if we attempt to dial our own peer ErrDialToSelf = errors.New("dial to self attempted") + + // ErrNoTransport is returned when we don't know a transport for the + // given multiaddr. + ErrNoTransport = errors.New("no transport for protocol") ) -// dialAttempts governs how many times a goroutine will try to dial a given peer. +// DialAttempts governs how many times a goroutine will try to dial a given peer. // Note: this is down to one, as we have _too many dials_ atm. To add back in, // add loop back in Dial(.) -const dialAttempts = 1 +const DialAttempts = 1 -// number of concurrent outbound dials over transports that consume file descriptors -const concurrentFdDials = 160 +// ConcurrentFdDials is the number of concurrent outbound dials over transports +// that consume file descriptors +const ConcurrentFdDials = 160 -// number of concurrent outbound dials to make per peer -const defaultPerPeerRateLimit = 8 +// DefaultPerPeerRateLimit is the number of concurrent outbound dials to make +// per peer +const DefaultPerPeerRateLimit = 8 // dialbackoff is a struct used to avoid over-dialing the same, dead peers. // Whenever we totally time out on a peer (all three attempts), we add them @@ -74,7 +82,12 @@ const defaultPerPeerRateLimit = 8 // } // -type dialbackoff struct { +// DialBackoff is a type for tracking peer dial backoffs. +// +// * It's safe to use it's zero value. +// * It's thread-safe. +// * It's *not* safe to move this type after using. +type DialBackoff struct { entries map[peer.ID]*backoffPeer lock sync.RWMutex } @@ -84,7 +97,7 @@ type backoffPeer struct { until time.Time } -func (db *dialbackoff) init() { +func (db *DialBackoff) init() { if db.entries == nil { db.entries = make(map[peer.ID]*backoffPeer) } @@ -92,7 +105,7 @@ func (db *dialbackoff) init() { // Backoff returns whether the client should backoff from dialing // peer p -func (db *dialbackoff) Backoff(p peer.ID) (backoff bool) { +func (db *DialBackoff) Backoff(p peer.ID) (backoff bool) { db.lock.Lock() defer db.lock.Unlock() db.init() @@ -104,13 +117,26 @@ func (db *dialbackoff) Backoff(p peer.ID) (backoff bool) { return false } -const baseBackoffTime = time.Second * 5 -const maxBackoffTime = time.Minute * 5 +// BackoffBase is the base amount of time to backoff (default: 5s). +var BackoffBase = time.Second * 5 + +// BackoffCoef is the backoff coefficient (default: 1s). +var BackoffCoef = time.Second + +// BackoffMax is the maximum backoff time (default: 5m). +var BackoffMax = time.Minute * 5 // AddBackoff lets other nodes know that we've entered backoff with // peer p, so dialers should not wait unnecessarily. We still will // attempt to dial with one goroutine, in case we get through. -func (db *dialbackoff) AddBackoff(p peer.ID) { +// +// Backoff is not exponential, it's quadratic and computed according to the +// following formula: +// +// BackoffBase + BakoffCoef * PriorBackoffs^2 +// +// Where PriorBackoffs is the number of previous backoffs. +func (db *DialBackoff) AddBackoff(p peer.ID) { db.lock.Lock() defer db.lock.Unlock() db.init() @@ -118,61 +144,54 @@ func (db *dialbackoff) AddBackoff(p peer.ID) { if !ok { db.entries[p] = &backoffPeer{ tries: 1, - until: time.Now().Add(baseBackoffTime), + until: time.Now().Add(BackoffBase), } return } - expTimeAdd := time.Second * time.Duration(bp.tries*bp.tries) - if expTimeAdd > maxBackoffTime { - expTimeAdd = maxBackoffTime + backoffTime := BackoffBase + BackoffCoef*time.Duration(bp.tries*bp.tries) + if backoffTime > BackoffMax { + backoffTime = BackoffMax } - bp.until = time.Now().Add(baseBackoffTime + expTimeAdd) + bp.until = time.Now().Add(backoffTime) bp.tries++ } // Clear removes a backoff record. Clients should call this after a // successful Dial. -func (db *dialbackoff) Clear(p peer.ID) { +func (db *DialBackoff) Clear(p peer.ID) { db.lock.Lock() defer db.lock.Unlock() db.init() delete(db.entries, p) } -// Dial connects to a peer. +// DialPeer connects to a peer. // // The idea is that the client of Swarm does not need to know what network // the connection will happen over. Swarm can use whichever it choses. // This allows us to use various transport protocols, do NAT traversal/relay, -// etc. to achive connection. -func (s *Swarm) Dial(ctx context.Context, p peer.ID) (*Conn, error) { +// etc. to achieve connection. +func (s *Swarm) DialPeer(ctx context.Context, p peer.ID) (inet.Conn, error) { + return s.dialPeer(ctx, p) +} + +// internal dial method that returns an unwrapped conn +// +// It is gated by the swarm's dial synchronization systems: dialsync and +// dialbackoff. +func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { + log.Debugf("[%s] swarm dialing peer [%s]", s.local, p) var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) if p == s.local { log.Event(ctx, "swarmDialSelf", logdial) return nil, ErrDialToSelf } - return s.gatedDialAttempt(ctx, p) -} - -func (s *Swarm) bestConnectionToPeer(p peer.ID) *Conn { - cs := s.ConnectionsToPeer(p) - for _, conn := range cs { - if conn != nil { // dump out the first one we find. (TODO pick better) - return conn - } - } - return nil -} - -// gatedDialAttempt is an attempt to dial a node. It is gated by the swarm's -// dial synchronization systems: dialsync and dialbackoff. -func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) { defer log.EventBegin(ctx, "swarmDialAttemptSync", p).Done() // check if we already have an open connection first - conn := s.bestConnectionToPeer(p) + conn := s.bestConnToPeer(p) if conn != nil { return conn, nil } @@ -183,13 +202,26 @@ func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) return nil, ErrDialBackoff } - return s.dsync.DialLock(ctx, p) + conn, err := s.dsync.DialLock(ctx, p) + if err != nil { + return nil, err + } + log.Debugf("network for %s finished dialing %s", s.local, p) + return conn, err } // doDial is an ugly shim method to retain all the logging and backoff logic // of the old dialsync code func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { - ctx, cancel := context.WithTimeout(ctx, s.dialT) + // Short circuit. + // By the time we take the dial lock, we may already *have* a connection + // to the peer. + c := s.bestConnToPeer(p) + if c != nil { + return c, nil + } + + ctx, cancel := context.WithTimeout(ctx, DialTimeout) defer cancel() logdial := lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) @@ -200,17 +232,31 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { conn, err := s.dial(ctx, p) if err != nil { + conn = s.bestConnToPeer(p) + if conn != nil { + // Hm? What error? + // Could have canceled the dial because we received a + // connection or some other random reason. + // Just ignore the error and return the connection. + log.Debugf("ignoring dial error because we have a connection: %s", err) + return conn, nil + } if err != context.Canceled { log.Event(ctx, "swarmDialBackoffAdd", logdial) s.backf.AddBackoff(p) // let others know to backoff } - // ok, we failed. try again. (if loop is done, our error is output) + // ok, we failed. return nil, fmt.Errorf("dial attempt failed: %s", err) } return conn, nil } +func (s *Swarm) canDial(addr ma.Multiaddr) bool { + t := s.TransportForDialing(addr) + return t != nil && t.CanDial(addr) +} + // dial is the actual swarm's dial logic, gated by Dial. func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) @@ -249,8 +295,10 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { */ paddrs := s.peers.Addrs(p) goodAddrs := addrutil.FilterAddrs(paddrs, - addrutil.AddrUsableFunc, subtractFilter, + s.canDial, + // TODO: Consider allowing this? + addrutil.AddrOverNonLocalIP, addrutil.FilterNeg(s.Filters.AddrBlocked), ) remoteAddrChan := make(chan ma.Multiaddr, len(goodAddrs)) @@ -266,11 +314,11 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { logdial["error"] = err.Error() return nil, err } - logdial["netconn"] = lgbl.NetConn(connC) - - // ok try to setup the new connection. - defer log.EventBegin(ctx, "swarmDialDoSetup", logdial, lgbl.NetConn(connC)).Done() - swarmC, err := dialConnSetup(ctx, s, connC) + logdial["conn"] = logging.Metadata{ + "localAddr": connC.LocalMultiaddr(), + "remoteAddr": connC.RemoteMultiaddr(), + } + swarmC, err := s.addConn(connC) if err != nil { logdial["error"] = err.Error() connC.Close() // close the connection. didn't work out :( @@ -281,7 +329,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return swarmC, nil } -func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (iconn.Conn, error) { +func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (transport.Conn, error) { log.Debugf("%s swarm dialing %s", s.local, p) ctx, cancel := context.WithCancel(ctx) @@ -343,64 +391,31 @@ func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp }) } -func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (iconn.Conn, error) { +func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.Conn, error) { + // Just to double check. Costs nothing. + if s.local == p { + return nil, ErrDialToSelf + } log.Debugf("%s swarm dialing %s %s", s.local, p, addr) - connC, err := s.dialer.Dial(ctx, addr, p) - if err != nil { - return nil, fmt.Errorf("%s --> %s dial attempt failed: %s", s.local, p, err) + transport := s.TransportForDialing(addr) + if transport == nil { + return nil, ErrNoTransport } - // if the connection is not to whom we thought it would be... - remotep := connC.RemotePeer() - if remotep != p { - connC.Close() - _, err := connC.Read(nil) // should return any potential errors (ex: from secio) - return nil, fmt.Errorf("misdial to %s through %s (got %s): %s", p, addr, remotep, err) + connC, err := transport.Dial(ctx, addr, p) + if err != nil { + return nil, fmt.Errorf("%s --> %s dial attempt failed: %s", s.local, p, err) } - // if the connection is to ourselves... - // this can happen TONS when Loopback addrs are advertized. - // (this should be caught by two checks above, but let's just make sure.) - if remotep == s.local { + // Trust the transport? Yeah... right. + if connC.RemotePeer() != p { connC.Close() - return nil, fmt.Errorf("misdial to %s through %s (got self)", p, addr) + err = fmt.Errorf("BUG in transport %T: tried to dial %s, dialed %s", p, connC.RemotePeer(), transport) + log.Error(err) + return nil, err } // success! we got one! return connC, nil } - -var ConnSetupTimeout = time.Minute * 5 - -// dialConnSetup is the setup logic for a connection from the dial side. -func dialConnSetup(ctx context.Context, s *Swarm, connC iconn.Conn) (*Conn, error) { - deadline, ok := ctx.Deadline() - if !ok { - deadline = time.Now().Add(ConnSetupTimeout) - } - - if err := connC.SetDeadline(deadline); err != nil { - return nil, err - } - - // Add conn to ps swarm. Setup will be done by the connection handler. - psC, err := s.swarm.AddConn(connC) - if err != nil { - // connC is closed by caller if we fail. - return nil, fmt.Errorf("failed to add conn to ps.Swarm: %s", err) - } - - swarmC, err := wrapConn(psC) - if err != nil { - psC.Close() // we need to make sure psC is Closed. - return nil, err - } - - if err := connC.SetDeadline(time.Time{}); err != nil { - log.Error("failed to reset connection deadline after setup: ", err) - return nil, err - } - - return swarmC, err -} diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 148d9a8881..df0a97f248 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -1,47 +1,15 @@ package swarm import ( - "context" "fmt" - conn "github.com/libp2p/go-libp2p-conn" - iconn "github.com/libp2p/go-libp2p-interface-conn" - lgbl "github.com/libp2p/go-libp2p-loggables" - mconn "github.com/libp2p/go-libp2p-metrics/conn" inet "github.com/libp2p/go-libp2p-net" - transport "github.com/libp2p/go-libp2p-transport" - ps "github.com/libp2p/go-peerstream" ma "github.com/multiformats/go-multiaddr" ) -func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { - tpt := s.transportForAddr(a) - if tpt == nil { - return fmt.Errorf("no transport for address: %s", a) - } - - d, err := tpt.Dialer(a, transport.ReusePorts) - if err != nil { - return err - } - - s.dialer.AddDialer(d) - - list, err := tpt.Listen(a) - if err != nil { - return err - } - - err = s.addListener(list) - if err != nil { - return err - } - - return nil -} - -// Open listeners and reuse-dialers for the given addresses -func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { +// Listen sets up listeners for all of the given addresses. +// It returns as long as we successfully listen on at least *one* address. +func (s *Swarm) Listen(addrs ...ma.Multiaddr) error { errs := make([]error, len(addrs)) var succeeded int for i, a := range addrs { @@ -65,107 +33,62 @@ func (s *Swarm) setupInterfaces(addrs []ma.Multiaddr) error { return nil } -func (s *Swarm) transportForAddr(a ma.Multiaddr) transport.Transport { - for _, t := range s.transports { - if t.Matches(a) { - return t - } - } - - return nil -} - -func (s *Swarm) addListener(tptlist transport.Listener) error { - - sk := s.peers.PrivKey(s.local) - if sk == nil { - // may be fine for sk to be nil, just log a warning. - log.Warning("Listener not given PrivateKey, so WILL NOT SECURE conns.") +// AddListenAddr tells the swarm to listen on a single address. Unlike Listen, +// this method does not attempt to filter out bad addresses. +func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { + tpt := s.TransportForListening(a) + if tpt == nil { + return ErrNoTransport } - list, err := conn.WrapTransportListenerWithProtector(s.Context(), tptlist, s.local, sk, s.protec) + list, err := tpt.Listen(a) if err != nil { return err } - list.SetAddrFilters(s.Filters) - - if cw, ok := list.(conn.ListenerConnWrapper); ok && s.bwc != nil { - cw.SetConnWrapper(func(c transport.Conn) transport.Conn { - return mconn.WrapConn(s.bwc, c) - }) - } - - return s.addConnListener(list) -} - -func (s *Swarm) addConnListener(list iconn.Listener) error { - // AddListener to the peerstream Listener. this will begin accepting connections - // and streams! - sl, err := s.swarm.AddListener(list) - if err != nil { - return err + s.listeners.Lock() + if s.listeners.m == nil { + s.listeners.Unlock() + list.Close() + return ErrSwarmClosed } - log.Debugf("Swarm Listeners at %s", s.ListenAddresses()) + s.refs.Add(1) + s.listeners.m[list] = struct{}{} + s.listeners.Unlock() maddr := list.Multiaddr() // signal to our notifiees on successful conn. s.notifyAll(func(n inet.Notifiee) { - n.Listen((*Network)(s), maddr) + n.Listen(s, maddr) }) - // go consume peerstream's listen accept errors. note, these ARE errors. - // they may be killing the listener, and if we get _any_ we should be - // fixing this in our conn.Listener (to ignore them or handle them - // differently.) - go func(ctx context.Context, sl *ps.Listener) { - - // signal to our notifiees closing - defer s.notifyAll(func(n inet.Notifiee) { - n.ListenClose((*Network)(s), maddr) - }) - + go func() { + defer func() { + list.Close() + s.listeners.Lock() + delete(s.listeners.m, list) + s.listeners.Unlock() + s.refs.Done() + }() for { - select { - case err, more := <-sl.AcceptErrors(): - if !more { - return - } + c, err := list.Accept() + if err != nil { log.Warningf("swarm listener accept error: %s", err) - case <-ctx.Done(): return } + log.Debugf("swarm listener accepted connection: %s", c) + s.refs.Add(1) + go func() { + defer s.refs.Done() + _, err := s.addConn(c) + if err != nil { + // Probably just means that the swarm has been closed. + log.Warningf("add conn failed: ", err) + return + } + }() } - }(s.Context(), sl) - + }() return nil } - -// connHandler is called by the StreamSwarm whenever a new connection is added -// here we configure it slightly. Note that this is sequential, so if anything -// will take a while do it in a goroutine. -// See https://godoc.org/github.com/libp2p/go-peerstream for more information -func (s *Swarm) connHandler(c *ps.Conn) *Conn { - sc, err := wrapConn(c) - if err != nil { - log.Event(s.Context(), "newConnHandlerDisconnect", lgbl.NetConn(c.NetConn()), lgbl.Error(err)) - c.Close() // boom. close it. - return nil - } - - // Add the public key. - if pk := sc.RemotePublicKey(); pk != nil { - s.peers.AddPubKey(sc.RemotePeer(), pk) - } - - // Add the group - c.AddGroup(sc.RemotePeer()) - - // clear backoff on successful connection. - logdial := lgbl.Dial("swarm", sc.LocalPeer(), sc.RemotePeer(), nil, nil) - log.Event(s.Context(), "swarmDialBackoffClear", logdial) - s.backf.Clear(sc.RemotePeer()) - - return sc -} diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go deleted file mode 100644 index eb11ecb7da..0000000000 --- a/p2p/net/swarm/swarm_net.go +++ /dev/null @@ -1,177 +0,0 @@ -package swarm - -import ( - "context" - "fmt" - - "github.com/jbenet/goprocess" - ipnet "github.com/libp2p/go-libp2p-interface-pnet" - metrics "github.com/libp2p/go-libp2p-metrics" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - ma "github.com/multiformats/go-multiaddr" -) - -// Network implements the inet.Network interface. -// It is simply a swarm, with a few different functions -// to implement inet.Network. -type Network Swarm - -func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, - peers pstore.Peerstore, bwc metrics.Reporter) (*Network, error) { - - return NewNetworkWithProtector(ctx, listen, local, peers, nil, bwc) -} - -// NewNetwork constructs a new network and starts listening on given addresses. -func NewNetworkWithProtector(ctx context.Context, listen []ma.Multiaddr, local peer.ID, - peers pstore.Peerstore, protec ipnet.Protector, bwc metrics.Reporter) (*Network, error) { - - s, err := NewSwarmWithProtector(ctx, listen, local, peers, protec, PSTransport, bwc) - if err != nil { - return nil, err - } - - return (*Network)(s), nil -} - -// DialPeer attempts to establish a connection to a given peer. -// Respects the context. -func (n *Network) DialPeer(ctx context.Context, p peer.ID) (inet.Conn, error) { - log.Debugf("[%s] network dialing peer [%s]", n.local, p) - sc, err := n.Swarm().Dial(ctx, p) - if err != nil { - return nil, err - } - - log.Debugf("network for %s finished dialing %s", n.local, p) - return inet.Conn(sc), nil -} - -// Process returns the network's Process -func (n *Network) Process() goprocess.Process { - return n.proc -} - -// Swarm returns the network's peerstream.Swarm -func (n *Network) Swarm() *Swarm { - return (*Swarm)(n) -} - -// LocalPeer the network's LocalPeer -func (n *Network) LocalPeer() peer.ID { - return n.Swarm().LocalPeer() -} - -// Peers returns the known peer IDs from the Peerstore -func (n *Network) Peers() []peer.ID { - return n.Swarm().Peers() -} - -// Peerstore returns the Peerstore, which tracks known peers -func (n *Network) Peerstore() pstore.Peerstore { - return n.Swarm().peers -} - -// Conns returns the connected peers -func (n *Network) Conns() []inet.Conn { - conns1 := n.Swarm().Connections() - out := make([]inet.Conn, len(conns1)) - for i, c := range conns1 { - out[i] = inet.Conn(c) - } - return out -} - -// ConnsToPeer returns the connections in this Netowrk for given peer. -func (n *Network) ConnsToPeer(p peer.ID) []inet.Conn { - conns1 := n.Swarm().ConnectionsToPeer(p) - out := make([]inet.Conn, len(conns1)) - for i, c := range conns1 { - out[i] = inet.Conn(c) - } - return out -} - -// ClosePeer connection to peer -func (n *Network) ClosePeer(p peer.ID) error { - return n.Swarm().CloseConnection(p) -} - -// close is the real teardown function -func (n *Network) close() error { - return n.Swarm().Close() -} - -// Close calls the ContextCloser func -func (n *Network) Close() error { - return n.Swarm().proc.Close() -} - -// Listen tells the network to start listening on given multiaddrs. -func (n *Network) Listen(addrs ...ma.Multiaddr) error { - return n.Swarm().Listen(addrs...) -} - -// ListenAddresses returns a list of addresses at which this network listens. -func (n *Network) ListenAddresses() []ma.Multiaddr { - return n.Swarm().ListenAddresses() -} - -// InterfaceListenAddresses returns a list of addresses at which this network -// listens. It expands "any interface" addresses (/ip4/0.0.0.0, /ip6/::) to -// use the known local interfaces. -func (n *Network) InterfaceListenAddresses() ([]ma.Multiaddr, error) { - return n.Swarm().InterfaceListenAddresses() -} - -// Connectedness returns a state signaling connection capabilities -// For now only returns Connected || NotConnected. Expand into more later. -func (n *Network) Connectedness(p peer.ID) inet.Connectedness { - if n.Swarm().HaveConnsToPeer(p) { - return inet.Connected - } - return inet.NotConnected -} - -// NewStream returns a new stream to given peer p. -// If there is no connection to p, attempts to create one. -func (n *Network) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) { - log.Debugf("[%s] network opening stream to peer [%s]", n.local, p) - s, err := n.Swarm().NewStreamWithPeer(ctx, p) - if err != nil { - return nil, err - } - - return inet.Stream(s), nil -} - -// SetStreamHandler sets the protocol handler on the Network's Muxer. -// This operation is threadsafe. -func (n *Network) SetStreamHandler(h inet.StreamHandler) { - n.Swarm().SetStreamHandler(h) -} - -// SetConnHandler sets the conn handler on the Network. -// This operation is threadsafe. -func (n *Network) SetConnHandler(h inet.ConnHandler) { - n.Swarm().SetConnHandler(func(c *Conn) { - h(inet.Conn(c)) - }) -} - -// String returns a string representation of Network. -func (n *Network) String() string { - return fmt.Sprintf("", n.LocalPeer()) -} - -// Notify signs up Notifiee to receive signals when events happen -func (n *Network) Notify(f inet.Notifiee) { - n.Swarm().Notify(f) -} - -// StopNotify unregisters Notifiee fromr receiving signals -func (n *Network) StopNotify(f inet.Notifiee) { - n.Swarm().StopNotify(f) -} diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 95bc268307..425a1707d1 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -1,4 +1,4 @@ -package swarm +package swarm_test import ( "context" @@ -7,29 +7,9 @@ import ( "time" inet "github.com/libp2p/go-libp2p-net" - pstore "github.com/libp2p/go-libp2p-peerstore" - tu "github.com/libp2p/go-testutil" - ma "github.com/multiformats/go-multiaddr" -) - -func GenSwarmNetwork(t *testing.T, ctx context.Context) *Network { - p := tu.RandPeerNetParamsOrFatal(t) - ps := pstore.NewPeerstore() - ps.AddPubKey(p.ID, p.PubKey) - ps.AddPrivKey(p.ID, p.PrivKey) - n, err := NewNetwork(ctx, []ma.Multiaddr{p.Addr}, p.ID, ps, nil) - if err != nil { - t.Fatal(err) - } - ps.AddAddrs(p.ID, n.ListenAddresses(), pstore.PermanentAddrTTL) - return n -} -func DivulgeAddresses(a, b inet.Network) { - id := a.LocalPeer() - addrs := a.Peerstore().Addrs(id) - b.Peerstore().AddAddrs(id, addrs, pstore.PermanentAddrTTL) -} + . "github.com/libp2p/go-libp2p-swarm/testing" +) // TestConnectednessCorrect starts a few networks, connects a few // and tests Connectedness value is correct. @@ -39,7 +19,7 @@ func TestConnectednessCorrect(t *testing.T) { nets := make([]inet.Network, 4) for i := 0; i < 4; i++ { - nets[i] = GenSwarmNetwork(t, ctx) + nets[i] = GenSwarm(t, ctx) } // connect 0-1, 0-2, 0-3, 1-2, 2-3 @@ -130,7 +110,7 @@ func TestNetworkOpenStream(t *testing.T) { nets := make([]inet.Network, 4) for i := 0; i < 4; i++ { - nets[i] = GenSwarmNetwork(t, ctx) + nets[i] = GenSwarm(t, ctx) } dial := func(a, b inet.Network) { @@ -165,7 +145,7 @@ func TestNetworkOpenStream(t *testing.T) { t.Fatal(err) } - streams, err := nets[0].ConnsToPeer(nets[1].LocalPeer())[0].GetStreams() + streams := nets[0].ConnsToPeer(nets[1].LocalPeer())[0].GetStreams() if err != nil { t.Fatal(err) } diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 8a574aa070..d05766d6c7 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -1,4 +1,4 @@ -package swarm +package swarm_test import ( "testing" @@ -8,13 +8,9 @@ import ( inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" ma "github.com/multiformats/go-multiaddr" -) -func streamsSame(a, b inet.Stream) bool { - sa := a.(*Stream) - sb := b.(*Stream) - return sa.Stream() == sb.Stream() -} + . "github.com/libp2p/go-libp2p-swarm" +) func TestNotifications(t *testing.T) { const swarmSize = 5 @@ -52,7 +48,7 @@ func TestNotifications(t *testing.T) { } // this feels a little sketchy, but its probably okay - for len(s.ConnectionsToPeer(s2.LocalPeer())) != len(notifs[s2.LocalPeer()]) { + for len(s.ConnsToPeer(s2.LocalPeer())) != len(notifs[s2.LocalPeer()]) { select { case c := <-n.connected: nfp := notifs[c.RemotePeer()] @@ -64,7 +60,7 @@ func TestNotifications(t *testing.T) { } for p, cons := range notifs { - expect := s.ConnectionsToPeer(p) + expect := s.ConnsToPeer(p) if len(expect) != len(cons) { t.Fatal("got different number of connections") } @@ -87,10 +83,10 @@ func TestNotifications(t *testing.T) { complement := func(c inet.Conn) (*Swarm, *netNotifiee, *Conn) { for i, s := range swarms { - for _, c2 := range s.Connections() { + for _, c2 := range s.Conns() { if c.LocalMultiaddr().Equal(c2.RemoteMultiaddr()) && c2.LocalMultiaddr().Equal(c.RemoteMultiaddr()) { - return s, notifiees[i], c2 + return s, notifiees[i], c2.(*Conn) } } } @@ -106,7 +102,7 @@ func TestNotifications(t *testing.T) { case <-time.After(timeout): t.Fatal("timeout") } - if !streamsSame(s, s2) { + if s != s2 { t.Fatal("got incorrect stream", s.Conn(), s2.Conn()) } @@ -116,7 +112,7 @@ func TestNotifications(t *testing.T) { case <-time.After(timeout): t.Fatal("timeout") } - if !streamsSame(s, s2) { + if s != s2 { t.Fatal("got incorrect stream", s.Conn(), s2.Conn()) } } @@ -131,7 +127,7 @@ func TestNotifications(t *testing.T) { // open a streams in each conn for i, s := range swarms { - for _, c := range s.Connections() { + for _, c := range s.Conns() { _, n2, _ := complement(c) st1, err := c.NewStream() @@ -150,7 +146,7 @@ func TestNotifications(t *testing.T) { // close conns for i, s := range swarms { n := notifiees[i] - for _, c := range s.Connections() { + for _, c := range s.Conns() { _, n2, c2 := complement(c) c.Close() c2.Close() diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 7e83dde1d8..30b44bd28e 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -1,69 +1,167 @@ package swarm import ( + "fmt" + "io" + "sync" + "sync/atomic" "time" inet "github.com/libp2p/go-libp2p-net" protocol "github.com/libp2p/go-libp2p-protocol" - ps "github.com/libp2p/go-peerstream" + smux "github.com/libp2p/go-stream-muxer" ) -// Stream is a wrapper around a ps.Stream that exposes a way to get -// our Conn and Swarm (instead of just the ps.Conn and ps.Swarm) -type Stream ps.Stream +type streamState int -// Stream returns the underlying peerstream.Stream -func (s *Stream) Stream() *ps.Stream { - return (*ps.Stream)(s) +const ( + streamOpen streamState = iota + streamCloseRead + streamCloseWrite + streamCloseBoth + streamReset +) + +// Stream is the stream type used by swarm. In general, you won't use this type +// directly. +type Stream struct { + stream smux.Stream + conn *Conn + + state struct { + sync.Mutex + v streamState + } + + notifyLk sync.Mutex + + protocol atomic.Value } -// Conn returns the Conn associated with this Stream, as an inet.Conn -func (s *Stream) Conn() inet.Conn { - return s.SwarmConn() +func (s *Stream) String() string { + return fmt.Sprintf( + " %s (%s)>", + s.conn.conn.Transport(), + s.conn.LocalMultiaddr(), + s.conn.LocalPeer(), + s.conn.RemoteMultiaddr(), + s.conn.RemotePeer(), + ) } -// SwarmConn returns the Conn associated with this Stream, as a *Conn -func (s *Stream) SwarmConn() *Conn { - return (*Conn)(s.Stream().Conn()) +// Conn returns the Conn associated with this stream, as an inet.Conn +func (s *Stream) Conn() inet.Conn { + return s.conn } // Read reads bytes from a stream. -func (s *Stream) Read(p []byte) (n int, err error) { - return s.Stream().Read(p) +func (s *Stream) Read(p []byte) (int, error) { + n, err := s.stream.Read(p) + // TODO: push this down to a lower level for better accuracy. + if s.conn.swarm.bwc != nil { + s.conn.swarm.bwc.LogRecvMessage(int64(n)) + s.conn.swarm.bwc.LogRecvMessageStream(int64(n), s.Protocol(), s.Conn().RemotePeer()) + } + // If we observe an EOF, this stream is now closed for reading. + // If we're already closed for writing, this stream is now fully closed. + if err == io.EOF { + s.state.Lock() + switch s.state.v { + case streamCloseWrite: + s.state.v = streamCloseBoth + s.remove() + case streamOpen: + s.state.v = streamCloseRead + } + s.state.Unlock() + } + return n, err } // Write writes bytes to a stream, flushing for each call. -func (s *Stream) Write(p []byte) (n int, err error) { - return s.Stream().Write(p) +func (s *Stream) Write(p []byte) (int, error) { + n, err := s.stream.Write(p) + // TODO: push this down to a lower level for better accuracy. + if s.conn.swarm.bwc != nil { + s.conn.swarm.bwc.LogSentMessage(int64(n)) + s.conn.swarm.bwc.LogSentMessageStream(int64(n), s.Protocol(), s.Conn().RemotePeer()) + } + return n, err } // Close closes the stream, indicating this side is finished // with the stream. func (s *Stream) Close() error { - return s.Stream().Close() + err := s.stream.Close() + + s.state.Lock() + switch s.state.v { + case streamCloseRead: + s.state.v = streamCloseBoth + s.remove() + case streamOpen: + s.state.v = streamCloseWrite + } + s.state.Unlock() + return err } // Reset resets the stream, closing both ends. func (s *Stream) Reset() error { - return s.Stream().Reset() + err := s.stream.Reset() + s.state.Lock() + switch s.state.v { + case streamOpen, streamCloseRead, streamCloseWrite: + s.state.v = streamReset + s.remove() + } + s.state.Unlock() + return err +} + +func (s *Stream) remove() { + s.conn.removeStream(s) + + // We *must* do this in a goroutine. This can be called during a + // an open notification and will block until that notification is done. + go func() { + s.notifyLk.Lock() + defer s.notifyLk.Unlock() + + s.conn.swarm.notifyAll(func(f inet.Notifiee) { + f.ClosedStream(s.conn.swarm, s) + }) + s.conn.swarm.refs.Done() + }() } +// Protocol returns the protocol negotiated on this stream (if set). func (s *Stream) Protocol() protocol.ID { - return (*ps.Stream)(s).Protocol() + // Ignore type error. It means that the protocol is unset. + p, _ := s.protocol.Load().(protocol.ID) + return p } +// SetProtocol sets the protocol for this stream. +// +// This doesn't actually *do* anything other than record the fact that we're +// speaking the given protocol over this stream. It's still up to the user to +// negotiate the protocol. This is usually done by the Host. func (s *Stream) SetProtocol(p protocol.ID) { - (*ps.Stream)(s).SetProtocol(p) + s.protocol.Store(p) } +// SetDeadline sets the read and write deadlines for this stream. func (s *Stream) SetDeadline(t time.Time) error { - return s.Stream().SetDeadline(t) + return s.stream.SetDeadline(t) } +// SetReadDeadline sets the read deadline for this stream. func (s *Stream) SetReadDeadline(t time.Time) error { - return s.Stream().SetReadDeadline(t) + return s.stream.SetReadDeadline(t) } +// SetWriteDeadline sets the write deadline for this stream. func (s *Stream) SetWriteDeadline(t time.Time) error { - return s.Stream().SetWriteDeadline(t) + return s.stream.SetWriteDeadline(t) } diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index e14de3faad..4911fc96a1 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -1,4 +1,4 @@ -package swarm +package swarm_test import ( "bytes" @@ -10,14 +10,18 @@ import ( "testing" "time" - metrics "github.com/libp2p/go-libp2p-metrics" + logging "github.com/ipfs/go-log" inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" pstore "github.com/libp2p/go-libp2p-peerstore" - testutil "github.com/libp2p/go-testutil" ma "github.com/multiformats/go-multiaddr" + + . "github.com/libp2p/go-libp2p-swarm" + . "github.com/libp2p/go-libp2p-swarm/testing" ) +var log = logging.Logger("swarm_test") + func EchoStreamHandler(stream inet.Stream) { go func() { defer stream.Close() @@ -50,38 +54,17 @@ func EchoStreamHandler(stream inet.Stream) { } func makeDialOnlySwarm(ctx context.Context, t *testing.T) *Swarm { - id := testutil.RandIdentityOrFatal(t) - - peerstore := pstore.NewPeerstore() - peerstore.AddPubKey(id.ID(), id.PublicKey()) - peerstore.AddPrivKey(id.ID(), id.PrivateKey()) - - swarm, err := NewSwarm(ctx, nil, id.ID(), peerstore, metrics.NewBandwidthCounter()) - if err != nil { - t.Fatal(err) - } - + swarm := GenSwarm(t, ctx, OptDialOnly) swarm.SetStreamHandler(EchoStreamHandler) return swarm } -func makeSwarms(ctx context.Context, t *testing.T, num int) []*Swarm { +func makeSwarms(ctx context.Context, t *testing.T, num int, opts ...Option) []*Swarm { swarms := make([]*Swarm, 0, num) for i := 0; i < num; i++ { - localnp := testutil.RandPeerNetParamsOrFatal(t) - - peerstore := pstore.NewPeerstore() - peerstore.AddPubKey(localnp.ID, localnp.PubKey) - peerstore.AddPrivKey(localnp.ID, localnp.PrivKey) - - addrs := []ma.Multiaddr{localnp.Addr} - swarm, err := NewSwarm(ctx, addrs, localnp.ID, peerstore, metrics.NewBandwidthCounter()) - if err != nil { - t.Fatal(err) - } - + swarm := GenSwarm(t, ctx, opts...) swarm.SetStreamHandler(EchoStreamHandler) swarms = append(swarms, swarm) } @@ -94,8 +77,8 @@ func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { var wg sync.WaitGroup connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. - s.peers.AddAddr(dst, addr, pstore.PermanentAddrTTL) - if _, err := s.Dial(ctx, dst); err != nil { + s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL) + if _, err := s.DialPeer(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } wg.Done() @@ -111,7 +94,7 @@ func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { wg.Wait() for _, s := range swarms { - log.Infof("%s swarm routing table: %s", s.local, s.Peers()) + log.Infof("%s swarm routing table: %s", s.LocalPeer(), s.Peers()) } } @@ -119,7 +102,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { // t.Skip("skipping for another test") ctx := context.Background() - swarms := makeSwarms(ctx, t, SwarmNum) + swarms := makeSwarms(ctx, t, SwarmNum, OptDisableReuseport) // connect everyone connectSwarms(t, ctx, swarms) @@ -127,13 +110,13 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { // ping/pong for _, s1 := range swarms { log.Debugf("-------------------------------------------------------") - log.Debugf("%s ping pong round", s1.local) + log.Debugf("%s ping pong round", s1.LocalPeer()) log.Debugf("-------------------------------------------------------") _, cancel := context.WithCancel(ctx) got := map[peer.ID]int{} errChan := make(chan error, MsgNum*len(swarms)) - streamChan := make(chan *Stream, MsgNum) + streamChan := make(chan inet.Stream, MsgNum) // send out "ping" x MsgNum to every peer go func() { @@ -144,7 +127,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { defer wg.Done() // first, one stream per peer (nice) - stream, err := s1.NewStreamWithPeer(ctx, p) + stream, err := s1.NewStream(ctx, p) if err != nil { errChan <- err return @@ -153,7 +136,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { // send out ping! for k := 0; k < MsgNum; k++ { // with k messages msg := "ping" - log.Debugf("%s %s %s (%d)", s1.local, msg, p, k) + log.Debugf("%s %s %s (%d)", s1.LocalPeer(), msg, p, k) if _, err := stream.Write([]byte(msg)); err != nil { errChan <- err continue @@ -165,12 +148,12 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { } for _, s2 := range swarms { - if s2.local == s1.local { + if s2.LocalPeer() == s1.LocalPeer() { continue // dont send to self... } wg.Add(1) - go send(s2.local) + go send(s2.LocalPeer()) } wg.Wait() }() @@ -202,7 +185,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { continue } - log.Debugf("%s %s %s (%d)", s1.local, msg, p, k) + log.Debugf("%s %s %s (%d)", s1.LocalPeer(), msg, p, k) msgCount++ } @@ -222,7 +205,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { } } - log.Debugf("%s got pongs", s1.local) + log.Debugf("%s got pongs", s1.LocalPeer()) if (len(swarms) - 1) != len(got) { t.Errorf("got (%d) less messages than sent (%d).", len(got), len(swarms)) } @@ -269,7 +252,7 @@ func TestConnHandler(t *testing.T) { swarms := makeSwarms(ctx, t, 5) gotconn := make(chan struct{}, 10) - swarms[0].SetConnHandler(func(conn *Conn) { + swarms[0].SetConnHandler(func(conn inet.Conn) { gotconn <- struct{}{} }) @@ -300,7 +283,7 @@ func TestAddrBlocking(t *testing.T) { ctx := context.Background() swarms := makeSwarms(ctx, t, 2) - swarms[0].SetConnHandler(func(conn *Conn) { + swarms[0].SetConnHandler(func(conn inet.Conn) { t.Errorf("no connections should happen! -- %s", conn) }) @@ -311,14 +294,14 @@ func TestAddrBlocking(t *testing.T) { swarms[1].Filters.AddDialFilter(block) - swarms[1].peers.AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], pstore.PermanentAddrTTL) - _, err = swarms[1].Dial(ctx, swarms[0].LocalPeer()) + swarms[1].Peerstore().AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], pstore.PermanentAddrTTL) + _, err = swarms[1].DialPeer(ctx, swarms[0].LocalPeer()) if err == nil { t.Fatal("dial should have failed") } - swarms[0].peers.AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], pstore.PermanentAddrTTL) - _, err = swarms[0].Dial(ctx, swarms[1].LocalPeer()) + swarms[0].Peerstore().AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], pstore.PermanentAddrTTL) + _, err = swarms[0].DialPeer(ctx, swarms[1].LocalPeer()) if err == nil { t.Fatal("dial should have failed") } @@ -329,7 +312,7 @@ func TestFilterBounds(t *testing.T) { swarms := makeSwarms(ctx, t, 2) conns := make(chan struct{}, 8) - swarms[0].SetConnHandler(func(conn *Conn) { + swarms[0].SetConnHandler(func(conn inet.Conn) { conns <- struct{}{} }) diff --git a/p2p/net/swarm/swarm_transport.go b/p2p/net/swarm/swarm_transport.go new file mode 100644 index 0000000000..bc60dcf993 --- /dev/null +++ b/p2p/net/swarm/swarm_transport.go @@ -0,0 +1,87 @@ +package swarm + +import ( + "fmt" + "strings" + + transport "github.com/libp2p/go-libp2p-transport" + ma "github.com/multiformats/go-multiaddr" +) + +// TransportForDialing retrieves the appropriate transport for dialing the given +// multiaddr. +func (s *Swarm) TransportForDialing(a ma.Multiaddr) transport.Transport { + protocols := a.Protocols() + if len(protocols) == 0 { + return nil + } + + s.transports.RLock() + defer s.transports.RUnlock() + for _, p := range protocols { + transport, ok := s.transports.m[p.Code] + if !ok { + continue + } + if transport.Proxy() { + return transport + } + } + + return s.transports.m[protocols[len(protocols)-1].Code] +} + +// TransportForListening retrieves the appropriate transport for listening on +// the given multiaddr. +func (s *Swarm) TransportForListening(a ma.Multiaddr) transport.Transport { + protocols := a.Protocols() + if len(protocols) == 0 { + return nil + } + + s.transports.RLock() + defer s.transports.RUnlock() + selected := s.transports.m[protocols[len(protocols)-1].Code] + for _, p := range protocols { + transport, ok := s.transports.m[p.Code] + if !ok { + continue + } + if transport.Proxy() { + selected = transport + } + } + return selected +} + +// AddTransport adds a transport to this swarm. +// +// Satisfies the Network interface from go-libp2p-transport. +func (s *Swarm) AddTransport(t transport.Transport) error { + protocols := t.Protocols() + + s.transports.Lock() + defer s.transports.Unlock() + var registered []string + for _, p := range protocols { + if _, ok := s.transports.m[p]; ok { + proto := ma.ProtocolWithCode(p) + name := proto.Name + if name == "" { + name = fmt.Sprintf("unknown (%d)", p) + } + registered = append(registered, name) + } + } + if len(registered) > 0 { + return fmt.Errorf( + "transports already registered for protocol(s): %s", + strings.Join(registered, ", "), + ) + } + + for _, p := range protocols { + s.transports.m[p] = t + } + return nil +} diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go new file mode 100644 index 0000000000..29017df808 --- /dev/null +++ b/p2p/net/swarm/testing/testing.go @@ -0,0 +1,97 @@ +package testing + +import ( + "context" + "testing" + + csms "github.com/libp2p/go-conn-security-multistream" + metrics "github.com/libp2p/go-libp2p-metrics" + inet "github.com/libp2p/go-libp2p-net" + pstore "github.com/libp2p/go-libp2p-peerstore" + secio "github.com/libp2p/go-libp2p-secio" + tptu "github.com/libp2p/go-libp2p-transport-upgrader" + tcp "github.com/libp2p/go-tcp-transport" + tu "github.com/libp2p/go-testutil" + msmux "github.com/whyrusleeping/go-smux-multistream" + yamux "github.com/whyrusleeping/go-smux-yamux" + + swarm "github.com/libp2p/go-libp2p-swarm" +) + +type config struct { + disableReuseport bool + dialOnly bool +} + +// Option is an option that can be passed when constructing a test swarm. +type Option func(*testing.T, *config) + +// OptDisableReuseport disables reuseport in this test swarm. +var OptDisableReuseport Option = func(_ *testing.T, c *config) { + c.disableReuseport = true +} + +// OptDialOnly prevents the test swarm from listening. +var OptDialOnly Option = func(_ *testing.T, c *config) { + c.dialOnly = true +} + +// GenUpgrader creates a new connection upgrader for use with this swarm. +func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { + id := n.LocalPeer() + pk := n.Peerstore().PrivKey(id) + secMuxer := new(csms.SSMuxer) + secMuxer.AddTransport(secio.ID, &secio.Transport{ + LocalID: id, + PrivateKey: pk, + }) + + stMuxer := msmux.NewBlankTransport() + stMuxer.AddTransport("/yamux/1.0.0", yamux.DefaultTransport) + + return &tptu.Upgrader{ + Secure: secMuxer, + Muxer: stMuxer, + Filters: n.Filters, + } + +} + +// GenSwarm generates a new test swarm. +func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { + var cfg config + for _, o := range opts { + o(t, &cfg) + } + + p := tu.RandPeerNetParamsOrFatal(t) + + ps := pstore.NewPeerstore() + ps.AddPubKey(p.ID, p.PubKey) + ps.AddPrivKey(p.ID, p.PrivKey) + s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter()) + + tcpTransport := tcp.NewTCPTransport(GenUpgrader(s)) + tcpTransport.DisableReuseport = cfg.disableReuseport + + if err := s.AddTransport(tcpTransport); err != nil { + t.Fatal(err) + } + + if !cfg.dialOnly { + if err := s.Listen(p.Addr); err != nil { + t.Fatal(err) + } + + s.Peerstore().AddAddrs(p.ID, s.ListenAddresses(), pstore.PermanentAddrTTL) + } + + return s +} + +// DivulgeAddresses adds swarm a's addresses to swarm b's peerstore. +func DivulgeAddresses(a, b inet.Network) { + id := a.LocalPeer() + addrs := a.Peerstore().Addrs(id) + b.Peerstore().AddAddrs(id, addrs, pstore.PermanentAddrTTL) +} From f70bce27236ec9fa16bc66ac5be91962c485e5be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Jun 2018 18:47:58 +0200 Subject: [PATCH 102/259] feat: lower timeout for local address dials --- p2p/net/swarm/addrs.go | 35 +++++++++++++++++++++++++++++++++++ p2p/net/swarm/limiter.go | 15 ++++++++++++++- p2p/net/swarm/swarm.go | 8 +++++++- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 p2p/net/swarm/addrs.go diff --git a/p2p/net/swarm/addrs.go b/p2p/net/swarm/addrs.go new file mode 100644 index 0000000000..ed510f2626 --- /dev/null +++ b/p2p/net/swarm/addrs.go @@ -0,0 +1,35 @@ +package swarm + +import ( + mafilter "github.com/libp2p/go-maddr-filter" + mamask "github.com/whyrusleeping/multiaddr-filter" +) + +// http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +var lowTimeoutFilters = mafilter.NewFilters() + +func init() { + for _, p := range []string{ + "/ip4/10.0.0.0/ipcidr/8", + "/ip4/100.64.0.0/ipcidr/10", + "/ip4/169.254.0.0/ipcidr/16", + "/ip4/172.16.0.0/ipcidr/12", + "/ip4/192.0.0.0/ipcidr/24", + "/ip4/192.0.0.0/ipcidr/29", + "/ip4/192.0.0.8/ipcidr/32", + "/ip4/192.0.0.170/ipcidr/32", + "/ip4/192.0.0.171/ipcidr/32", + "/ip4/192.0.2.0/ipcidr/24", + "/ip4/192.168.0.0/ipcidr/16", + "/ip4/198.18.0.0/ipcidr/15", + "/ip4/198.51.100.0/ipcidr/24", + "/ip4/203.0.113.0/ipcidr/24", + "/ip4/240.0.0.0/ipcidr/4", + } { + f, err := mamask.NewMask(p) + if err != nil { + panic("error in lowTimeoutFilters init: " + err.Error()) + } + lowTimeoutFilters.AddDialFilter(f) + } +} diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 3d3437b95f..a740e27efd 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -3,6 +3,7 @@ package swarm import ( "context" "sync" + "time" addrutil "github.com/libp2p/go-addr-util" peer "github.com/libp2p/go-libp2p-peer" @@ -33,6 +34,15 @@ func (dj *dialJob) cancelled() bool { } } +func (dj *dialJob) dialTimeout() time.Duration { + timeout := DialTimeout + if lowTimeoutFilters.AddrBlocked(dj.addr) { + timeout = DialTimeoutLocal + } + + return timeout +} + type dialLimiter struct { rllock sync.Mutex fdConsuming int @@ -162,7 +172,10 @@ func (dl *dialLimiter) executeDial(j *dialJob) { return } - con, err := dl.dialFunc(j.ctx, j.peer, j.addr) + dctx, cancel := context.WithTimeout(j.ctx, j.dialTimeout()) + defer cancel() + + con, err := dl.dialFunc(dctx, j.peer, j.addr) select { case j.resp <- dialResult{Conn: con, Addr: j.addr, Err: err}: case <-j.ctx.Done(): diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index dbf81613f9..002abd2d9f 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -22,9 +22,15 @@ import ( ) // DialTimeout is the maximum duration a Dial is allowed to take. +// This includes the time spent waiting in dial limiter, between dialing the raw +// network connection, protocol selection as well the handshake, if applicable. +var DialTimeout = 60 * time.Second + +// DialTimeoutLocal is the maximum duration a Dial to local network address +// is allowed to take. // This includes the time between dialing the raw network connection, // protocol selection as well the handshake, if applicable. -var DialTimeout = 60 * time.Second +var DialTimeoutLocal = 5 * time.Second var log = logging.Logger("swarm2") From af53412341665ec5c178caca54336e8dad2c5288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Jun 2018 00:30:55 +0200 Subject: [PATCH 103/259] limiter: cleanup the code --- p2p/net/swarm/limiter.go | 107 ++++++++++++++++++---------------- p2p/net/swarm/limiter_test.go | 4 +- 2 files changed, 59 insertions(+), 52 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 3d3437b95f..1d9ff1a2ab 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -34,12 +34,13 @@ func (dj *dialJob) cancelled() bool { } type dialLimiter struct { - rllock sync.Mutex + lk sync.Mutex + fdConsuming int fdLimit int waitingOnFd []*dialJob - dialFunc func(context.Context, peer.ID, ma.Multiaddr) (transport.Conn, error) + dialFunc dialfunc activePerPeer map[peer.ID]int perPeerLimit int @@ -62,26 +63,26 @@ func newDialLimiterWithParams(df dialfunc, fdLimit, perPeerLimit int) *dialLimit } } -func (dl *dialLimiter) finishedDial(dj *dialJob) { - dl.rllock.Lock() - defer dl.rllock.Unlock() - - if addrutil.IsFDCostlyTransport(dj.addr) { - dl.fdConsuming-- - - if len(dl.waitingOnFd) > 0 { - next := dl.waitingOnFd[0] - dl.waitingOnFd[0] = nil // clear out memory - dl.waitingOnFd = dl.waitingOnFd[1:] - if len(dl.waitingOnFd) == 0 { - dl.waitingOnFd = nil // clear out memory - } - dl.fdConsuming++ - - go dl.executeDial(next) +// freeFDToken frees FD token and if there are any schedules another waiting dialJob +// in it's place +func (dl *dialLimiter) freeFDToken() { + dl.fdConsuming-- + + if len(dl.waitingOnFd) > 0 { + next := dl.waitingOnFd[0] + dl.waitingOnFd[0] = nil // clear out memory + dl.waitingOnFd = dl.waitingOnFd[1:] + if len(dl.waitingOnFd) == 0 { + dl.waitingOnFd = nil // clear out memory } + dl.fdConsuming++ + + // we already have activePerPeer token at this point so we can just dial + go dl.executeDial(next) } +} +func (dl *dialLimiter) freePeerToken(dj *dialJob) { // release tokens in reverse order than we take them dl.activePerPeer[dj.peer]-- if dl.activePerPeer[dj.peer] == 0 { @@ -91,45 +92,31 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { waitlist := dl.waitingOnPeerLimit[dj.peer] if !dj.success && len(waitlist) > 0 { next := waitlist[0] - if len(waitlist) == 1 { - delete(dl.waitingOnPeerLimit, dj.peer) + delete(dl.waitingOnPeerLimit, next.peer) } else { waitlist[0] = nil // clear out memory - dl.waitingOnPeerLimit[dj.peer] = waitlist[1:] + dl.waitingOnPeerLimit[next.peer] = waitlist[1:] } - dl.activePerPeer[dj.peer]++ // just kidding, we still want this token - if addrutil.IsFDCostlyTransport(next.addr) { - if dl.fdConsuming >= dl.fdLimit { - dl.waitingOnFd = append(dl.waitingOnFd, next) - return - } - - // take token - dl.fdConsuming++ - } + dl.activePerPeer[next.peer]++ // just kidding, we still want this token - // can kick this off right here, dials in this list already - // have the other tokens needed - go dl.executeDial(next) + dl.addCheckFdLimit(next) } } -// AddDialJob tries to take the needed tokens for starting the given dial job. -// If it acquires all needed tokens, it immediately starts the dial, otherwise -// it will put it on the waitlist for the requested token. -func (dl *dialLimiter) AddDialJob(dj *dialJob) { - dl.rllock.Lock() - defer dl.rllock.Unlock() +func (dl *dialLimiter) finishedDial(dj *dialJob) { + dl.lk.Lock() + defer dl.lk.Unlock() - if dl.activePerPeer[dj.peer] >= dl.perPeerLimit { - wlist := dl.waitingOnPeerLimit[dj.peer] - dl.waitingOnPeerLimit[dj.peer] = append(wlist, dj) - return + if addrutil.IsFDCostlyTransport(dj.addr) { + dl.freeFDToken() } - dl.activePerPeer[dj.peer]++ + dl.freePeerToken(dj) +} + +func (dl *dialLimiter) addCheckFdLimit(dj *dialJob) { if addrutil.IsFDCostlyTransport(dj.addr) { if dl.fdConsuming >= dl.fdLimit { dl.waitingOnFd = append(dl.waitingOnFd, dj) @@ -140,15 +127,35 @@ func (dl *dialLimiter) AddDialJob(dj *dialJob) { dl.fdConsuming++ } - // take second needed token and start dial! go dl.executeDial(dj) } +func (dl *dialLimiter) addCheckPeerLimit(dj *dialJob) { + if dl.activePerPeer[dj.peer] >= dl.perPeerLimit { + wlist := dl.waitingOnPeerLimit[dj.peer] + dl.waitingOnPeerLimit[dj.peer] = append(wlist, dj) + return + } + dl.activePerPeer[dj.peer]++ + + dl.addCheckFdLimit(dj) +} + +// AddDialJob tries to take the needed tokens for starting the given dial job. +// If it acquires all needed tokens, it immediately starts the dial, otherwise +// it will put it on the waitlist for the requested token. +func (dl *dialLimiter) AddDialJob(dj *dialJob) { + dl.lk.Lock() + defer dl.lk.Unlock() + + dl.addCheckPeerLimit(dj) +} + func (dl *dialLimiter) clearAllPeerDials(p peer.ID) { - dl.rllock.Lock() - defer dl.rllock.Unlock() + dl.lk.Lock() + defer dl.lk.Unlock() delete(dl.waitingOnPeerLimit, p) - // NB: the waitingOnFd list doesnt need to be cleaned out here, we will + // NB: the waitingOnFd list doesn't need to be cleaned out here, we will // remove them as we encounter them because they are 'cancelled' at this // point } diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 6ae3935caf..4338cad50a 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -378,9 +378,9 @@ func TestFDLimitUnderflow(t *testing.T) { time.Sleep(time.Second * 3) - l.rllock.Lock() + l.lk.Lock() fdConsuming := l.fdConsuming - l.rllock.Unlock() + l.lk.Unlock() if fdConsuming < 0 { t.Fatalf("l.fdConsuming < 0") From ebc7440631bfeccc4cd4da7b92d1f597a7827691 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 8 Jun 2018 10:23:07 -0700 Subject: [PATCH 104/259] add a per-dial transport-level dial timeout --- p2p/net/swarm/dial_test.go | 27 ++++++++++++++------------- p2p/net/swarm/swarm.go | 10 +++++----- p2p/net/swarm/swarm_dial.go | 17 ++++++++++------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index a7eebcea1f..dd93bd7cec 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -11,6 +11,7 @@ import ( peer "github.com/libp2p/go-libp2p-peer" pstore "github.com/libp2p/go-libp2p-peerstore" swarmt "github.com/libp2p/go-libp2p-swarm/testing" + transport "github.com/libp2p/go-libp2p-transport" testutil "github.com/libp2p/go-testutil" ci "github.com/libp2p/go-testutil/ci" ma "github.com/multiformats/go-multiaddr" @@ -20,7 +21,7 @@ import ( ) func init() { - DialTimeout = time.Second + transport.DialTimeout = time.Second } func closeSwarms(swarms []*Swarm) { @@ -190,11 +191,11 @@ func TestDialWait(t *testing.T) { } duration := time.Since(before) - if duration < DialTimeout*DialAttempts { - t.Error("< DialTimeout * DialAttempts not being respected", duration, DialTimeout*DialAttempts) + if duration < transport.DialTimeout*DialAttempts { + t.Error("< transport.DialTimeout * DialAttempts not being respected", duration, transport.DialTimeout*DialAttempts) } - if duration > 2*DialTimeout*DialAttempts { - t.Error("> 2*DialTimeout * DialAttempts not being respected", duration, 2*DialTimeout*DialAttempts) + if duration > 2*transport.DialTimeout*DialAttempts { + t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) } if !s1.Backoff().Backoff(s2p) { @@ -278,8 +279,8 @@ func TestDialBackoff(t *testing.T) { s3done := dialOfflineNode(s3p, N) // when all dials should be done by: - dialTimeout1x := time.After(DialTimeout) - dialTimeout10Ax := time.After(DialTimeout * 2 * 10) // DialAttempts * 10) + dialTimeout1x := time.After(transport.DialTimeout) + dialTimeout10Ax := time.After(transport.DialTimeout * 2 * 10) // DialAttempts * 10) // 2) all dials should hang select { @@ -361,8 +362,8 @@ func TestDialBackoff(t *testing.T) { s3done := dialOfflineNode(s3p, N) // when all dials should be done by: - dialTimeout1x := time.After(DialTimeout) - dialTimeout10Ax := time.After(DialTimeout * 2 * 10) // DialAttempts * 10) + dialTimeout1x := time.After(transport.DialTimeout) + dialTimeout10Ax := time.After(transport.DialTimeout * 2 * 10) // DialAttempts * 10) // 7) s3 dials should all return immediately (except 1) for i := 0; i < N-1; i++ { @@ -441,11 +442,11 @@ func TestDialBackoffClears(t *testing.T) { } duration := time.Since(before) - if duration < DialTimeout*DialAttempts { - t.Error("< DialTimeout * DialAttempts not being respected", duration, DialTimeout*DialAttempts) + if duration < transport.DialTimeout*DialAttempts { + t.Error("< transport.DialTimeout * DialAttempts not being respected", duration, transport.DialTimeout*DialAttempts) } - if duration > 2*DialTimeout*DialAttempts { - t.Error("> 2*DialTimeout * DialAttempts not being respected", duration, 2*DialTimeout*DialAttempts) + if duration > 2*transport.DialTimeout*DialAttempts { + t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) } if !s1.Backoff().Backoff(s2.LocalPeer()) { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 002abd2d9f..0a50bed9e6 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -21,11 +21,6 @@ import ( mafilter "github.com/whyrusleeping/multiaddr-filter" ) -// DialTimeout is the maximum duration a Dial is allowed to take. -// This includes the time spent waiting in dial limiter, between dialing the raw -// network connection, protocol selection as well the handshake, if applicable. -var DialTimeout = 60 * time.Second - // DialTimeoutLocal is the maximum duration a Dial to local network address // is allowed to take. // This includes the time between dialing the raw network connection, @@ -42,6 +37,11 @@ var ErrSwarmClosed = errors.New("swarm closed") // transport is misbehaving. var ErrAddrFiltered = errors.New("address filtered") +// DialTimeout is the maximum duration a Dial is allowed to take. +// This includes the time between dialing the raw network connection, +// protocol selection as well the handshake, if applicable. +var DialTimeout = 60 * time.Second + // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 7125bc3cb2..fe7a4cbfd5 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -221,9 +221,6 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { return c, nil } - ctx, cancel := context.WithTimeout(ctx, DialTimeout) - defer cancel() - logdial := lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) // ok, we have been charged to dial! let's do it. @@ -259,6 +256,9 @@ func (s *Swarm) canDial(addr ma.Multiaddr) bool { // dial is the actual swarm's dial logic, gated by Dial. func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { + ctx, cancel := context.WithTimeout(ctx, DialTimeout) + defer cancel() + var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) if p == s.local { log.Event(ctx, "swarmDialDoDialSelf", logdial) @@ -398,12 +398,15 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (tra } log.Debugf("%s swarm dialing %s %s", s.local, p, addr) - transport := s.TransportForDialing(addr) - if transport == nil { + tpt := s.TransportForDialing(addr) + if tpt == nil { return nil, ErrNoTransport } - connC, err := transport.Dial(ctx, addr, p) + ctx, cancel := context.WithTimeout(ctx, transport.DialTimeout) + defer cancel() + + connC, err := tpt.Dial(ctx, addr, p) if err != nil { return nil, fmt.Errorf("%s --> %s dial attempt failed: %s", s.local, p, err) } @@ -411,7 +414,7 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (tra // Trust the transport? Yeah... right. if connC.RemotePeer() != p { connC.Close() - err = fmt.Errorf("BUG in transport %T: tried to dial %s, dialed %s", p, connC.RemotePeer(), transport) + err = fmt.Errorf("BUG in transport %T: tried to dial %s, dialed %s", p, connC.RemotePeer(), tpt) log.Error(err) return nil, err } From d2b407f84ddaf3a2572b3242b147b1e36ecf9a94 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 11 Jun 2018 09:14:37 -0700 Subject: [PATCH 105/259] don't hang tests on dial failure --- p2p/net/swarm/simul_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 22b0dbf330..b162ab380f 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -27,13 +27,13 @@ func TestSimultOpen(t *testing.T) { { var wg sync.WaitGroup connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + defer wg.Done() // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL) if _, err := s.DialPeer(ctx, dst); err != nil { - t.Fatal("error swarm dialing to peer", err) + t.Error("error swarm dialing to peer", err) } - wg.Done() } log.Info("Connecting swarms simultaneously.") From cbd2028e00fb9926ee9b0f1b49b560b5a44e4868 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 14 Jun 2018 13:26:42 -0700 Subject: [PATCH 106/259] remove unused success boolean --- p2p/net/swarm/limiter.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 32b75cf6b6..391b828f01 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -18,11 +18,10 @@ type dialResult struct { } type dialJob struct { - addr ma.Multiaddr - peer peer.ID - ctx context.Context - resp chan dialResult - success bool + addr ma.Multiaddr + peer peer.ID + ctx context.Context + resp chan dialResult } func (dj *dialJob) cancelled() bool { @@ -100,7 +99,7 @@ func (dl *dialLimiter) freePeerToken(dj *dialJob) { } waitlist := dl.waitingOnPeerLimit[dj.peer] - if !dj.success && len(waitlist) > 0 { + if len(waitlist) > 0 { next := waitlist[0] if len(waitlist) == 1 { delete(dl.waitingOnPeerLimit, next.peer) From c30ac0e7437909a2c0db367a37b05f59d76c7013 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 14 Jun 2018 15:48:25 -0700 Subject: [PATCH 107/259] switch to a per DialPeer/NewStream timeout defined in go-libp2p-net The global per-peer dial timeout had a significant drawback: When dialing many peers, this timeout could cause libp2p to cancel dials while they were still stuck in the limiter. A better but more complicated approach is a time budget system but we can implement that later. This change simply applies the limit to each `DialPeer`/`NewStream` call independently and makes it easy to override. While old timeout tried to account for how much we're willing to spend dialing a single peer, this new timeout tries to account for the amount of time a single "client" is willing to wait for a dial to complete before they no longer care. --- p2p/net/swarm/limiter.go | 2 +- p2p/net/swarm/swarm.go | 5 ----- p2p/net/swarm/swarm_dial.go | 8 +++++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 391b828f01..02aed50a2e 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -34,7 +34,7 @@ func (dj *dialJob) cancelled() bool { } func (dj *dialJob) dialTimeout() time.Duration { - timeout := DialTimeout + timeout := transport.DialTimeout if lowTimeoutFilters.AddrBlocked(dj.addr) { timeout = DialTimeoutLocal } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 0a50bed9e6..7d795abd86 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -37,11 +37,6 @@ var ErrSwarmClosed = errors.New("swarm closed") // transport is misbehaving. var ErrAddrFiltered = errors.New("address filtered") -// DialTimeout is the maximum duration a Dial is allowed to take. -// This includes the time between dialing the raw network connection, -// protocol selection as well the handshake, if applicable. -var DialTimeout = 60 * time.Second - // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index fe7a4cbfd5..3e0cfafac2 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -202,10 +202,15 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { return nil, ErrDialBackoff } + // apply the DialPeer timeout + ctx, cancel := context.WithTimeout(ctx, inet.GetDialPeerTimeout(ctx)) + defer cancel() + conn, err := s.dsync.DialLock(ctx, p) if err != nil { return nil, err } + log.Debugf("network for %s finished dialing %s", s.local, p) return conn, err } @@ -256,9 +261,6 @@ func (s *Swarm) canDial(addr ma.Multiaddr) bool { // dial is the actual swarm's dial logic, gated by Dial. func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { - ctx, cancel := context.WithTimeout(ctx, DialTimeout) - defer cancel() - var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) if p == s.local { log.Event(ctx, "swarmDialDoDialSelf", logdial) From eb986b0c00f21555e6916eb437ed123dcd5b09dc Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 25 Jun 2018 20:47:44 -0700 Subject: [PATCH 108/259] remove redundant dial timeout This timeout is already set by the dial limiter. --- p2p/net/swarm/swarm_dial.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 3e0cfafac2..59adb037cb 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -405,9 +405,6 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (tra return nil, ErrNoTransport } - ctx, cancel := context.WithTimeout(ctx, transport.DialTimeout) - defer cancel() - connC, err := tpt.Dial(ctx, addr, p) if err != nil { return nil, fmt.Errorf("%s --> %s dial attempt failed: %s", s.local, p, err) From 5a9655ca4de3d57f8cad3a7da35dee6413bb6b18 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Thu, 16 Aug 2018 18:12:11 +0530 Subject: [PATCH 109/259] Issue #370 Check if empty peer ID On trying to open a new stream against an empty peer ID, we get an error saying that failed to dial a peer. This commit changes that to a more informative error message: `empty peer ID` --- p2p/net/swarm/swarm_dial.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 59adb037cb..2af26ed1a2 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -183,6 +183,11 @@ func (s *Swarm) DialPeer(ctx context.Context, p peer.ID) (inet.Conn, error) { func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { log.Debugf("[%s] swarm dialing peer [%s]", s.local, p) var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + err := p.Validate() + if err != nil { + return nil, err + } + if p == s.local { log.Event(ctx, "swarmDialSelf", logdial) return nil, ErrDialToSelf @@ -206,7 +211,7 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { ctx, cancel := context.WithTimeout(ctx, inet.GetDialPeerTimeout(ctx)) defer cancel() - conn, err := s.dsync.DialLock(ctx, p) + conn, err = s.dsync.DialLock(ctx, p) if err != nil { return nil, err } From 6d836d558002287beaea3202f2a4366399ffb5a1 Mon Sep 17 00:00:00 2001 From: Cole Brown Date: Fri, 15 Jun 2018 19:07:37 -0400 Subject: [PATCH 110/259] Add support for Stat() to Swarm Stream and Conn --- p2p/net/swarm/swarm.go | 3 ++- p2p/net/swarm/swarm_conn.go | 16 +++++++++++++--- p2p/net/swarm/swarm_dial.go | 3 ++- p2p/net/swarm/swarm_listen.go | 3 ++- p2p/net/swarm/swarm_stream.go | 10 ++++++++++ 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 7d795abd86..78d8e127dc 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -165,7 +165,7 @@ func (s *Swarm) Process() goprocess.Process { return s.proc } -func (s *Swarm) addConn(tc transport.Conn) (*Conn, error) { +func (s *Swarm) addConn(tc transport.Conn, stat inet.Stat) (*Conn, error) { // The underlying transport (or the dialer) *should* filter it's own // connections but we should double check anyways. raddr := tc.RemoteMultiaddr() @@ -197,6 +197,7 @@ func (s *Swarm) addConn(tc transport.Conn) (*Conn, error) { c := &Conn{ conn: tc, swarm: s, + stat: stat, } c.streams.m = make(map[*Stream]struct{}) s.conns.m[p] = append(s.conns.m[p], c) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 5b2420cd5f..05e6bfca1a 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -33,6 +33,8 @@ type Conn struct { sync.Mutex m map[*Stream]struct{} } + + stat inet.Stat } // Close closes this connection. @@ -98,7 +100,8 @@ func (c *Conn) start() { } c.swarm.refs.Add(1) go func() { - s, err := c.addStream(ts) + stat := inet.Stat{Direction: inet.DirInbound} + s, err := c.addStream(ts, stat) // Don't defer this. We don't want to block // swarm shutdown on the connection handler. @@ -158,16 +161,22 @@ func (c *Conn) RemotePublicKey() ic.PubKey { return c.conn.RemotePublicKey() } +// Stat returns metadata pertaining to this connection +func (c *Conn) Stat() inet.Stat { + return c.stat +} + // NewStream returns a new Stream from this connection func (c *Conn) NewStream() (inet.Stream, error) { ts, err := c.conn.OpenStream() if err != nil { return nil, err } - return c.addStream(ts) + stat := inet.Stat{Direction: inet.DirOutbound} + return c.addStream(ts, stat) } -func (c *Conn) addStream(ts smux.Stream) (*Stream, error) { +func (c *Conn) addStream(ts smux.Stream, stat inet.Stat) (*Stream, error) { c.streams.Lock() // Are we still online? if c.streams.m == nil { @@ -180,6 +189,7 @@ func (c *Conn) addStream(ts smux.Stream) (*Stream, error) { s := &Stream{ stream: ts, conn: c, + stat: stat, } c.streams.m[s] = struct{}{} diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 2af26ed1a2..246935ab58 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -325,7 +325,8 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { "localAddr": connC.LocalMultiaddr(), "remoteAddr": connC.RemoteMultiaddr(), } - swarmC, err := s.addConn(connC) + stat := inet.Stat{Direction: inet.DirOutbound} + swarmC, err := s.addConn(connC, stat) if err != nil { logdial["error"] = err.Error() connC.Close() // close the connection. didn't work out :( diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index df0a97f248..cb56b42f06 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -81,7 +81,8 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { s.refs.Add(1) go func() { defer s.refs.Done() - _, err := s.addConn(c) + stat := inet.Stat{Direction: inet.DirInbound} + _, err := s.addConn(c, stat) if err != nil { // Probably just means that the swarm has been closed. log.Warningf("add conn failed: ", err) diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 30b44bd28e..754dbd50e7 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -22,6 +22,9 @@ const ( streamReset ) +// Validate Stream conforms to the go-libp2p-net Stream interface +var _ inet.Stream = &Stream{} + // Stream is the stream type used by swarm. In general, you won't use this type // directly. type Stream struct { @@ -36,6 +39,8 @@ type Stream struct { notifyLk sync.Mutex protocol atomic.Value + + stat inet.Stat } func (s *Stream) String() string { @@ -165,3 +170,8 @@ func (s *Stream) SetReadDeadline(t time.Time) error { func (s *Stream) SetWriteDeadline(t time.Time) error { return s.stream.SetWriteDeadline(t) } + +// Stat returns metadata information for this stream. +func (s *Stream) Stat() inet.Stat { + return s.stat +} From 3f9733aeb6bd764420c91d5fc1c31b6e871acaef Mon Sep 17 00:00:00 2001 From: Cole Brown Date: Sun, 24 Jun 2018 20:54:56 -0400 Subject: [PATCH 111/259] Update addStream/Conn to accept a Direction --- p2p/net/swarm/swarm.go | 3 ++- p2p/net/swarm/swarm_conn.go | 9 ++++----- p2p/net/swarm/swarm_dial.go | 3 +-- p2p/net/swarm/swarm_listen.go | 3 +-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 78d8e127dc..c8d6fa02d1 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -165,7 +165,7 @@ func (s *Swarm) Process() goprocess.Process { return s.proc } -func (s *Swarm) addConn(tc transport.Conn, stat inet.Stat) (*Conn, error) { +func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { // The underlying transport (or the dialer) *should* filter it's own // connections but we should double check anyways. raddr := tc.RemoteMultiaddr() @@ -194,6 +194,7 @@ func (s *Swarm) addConn(tc transport.Conn, stat inet.Stat) (*Conn, error) { } // Wrap and register the connection. + stat := inet.Stat{Direction: dir} c := &Conn{ conn: tc, swarm: s, diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 05e6bfca1a..26a7794d15 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -100,8 +100,7 @@ func (c *Conn) start() { } c.swarm.refs.Add(1) go func() { - stat := inet.Stat{Direction: inet.DirInbound} - s, err := c.addStream(ts, stat) + s, err := c.addStream(ts, inet.DirInbound) // Don't defer this. We don't want to block // swarm shutdown on the connection handler. @@ -172,11 +171,10 @@ func (c *Conn) NewStream() (inet.Stream, error) { if err != nil { return nil, err } - stat := inet.Stat{Direction: inet.DirOutbound} - return c.addStream(ts, stat) + return c.addStream(ts, inet.DirOutbound) } -func (c *Conn) addStream(ts smux.Stream, stat inet.Stat) (*Stream, error) { +func (c *Conn) addStream(ts smux.Stream, dir inet.Direction) (*Stream, error) { c.streams.Lock() // Are we still online? if c.streams.m == nil { @@ -186,6 +184,7 @@ func (c *Conn) addStream(ts smux.Stream, stat inet.Stat) (*Stream, error) { } // Wrap and register the stream. + stat := inet.Stat{Direction: dir} s := &Stream{ stream: ts, conn: c, diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 246935ab58..64971b07e9 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -325,8 +325,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { "localAddr": connC.LocalMultiaddr(), "remoteAddr": connC.RemoteMultiaddr(), } - stat := inet.Stat{Direction: inet.DirOutbound} - swarmC, err := s.addConn(connC, stat) + swarmC, err := s.addConn(connC, inet.DirOutbound) if err != nil { logdial["error"] = err.Error() connC.Close() // close the connection. didn't work out :( diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index cb56b42f06..31b4a04208 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -81,8 +81,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { s.refs.Add(1) go func() { defer s.refs.Done() - stat := inet.Stat{Direction: inet.DirInbound} - _, err := s.addConn(c, stat) + _, err := s.addConn(c, inet.DirInbound) if err != nil { // Probably just means that the swarm has been closed. log.Warningf("add conn failed: ", err) From 0bc531b8860fdd84518b540c227de96dc77852a2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 13 Jun 2018 11:20:29 -0700 Subject: [PATCH 112/259] log an error when we have no transports configured Modify our two transport lookup functions to log loudly when there are no transports registered. --- p2p/net/swarm/swarm_transport.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/p2p/net/swarm/swarm_transport.go b/p2p/net/swarm/swarm_transport.go index bc60dcf993..c96a079d9e 100644 --- a/p2p/net/swarm/swarm_transport.go +++ b/p2p/net/swarm/swarm_transport.go @@ -18,6 +18,11 @@ func (s *Swarm) TransportForDialing(a ma.Multiaddr) transport.Transport { s.transports.RLock() defer s.transports.RUnlock() + if len(s.transports.m) == 0 { + log.Error("you have no transports configured") + return nil + } + for _, p := range protocols { transport, ok := s.transports.m[p.Code] if !ok { @@ -41,6 +46,11 @@ func (s *Swarm) TransportForListening(a ma.Multiaddr) transport.Transport { s.transports.RLock() defer s.transports.RUnlock() + if len(s.transports.m) == 0 { + log.Error("you have no transports configured") + return nil + } + selected := s.transports.m[protocols[len(protocols)-1].Code] for _, p := range protocols { transport, ok := s.transports.m[p.Code] From 602f9d4cef57928d948e79adb693f5f54c3ea2c6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 7 Sep 2018 22:05:41 -0700 Subject: [PATCH 113/259] gx: update go-libp2p-peerstore --- p2p/net/swarm/testing/testing.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 29017df808..e3a62c5283 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -8,6 +8,7 @@ import ( metrics "github.com/libp2p/go-libp2p-metrics" inet "github.com/libp2p/go-libp2p-net" pstore "github.com/libp2p/go-libp2p-peerstore" + pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" secio "github.com/libp2p/go-libp2p-secio" tptu "github.com/libp2p/go-libp2p-transport-upgrader" tcp "github.com/libp2p/go-tcp-transport" @@ -66,7 +67,7 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { p := tu.RandPeerNetParamsOrFatal(t) - ps := pstore.NewPeerstore() + ps := pstoremem.NewPeerstore() ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter()) From c8096184f855f18a5df4565275f841c1f74cd731 Mon Sep 17 00:00:00 2001 From: Lars Gierth Date: Tue, 2 Oct 2018 04:59:24 +0200 Subject: [PATCH 114/259] Refactor and relax filtering of known undialable addresses This commit moves filtering of dial candidates into its own little function. Things that are being filtered: addresses configured to be blocked, IPv6 link-local addresses, addresses without a dial-capable transport, and addresses that we know to be our own. It's is an optimization to avoid wasting time on dials that we know are going to fail. This also relaxes the filtering of addresses that we consider our own. Previously, any address would get filtered that's registered in peerstore for or own PeerID. For e.g. /ip4/1.2.3.4/tcp/4001 that's fine, but for ephemeral ports it can already cause problems. In addition, now that go-libp2p-circuit is being fixed to handle its multiaddrs slightly differently, /p2p-circuit addresses won't contain the PeerID anymore. That means they stand for themselves, and would get filtered too. (/p2p-circuit is the address we want to dial, but it's also on of "our own addresses"). In the future we'd want to use the mafmt package here, and also consider /quic, /ws, etc. addresses as our own. --- p2p/net/swarm/swarm_dial.go | 55 +++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 64971b07e9..6030ca79d7 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -281,42 +281,24 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { log.Debug("Dial not given PrivateKey, so WILL NOT SECURE conn.") } - ila, _ := s.InterfaceListenAddresses() - subtractFilter := addrutil.SubtractFilter(append(ila, s.peers.Addrs(s.local)...)...) - - // get live channel of addresses for peer, filtered by the given filters - /* - remoteAddrChan := s.peers.AddrsChan(ctx, p, - addrutil.AddrUsableFilter, - subtractFilter, - s.Filters.AddrBlocked) - */ - ////// /* - This code is temporary, the peerstore can currently provide + This slice-to-chan code is temporary, the peerstore can currently provide a channel as an interface for receiving addresses, but more thought needs to be put into the execution. For now, this allows us to use the improved rate limiter, while maintaining the outward behaviour that we previously had (halting a dial when we run out of addrs) */ - paddrs := s.peers.Addrs(p) - goodAddrs := addrutil.FilterAddrs(paddrs, - subtractFilter, - s.canDial, - // TODO: Consider allowing this? - addrutil.AddrOverNonLocalIP, - addrutil.FilterNeg(s.Filters.AddrBlocked), - ) - remoteAddrChan := make(chan ma.Multiaddr, len(goodAddrs)) + goodAddrs := s.filterKnownUndialables(s.peers.Addrs(p)) + goodAddrsChan := make(chan ma.Multiaddr, len(goodAddrs)) for _, a := range goodAddrs { - remoteAddrChan <- a + goodAddrsChan <- a } - close(remoteAddrChan) + close(goodAddrsChan) ///////// // try to get a connection to any addr - connC, err := s.dialAddrs(ctx, p, remoteAddrChan) + connC, err := s.dialAddrs(ctx, p, goodAddrsChan) if err != nil { logdial["error"] = err.Error() return nil, err @@ -336,6 +318,31 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return swarmC, nil } +// filterKnownUndialables takes a list of multiaddrs, and removes those +// that we definitely don't want to dial: addresses configured to be blocked, +// IPv6 link-local addresses, addresses without a dial-capable transport, +// and addresses that we know to be our own. +// This is an optimization to avoid wasting time on dials that we know are going to fail. +func (s *Swarm) filterKnownUndialables(addrs []ma.Multiaddr) []ma.Multiaddr { + lisAddrs, _ := s.InterfaceListenAddresses() + var ourAddrs []ma.Multiaddr + for _, addr := range lisAddrs { + protos := addr.Protocols() + // we're only sure about filtering out /ip4 and /ip6 addresses, so far + if len(protos) == 2 && (protos[0].Code == ma.P_IP4 || protos[0].Code == ma.P_IP6) { + ourAddrs = append(ourAddrs, addr) + } + } + + return addrutil.FilterAddrs(addrs, + addrutil.SubtractFilter(ourAddrs...), + s.canDial, + // TODO: Consider allowing link-local addresses + addrutil.AddrOverNonLocalIP, + addrutil.FilterNeg(s.Filters.AddrBlocked), + ) +} + func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (transport.Conn, error) { log.Debugf("%s swarm dialing %s", s.local, p) From 374d8d9fa4bf12961fc804f5c0479225a5405d03 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 15 Nov 2018 21:11:10 -0800 Subject: [PATCH 115/259] warn when we encounter a useless transport --- p2p/net/swarm/swarm_transport.go | 4 +++ p2p/net/swarm/transport_test.go | 47 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 p2p/net/swarm/transport_test.go diff --git a/p2p/net/swarm/swarm_transport.go b/p2p/net/swarm/swarm_transport.go index c96a079d9e..0431488777 100644 --- a/p2p/net/swarm/swarm_transport.go +++ b/p2p/net/swarm/swarm_transport.go @@ -70,6 +70,10 @@ func (s *Swarm) TransportForListening(a ma.Multiaddr) transport.Transport { func (s *Swarm) AddTransport(t transport.Transport) error { protocols := t.Protocols() + if len(protocols) == 0 { + return fmt.Errorf("useless transport handles no protocols: %T", t) + } + s.transports.Lock() defer s.transports.Unlock() var registered []string diff --git a/p2p/net/swarm/transport_test.go b/p2p/net/swarm/transport_test.go new file mode 100644 index 0000000000..018775b261 --- /dev/null +++ b/p2p/net/swarm/transport_test.go @@ -0,0 +1,47 @@ +package swarm_test + +import ( + "context" + "testing" + + swarmt "github.com/libp2p/go-libp2p-swarm/testing" + + peer "github.com/libp2p/go-libp2p-peer" + transport "github.com/libp2p/go-libp2p-transport" + ma "github.com/multiformats/go-multiaddr" +) + +type dummyTransport struct { + protocols []int + proxy bool +} + +func (dt *dummyTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.Conn, error) { + panic("unimplemented") +} + +func (dt *dummyTransport) CanDial(addr ma.Multiaddr) bool { + panic("unimplemented") +} + +func (dt *dummyTransport) Listen(laddr ma.Multiaddr) (transport.Listener, error) { + panic("unimplemented") +} + +func (dt *dummyTransport) Proxy() bool { + return dt.proxy +} + +func (dt *dummyTransport) Protocols() []int { + return dt.protocols +} + +func TestUselessTransport(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + swarm := swarmt.GenSwarm(t, ctx) + err := swarm.AddTransport(new(dummyTransport)) + if err == nil { + t.Fatal("adding a transport that supports no protocols should have failed") + } +} From 70611007e5cbe410794edde1f65c2e04928f6e9a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Jan 2019 07:43:12 -0800 Subject: [PATCH 116/259] avoid spawning goroutines for canceled dials This may speed up working through our dial queue a bit (bit probably isn't the main issue). --- p2p/net/swarm/limiter.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 02aed50a2e..11c20bc100 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -77,17 +77,26 @@ func newDialLimiterWithParams(df dialfunc, fdLimit, perPeerLimit int) *dialLimit func (dl *dialLimiter) freeFDToken() { dl.fdConsuming-- - if len(dl.waitingOnFd) > 0 { + for len(dl.waitingOnFd) > 0 { next := dl.waitingOnFd[0] dl.waitingOnFd[0] = nil // clear out memory dl.waitingOnFd = dl.waitingOnFd[1:] + if len(dl.waitingOnFd) == 0 { - dl.waitingOnFd = nil // clear out memory + // clear out memory. + dl.waitingOnFd = nil + } + + // Skip over canceled dials instead of queuing up a goroutine. + if next.cancelled() { + dl.freePeerToken(next) + continue } dl.fdConsuming++ // we already have activePerPeer token at this point so we can just dial go dl.executeDial(next) + return } } @@ -99,18 +108,25 @@ func (dl *dialLimiter) freePeerToken(dj *dialJob) { } waitlist := dl.waitingOnPeerLimit[dj.peer] - if len(waitlist) > 0 { + for len(waitlist) > 0 { next := waitlist[0] - if len(waitlist) == 1 { + waitlist[0] = nil // clear out memory + waitlist = waitlist[1:] + + if len(waitlist) == 0 { delete(dl.waitingOnPeerLimit, next.peer) } else { - waitlist[0] = nil // clear out memory - dl.waitingOnPeerLimit[next.peer] = waitlist[1:] + dl.waitingOnPeerLimit[next.peer] = waitlist + } + + if next.cancelled() { + continue } dl.activePerPeer[next.peer]++ // just kidding, we still want this token dl.addCheckFdLimit(next) + return } } From 8d093f29d2a00a300f4a6ca02940b8a3e8d28277 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Jan 2019 09:06:37 -0800 Subject: [PATCH 117/259] dialer: handle dial cancel and/or completion before trying new addresses --- p2p/net/swarm/swarm_dial.go | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 6030ca79d7..37f1234cc4 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -358,14 +358,34 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. defer s.limiter.clearAllPeerDials(p) var active int - for { + for remoteAddrs != nil || active > 0 { + // Check for context cancellations and/or responses first. + select { + case <-ctx.Done(): + if exitErr == defaultDialFail { + exitErr = ctx.Err() + } + return nil, exitErr + case resp := <-respch: + active-- + if resp.Err != nil { + log.Infof("got error on dial to %s: %s", resp.Addr, resp.Err) + // Errors are normal, lots of dials will fail + exitErr = resp.Err + } else if resp.Conn != nil { + return resp.Conn, nil + } + + // We got a result, try again from the top. + continue + default: + } + + // Now, attempt to dial. select { case addr, ok := <-remoteAddrs: if !ok { remoteAddrs = nil - if active == 0 { - return nil, exitErr - } continue } @@ -382,15 +402,12 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. log.Infof("got error on dial to %s: %s", resp.Addr, resp.Err) // Errors are normal, lots of dials will fail exitErr = resp.Err - - if remoteAddrs == nil && active == 0 { - return nil, exitErr - } } else if resp.Conn != nil { return resp.Conn, nil } } } + return nil, exitErr } // limitedDial will start a dial to the given peer when From addf6f1a11b99e26b50dd46043c87d4a5a40ce88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 29 Jan 2019 16:06:09 +0000 Subject: [PATCH 118/259] enhance debug logging in limiter. --- p2p/net/swarm/limiter.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 11c20bc100..2def638c3d 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -75,6 +75,7 @@ func newDialLimiterWithParams(df dialfunc, fdLimit, perPeerLimit int) *dialLimit // freeFDToken frees FD token and if there are any schedules another waiting dialJob // in it's place func (dl *dialLimiter) freeFDToken() { + log.Debugf("[limiter] freeing FD token; waiting: %d; consuming: %d", len(dl.waitingOnFd), dl.fdConsuming) dl.fdConsuming-- for len(dl.waitingOnFd) > 0 { @@ -101,6 +102,8 @@ func (dl *dialLimiter) freeFDToken() { } func (dl *dialLimiter) freePeerToken(dj *dialJob) { + log.Debugf("[limiter] freeing peer token; peer %s; addr: %s; active for peer: %d; waiting on peer limit: %d", + dj.peer, dj.addr, dl.activePerPeer[dj.peer], len(dl.waitingOnPeerLimit[dj.peer])) // release tokens in reverse order than we take them dl.activePerPeer[dj.peer]-- if dl.activePerPeer[dj.peer] == 0 { @@ -144,19 +147,28 @@ func (dl *dialLimiter) finishedDial(dj *dialJob) { func (dl *dialLimiter) addCheckFdLimit(dj *dialJob) { if addrutil.IsFDCostlyTransport(dj.addr) { if dl.fdConsuming >= dl.fdLimit { + log.Debugf("[limiter] blocked dial waiting on FD token; peer: %s; addr: %s; consuming: %d; "+ + "limit: %d; waiting: %d", dj.peer, dj.addr, dl.fdConsuming, dl.fdLimit, len(dl.waitingOnFd)) dl.waitingOnFd = append(dl.waitingOnFd, dj) return } + log.Debugf("[limiter] taking FD token: peer: %s; addr: %s; prev consuming: %d", + dj.peer, dj.addr, dl.fdConsuming) // take token dl.fdConsuming++ } + log.Debugf("[limiter] executing dial; peer: %s; addr: %s; FD consuming: %d; waiting: %d", + dj.peer, dj.addr, dl.fdConsuming, len(dl.waitingOnFd)) go dl.executeDial(dj) } func (dl *dialLimiter) addCheckPeerLimit(dj *dialJob) { if dl.activePerPeer[dj.peer] >= dl.perPeerLimit { + log.Debugf("[limiter] blocked dial waiting on peer limit; peer: %s; addr: %s; active: %d; "+ + "peer limit: %d; waiting: %d", dj.peer, dj.addr, dl.activePerPeer[dj.peer], dl.perPeerLimit, + len(dl.waitingOnPeerLimit[dj.peer])) wlist := dl.waitingOnPeerLimit[dj.peer] dl.waitingOnPeerLimit[dj.peer] = append(wlist, dj) return @@ -173,6 +185,7 @@ func (dl *dialLimiter) AddDialJob(dj *dialJob) { dl.lk.Lock() defer dl.lk.Unlock() + log.Debugf("[limiter] adding a dial job through limiter: %v", dj.addr) dl.addCheckPeerLimit(dj) } @@ -180,6 +193,7 @@ func (dl *dialLimiter) clearAllPeerDials(p peer.ID) { dl.lk.Lock() defer dl.lk.Unlock() delete(dl.waitingOnPeerLimit, p) + log.Debugf("[limiter] clearing all peer dials: %v", p) // NB: the waitingOnFd list doesn't need to be cleaned out here, we will // remove them as we encounter them because they are 'cancelled' at this // point @@ -189,6 +203,8 @@ func (dl *dialLimiter) clearAllPeerDials(p peer.ID) { // channel when finished. Once the response is sent it also releases all tokens // it held during the dial. func (dl *dialLimiter) executeDial(j *dialJob) { + log.Debugf("[limiter] executing dial (dialfunc); peer: %s; addr: %s; FD consuming: %d; waiting: %d", + j.peer, j.addr, dl.fdConsuming, len(dl.waitingOnFd)) defer dl.finishedDial(j) if j.cancelled() { return From 1c5f2407cb89f2bc5003a7e5cb2c224eea94d20b Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 4 Feb 2019 15:18:27 +1100 Subject: [PATCH 119/259] Add CircleCI config --- p2p/net/swarm/.circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 p2p/net/swarm/.circleci/config.yml diff --git a/p2p/net/swarm/.circleci/config.yml b/p2p/net/swarm/.circleci/config.yml new file mode 100644 index 0000000000..68d7e72668 --- /dev/null +++ b/p2p/net/swarm/.circleci/config.yml @@ -0,0 +1,10 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/golang:1.11 + environment: + GO111MODULE: "on" + steps: + - checkout + - run: go test -v -race -bench . ./... From 27f08059c5ebf66c56e4c871a33e2d2cd632d4e6 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 4 Feb 2019 15:20:35 +1100 Subject: [PATCH 120/259] Fix logging race --- p2p/net/swarm/limiter.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 2def638c3d..ce9e2448ca 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -203,8 +203,10 @@ func (dl *dialLimiter) clearAllPeerDials(p peer.ID) { // channel when finished. Once the response is sent it also releases all tokens // it held during the dial. func (dl *dialLimiter) executeDial(j *dialJob) { + dl.lk.Lock() log.Debugf("[limiter] executing dial (dialfunc); peer: %s; addr: %s; FD consuming: %d; waiting: %d", j.peer, j.addr, dl.fdConsuming, len(dl.waitingOnFd)) + dl.lk.Unlock() defer dl.finishedDial(j) if j.cancelled() { return From 9d153e8ea7414170b086b49c972abf0f92da153e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 4 Feb 2019 17:47:51 +0000 Subject: [PATCH 121/259] remove superfluous log statement. --- p2p/net/swarm/limiter.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index ce9e2448ca..ef78f0eeb8 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -203,10 +203,6 @@ func (dl *dialLimiter) clearAllPeerDials(p peer.ID) { // channel when finished. Once the response is sent it also releases all tokens // it held during the dial. func (dl *dialLimiter) executeDial(j *dialJob) { - dl.lk.Lock() - log.Debugf("[limiter] executing dial (dialfunc); peer: %s; addr: %s; FD consuming: %d; waiting: %d", - j.peer, j.addr, dl.fdConsuming, len(dl.waitingOnFd)) - dl.lk.Unlock() defer dl.finishedDial(j) if j.cancelled() { return From 96a640d5913618166a8c665729c341806373a17a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 31 Jan 2019 13:52:54 +0000 Subject: [PATCH 122/259] make FD limits configurable by env prop. --- p2p/net/swarm/limiter.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index ef78f0eeb8..fa9582bbe6 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -2,6 +2,8 @@ package swarm import ( "context" + "os" + "strconv" "sync" "time" @@ -59,7 +61,13 @@ type dialLimiter struct { type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (transport.Conn, error) func newDialLimiter(df dialfunc) *dialLimiter { - return newDialLimiterWithParams(df, ConcurrentFdDials, DefaultPerPeerRateLimit) + fd := ConcurrentFdDials + if env := os.Getenv("LIBP2P_SWARM_FD_LIMIT"); env != "" { + if n, err := strconv.ParseInt(env, 10, 32); err == nil { + fd = int(n) + } + } + return newDialLimiterWithParams(df, fd, DefaultPerPeerRateLimit) } func newDialLimiterWithParams(df dialfunc, fdLimit, perPeerLimit int) *dialLimiter { From b3789be9f912eb6ea1c65709bef9278a0b62de04 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 15 Feb 2019 15:46:50 +1100 Subject: [PATCH 123/259] Improve context use --- p2p/net/swarm/limiter.go | 7 +------ p2p/net/swarm/swarm_listen.go | 4 +++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index fa9582bbe6..bd4ab57256 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -27,12 +27,7 @@ type dialJob struct { } func (dj *dialJob) cancelled() bool { - select { - case <-dj.ctx.Done(): - return true - default: - return false - } + return dj.ctx.Err() != nil } func (dj *dialJob) dialTimeout() time.Duration { diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 31b4a04208..7b37d53497 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -74,7 +74,9 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { for { c, err := list.Accept() if err != nil { - log.Warningf("swarm listener accept error: %s", err) + if s.ctx.Err() == nil { + log.Errorf("swarm listener accept error: %s", err) + } return } log.Debugf("swarm listener accepted connection: %s", c) From a65b8ab2d149b529d1d7616565cd2affb40b23a5 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 15 Feb 2019 15:47:27 +1100 Subject: [PATCH 124/259] Reflow comments --- p2p/net/swarm/swarm.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c8d6fa02d1..cc77e80ec6 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -121,8 +121,8 @@ func (s *Swarm) teardown() error { s.conns.m = nil s.conns.Unlock() - // Lots of go routines but we might as well do this in parallel. We want - // to shut down as fast as possible. + // Lots of goroutines but we might as well do this in parallel. We want to shut down as fast as + // possible. for l := range listeners { go func(l transport.Listener) { @@ -148,8 +148,8 @@ func (s *Swarm) teardown() error { return nil } -// AddAddrFilter adds a multiaddr filter to the set of filters the swarm will -// use to determine which addresses not to dial to. +// AddAddrFilter adds a multiaddr filter to the set of filters the swarm will use to determine which +// addresses not to dial to. func (s *Swarm) AddAddrFilter(f string) error { m, err := mafilter.NewMask(f) if err != nil { From 6ee2ac3e464ddf557f44d8e10584d552ae33f879 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 15 Feb 2019 15:47:55 +1100 Subject: [PATCH 125/259] Do notifications synchronously --- p2p/net/swarm/swarm.go | 14 +------------- p2p/net/swarm/swarm_conn.go | 15 ++++----------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index cc77e80ec6..85c5680c6b 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -208,9 +208,6 @@ func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { // * The other will be decremented when Conn.start exits. s.refs.Add(2) - // Take the notification lock before releasing the conns lock to block - // Disconnect notifications until after the Connect notifications done. - c.notifyLk.Lock() s.conns.Unlock() // We have a connection now. Cancel all other in-progress dials. @@ -220,7 +217,6 @@ func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { s.notifyAll(func(f inet.Notifiee) { f.Connected(s, c) }) - c.notifyLk.Unlock() c.start() @@ -438,18 +434,10 @@ func (s *Swarm) Backoff() *DialBackoff { // notifyAll sends a signal to all Notifiees func (s *Swarm) notifyAll(notify func(inet.Notifiee)) { - var wg sync.WaitGroup - s.notifs.RLock() - wg.Add(len(s.notifs.m)) for f := range s.notifs.m { - go func(f inet.Notifiee) { - defer wg.Done() - notify(f) - }(f) + notify(f) } - - wg.Wait() s.notifs.RUnlock() } diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 26a7794d15..4e837cd934 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -65,17 +65,10 @@ func (c *Conn) doClose() { s.Reset() } - // do this in a goroutine to avoid deadlocking if we call close in an open notification. - go func() { - // prevents us from issuing close notifications before finishing the open notifications - c.notifyLk.Lock() - defer c.notifyLk.Unlock() - - c.swarm.notifyAll(func(f inet.Notifiee) { - f.Disconnected(c.swarm, c) - }) - c.swarm.refs.Done() // taken in Swarm.addConn - }() + c.swarm.notifyAll(func(f inet.Notifiee) { + f.Disconnected(c.swarm, c) + }) + c.swarm.refs.Done() // taken in Swarm.addConn } func (c *Conn) removeStream(s *Stream) { From 76866ef9f3fc93854335b6e10be3d821606ec560 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 15 Feb 2019 15:48:11 +1100 Subject: [PATCH 126/259] Fix code format --- p2p/net/swarm/swarm_dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 37f1234cc4..56d8420e73 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -275,7 +275,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { logdial["dial"] = "failure" // start off with failure. set to "success" at the end. sk := s.peers.PrivKey(s.local) - logdial["encrypted"] = (sk != nil) // log wether this will be an encrypted dial or not. + logdial["encrypted"] = sk != nil // log whether this will be an encrypted dial or not. if sk == nil { // fine for sk to be nil, just log. log.Debug("Dial not given PrivateKey, so WILL NOT SECURE conn.") From c8c717cc32310db4334160298d9c535a67163bf6 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 19 Feb 2019 16:31:52 +1100 Subject: [PATCH 127/259] Revert "Do notifications synchronously" This reverts commit 6ee2ac3e464ddf557f44d8e10584d552ae33f879. --- p2p/net/swarm/swarm.go | 14 +++++++++++++- p2p/net/swarm/swarm_conn.go | 15 +++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 85c5680c6b..cc77e80ec6 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -208,6 +208,9 @@ func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { // * The other will be decremented when Conn.start exits. s.refs.Add(2) + // Take the notification lock before releasing the conns lock to block + // Disconnect notifications until after the Connect notifications done. + c.notifyLk.Lock() s.conns.Unlock() // We have a connection now. Cancel all other in-progress dials. @@ -217,6 +220,7 @@ func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { s.notifyAll(func(f inet.Notifiee) { f.Connected(s, c) }) + c.notifyLk.Unlock() c.start() @@ -434,10 +438,18 @@ func (s *Swarm) Backoff() *DialBackoff { // notifyAll sends a signal to all Notifiees func (s *Swarm) notifyAll(notify func(inet.Notifiee)) { + var wg sync.WaitGroup + s.notifs.RLock() + wg.Add(len(s.notifs.m)) for f := range s.notifs.m { - notify(f) + go func(f inet.Notifiee) { + defer wg.Done() + notify(f) + }(f) } + + wg.Wait() s.notifs.RUnlock() } diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 4e837cd934..26a7794d15 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -65,10 +65,17 @@ func (c *Conn) doClose() { s.Reset() } - c.swarm.notifyAll(func(f inet.Notifiee) { - f.Disconnected(c.swarm, c) - }) - c.swarm.refs.Done() // taken in Swarm.addConn + // do this in a goroutine to avoid deadlocking if we call close in an open notification. + go func() { + // prevents us from issuing close notifications before finishing the open notifications + c.notifyLk.Lock() + defer c.notifyLk.Unlock() + + c.swarm.notifyAll(func(f inet.Notifiee) { + f.Disconnected(c.swarm, c) + }) + c.swarm.refs.Done() // taken in Swarm.addConn + }() } func (c *Conn) removeStream(s *Stream) { From e7afaa3f50995030dcd06b1583ef3fc6e1858845 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 19 Feb 2019 20:36:36 +1100 Subject: [PATCH 128/259] Fix failing tests? --- p2p/net/swarm/dial_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index dd93bd7cec..c9d0d5dda0 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -150,7 +150,7 @@ func TestSimultDials(t *testing.T) { func newSilentPeer(t *testing.T) (peer.ID, ma.Multiaddr, net.Listener) { dst := testutil.RandPeerIDFatal(t) - lst, err := net.Listen("tcp", ":0") + lst, err := net.Listen("tcp4", ":0") if err != nil { t.Fatal(err) } @@ -168,7 +168,6 @@ func newSilentPeer(t *testing.T) (peer.ID, ma.Multiaddr, net.Listener) { } func TestDialWait(t *testing.T) { - // t.Skip("skipping for another test") t.Parallel() ctx := context.Background() From f22b6c7f7339d439e8ba222b7a4af77830a0c595 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Thu, 28 Feb 2019 11:17:08 +1100 Subject: [PATCH 129/259] Improve dial error messages --- p2p/net/swarm/swarm_dial.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 56d8420e73..1bed042267 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -290,6 +290,9 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { that we previously had (halting a dial when we run out of addrs) */ goodAddrs := s.filterKnownUndialables(s.peers.Addrs(p)) + if len(goodAddrs) == 0 { + return nil, errors.New("no good addresses") + } goodAddrsChan := make(chan ma.Multiaddr, len(goodAddrs)) for _, a := range goodAddrs { goodAddrsChan <- a @@ -352,7 +355,7 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. // use a single response type instead of errs and conns, reduces complexity *a ton* respch := make(chan dialResult) - defaultDialFail := fmt.Errorf("failed to dial %s (default failure)", p) + defaultDialFail := inet.ErrNoRemoteAddrs exitErr := defaultDialFail defer s.limiter.clearAllPeerDials(p) From 3899936deea191b565fbab337c7fea6711f10f27 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 8 Mar 2019 15:23:17 +1100 Subject: [PATCH 130/259] Add an error for no addresses --- p2p/net/swarm/swarm_dial.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 1bed042267..db7d88894d 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -289,7 +289,11 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { the improved rate limiter, while maintaining the outward behaviour that we previously had (halting a dial when we run out of addrs) */ - goodAddrs := s.filterKnownUndialables(s.peers.Addrs(p)) + peerAddrs := s.peers.Addrs(p) + if len(peerAddrs) == 0 { + return nil, errors.New("no addresses") + } + goodAddrs := s.filterKnownUndialables(peerAddrs) if len(goodAddrs) == 0 { return nil, errors.New("no good addresses") } From 31088db32d8daded300eed170e01f2dd0e9d7112 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 8 Mar 2019 15:24:02 +1100 Subject: [PATCH 131/259] Check for context failure, rather than a specific downstream error --- p2p/net/swarm/swarm_dial.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index db7d88894d..c0c5e69d74 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -84,7 +84,7 @@ const DefaultPerPeerRateLimit = 8 // DialBackoff is a type for tracking peer dial backoffs. // -// * It's safe to use it's zero value. +// * It's safe to use its zero value. // * It's thread-safe. // * It's *not* safe to move this type after using. type DialBackoff struct { @@ -248,7 +248,7 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { log.Debugf("ignoring dial error because we have a connection: %s", err) return conn, nil } - if err != context.Canceled { + if ctx.Err() == nil { log.Event(ctx, "swarmDialBackoffAdd", logdial) s.backf.AddBackoff(p) // let others know to backoff } From 86b63ef35c8075dbbb6b9381ea973105660f22ea Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 13 Mar 2019 14:22:47 +1100 Subject: [PATCH 132/259] Don't modify the error handling --- p2p/net/swarm/swarm_dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index c0c5e69d74..8a3ba49172 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -248,7 +248,7 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { log.Debugf("ignoring dial error because we have a connection: %s", err) return conn, nil } - if ctx.Err() == nil { + if err != context.Canceled { log.Event(ctx, "swarmDialBackoffAdd", logdial) s.backf.AddBackoff(p) // let others know to backoff } From 2df36e51fcf6682634f85771c577d313486d09a8 Mon Sep 17 00:00:00 2001 From: tg Date: Tue, 26 Mar 2019 21:39:22 +0300 Subject: [PATCH 133/259] return all dial errors if dial has failed License: MIT Signed-off-by: Georgij Tolstov --- p2p/net/swarm/dial_test.go | 48 +++++++++++++++++++++++++++++++++++++ p2p/net/swarm/swarm_dial.go | 37 ++++++++++++++++------------ 2 files changed, 70 insertions(+), 15 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index c9d0d5dda0..adb3438ac4 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -2,7 +2,9 @@ package swarm_test import ( "context" + "fmt" "net" + "regexp" "sync" "testing" "time" @@ -480,3 +482,49 @@ func TestDialBackoffClears(t *testing.T) { t.Log("correctly cleared backoff") } } + +func TestDialPeerFailed(t *testing.T) { + t.Parallel() + ctx := context.Background() + + swarms := makeSwarms(ctx, t, 2) + defer closeSwarms(swarms) + testedSwarm, targetSwarm := swarms[0], swarms[1] + + exceptedErrorsCount := 5 + for i := 0; i < exceptedErrorsCount; i++ { + _, silentPeerAddress, silentPeerListener := newSilentPeer(t) + go acceptAndHang(silentPeerListener) + defer silentPeerListener.Close() + + testedSwarm.Peerstore().AddAddr( + targetSwarm.LocalPeer(), + silentPeerAddress, + pstore.PermanentAddrTTL) + } + + _, err := testedSwarm.DialPeer(ctx, targetSwarm.LocalPeer()) + if err == nil { + t.Fatal(err) + } + + // dial_test.go:508: correctly get a combined error: dial attempt failed: 10 errors occurred: + // * --> (/ip4/127.0.0.1/tcp/46485) dial attempt failed: failed to negotiate security protocol: context deadline exceeded + // * --> (/ip4/127.0.0.1/tcp/34881) dial attempt failed: failed to negotiate security protocol: context deadline exceeded + // ... + + errorCountRegexpString := fmt.Sprintf("%d errors occurred", exceptedErrorsCount) + errorCountRegexp := regexp.MustCompile(errorCountRegexpString) + if !errorCountRegexp.MatchString(err.Error()) { + t.Fatalf("can't find total err count: `%s' in `%s'", errorCountRegexpString, err.Error()) + } + + connectErrorsRegexpString := `\* --> \(.+?\) dial attempt failed:.+` + connectErrorsRegexp := regexp.MustCompile(connectErrorsRegexpString) + connectErrors := connectErrorsRegexp.FindAll([]byte(err.Error()), -1) + if len(connectErrors) != exceptedErrorsCount { + t.Fatalf("connectErrors must contain %d errros; "+ + "but `%s' was found in `%s' %d times", + exceptedErrorsCount, connectErrorsRegexpString, err.Error(), len(connectErrors)) + } +} diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 8a3ba49172..d4b315525f 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/hashicorp/go-multierror" + logging "github.com/ipfs/go-log" addrutil "github.com/libp2p/go-addr-util" lgbl "github.com/libp2p/go-libp2p-loggables" @@ -358,9 +360,7 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. // use a single response type instead of errs and conns, reduces complexity *a ton* respch := make(chan dialResult) - - defaultDialFail := inet.ErrNoRemoteAddrs - exitErr := defaultDialFail + var dialErrors *multierror.Error defer s.limiter.clearAllPeerDials(p) @@ -369,16 +369,17 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. // Check for context cancellations and/or responses first. select { case <-ctx.Done(): - if exitErr == defaultDialFail { - exitErr = ctx.Err() + if dialError := dialErrors.ErrorOrNil(); dialError != nil { + return nil, dialError } - return nil, exitErr + + return nil, ctx.Err() case resp := <-respch: active-- if resp.Err != nil { - log.Infof("got error on dial to %s: %s", resp.Addr, resp.Err) // Errors are normal, lots of dials will fail - exitErr = resp.Err + log.Infof("got error on dial: %s", resp.Err) + dialErrors = multierror.Append(dialErrors, resp.Err) } else if resp.Conn != nil { return resp.Conn, nil } @@ -399,22 +400,28 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. s.limitedDial(ctx, p, addr, respch) active++ case <-ctx.Done(): - if exitErr == defaultDialFail { - exitErr = ctx.Err() + if dialError := dialErrors.ErrorOrNil(); dialError != nil { + return nil, dialError } - return nil, exitErr + + return nil, ctx.Err() case resp := <-respch: active-- if resp.Err != nil { - log.Infof("got error on dial to %s: %s", resp.Addr, resp.Err) // Errors are normal, lots of dials will fail - exitErr = resp.Err + log.Infof("got error on dial: %s", resp.Err) + dialErrors = multierror.Append(dialErrors, resp.Err) } else if resp.Conn != nil { return resp.Conn, nil } } } - return nil, exitErr + + if dialError := dialErrors.ErrorOrNil(); dialError != nil { + return nil, dialError + } + + return nil, inet.ErrNoRemoteAddrs } // limitedDial will start a dial to the given peer when @@ -443,7 +450,7 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (tra connC, err := tpt.Dial(ctx, addr, p) if err != nil { - return nil, fmt.Errorf("%s --> %s dial attempt failed: %s", s.local, p, err) + return nil, fmt.Errorf("%s --> %s (%s) dial attempt failed: %s", s.local, p, addr, err) } // Trust the transport? Yeah... right. From 02d01da08cad077ae1518c894d403fe32e17e409 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 10 Apr 2019 20:24:32 +0300 Subject: [PATCH 134/259] context option to disable dialing when opening a new stream --- p2p/net/swarm/swarm.go | 16 ++++++++++++++++ p2p/net/swarm/swarm_test.go | 11 +++++++++++ 2 files changed, 27 insertions(+) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index cc77e80ec6..c4661d4b33 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -37,6 +37,18 @@ var ErrSwarmClosed = errors.New("swarm closed") // transport is misbehaving. var ErrAddrFiltered = errors.New("address filtered") +// ErrNoConn is returned when attempting to open a stream to a peer with the NoDial +// option and no usable connection is available. +var ErrNoConn = errors.New("no usable connection to peer") + +// ContextOption is the type of context options understood by the swarm +type ContextOption string + +// NoDial is a context option that instructs the swarm to not attempt a new +// dial when opening a stream. The value of the key should be a string indicating +// the source of the option. +var NoDial = ContextOption("swarm.NoDial") + // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the @@ -295,6 +307,10 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) { for { c := s.bestConnToPeer(p) if c == nil { + if nodial := ctx.Value(NoDial); nodial != nil { + return nil, ErrNoConn + } + if dials >= DialAttempts { return nil, errors.New("max dial attempts exceeded") } diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 4911fc96a1..aa5abeecf5 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -50,6 +50,7 @@ func EchoStreamHandler(stream inet.Stream) { return } } + }() } @@ -335,3 +336,13 @@ func TestFilterBounds(t *testing.T) { t.Log("got connect") } } + +func TestNoDial(t *testing.T) { + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + + _, err := swarms[0].NewStream(context.WithValue(ctx, NoDial, "swarm.test"), swarms[1].LocalPeer()) + if err != ErrNoConn { + t.Fatal("should have failed with ErrNoConn") + } +} From b355bce1630d240bd72c020b72028fce7e424bf2 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 10 Apr 2019 21:46:51 +0300 Subject: [PATCH 135/259] update to use NoDial option from go-libp2p-net --- p2p/net/swarm/swarm.go | 16 ++-------------- p2p/net/swarm/swarm_test.go | 4 ++-- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c4661d4b33..1094e72e19 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -37,18 +37,6 @@ var ErrSwarmClosed = errors.New("swarm closed") // transport is misbehaving. var ErrAddrFiltered = errors.New("address filtered") -// ErrNoConn is returned when attempting to open a stream to a peer with the NoDial -// option and no usable connection is available. -var ErrNoConn = errors.New("no usable connection to peer") - -// ContextOption is the type of context options understood by the swarm -type ContextOption string - -// NoDial is a context option that instructs the swarm to not attempt a new -// dial when opening a stream. The value of the key should be a string indicating -// the source of the option. -var NoDial = ContextOption("swarm.NoDial") - // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the @@ -307,8 +295,8 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) { for { c := s.bestConnToPeer(p) if c == nil { - if nodial := ctx.Value(NoDial); nodial != nil { - return nil, ErrNoConn + if nodial, _ := inet.GetNoDial(ctx); nodial { + return nil, inet.ErrNoConn } if dials >= DialAttempts { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index aa5abeecf5..89bf909295 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -341,8 +341,8 @@ func TestNoDial(t *testing.T) { ctx := context.Background() swarms := makeSwarms(ctx, t, 2) - _, err := swarms[0].NewStream(context.WithValue(ctx, NoDial, "swarm.test"), swarms[1].LocalPeer()) - if err != ErrNoConn { + _, err := swarms[0].NewStream(inet.WithNoDial(ctx, "swarm test"), swarms[1].LocalPeer()) + if err != inet.ErrNoConn { t.Fatal("should have failed with ErrNoConn") } } From fb0dd0ed76b768836be2d5a8286291bc375769e9 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 10 Apr 2019 21:48:47 +0300 Subject: [PATCH 136/259] kill stray line --- p2p/net/swarm/swarm_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 89bf909295..69eceb94c4 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -50,7 +50,6 @@ func EchoStreamHandler(stream inet.Stream) { return } } - }() } From 3d252fa3062e8e7003b3784eb59bf96eab9aa1b8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 24 Apr 2019 09:02:51 -0700 Subject: [PATCH 137/259] dial: return a nice custom dial error * Limits the number of dial errors we keep * Exposes the dial errors * Avoids string formatting unless we actually need to. Helps address https://github.com/libp2p/go-libp2p-swarm/issues/119 --- p2p/net/swarm/dial_error.go | 69 ++++++++++++++++++++++++++++++++++ p2p/net/swarm/dial_test.go | 28 +++++--------- p2p/net/swarm/swarm_dial.go | 74 +++++++++++++++++++++---------------- 3 files changed, 121 insertions(+), 50 deletions(-) create mode 100644 p2p/net/swarm/dial_error.go diff --git a/p2p/net/swarm/dial_error.go b/p2p/net/swarm/dial_error.go new file mode 100644 index 0000000000..6d5bb14929 --- /dev/null +++ b/p2p/net/swarm/dial_error.go @@ -0,0 +1,69 @@ +package swarm + +import ( + "fmt" + "strings" + + peer "github.com/libp2p/go-libp2p-peer" + ma "github.com/multiformats/go-multiaddr" +) + +// maxDialDialErrors is the maximum number of dial errors we record +const maxDialDialErrors = 16 + +// DialError is the error type returned when dialing. +type DialError struct { + Peer peer.ID + DialErrors []TransportError + Cause error + Skipped int +} + +func (e *DialError) recordErr(addr ma.Multiaddr, err error) { + if len(e.DialErrors) >= maxDialDialErrors { + e.Skipped++ + return + } + e.DialErrors = append(e.DialErrors, TransportError{ + Address: addr, + Cause: err, + }) +} + +func (e *DialError) Error() string { + var builder strings.Builder + fmt.Fprintf(&builder, "failed to dial %s:", e.Peer) + if e.Cause != nil { + fmt.Fprintf(&builder, " %s", e.Cause) + } + for _, te := range e.DialErrors { + fmt.Fprintf(&builder, "\n * [%s] %s", te.Address, te.Cause) + } + if e.Skipped > 0 { + fmt.Fprintf(&builder, "\n ... skipping %d errors ...", e.Skipped) + } + return builder.String() +} + +// Unwrap implements https://godoc.org/golang.org/x/xerrors#Wrapper. +func (e *DialError) Unwrap() error { + // If we have a context error, that's the "ultimate" error. + if e.Cause != nil { + return e.Cause + } + return nil +} + +var _ error = (*DialError)(nil) + +// TransportError is the error returned when dialing a specific address. +type TransportError struct { + Address ma.Multiaddr + Cause error +} + +func (e *TransportError) Error() string { + return fmt.Sprintf("failed to dial %s: %s", e.Address, e.Cause) +} + +var _ error = (*TransportError)(nil) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index adb3438ac4..99585d2ce7 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -2,9 +2,7 @@ package swarm_test import ( "context" - "fmt" "net" - "regexp" "sync" "testing" "time" @@ -491,8 +489,8 @@ func TestDialPeerFailed(t *testing.T) { defer closeSwarms(swarms) testedSwarm, targetSwarm := swarms[0], swarms[1] - exceptedErrorsCount := 5 - for i := 0; i < exceptedErrorsCount; i++ { + expectedErrorsCount := 5 + for i := 0; i < expectedErrorsCount; i++ { _, silentPeerAddress, silentPeerListener := newSilentPeer(t) go acceptAndHang(silentPeerListener) defer silentPeerListener.Close() @@ -508,23 +506,17 @@ func TestDialPeerFailed(t *testing.T) { t.Fatal(err) } - // dial_test.go:508: correctly get a combined error: dial attempt failed: 10 errors occurred: - // * --> (/ip4/127.0.0.1/tcp/46485) dial attempt failed: failed to negotiate security protocol: context deadline exceeded - // * --> (/ip4/127.0.0.1/tcp/34881) dial attempt failed: failed to negotiate security protocol: context deadline exceeded + // dial_test.go:508: correctly get a combined error: failed to dial PEER: all dials failed + // * [/ip4/127.0.0.1/tcp/46485] failed to negotiate security protocol: context deadline exceeded + // * [/ip4/127.0.0.1/tcp/34881] failed to negotiate security protocol: context deadline exceeded // ... - errorCountRegexpString := fmt.Sprintf("%d errors occurred", exceptedErrorsCount) - errorCountRegexp := regexp.MustCompile(errorCountRegexpString) - if !errorCountRegexp.MatchString(err.Error()) { - t.Fatalf("can't find total err count: `%s' in `%s'", errorCountRegexpString, err.Error()) + dialErr, ok := err.(*DialError) + if !ok { + t.Fatalf("expected *DialError, got %T", err) } - connectErrorsRegexpString := `\* --> \(.+?\) dial attempt failed:.+` - connectErrorsRegexp := regexp.MustCompile(connectErrorsRegexpString) - connectErrors := connectErrorsRegexp.FindAll([]byte(err.Error()), -1) - if len(connectErrors) != exceptedErrorsCount { - t.Fatalf("connectErrors must contain %d errros; "+ - "but `%s' was found in `%s' %d times", - exceptedErrorsCount, connectErrorsRegexpString, err.Error(), len(connectErrors)) + if len(dialErr.DialErrors) != expectedErrorsCount { + t.Errorf("expected %d errors, got %d", expectedErrorsCount, len(dialErr.DialErrors)) } } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index d4b315525f..3feeffe3fe 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -7,8 +7,6 @@ import ( "sync" "time" - "github.com/hashicorp/go-multierror" - logging "github.com/ipfs/go-log" addrutil "github.com/libp2p/go-addr-util" lgbl "github.com/libp2p/go-libp2p-loggables" @@ -34,15 +32,23 @@ var ( // been dialed too frequently ErrDialBackoff = errors.New("dial backoff") - // ErrDialFailed is returned when connecting to a peer has ultimately failed - ErrDialFailed = errors.New("dial attempt failed") - // ErrDialToSelf is returned if we attempt to dial our own peer ErrDialToSelf = errors.New("dial to self attempted") // ErrNoTransport is returned when we don't know a transport for the // given multiaddr. ErrNoTransport = errors.New("no transport for protocol") + + // ErrAllDialsFailed is returned when connecting to a peer has ultimately failed + ErrAllDialsFailed = errors.New("all dials failed") + + // ErrNoAddresses is returned when we fail to find any addresses for a + // peer we're trying to dial. + ErrNoAddresses = errors.New("no addresses") + + // ErrNoAddresses is returned when we find addresses for a peer but + // can't use any of them. + ErrNoGoodAddresses = errors.New("no good addresses") ) // DialAttempts governs how many times a goroutine will try to dial a given peer. @@ -256,7 +262,7 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { } // ok, we failed. - return nil, fmt.Errorf("dial attempt failed: %s", err) + return nil, err } return conn, nil } @@ -293,11 +299,11 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { */ peerAddrs := s.peers.Addrs(p) if len(peerAddrs) == 0 { - return nil, errors.New("no addresses") + return nil, &DialError{Peer: p, Cause: ErrNoAddresses} } goodAddrs := s.filterKnownUndialables(peerAddrs) if len(goodAddrs) == 0 { - return nil, errors.New("no good addresses") + return nil, &DialError{Peer: p, Cause: ErrNoGoodAddresses} } goodAddrsChan := make(chan ma.Multiaddr, len(goodAddrs)) for _, a := range goodAddrs { @@ -307,10 +313,18 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { ///////// // try to get a connection to any addr - connC, err := s.dialAddrs(ctx, p, goodAddrsChan) - if err != nil { - logdial["error"] = err.Error() - return nil, err + connC, dialErr := s.dialAddrs(ctx, p, goodAddrsChan) + if dialErr != nil { + logdial["error"] = dialErr.Cause.Error() + if dialErr.Cause == context.Canceled { + // always prefer the "context canceled" error. + // we rely on behing able to check `err == context.Canceled` + // + // Removing this will BREAK backoff (causing us to + // backoff when canceling dials). + return nil, context.Canceled + } + return nil, dialErr } logdial["conn"] = logging.Metadata{ "localAddr": connC.LocalMultiaddr(), @@ -320,7 +334,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { if err != nil { logdial["error"] = err.Error() connC.Close() // close the connection. didn't work out :( - return nil, err + return nil, &DialError{Peer: p, Cause: err} } logdial["dial"] = "success" @@ -352,7 +366,7 @@ func (s *Swarm) filterKnownUndialables(addrs []ma.Multiaddr) []ma.Multiaddr { ) } -func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (transport.Conn, error) { +func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (transport.Conn, *DialError) { log.Debugf("%s swarm dialing %s", s.local, p) ctx, cancel := context.WithCancel(ctx) @@ -360,26 +374,23 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. // use a single response type instead of errs and conns, reduces complexity *a ton* respch := make(chan dialResult) - var dialErrors *multierror.Error + err := new(DialError) defer s.limiter.clearAllPeerDials(p) var active int +dialLoop: for remoteAddrs != nil || active > 0 { // Check for context cancellations and/or responses first. select { case <-ctx.Done(): - if dialError := dialErrors.ErrorOrNil(); dialError != nil { - return nil, dialError - } - - return nil, ctx.Err() + break dialLoop case resp := <-respch: active-- if resp.Err != nil { // Errors are normal, lots of dials will fail log.Infof("got error on dial: %s", resp.Err) - dialErrors = multierror.Append(dialErrors, resp.Err) + err.recordErr(resp.Addr, resp.Err) } else if resp.Conn != nil { return resp.Conn, nil } @@ -400,28 +411,27 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. s.limitedDial(ctx, p, addr, respch) active++ case <-ctx.Done(): - if dialError := dialErrors.ErrorOrNil(); dialError != nil { - return nil, dialError - } - - return nil, ctx.Err() + break dialLoop case resp := <-respch: active-- if resp.Err != nil { // Errors are normal, lots of dials will fail log.Infof("got error on dial: %s", resp.Err) - dialErrors = multierror.Append(dialErrors, resp.Err) + err.recordErr(resp.Addr, resp.Err) } else if resp.Conn != nil { return resp.Conn, nil } } } - if dialError := dialErrors.ErrorOrNil(); dialError != nil { - return nil, dialError + if ctxErr := ctx.Err(); ctxErr != nil { + err.Cause = ctxErr + } else if len(err.DialErrors) == 0 { + err.Cause = inet.ErrNoRemoteAddrs + } else { + err.Cause = ErrAllDialsFailed } - - return nil, inet.ErrNoRemoteAddrs + return nil, err } // limitedDial will start a dial to the given peer when @@ -450,7 +460,7 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (tra connC, err := tpt.Dial(ctx, addr, p) if err != nil { - return nil, fmt.Errorf("%s --> %s (%s) dial attempt failed: %s", s.local, p, addr, err) + return nil, err } // Trust the transport? Yeah... right. From cf873f59f686bfe50c2e240e4ff5f3c4ac1556b6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 21 May 2019 19:33:31 -0700 Subject: [PATCH 138/259] dep: import go-smux-* into the libp2p org --- p2p/net/swarm/testing/testing.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index e3a62c5283..edaaf5b274 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -11,10 +11,10 @@ import ( pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" secio "github.com/libp2p/go-libp2p-secio" tptu "github.com/libp2p/go-libp2p-transport-upgrader" + yamux "github.com/libp2p/go-libp2p-yamux" + msmux "github.com/libp2p/go-stream-muxer-multistream" tcp "github.com/libp2p/go-tcp-transport" tu "github.com/libp2p/go-testutil" - msmux "github.com/whyrusleeping/go-smux-multistream" - yamux "github.com/whyrusleeping/go-smux-yamux" swarm "github.com/libp2p/go-libp2p-swarm" ) From e132f6af3ffe51f271e3cd9e697ccde202e9d5e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 26 May 2019 15:55:50 +0100 Subject: [PATCH 139/259] migrate to consolidated types; remove CircleCI config (#127) --- p2p/net/swarm/.circleci/config.yml | 10 ---- p2p/net/swarm/dial_error.go | 3 +- p2p/net/swarm/dial_sync.go | 2 +- p2p/net/swarm/dial_sync_test.go | 2 +- p2p/net/swarm/dial_test.go | 31 ++++++------ p2p/net/swarm/limiter.go | 9 ++-- p2p/net/swarm/limiter_test.go | 20 ++++---- p2p/net/swarm/peers_test.go | 16 +++---- p2p/net/swarm/simul_test.go | 9 ++-- p2p/net/swarm/swarm.go | 76 +++++++++++++++--------------- p2p/net/swarm/swarm_addr_test.go | 9 ++-- p2p/net/swarm/swarm_conn.go | 35 +++++++------- p2p/net/swarm/swarm_dial.go | 21 +++++---- p2p/net/swarm/swarm_listen.go | 7 +-- p2p/net/swarm/swarm_net_test.go | 30 ++++++------ p2p/net/swarm/swarm_notif_test.go | 49 +++++++++---------- p2p/net/swarm/swarm_stream.go | 20 ++++---- p2p/net/swarm/swarm_test.go | 27 ++++++----- p2p/net/swarm/swarm_transport.go | 3 +- p2p/net/swarm/testing/testing.go | 19 ++++---- p2p/net/swarm/transport_test.go | 6 +-- 21 files changed, 205 insertions(+), 199 deletions(-) delete mode 100644 p2p/net/swarm/.circleci/config.yml diff --git a/p2p/net/swarm/.circleci/config.yml b/p2p/net/swarm/.circleci/config.yml deleted file mode 100644 index 68d7e72668..0000000000 --- a/p2p/net/swarm/.circleci/config.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/golang:1.11 - environment: - GO111MODULE: "on" - steps: - - checkout - - run: go test -v -race -bench . ./... diff --git a/p2p/net/swarm/dial_error.go b/p2p/net/swarm/dial_error.go index 6d5bb14929..d21796172f 100644 --- a/p2p/net/swarm/dial_error.go +++ b/p2p/net/swarm/dial_error.go @@ -4,7 +4,8 @@ import ( "fmt" "strings" - peer "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" + ma "github.com/multiformats/go-multiaddr" ) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 82256eca48..f746b9a9a3 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -4,7 +4,7 @@ import ( "context" "sync" - peer "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) // DialFunc is the type of function expected by DialSync. diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 53c3fc25b4..485d1a3171 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -9,7 +9,7 @@ import ( . "github.com/libp2p/go-libp2p-swarm" - peer "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) func getMockDialFunc() (DialFunc, func(), context.Context, <-chan struct{}) { diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 99585d2ce7..73bd02c674 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -8,12 +8,15 @@ import ( "time" addrutil "github.com/libp2p/go-addr-util" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" + + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/transport" + + testutil "github.com/libp2p/go-libp2p-core/test" swarmt "github.com/libp2p/go-libp2p-swarm/testing" - transport "github.com/libp2p/go-libp2p-transport" - testutil "github.com/libp2p/go-testutil" - ci "github.com/libp2p/go-testutil/ci" + "github.com/libp2p/go-libp2p-testing/ci" + ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -39,7 +42,7 @@ func TestBasicDialPeer(t *testing.T) { s1 := swarms[0] s2 := swarms[1] - s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), pstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) c, err := s1.DialPeer(ctx, s2.LocalPeer()) if err != nil { @@ -64,7 +67,7 @@ func TestDialWithNoListeners(t *testing.T) { defer closeSwarms(swarms) s2 := swarms[0] - s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), pstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) c, err := s1.DialPeer(ctx, s2.LocalPeer()) if err != nil { @@ -108,7 +111,7 @@ func TestSimultDials(t *testing.T) { connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) - s.Peerstore().AddAddr(dst, addr, pstore.TempAddrTTL) + s.Peerstore().AddAddr(dst, addr, peerstore.TempAddrTTL) if _, err := s.DialPeer(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } @@ -179,7 +182,7 @@ func TestDialWait(t *testing.T) { s2p, s2addr, s2l := newSilentPeer(t) go acceptAndHang(s2l) defer s2l.Close() - s1.Peerstore().AddAddr(s2p, s2addr, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddr(s2p, s2addr, peerstore.PermanentAddrTTL) before := time.Now() if c, err := s1.DialPeer(ctx, s2p); err == nil { @@ -221,13 +224,13 @@ func TestDialBackoff(t *testing.T) { if err != nil { t.Fatal(err) } - s1.Peerstore().AddAddrs(s2.LocalPeer(), s2addrs, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2addrs, peerstore.PermanentAddrTTL) // dial to a non-existent peer. s3p, s3addr, s3l := newSilentPeer(t) go acceptAndHang(s3l) defer s3l.Close() - s1.Peerstore().AddAddr(s3p, s3addr, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddr(s3p, s3addr, peerstore.PermanentAddrTTL) // in this test we will: // 1) dial 10x to each node. @@ -430,7 +433,7 @@ func TestDialBackoffClears(t *testing.T) { defer s2l.Close() // phase 1 -- dial to non-operational addresses - s1.Peerstore().AddAddr(s2.LocalPeer(), s2bad, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddr(s2.LocalPeer(), s2bad, peerstore.PermanentAddrTTL) before := time.Now() if c, err := s1.DialPeer(ctx, s2.LocalPeer()); err == nil { @@ -459,7 +462,7 @@ func TestDialBackoffClears(t *testing.T) { if err != nil { t.Fatal(err) } - s1.Peerstore().AddAddrs(s2.LocalPeer(), ifaceAddrs1, pstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), ifaceAddrs1, peerstore.PermanentAddrTTL) if _, err := s1.DialPeer(ctx, s2.LocalPeer()); err == nil { t.Fatal("should have failed to dial backed off peer") @@ -498,7 +501,7 @@ func TestDialPeerFailed(t *testing.T) { testedSwarm.Peerstore().AddAddr( targetSwarm.LocalPeer(), silentPeerAddress, - pstore.PermanentAddrTTL) + peerstore.PermanentAddrTTL) } _, err := testedSwarm.DialPeer(ctx, targetSwarm.LocalPeer()) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index bd4ab57256..6808dd71a6 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -7,14 +7,15 @@ import ( "sync" "time" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/transport" + addrutil "github.com/libp2p/go-addr-util" - peer "github.com/libp2p/go-libp2p-peer" - transport "github.com/libp2p/go-libp2p-transport" ma "github.com/multiformats/go-multiaddr" ) type dialResult struct { - Conn transport.Conn + Conn transport.CapableConn Addr ma.Multiaddr Err error } @@ -53,7 +54,7 @@ type dialLimiter struct { waitingOnPeerLimit map[peer.ID][]*dialJob } -type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (transport.Conn, error) +type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (transport.CapableConn, error) func newDialLimiter(df dialfunc) *dialLimiter { fd := ConcurrentFdDials diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 4338cad50a..32d87fa13d 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - peer "github.com/libp2p/go-libp2p-peer" - transport "github.com/libp2p/go-libp2p-transport" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/transport" ma "github.com/multiformats/go-multiaddr" mafmt "github.com/whyrusleeping/mafmt" ) @@ -56,13 +56,13 @@ func tryDialAddrs(ctx context.Context, l *dialLimiter, p peer.ID, addrs []ma.Mul } func hangDialFunc(hang chan struct{}) dialfunc { - return func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.Conn, error) { + return func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.CapableConn, error) { if mafmt.UTP.Matches(a) { - return transport.Conn(nil), nil + return transport.CapableConn(nil), nil } if tcpPortOver(a, 10) { - return transport.Conn(nil), nil + return transport.CapableConn(nil), nil } <-hang @@ -173,9 +173,9 @@ func TestFDLimiting(t *testing.T) { func TestTokenRedistribution(t *testing.T) { var lk sync.Mutex hangchs := make(map[peer.ID]chan struct{}) - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.Conn, error) { + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.CapableConn, error) { if tcpPortOver(a, 10) { - return (transport.Conn)(nil), nil + return (transport.CapableConn)(nil), nil } lk.Lock() @@ -268,9 +268,9 @@ func TestTokenRedistribution(t *testing.T) { } func TestStressLimiter(t *testing.T) { - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.Conn, error) { + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.CapableConn, error) { if tcpPortOver(a, 1000) { - return transport.Conn(nil), nil + return transport.CapableConn(nil), nil } time.Sleep(time.Millisecond * time.Duration(5+rand.Intn(100))) @@ -322,7 +322,7 @@ func TestStressLimiter(t *testing.T) { } func TestFDLimitUnderflow(t *testing.T) { - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.Conn, error) { + df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.CapableConn, error) { timeout := make(chan bool, 1) go func() { time.Sleep(time.Second * 5) diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 76cc00b022..fd1984d072 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -1,19 +1,19 @@ package swarm_test import ( + "context" "testing" - "context" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peerstore" + ma "github.com/multiformats/go-multiaddr" . "github.com/libp2p/go-libp2p-swarm" ) func TestPeers(t *testing.T) { - ctx := context.Background() swarms := makeSwarms(ctx, t, 2) s1 := swarms[0] @@ -21,7 +21,7 @@ func TestPeers(t *testing.T) { connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. - s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL) + s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) // t.Logf("connections from %s", s.LocalPeer()) // for _, c := range s.ConnsToPeer(dst) { // t.Logf("connection from %s to %s: %v", s.LocalPeer(), dst, c) @@ -35,10 +35,10 @@ func TestPeers(t *testing.T) { s1GotConn := make(chan struct{}, 0) s2GotConn := make(chan struct{}, 0) - s1.SetConnHandler(func(c inet.Conn) { + s1.SetConnHandler(func(c network.Conn) { s1GotConn <- struct{}{} }) - s2.SetConnHandler(func(c inet.Conn) { + s2.SetConnHandler(func(c network.Conn) { s2GotConn <- struct{}{} }) diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index b162ab380f..0373e37dfb 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,13 +7,14 @@ import ( "testing" "time" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - ci "github.com/libp2p/go-testutil/ci" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peerstore" + ma "github.com/multiformats/go-multiaddr" . "github.com/libp2p/go-libp2p-swarm" swarmt "github.com/libp2p/go-libp2p-swarm/testing" + "github.com/libp2p/go-libp2p-testing/ci" ) func TestSimultOpen(t *testing.T) { @@ -30,7 +31,7 @@ func TestSimultOpen(t *testing.T) { defer wg.Done() // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) - s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL) + s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) if _, err := s.DialPeer(ctx, dst); err != nil { t.Error("error swarm dialing to peer", err) } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 1094e72e19..b14e9cf0b9 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -9,14 +9,16 @@ import ( "sync/atomic" "time" + "github.com/libp2p/go-libp2p-core/metrics" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/transport" + logging "github.com/ipfs/go-log" "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" - metrics "github.com/libp2p/go-libp2p-metrics" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" - transport "github.com/libp2p/go-libp2p-transport" + filter "github.com/libp2p/go-maddr-filter" mafilter "github.com/whyrusleeping/multiaddr-filter" ) @@ -47,7 +49,7 @@ type Swarm struct { refs sync.WaitGroup local peer.ID - peers pstore.Peerstore + peers peerstore.Peerstore conns struct { sync.RWMutex @@ -61,7 +63,7 @@ type Swarm struct { notifs struct { sync.RWMutex - m map[inet.Notifiee]struct{} + m map[network.Notifiee]struct{} } transports struct { @@ -87,7 +89,7 @@ type Swarm struct { } // NewSwarm constructs a Swarm -func NewSwarm(ctx context.Context, local peer.ID, peers pstore.Peerstore, bwc metrics.Reporter) *Swarm { +func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter) *Swarm { s := &Swarm{ local: local, peers: peers, @@ -98,7 +100,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers pstore.Peerstore, bwc me s.conns.m = make(map[peer.ID][]*Conn) s.listeners.m = make(map[transport.Listener]struct{}) s.transports.m = make(map[int]transport.Transport) - s.notifs.m = make(map[inet.Notifiee]struct{}) + s.notifs.m = make(map[network.Notifiee]struct{}) s.dsync = NewDialSync(s.doDial) s.limiter = newDialLimiter(s.dialAddr) @@ -165,7 +167,7 @@ func (s *Swarm) Process() goprocess.Process { return s.proc } -func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { +func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, error) { // The underlying transport (or the dialer) *should* filter it's own // connections but we should double check anyways. raddr := tc.RemoteMultiaddr() @@ -194,7 +196,7 @@ func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { } // Wrap and register the connection. - stat := inet.Stat{Direction: dir} + stat := network.Stat{Direction: dir} c := &Conn{ conn: tc, swarm: s, @@ -217,7 +219,7 @@ func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { // This should be fast, no reason to wait till later. s.dsync.CancelDial(p) - s.notifyAll(func(f inet.Notifiee) { + s.notifyAll(func(f network.Notifiee) { f.Connected(s, c) }) c.notifyLk.Unlock() @@ -235,7 +237,7 @@ func (s *Swarm) addConn(tc transport.Conn, dir inet.Direction) (*Conn, error) { } // Peerstore returns this swarms internal Peerstore. -func (s *Swarm) Peerstore() pstore.Peerstore { +func (s *Swarm) Peerstore() peerstore.Peerstore { return s.peers } @@ -253,30 +255,30 @@ func (s *Swarm) Close() error { // SetConnHandler assigns the handler for new connections. // You will rarely use this. See SetStreamHandler -func (s *Swarm) SetConnHandler(handler inet.ConnHandler) { +func (s *Swarm) SetConnHandler(handler network.ConnHandler) { s.connh.Store(handler) } // ConnHandler gets the handler for new connections. -func (s *Swarm) ConnHandler() inet.ConnHandler { - handler, _ := s.connh.Load().(inet.ConnHandler) +func (s *Swarm) ConnHandler() network.ConnHandler { + handler, _ := s.connh.Load().(network.ConnHandler) return handler } // SetStreamHandler assigns the handler for new streams. -func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) { +func (s *Swarm) SetStreamHandler(handler network.StreamHandler) { s.streamh.Store(handler) } // StreamHandler gets the handler for new streams. -func (s *Swarm) StreamHandler() inet.StreamHandler { - handler, _ := s.streamh.Load().(inet.StreamHandler) +func (s *Swarm) StreamHandler() network.StreamHandler { + handler, _ := s.streamh.Load().(network.StreamHandler) return handler } // NewStream creates a new stream on any available connection to peer, dialing // if necessary. -func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) { +func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (network.Stream, error) { log.Debugf("[%s] opening stream to peer [%s]", s.local, p) // Algorithm: @@ -295,8 +297,8 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) { for { c := s.bestConnToPeer(p) if c == nil { - if nodial, _ := inet.GetNoDial(ctx); nodial { - return nil, inet.ErrNoConn + if nodial, _ := network.GetNoDial(ctx); nodial { + return nil, network.ErrNoConn } if dials >= DialAttempts { @@ -322,13 +324,13 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (inet.Stream, error) { } // ConnsToPeer returns all the live connections to peer. -func (s *Swarm) ConnsToPeer(p peer.ID) []inet.Conn { +func (s *Swarm) ConnsToPeer(p peer.ID) []network.Conn { // TODO: Consider sorting the connection list best to worst. Currently, // it's sorted oldest to newest. s.conns.RLock() defer s.conns.RUnlock() conns := s.conns.m[p] - output := make([]inet.Conn, len(conns)) + output := make([]network.Conn, len(conns)) for i, c := range conns { output[i] = c } @@ -366,20 +368,20 @@ func (s *Swarm) bestConnToPeer(p peer.ID) *Conn { // Connectedness returns our "connectedness" state with the given peer. // // To check if we have an open connection, use `s.Connectedness(p) == -// inet.Connected`. -func (s *Swarm) Connectedness(p peer.ID) inet.Connectedness { +// network.Connected`. +func (s *Swarm) Connectedness(p peer.ID) network.Connectedness { if s.bestConnToPeer(p) != nil { - return inet.Connected + return network.Connected } - return inet.NotConnected + return network.NotConnected } // Conns returns a slice of all connections. -func (s *Swarm) Conns() []inet.Conn { +func (s *Swarm) Conns() []network.Conn { s.conns.RLock() defer s.conns.RUnlock() - conns := make([]inet.Conn, 0, len(s.conns.m)) + conns := make([]network.Conn, 0, len(s.conns.m)) for _, cs := range s.conns.m { for _, c := range cs { conns = append(conns, c) @@ -399,7 +401,7 @@ func (s *Swarm) ClosePeer(p peer.ID) error { default: errCh := make(chan error) for _, c := range conns { - go func(c inet.Conn) { + go func(c network.Conn) { errCh <- c.Close() }(c) } @@ -441,13 +443,13 @@ func (s *Swarm) Backoff() *DialBackoff { } // notifyAll sends a signal to all Notifiees -func (s *Swarm) notifyAll(notify func(inet.Notifiee)) { +func (s *Swarm) notifyAll(notify func(network.Notifiee)) { var wg sync.WaitGroup s.notifs.RLock() wg.Add(len(s.notifs.m)) for f := range s.notifs.m { - go func(f inet.Notifiee) { + go func(f network.Notifiee) { defer wg.Done() notify(f) }(f) @@ -458,14 +460,14 @@ func (s *Swarm) notifyAll(notify func(inet.Notifiee)) { } // Notify signs up Notifiee to receive signals when events happen -func (s *Swarm) Notify(f inet.Notifiee) { +func (s *Swarm) Notify(f network.Notifiee) { s.notifs.Lock() s.notifs.m[f] = struct{}{} s.notifs.Unlock() } // StopNotify unregisters Notifiee fromr receiving signals -func (s *Swarm) StopNotify(f inet.Notifiee) { +func (s *Swarm) StopNotify(f network.Notifiee) { s.notifs.Lock() delete(s.notifs.m, f) s.notifs.Unlock() @@ -500,5 +502,5 @@ func (s *Swarm) String() string { } // Swarm is a Network. -var _ inet.Network = (*Swarm)(nil) -var _ transport.Network = (*Swarm)(nil) +var _ network.Network = (*Swarm)(nil) +var _ transport.TransportNetwork = (*Swarm)(nil) diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index bafcfb6697..e75d660615 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -4,8 +4,9 @@ import ( "context" "testing" - pstore "github.com/libp2p/go-libp2p-peerstore" - testutil "github.com/libp2p/go-testutil" + "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/test" + ma "github.com/multiformats/go-multiaddr" ) @@ -23,8 +24,8 @@ func TestDialBadAddrs(t *testing.T) { s := makeSwarms(ctx, t, 1)[0] test := func(a ma.Multiaddr) { - p := testutil.RandPeerIDFatal(t) - s.Peerstore().AddAddr(p, a, pstore.PermanentAddrTTL) + p := test.RandPeerIDFatal(t) + s.Peerstore().AddAddr(p, a, peerstore.PermanentAddrTTL) if _, err := s.DialPeer(ctx, p); err == nil { t.Errorf("swarm should not dial: %s", p) } diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 26a7794d15..cdb9866a2a 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -5,11 +5,12 @@ import ( "fmt" "sync" - ic "github.com/libp2p/go-libp2p-crypto" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - transport "github.com/libp2p/go-libp2p-transport" - smux "github.com/libp2p/go-stream-muxer" + ic "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/mux" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/transport" + ma "github.com/multiformats/go-multiaddr" ) @@ -21,7 +22,7 @@ var ErrConnClosed = errors.New("connection closed") // Conn is the connection type used by swarm. In general, you won't use this // type directly. type Conn struct { - conn transport.Conn + conn transport.CapableConn swarm *Swarm closeOnce sync.Once @@ -34,7 +35,7 @@ type Conn struct { m map[*Stream]struct{} } - stat inet.Stat + stat network.Stat } // Close closes this connection. @@ -71,7 +72,7 @@ func (c *Conn) doClose() { c.notifyLk.Lock() defer c.notifyLk.Unlock() - c.swarm.notifyAll(func(f inet.Notifiee) { + c.swarm.notifyAll(func(f network.Notifiee) { f.Disconnected(c.swarm, c) }) c.swarm.refs.Done() // taken in Swarm.addConn @@ -100,7 +101,7 @@ func (c *Conn) start() { } c.swarm.refs.Add(1) go func() { - s, err := c.addStream(ts, inet.DirInbound) + s, err := c.addStream(ts, network.DirInbound) // Don't defer this. We don't want to block // swarm shutdown on the connection handler. @@ -161,20 +162,20 @@ func (c *Conn) RemotePublicKey() ic.PubKey { } // Stat returns metadata pertaining to this connection -func (c *Conn) Stat() inet.Stat { +func (c *Conn) Stat() network.Stat { return c.stat } // NewStream returns a new Stream from this connection -func (c *Conn) NewStream() (inet.Stream, error) { +func (c *Conn) NewStream() (network.Stream, error) { ts, err := c.conn.OpenStream() if err != nil { return nil, err } - return c.addStream(ts, inet.DirOutbound) + return c.addStream(ts, network.DirOutbound) } -func (c *Conn) addStream(ts smux.Stream, dir inet.Direction) (*Stream, error) { +func (c *Conn) addStream(ts mux.MuxedStream, dir network.Direction) (*Stream, error) { c.streams.Lock() // Are we still online? if c.streams.m == nil { @@ -184,7 +185,7 @@ func (c *Conn) addStream(ts smux.Stream, dir inet.Direction) (*Stream, error) { } // Wrap and register the stream. - stat := inet.Stat{Direction: dir} + stat := network.Stat{Direction: dir} s := &Stream{ stream: ts, conn: c, @@ -202,7 +203,7 @@ func (c *Conn) addStream(ts smux.Stream, dir inet.Direction) (*Stream, error) { s.notifyLk.Lock() c.streams.Unlock() - c.swarm.notifyAll(func(f inet.Notifiee) { + c.swarm.notifyAll(func(f network.Notifiee) { f.OpenedStream(c.swarm, s) }) s.notifyLk.Unlock() @@ -211,10 +212,10 @@ func (c *Conn) addStream(ts smux.Stream, dir inet.Direction) (*Stream, error) { } // GetStreams returns the streams associated with this connection. -func (c *Conn) GetStreams() []inet.Stream { +func (c *Conn) GetStreams() []network.Stream { c.streams.Lock() defer c.streams.Unlock() - streams := make([]inet.Stream, 0, len(c.streams.m)) + streams := make([]network.Stream, 0, len(c.streams.m)) for s := range c.streams.m { streams = append(streams, s) } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 3feeffe3fe..40faad7848 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -7,12 +7,13 @@ import ( "sync" "time" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/transport" + lgbl "github.com/libp2p/go-libp2p-loggables" + logging "github.com/ipfs/go-log" addrutil "github.com/libp2p/go-addr-util" - lgbl "github.com/libp2p/go-libp2p-loggables" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - transport "github.com/libp2p/go-libp2p-transport" ma "github.com/multiformats/go-multiaddr" ) @@ -180,7 +181,7 @@ func (db *DialBackoff) Clear(p peer.ID) { // the connection will happen over. Swarm can use whichever it choses. // This allows us to use various transport protocols, do NAT traversal/relay, // etc. to achieve connection. -func (s *Swarm) DialPeer(ctx context.Context, p peer.ID) (inet.Conn, error) { +func (s *Swarm) DialPeer(ctx context.Context, p peer.ID) (network.Conn, error) { return s.dialPeer(ctx, p) } @@ -216,7 +217,7 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { } // apply the DialPeer timeout - ctx, cancel := context.WithTimeout(ctx, inet.GetDialPeerTimeout(ctx)) + ctx, cancel := context.WithTimeout(ctx, network.GetDialPeerTimeout(ctx)) defer cancel() conn, err = s.dsync.DialLock(ctx, p) @@ -330,7 +331,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { "localAddr": connC.LocalMultiaddr(), "remoteAddr": connC.RemoteMultiaddr(), } - swarmC, err := s.addConn(connC, inet.DirOutbound) + swarmC, err := s.addConn(connC, network.DirOutbound) if err != nil { logdial["error"] = err.Error() connC.Close() // close the connection. didn't work out :( @@ -366,7 +367,7 @@ func (s *Swarm) filterKnownUndialables(addrs []ma.Multiaddr) []ma.Multiaddr { ) } -func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (transport.Conn, *DialError) { +func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (transport.CapableConn, *DialError) { log.Debugf("%s swarm dialing %s", s.local, p) ctx, cancel := context.WithCancel(ctx) @@ -427,7 +428,7 @@ dialLoop: if ctxErr := ctx.Err(); ctxErr != nil { err.Cause = ctxErr } else if len(err.DialErrors) == 0 { - err.Cause = inet.ErrNoRemoteAddrs + err.Cause = network.ErrNoRemoteAddrs } else { err.Cause = ErrAllDialsFailed } @@ -446,7 +447,7 @@ func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp }) } -func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.Conn, error) { +func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.CapableConn, error) { // Just to double check. Costs nothing. if s.local == p { return nil, ErrDialToSelf diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 7b37d53497..85060c4e2c 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -3,7 +3,8 @@ package swarm import ( "fmt" - inet "github.com/libp2p/go-libp2p-net" + "github.com/libp2p/go-libp2p-core/network" + ma "github.com/multiformats/go-multiaddr" ) @@ -59,7 +60,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { maddr := list.Multiaddr() // signal to our notifiees on successful conn. - s.notifyAll(func(n inet.Notifiee) { + s.notifyAll(func(n network.Notifiee) { n.Listen(s, maddr) }) @@ -83,7 +84,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { s.refs.Add(1) go func() { defer s.refs.Done() - _, err := s.addConn(c, inet.DirInbound) + _, err := s.addConn(c, network.DirInbound) if err != nil { // Probably just means that the swarm has been closed. log.Warningf("add conn failed: ", err) diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 425a1707d1..2ba64edb96 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - inet "github.com/libp2p/go-libp2p-net" + "github.com/libp2p/go-libp2p-core/network" . "github.com/libp2p/go-libp2p-swarm/testing" ) @@ -17,14 +17,14 @@ func TestConnectednessCorrect(t *testing.T) { ctx := context.Background() - nets := make([]inet.Network, 4) + nets := make([]network.Network, 4) for i := 0; i < 4; i++ { nets[i] = GenSwarm(t, ctx) } // connect 0-1, 0-2, 0-3, 1-2, 2-3 - dial := func(a, b inet.Network) { + dial := func(a, b network.Network) { DivulgeAddresses(b, a) if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { t.Fatalf("Failed to dial: %s", err) @@ -44,14 +44,14 @@ func TestConnectednessCorrect(t *testing.T) { // test those connected show up correctly // test connected - expectConnectedness(t, nets[0], nets[1], inet.Connected) - expectConnectedness(t, nets[0], nets[3], inet.Connected) - expectConnectedness(t, nets[1], nets[2], inet.Connected) - expectConnectedness(t, nets[3], nets[2], inet.Connected) + expectConnectedness(t, nets[0], nets[1], network.Connected) + expectConnectedness(t, nets[0], nets[3], network.Connected) + expectConnectedness(t, nets[1], nets[2], network.Connected) + expectConnectedness(t, nets[3], nets[2], network.Connected) // test not connected - expectConnectedness(t, nets[0], nets[2], inet.NotConnected) - expectConnectedness(t, nets[1], nets[3], inet.NotConnected) + expectConnectedness(t, nets[0], nets[2], network.NotConnected) + expectConnectedness(t, nets[1], nets[3], network.NotConnected) if len(nets[0].Peers()) != 2 { t.Fatal("expected net 0 to have two peers") @@ -71,7 +71,7 @@ func TestConnectednessCorrect(t *testing.T) { time.Sleep(time.Millisecond * 50) - expectConnectedness(t, nets[2], nets[1], inet.NotConnected) + expectConnectedness(t, nets[2], nets[1], network.NotConnected) for _, n := range nets { n.Close() @@ -82,7 +82,7 @@ func TestConnectednessCorrect(t *testing.T) { } } -func expectConnectedness(t *testing.T, a, b inet.Network, expected inet.Connectedness) { +func expectConnectedness(t *testing.T, a, b network.Network, expected network.Connectedness) { es := "%s is connected to %s, but Connectedness incorrect. %s %s %s" atob := a.Connectedness(b.LocalPeer()) btoa := b.Connectedness(a.LocalPeer()) @@ -96,7 +96,7 @@ func expectConnectedness(t *testing.T, a, b inet.Network, expected inet.Connecte } } -func printConns(n inet.Network) string { +func printConns(n network.Network) string { s := fmt.Sprintf("Connections in %s:\n", n) for _, c := range n.Conns() { s = s + fmt.Sprintf("- %s\n", c) @@ -108,12 +108,12 @@ func TestNetworkOpenStream(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - nets := make([]inet.Network, 4) + nets := make([]network.Network, 4) for i := 0; i < 4; i++ { nets[i] = GenSwarm(t, ctx) } - dial := func(a, b inet.Network) { + dial := func(a, b network.Network) { DivulgeAddresses(b, a) if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { t.Fatalf("Failed to dial: %s", err) @@ -125,7 +125,7 @@ func TestNetworkOpenStream(t *testing.T) { dial(nets[1], nets[2]) done := make(chan bool) - nets[1].SetStreamHandler(func(s inet.Stream) { + nets[1].SetStreamHandler(func(s network.Stream) { defer close(done) defer s.Close() diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index d05766d6c7..f2c91ebb80 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -1,12 +1,13 @@ package swarm_test import ( + "context" "testing" "time" - "context" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + ma "github.com/multiformats/go-multiaddr" . "github.com/libp2p/go-libp2p-swarm" @@ -41,7 +42,7 @@ func TestNotifications(t *testing.T) { // test everyone got the correct connection opened calls for i, s := range swarms { n := notifiees[i] - notifs := make(map[peer.ID][]inet.Conn) + notifs := make(map[peer.ID][]network.Conn) for j, s2 := range swarms { if i == j { continue @@ -81,7 +82,7 @@ func TestNotifications(t *testing.T) { } } - complement := func(c inet.Conn) (*Swarm, *netNotifiee, *Conn) { + complement := func(c network.Conn) (*Swarm, *netNotifiee, *Conn) { for i, s := range swarms { for _, c2 := range s.Conns() { if c.LocalMultiaddr().Equal(c2.RemoteMultiaddr()) && @@ -94,8 +95,8 @@ func TestNotifications(t *testing.T) { return nil, nil, nil } - testOCStream := func(n *netNotifiee, s inet.Stream) { - var s2 inet.Stream + testOCStream := func(n *netNotifiee, s network.Stream) { + var s2 network.Stream select { case s2 = <-n.openedStream: t.Log("got notif for opened stream") @@ -117,9 +118,9 @@ func TestNotifications(t *testing.T) { } } - streams := make(chan inet.Stream) + streams := make(chan network.Stream) for _, s := range swarms { - s.SetStreamHandler(func(s inet.Stream) { + s.SetStreamHandler(func(s network.Stream) { streams <- s s.Reset() }) @@ -151,7 +152,7 @@ func TestNotifications(t *testing.T) { c.Close() c2.Close() - var c3, c4 inet.Conn + var c3, c4 network.Conn select { case c3 = <-n.disconnected: case <-time.After(timeout): @@ -176,38 +177,38 @@ func TestNotifications(t *testing.T) { type netNotifiee struct { listen chan ma.Multiaddr listenClose chan ma.Multiaddr - connected chan inet.Conn - disconnected chan inet.Conn - openedStream chan inet.Stream - closedStream chan inet.Stream + connected chan network.Conn + disconnected chan network.Conn + openedStream chan network.Stream + closedStream chan network.Stream } func newNetNotifiee(buffer int) *netNotifiee { return &netNotifiee{ listen: make(chan ma.Multiaddr, buffer), listenClose: make(chan ma.Multiaddr, buffer), - connected: make(chan inet.Conn, buffer), - disconnected: make(chan inet.Conn, buffer), - openedStream: make(chan inet.Stream, buffer), - closedStream: make(chan inet.Stream, buffer), + connected: make(chan network.Conn, buffer), + disconnected: make(chan network.Conn, buffer), + openedStream: make(chan network.Stream, buffer), + closedStream: make(chan network.Stream, buffer), } } -func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr) { +func (nn *netNotifiee) Listen(n network.Network, a ma.Multiaddr) { nn.listen <- a } -func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) { +func (nn *netNotifiee) ListenClose(n network.Network, a ma.Multiaddr) { nn.listenClose <- a } -func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) { +func (nn *netNotifiee) Connected(n network.Network, v network.Conn) { nn.connected <- v } -func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) { +func (nn *netNotifiee) Disconnected(n network.Network, v network.Conn) { nn.disconnected <- v } -func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) { +func (nn *netNotifiee) OpenedStream(n network.Network, v network.Stream) { nn.openedStream <- v } -func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) { +func (nn *netNotifiee) ClosedStream(n network.Network, v network.Stream) { nn.closedStream <- v } diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 754dbd50e7..9dded2a9d7 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -7,9 +7,9 @@ import ( "sync/atomic" "time" - inet "github.com/libp2p/go-libp2p-net" - protocol "github.com/libp2p/go-libp2p-protocol" - smux "github.com/libp2p/go-stream-muxer" + "github.com/libp2p/go-libp2p-core/mux" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/protocol" ) type streamState int @@ -23,12 +23,12 @@ const ( ) // Validate Stream conforms to the go-libp2p-net Stream interface -var _ inet.Stream = &Stream{} +var _ network.Stream = &Stream{} // Stream is the stream type used by swarm. In general, you won't use this type // directly. type Stream struct { - stream smux.Stream + stream mux.MuxedStream conn *Conn state struct { @@ -40,7 +40,7 @@ type Stream struct { protocol atomic.Value - stat inet.Stat + stat network.Stat } func (s *Stream) String() string { @@ -54,8 +54,8 @@ func (s *Stream) String() string { ) } -// Conn returns the Conn associated with this stream, as an inet.Conn -func (s *Stream) Conn() inet.Conn { +// Conn returns the Conn associated with this stream, as an network.Conn +func (s *Stream) Conn() network.Conn { return s.conn } @@ -133,7 +133,7 @@ func (s *Stream) remove() { s.notifyLk.Lock() defer s.notifyLk.Unlock() - s.conn.swarm.notifyAll(func(f inet.Notifiee) { + s.conn.swarm.notifyAll(func(f network.Notifiee) { f.ClosedStream(s.conn.swarm, s) }) s.conn.swarm.refs.Done() @@ -172,6 +172,6 @@ func (s *Stream) SetWriteDeadline(t time.Time) error { } // Stat returns metadata information for this stream. -func (s *Stream) Stat() inet.Stat { +func (s *Stream) Stat() network.Stat { return s.stat } diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 69eceb94c4..b155373f67 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -11,9 +11,10 @@ import ( "time" logging "github.com/ipfs/go-log" - inet "github.com/libp2p/go-libp2p-net" - peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peerstore" + ma "github.com/multiformats/go-multiaddr" . "github.com/libp2p/go-libp2p-swarm" @@ -22,7 +23,7 @@ import ( var log = logging.Logger("swarm_test") -func EchoStreamHandler(stream inet.Stream) { +func EchoStreamHandler(stream network.Stream) { go func() { defer stream.Close() @@ -77,7 +78,7 @@ func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { var wg sync.WaitGroup connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. - s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL) + s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) if _, err := s.DialPeer(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) } @@ -116,7 +117,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { _, cancel := context.WithCancel(ctx) got := map[peer.ID]int{} errChan := make(chan error, MsgNum*len(swarms)) - streamChan := make(chan inet.Stream, MsgNum) + streamChan := make(chan network.Stream, MsgNum) // send out "ping" x MsgNum to every peer go func() { @@ -252,7 +253,7 @@ func TestConnHandler(t *testing.T) { swarms := makeSwarms(ctx, t, 5) gotconn := make(chan struct{}, 10) - swarms[0].SetConnHandler(func(conn inet.Conn) { + swarms[0].SetConnHandler(func(conn network.Conn) { gotconn <- struct{}{} }) @@ -283,7 +284,7 @@ func TestAddrBlocking(t *testing.T) { ctx := context.Background() swarms := makeSwarms(ctx, t, 2) - swarms[0].SetConnHandler(func(conn inet.Conn) { + swarms[0].SetConnHandler(func(conn network.Conn) { t.Errorf("no connections should happen! -- %s", conn) }) @@ -294,13 +295,13 @@ func TestAddrBlocking(t *testing.T) { swarms[1].Filters.AddDialFilter(block) - swarms[1].Peerstore().AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], pstore.PermanentAddrTTL) + swarms[1].Peerstore().AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], peerstore.PermanentAddrTTL) _, err = swarms[1].DialPeer(ctx, swarms[0].LocalPeer()) if err == nil { t.Fatal("dial should have failed") } - swarms[0].Peerstore().AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], pstore.PermanentAddrTTL) + swarms[0].Peerstore().AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], peerstore.PermanentAddrTTL) _, err = swarms[0].DialPeer(ctx, swarms[1].LocalPeer()) if err == nil { t.Fatal("dial should have failed") @@ -312,7 +313,7 @@ func TestFilterBounds(t *testing.T) { swarms := makeSwarms(ctx, t, 2) conns := make(chan struct{}, 8) - swarms[0].SetConnHandler(func(conn inet.Conn) { + swarms[0].SetConnHandler(func(conn network.Conn) { conns <- struct{}{} }) @@ -340,8 +341,8 @@ func TestNoDial(t *testing.T) { ctx := context.Background() swarms := makeSwarms(ctx, t, 2) - _, err := swarms[0].NewStream(inet.WithNoDial(ctx, "swarm test"), swarms[1].LocalPeer()) - if err != inet.ErrNoConn { + _, err := swarms[0].NewStream(network.WithNoDial(ctx, "swarm test"), swarms[1].LocalPeer()) + if err != network.ErrNoConn { t.Fatal("should have failed with ErrNoConn") } } diff --git a/p2p/net/swarm/swarm_transport.go b/p2p/net/swarm/swarm_transport.go index 0431488777..307bfe641f 100644 --- a/p2p/net/swarm/swarm_transport.go +++ b/p2p/net/swarm/swarm_transport.go @@ -4,7 +4,8 @@ import ( "fmt" "strings" - transport "github.com/libp2p/go-libp2p-transport" + "github.com/libp2p/go-libp2p-core/transport" + ma "github.com/multiformats/go-multiaddr" ) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index edaaf5b274..6ffe7d2ae4 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -4,17 +4,18 @@ import ( "context" "testing" + "github.com/libp2p/go-libp2p-core/metrics" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-testing/net" + "github.com/libp2p/go-tcp-transport" + csms "github.com/libp2p/go-conn-security-multistream" - metrics "github.com/libp2p/go-libp2p-metrics" - inet "github.com/libp2p/go-libp2p-net" - pstore "github.com/libp2p/go-libp2p-peerstore" pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" secio "github.com/libp2p/go-libp2p-secio" tptu "github.com/libp2p/go-libp2p-transport-upgrader" yamux "github.com/libp2p/go-libp2p-yamux" msmux "github.com/libp2p/go-stream-muxer-multistream" - tcp "github.com/libp2p/go-tcp-transport" - tu "github.com/libp2p/go-testutil" swarm "github.com/libp2p/go-libp2p-swarm" ) @@ -65,7 +66,7 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { o(t, &cfg) } - p := tu.RandPeerNetParamsOrFatal(t) + p := tnet.RandPeerNetParamsOrFatal(t) ps := pstoremem.NewPeerstore() ps.AddPubKey(p.ID, p.PubKey) @@ -84,15 +85,15 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { t.Fatal(err) } - s.Peerstore().AddAddrs(p.ID, s.ListenAddresses(), pstore.PermanentAddrTTL) + s.Peerstore().AddAddrs(p.ID, s.ListenAddresses(), peerstore.PermanentAddrTTL) } return s } // DivulgeAddresses adds swarm a's addresses to swarm b's peerstore. -func DivulgeAddresses(a, b inet.Network) { +func DivulgeAddresses(a, b network.Network) { id := a.LocalPeer() addrs := a.Peerstore().Addrs(id) - b.Peerstore().AddAddrs(id, addrs, pstore.PermanentAddrTTL) + b.Peerstore().AddAddrs(id, addrs, peerstore.PermanentAddrTTL) } diff --git a/p2p/net/swarm/transport_test.go b/p2p/net/swarm/transport_test.go index 018775b261..f6090a6ef8 100644 --- a/p2p/net/swarm/transport_test.go +++ b/p2p/net/swarm/transport_test.go @@ -6,8 +6,8 @@ import ( swarmt "github.com/libp2p/go-libp2p-swarm/testing" - peer "github.com/libp2p/go-libp2p-peer" - transport "github.com/libp2p/go-libp2p-transport" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/transport" ma "github.com/multiformats/go-multiaddr" ) @@ -16,7 +16,7 @@ type dummyTransport struct { proxy bool } -func (dt *dummyTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.Conn, error) { +func (dt *dummyTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.CapableConn, error) { panic("unimplemented") } From 10d45bd76c399038233b7c8fb60613acff106075 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 3 Jun 2019 23:14:41 +0200 Subject: [PATCH 140/259] feat: cache interface addresses for 1 minute This can be quite an overhead in cases of high connection rates. The main overhead is thread blocking syscall causing a lot of context switching. License: MIT Signed-off-by: Jakub Sztandera --- p2p/net/swarm/swarm.go | 5 ++++ p2p/net/swarm/swarm_addr.go | 46 ++++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index b14e9cf0b9..642faea856 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -20,6 +20,7 @@ import ( goprocessctx "github.com/jbenet/goprocess/context" filter "github.com/libp2p/go-maddr-filter" + ma "github.com/multiformats/go-multiaddr" mafilter "github.com/whyrusleeping/multiaddr-filter" ) @@ -58,6 +59,10 @@ type Swarm struct { listeners struct { sync.RWMutex + + ifaceAddresses []ma.Multiaddr + cacheEOL time.Time + m map[transport.Listener]struct{} } diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index c86d58bb4d..10179fa3b9 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -1,6 +1,8 @@ package swarm import ( + "time" + addrutil "github.com/libp2p/go-addr-util" ma "github.com/multiformats/go-multiaddr" ) @@ -9,6 +11,10 @@ import ( func (s *Swarm) ListenAddresses() []ma.Multiaddr { s.listeners.RLock() defer s.listeners.RUnlock() + return s.listenAddressesNoLock() +} + +func (s *Swarm) listenAddressesNoLock() []ma.Multiaddr { addrs := make([]ma.Multiaddr, 0, len(s.listeners.m)) for l := range s.listeners.m { addrs = append(addrs, l.Multiaddr()) @@ -16,9 +22,47 @@ func (s *Swarm) ListenAddresses() []ma.Multiaddr { return addrs } +const ifaceAddrsCacheDuration = 1 * time.Minute + // InterfaceListenAddresses returns a list of addresses at which this swarm // listens. It expands "any interface" addresses (/ip4/0.0.0.0, /ip6/::) to // use the known local interfaces. func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { - return addrutil.ResolveUnspecifiedAddresses(s.ListenAddresses(), nil) + s.listeners.RLock() // RLock start + listenAddrs := s.listenAddressesNoLock() + + ifaceAddrs := s.listeners.ifaceAddresses + isEOL := time.Now().After(s.listeners.cacheEOL) + s.listeners.RUnlock() // RLock end + + if listenAddrs != nil && !isEOL { + // Cache is valid + return addrutil.ResolveUnspecifiedAddresses(listenAddrs, ifaceAddrs) + } + + // Cache is not valid + // Perfrom double checked locking + + s.listeners.Lock() // Lock start + + listenAddrs = s.listenAddressesNoLock() + + ifaceAddrs = s.listeners.ifaceAddresses + isEOL = time.Now().After(s.listeners.cacheEOL) + if listenAddrs == nil || isEOL { + // Cache is still invalid + var err error + ifaceAddrs, err = addrutil.InterfaceAddresses() + if err != nil { + s.listeners.Unlock() // Lock early exit + return nil, err + } + + s.listeners.ifaceAddresses = ifaceAddrs + s.listeners.cacheEOL = time.Now().Add(ifaceAddrsCacheDuration) + } + + s.listeners.Unlock() // Lock end + + return addrutil.ResolveUnspecifiedAddresses(listenAddrs, ifaceAddrs) } From c7245b745f049911ac170c853bd13ba55e2bfea6 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 4 Jun 2019 21:21:52 +0200 Subject: [PATCH 141/259] feat: cache full InterfaceListenAddresses License: MIT Signed-off-by: Jakub Sztandera --- p2p/net/swarm/swarm.go | 4 ++-- p2p/net/swarm/swarm_addr.go | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 642faea856..cc7eb978a6 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -60,8 +60,8 @@ type Swarm struct { listeners struct { sync.RWMutex - ifaceAddresses []ma.Multiaddr - cacheEOL time.Time + ifaceListenAddres []ma.Multiaddr + cacheEOL time.Time m map[transport.Listener]struct{} } diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 10179fa3b9..6210a4f809 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -29,15 +29,14 @@ const ifaceAddrsCacheDuration = 1 * time.Minute // use the known local interfaces. func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { s.listeners.RLock() // RLock start - listenAddrs := s.listenAddressesNoLock() - ifaceAddrs := s.listeners.ifaceAddresses + ifaceListenAddres := s.listeners.ifaceListenAddres isEOL := time.Now().After(s.listeners.cacheEOL) s.listeners.RUnlock() // RLock end - if listenAddrs != nil && !isEOL { + if ifaceListenAddres != nil && !isEOL { // Cache is valid - return addrutil.ResolveUnspecifiedAddresses(listenAddrs, ifaceAddrs) + return ifaceListenAddres, nil } // Cache is not valid @@ -45,24 +44,25 @@ func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { s.listeners.Lock() // Lock start - listenAddrs = s.listenAddressesNoLock() - - ifaceAddrs = s.listeners.ifaceAddresses + ifaceListenAddres = s.listeners.ifaceListenAddres isEOL = time.Now().After(s.listeners.cacheEOL) - if listenAddrs == nil || isEOL { + if ifaceListenAddres == nil || isEOL { // Cache is still invalid + var err error - ifaceAddrs, err = addrutil.InterfaceAddresses() + ifaceListenAddres, err = addrutil.ResolveUnspecifiedAddresses( + s.listenAddressesNoLock(), nil) + if err != nil { s.listeners.Unlock() // Lock early exit return nil, err } - s.listeners.ifaceAddresses = ifaceAddrs + s.listeners.ifaceListenAddres = ifaceListenAddres s.listeners.cacheEOL = time.Now().Add(ifaceAddrsCacheDuration) } s.listeners.Unlock() // Lock end - return addrutil.ResolveUnspecifiedAddresses(listenAddrs, ifaceAddrs) + return ifaceListenAddres, nil } From b80c6891d771fe207061e4a1d72ef2571df98d44 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 4 Jun 2019 23:10:02 +0200 Subject: [PATCH 142/259] invalidate cache for InterfaceListenAddresses on Listen License: MIT Signed-off-by: Jakub Sztandera --- p2p/net/swarm/swarm_listen.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 85060c4e2c..570acfedae 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -2,6 +2,7 @@ package swarm import ( "fmt" + "time" "github.com/libp2p/go-libp2p-core/network" @@ -55,6 +56,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { } s.refs.Add(1) s.listeners.m[list] = struct{}{} + s.listeners.cacheEOL = time.Time{} s.listeners.Unlock() maddr := list.Multiaddr() @@ -69,6 +71,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { list.Close() s.listeners.Lock() delete(s.listeners.m, list) + s.listeners.cacheEOL = time.Time{} s.listeners.Unlock() s.refs.Done() }() From c12ef6a31836b9fc1a49bb42338f6dd4a8017c3b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 6 Jun 2019 00:23:04 -0700 Subject: [PATCH 143/259] logging: make the swarm less noisy Avoid logging about closed listeners, etc., when shutting down. --- p2p/net/swarm/swarm.go | 5 +++++ p2p/net/swarm/swarm_listen.go | 11 ++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index cc7eb978a6..5366bdcc6f 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -116,6 +116,11 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc } func (s *Swarm) teardown() error { + // Wait for the context to be canceled. + // This allows other parts of the swarm to detect that we're shutting + // down. + <-s.ctx.Done() + // Prevents new connections and/or listeners from being added to the swarm. s.listeners.Lock() diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 570acfedae..f1cfa9e7f1 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -79,6 +79,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { c, err := list.Accept() if err != nil { if s.ctx.Err() == nil { + // only log if the swarm is still running. log.Errorf("swarm listener accept error: %s", err) } return @@ -88,9 +89,13 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { go func() { defer s.refs.Done() _, err := s.addConn(c, network.DirInbound) - if err != nil { - // Probably just means that the swarm has been closed. - log.Warningf("add conn failed: ", err) + switch err { + case nil: + case ErrSwarmClosed: + // ignore. + return + default: + log.Warningf("add conn %s failed: ", err) return } }() From c0a856ce8146329b485e49a44b050ef1715e6bcb Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 10 Jul 2019 03:19:22 -0700 Subject: [PATCH 144/259] fix: don't assume that transports implement stringer (#134) If they don't, this could end up reading a bunch of internal data, causing a race (and yes, this has happened). --- p2p/net/swarm/swarm_conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index cdb9866a2a..c09957c130 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -122,7 +122,7 @@ func (c *Conn) start() { func (c *Conn) String() string { return fmt.Sprintf( - " %s (%s)>", + " %s (%s)>", c.conn.Transport(), c.conn.LocalMultiaddr(), c.conn.LocalPeer().Pretty(), From 85cc7d8c92d03c5f9ad9feff64908bf7e030bd9f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 21 Aug 2019 10:47:31 -0700 Subject: [PATCH 145/259] fix listen addrs race Copy the listen address list before returning it. If you're wondering about the syntax: https://github.com/go101/go101/wiki/How-to-perfectly-clone-a-slice%3F --- p2p/net/swarm/swarm_addr.go | 6 +++--- p2p/net/swarm/swarm_addr_test.go | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 6210a4f809..650fb79d13 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -35,8 +35,8 @@ func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { s.listeners.RUnlock() // RLock end if ifaceListenAddres != nil && !isEOL { - // Cache is valid - return ifaceListenAddres, nil + // Cache is valid, clone the slice + return append(ifaceListenAddres[:0:0], ifaceListenAddres...), nil } // Cache is not valid @@ -64,5 +64,5 @@ func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { s.listeners.Unlock() // Lock end - return ifaceListenAddres, nil + return append(ifaceListenAddres[:0:0], ifaceListenAddres...), nil } diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index e75d660615..5009946c6d 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -35,3 +35,21 @@ func TestDialBadAddrs(t *testing.T) { test(m("/ip6/fe80::100")) // link local test(m("/ip4/127.0.0.1/udp/1234/utp")) // utp } + +func TestAddrRace(t *testing.T) { + ctx := context.Background() + s := makeSwarms(ctx, t, 1)[0] + + a1, err := s.InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + a2, err := s.InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + + if len(a1) > 0 && len(a2) > 0 && &a1[0] == &a2[0] { + t.Fatal("got the exact same address set twice; this could lead to data races") + } +} From 11ed2c35c62d450242847f74304ca56750130aeb Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 13 Sep 2019 13:37:04 -0700 Subject: [PATCH 146/259] test: close peerstore when closing the test swarm --- p2p/net/swarm/testing/testing.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 6ffe7d2ae4..5e39602426 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -10,6 +10,7 @@ import ( "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-tcp-transport" + goprocess "github.com/jbenet/goprocess" csms "github.com/libp2p/go-conn-security-multistream" pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" secio "github.com/libp2p/go-libp2p-secio" @@ -72,6 +73,7 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter()) + s.Process().AddChild(goprocess.WithTeardown(ps.Close)) tcpTransport := tcp.NewTCPTransport(GenUpgrader(s)) tcpTransport.DisableReuseport = cfg.disableReuseport From b8ac63238e27381b944a02df61fba0c04460347e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 18 Sep 2019 22:42:53 -0700 Subject: [PATCH 147/259] dep: update multiaddr for protocol migration Protocol definitions have been migrated to the main go-multiaddr package. --- p2p/net/swarm/limiter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 32d87fa13d..7373ddbb3e 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -12,7 +12,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" ma "github.com/multiformats/go-multiaddr" - mafmt "github.com/whyrusleeping/mafmt" + mafmt "github.com/multiformats/go-multiaddr-fmt" ) func mustAddr(t *testing.T, s string) ma.Multiaddr { From c96acd8c503f671109a75d2e6ec5f14c66b09d7d Mon Sep 17 00:00:00 2001 From: Aliabbas Merchant Date: Sun, 27 Oct 2019 16:39:24 +0530 Subject: [PATCH 148/259] Minor Docstring correction --- p2p/net/swarm/swarm_dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 40faad7848..054902e7f7 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -47,7 +47,7 @@ var ( // peer we're trying to dial. ErrNoAddresses = errors.New("no addresses") - // ErrNoAddresses is returned when we find addresses for a peer but + // ErrNoGoodAddresses is returned when we find addresses for a peer but // can't use any of them. ErrNoGoodAddresses = errors.New("no good addresses") ) From ad8e59543bb49b25d155822c1ed80e2dab280edd Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 4 Nov 2019 19:43:46 +0000 Subject: [PATCH 149/259] feat(dial): implement the Timeout interface on dial errors --- p2p/net/swarm/dial_error.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/p2p/net/swarm/dial_error.go b/p2p/net/swarm/dial_error.go index d21796172f..f2986348bf 100644 --- a/p2p/net/swarm/dial_error.go +++ b/p2p/net/swarm/dial_error.go @@ -2,6 +2,7 @@ package swarm import ( "fmt" + "os" "strings" "github.com/libp2p/go-libp2p-core/peer" @@ -20,6 +21,10 @@ type DialError struct { Skipped int } +func (e *DialError) Timeout() bool { + return os.IsTimeout(e.Cause) +} + func (e *DialError) recordErr(addr ma.Multiaddr, err error) { if len(e.DialErrors) >= maxDialDialErrors { e.Skipped++ @@ -48,11 +53,7 @@ func (e *DialError) Error() string { // Unwrap implements https://godoc.org/golang.org/x/xerrors#Wrapper. func (e *DialError) Unwrap() error { - // If we have a context error, that's the "ultimate" error. - if e.Cause != nil { - return e.Cause - } - return nil + return e.Cause } var _ error = (*DialError)(nil) From dbf68e09d2f4cb73fda6fe21d28430b65f1fc1fe Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 4 Nov 2019 19:45:09 +0000 Subject: [PATCH 150/259] feat(swarm): return unwrapped context deadline errors --- p2p/net/swarm/swarm_dial.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 054902e7f7..ffb7ab5e2c 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -317,13 +317,14 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { connC, dialErr := s.dialAddrs(ctx, p, goodAddrsChan) if dialErr != nil { logdial["error"] = dialErr.Cause.Error() - if dialErr.Cause == context.Canceled { - // always prefer the "context canceled" error. - // we rely on behing able to check `err == context.Canceled` + switch dialErr.Cause { + case context.Canceled, context.DeadlineExceeded: + // Always prefer the context errors as we rely on being + // able to check them. // // Removing this will BREAK backoff (causing us to // backoff when canceling dials). - return nil, context.Canceled + return nil, dialErr.Cause } return nil, dialErr } From 1263fb7faa4047aa6a8b9f16df38e69346a1e3e2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 4 Nov 2019 20:02:44 +0000 Subject: [PATCH 151/259] feat(swarm): improve dial context 1. Always return the caller's context error if relevant. 2. Don't return "context canceled" when we're just shutting down. 3. Don't claim that the context deadline has been exceeded when the dial timeout is canceled. --- p2p/net/swarm/dial_sync.go | 14 ++++++++++++++ p2p/net/swarm/swarm.go | 3 +++ p2p/net/swarm/swarm_dial.go | 17 ++++++++++++++--- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index f746b9a9a3..4c1230f5fc 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -2,11 +2,15 @@ package swarm import ( "context" + "errors" "sync" "github.com/libp2p/go-libp2p-core/peer" ) +// TODO: change this text when we fix the bug +var errDialCanceled = errors.New("dial was aborted internally, likely due to https://git.io/Je2wW") + // DialFunc is the type of function expected by DialSync. type DialFunc func(context.Context, peer.ID) (*Conn, error) @@ -78,6 +82,16 @@ func (ad *activeDial) decref() { func (ad *activeDial) start(ctx context.Context) { ad.conn, ad.err = ad.ds.dialFunc(ctx, ad.id) + + // This isn't the user's context so we should fix the error. + switch ad.err { + case context.Canceled: + // The dial was canceled with `CancelDial`. + ad.err = errDialCanceled + case context.DeadlineExceeded: + // We hit an internal timeout, not a context timeout. + ad.err = ErrDialTimeout + } close(ad.waitch) ad.cancel() } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 5366bdcc6f..f7c57717fa 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -40,6 +40,9 @@ var ErrSwarmClosed = errors.New("swarm closed") // transport is misbehaving. var ErrAddrFiltered = errors.New("address filtered") +// ErrDialTimeout is returned when one a dial times out due to the global timeout +var ErrDialTimeout = errors.New("dial timed out") + // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index ffb7ab5e2c..475a7e250e 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -221,12 +221,23 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { defer cancel() conn, err = s.dsync.DialLock(ctx, p) - if err != nil { - return nil, err + if err == nil { + return conn, nil } log.Debugf("network for %s finished dialing %s", s.local, p) - return conn, err + + if ctx.Err() != nil { + // Context error trumps any dial errors as it was likely the ultimate cause. + return nil, ctx.Err() + } + + if s.ctx.Err() != nil { + // Ok, so the swarm is shutting down. + return nil, ErrSwarmClosed + } + + return nil, err } // doDial is an ugly shim method to retain all the logging and backoff logic From 9d0e54866cc47c051e0403c9d43de48ac8919140 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 10 Feb 2020 10:13:53 -0500 Subject: [PATCH 152/259] only cancel dials when an outbound connection succeeds. this may result in duplicate connections, but it's better to have two connections than dropping both of them and ending up with zero connections. --- p2p/net/swarm/swarm.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index f7c57717fa..f85ef5234b 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -230,7 +230,9 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, // We have a connection now. Cancel all other in-progress dials. // This should be fast, no reason to wait till later. - s.dsync.CancelDial(p) + if dir == network.DirOutbound { + s.dsync.CancelDial(p) + } s.notifyAll(func(f network.Notifiee) { f.Connected(s, c) From a7319194056da86ba1f3656e173330a1fcff98ee Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 25 Feb 2020 19:41:48 -0800 Subject: [PATCH 153/259] fix: fire a listen close event when closing the listener fixes https://github.com/libp2p/go-libp2p-swarm/issues/163 --- p2p/net/swarm/swarm_listen.go | 7 ++++++- p2p/net/swarm/swarm_notif_test.go | 20 +++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index f1cfa9e7f1..09d411dfd8 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -61,7 +61,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { maddr := list.Multiaddr() - // signal to our notifiees on successful conn. + // signal to our notifiees on listen. s.notifyAll(func(n network.Notifiee) { n.Listen(s, maddr) }) @@ -73,6 +73,11 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { delete(s.listeners.m, list) s.listeners.cacheEOL = time.Time{} s.listeners.Unlock() + + // signal to our notifiees on listen close. + s.notifyAll(func(n network.Notifiee) { + n.ListenClose(s, maddr) + }) s.refs.Done() }() for { diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index f2c91ebb80..60cf461f5f 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -16,18 +16,32 @@ import ( func TestNotifications(t *testing.T) { const swarmSize = 5 + notifiees := make([]*netNotifiee, swarmSize) + ctx := context.Background() swarms := makeSwarms(ctx, t, swarmSize) defer func() { - for _, s := range swarms { - s.Close() + for i, s := range swarms { + select { + case <-notifiees[i].listenClose: + t.Error("should not have been closed") + default: + } + err := s.Close() + if err != nil { + t.Error(err) + } + select { + case <-notifiees[i].listenClose: + default: + t.Error("expected a listen close notification") + } } }() timeout := 5 * time.Second // signup notifs - notifiees := make([]*netNotifiee, len(swarms)) for i, swarm := range swarms { n := newNetNotifiee(swarmSize) swarm.Notify(n) From 791cb9dd289be2c93a3ef93f7061985c3edd7e17 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 17 Mar 2020 20:54:32 -0700 Subject: [PATCH 154/259] fix: make sure to include peer in dial error --- p2p/net/swarm/swarm_dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 475a7e250e..f2b21abf0a 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -387,7 +387,7 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. // use a single response type instead of errs and conns, reduces complexity *a ton* respch := make(chan dialResult) - err := new(DialError) + err := &DialError{Peer: p} defer s.limiter.clearAllPeerDials(p) From 3639cf81a136c029613050c667cf83f4d29dc932 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 29 Mar 2020 21:00:59 -0700 Subject: [PATCH 155/259] feat: handle no addresses If we aren't listening on any addresses, we should return no addresses, not an error. --- p2p/net/swarm/swarm_addr.go | 24 ++++++++++++++---------- p2p/net/swarm/swarm_addr_test.go | 20 +++++++++++++++++++- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 650fb79d13..88bc626bd2 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -34,7 +34,7 @@ func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { isEOL := time.Now().After(s.listeners.cacheEOL) s.listeners.RUnlock() // RLock end - if ifaceListenAddres != nil && !isEOL { + if !isEOL { // Cache is valid, clone the slice return append(ifaceListenAddres[:0:0], ifaceListenAddres...), nil } @@ -46,16 +46,20 @@ func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { ifaceListenAddres = s.listeners.ifaceListenAddres isEOL = time.Now().After(s.listeners.cacheEOL) - if ifaceListenAddres == nil || isEOL { + if isEOL { // Cache is still invalid - - var err error - ifaceListenAddres, err = addrutil.ResolveUnspecifiedAddresses( - s.listenAddressesNoLock(), nil) - - if err != nil { - s.listeners.Unlock() // Lock early exit - return nil, err + listenAddres := s.listenAddressesNoLock() + if len(listenAddres) > 0 { + // We're actually listening on addresses. + var err error + ifaceListenAddres, err = addrutil.ResolveUnspecifiedAddresses(listenAddres, nil) + + if err != nil { + s.listeners.Unlock() // Lock early exit + return nil, err + } + } else { + ifaceListenAddres = nil } s.listeners.ifaceListenAddres = ifaceListenAddres diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 5009946c6d..baeac46203 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -8,6 +8,8 @@ import ( "github.com/libp2p/go-libp2p-core/test" ma "github.com/multiformats/go-multiaddr" + + swarmt "github.com/libp2p/go-libp2p-swarm/testing" ) func TestDialBadAddrs(t *testing.T) { @@ -37,8 +39,10 @@ func TestDialBadAddrs(t *testing.T) { } func TestAddrRace(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() s := makeSwarms(ctx, t, 1)[0] + defer s.Close() a1, err := s.InterfaceListenAddresses() if err != nil { @@ -53,3 +57,17 @@ func TestAddrRace(t *testing.T) { t.Fatal("got the exact same address set twice; this could lead to data races") } } + +func TestAddressesWithoutListening(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := swarmt.GenSwarm(t, ctx, swarmt.OptDialOnly) + + a1, err := s.InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + if len(a1) != 0 { + t.Fatalf("expected to be listening on no addresses, was listening on %d", len(a1)) + } +} From 904165fd3170186b45ad044b9143533b7603c1ce Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 31 Mar 2020 08:57:02 -0700 Subject: [PATCH 156/259] fix: set teardown after storing the context. Otherwise, we can modify the context after/while the process is shutting down. fixes #189 --- p2p/net/swarm/swarm.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index f85ef5234b..fe4e1ee978 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -112,9 +112,13 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc s.dsync = NewDialSync(s.doDial) s.limiter = newDialLimiter(s.dialAddr) - s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown) + s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) + // Set teardown after setting the context/process so we don't start the + // teardown process early. + s.proc.SetTeardown(s.teardown) + return s } From 193a3191a480bdf62832eb6eac71b6669f0cace2 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 1 Apr 2020 14:39:49 -0700 Subject: [PATCH 157/259] change backoffs to per-address --- p2p/net/swarm/dial_test.go | 19 ++++++----- p2p/net/swarm/swarm_dial.go | 65 +++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 73bd02c674..49c21fd6d3 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -200,7 +200,7 @@ func TestDialWait(t *testing.T) { t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) } - if !s1.Backoff().Backoff(s2p) { + if !s1.Backoff().Backoff(s2p, s2addr) { t.Error("s2 should now be on backoff") } } @@ -337,10 +337,10 @@ func TestDialBackoff(t *testing.T) { } // check backoff state - if s1.Backoff().Backoff(s2.LocalPeer()) { + if s1.Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { t.Error("s2 should not be on backoff") } - if !s1.Backoff().Backoff(s3p) { + if !s1.Backoff().Backoff(s3p, s3addr) { t.Error("s3 should be on backoff") } @@ -407,10 +407,10 @@ func TestDialBackoff(t *testing.T) { } // check backoff state (the same) - if s1.Backoff().Backoff(s2.LocalPeer()) { + if s1.Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { t.Error("s2 should not be on backoff") } - if !s1.Backoff().Backoff(s3p) { + if !s1.Backoff().Backoff(s3p, s3addr) { t.Error("s3 should be on backoff") } } @@ -451,7 +451,7 @@ func TestDialBackoffClears(t *testing.T) { t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) } - if !s1.Backoff().Backoff(s2.LocalPeer()) { + if !s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { t.Error("s2 should now be on backoff") } else { t.Log("correctly added to backoff") @@ -464,8 +464,9 @@ func TestDialBackoffClears(t *testing.T) { } s1.Peerstore().AddAddrs(s2.LocalPeer(), ifaceAddrs1, peerstore.PermanentAddrTTL) - if _, err := s1.DialPeer(ctx, s2.LocalPeer()); err == nil { - t.Fatal("should have failed to dial backed off peer") + if c, err := s1.DialPeer(ctx, s2.LocalPeer()); err == nil { + c.Close() + t.Log("backoffs are per address, not peer") } time.Sleep(BackoffBase) @@ -477,7 +478,7 @@ func TestDialBackoffClears(t *testing.T) { t.Log("correctly connected") } - if s1.Backoff().Backoff(s2.LocalPeer()) { + if s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { t.Error("s2 should no longer be on backoff") } else { t.Log("correctly cleared backoff") diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index f2b21abf0a..40821644bd 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -101,7 +101,9 @@ type DialBackoff struct { lock sync.RWMutex } -type backoffPeer struct { +type backoffPeer map[ma.Multiaddr]*backoffAddr + +type backoffAddr struct { tries int until time.Time } @@ -113,14 +115,18 @@ func (db *DialBackoff) init() { } // Backoff returns whether the client should backoff from dialing -// peer p -func (db *DialBackoff) Backoff(p peer.ID) (backoff bool) { +// peer p at address addr +func (db *DialBackoff) Backoff(p peer.ID, addr ma.Multiaddr) (backoff bool) { db.lock.Lock() defer db.lock.Unlock() db.init() bp, found := db.entries[p] - if found && time.Now().Before(bp.until) { - return true + if found && bp != nil { + ap, found := (*bp)[addr] + // TODO: cleanup out of date entries. + if found && time.Now().Before(ap.until) { + return true + } } return false @@ -145,25 +151,36 @@ var BackoffMax = time.Minute * 5 // BackoffBase + BakoffCoef * PriorBackoffs^2 // // Where PriorBackoffs is the number of previous backoffs. -func (db *DialBackoff) AddBackoff(p peer.ID) { +func (db *DialBackoff) AddBackoff(p peer.ID, addr ma.Multiaddr) { db.lock.Lock() defer db.lock.Unlock() db.init() bp, ok := db.entries[p] if !ok { - db.entries[p] = &backoffPeer{ + bp := backoffPeer(make(map[ma.Multiaddr]*backoffAddr)) + db.entries[p] = &bp + bp[addr] = &backoffAddr{ + tries: 1, + until: time.Now().Add(BackoffBase), + } + return + } + // todo: cleanup out of date entries. + ba, ok := (*bp)[addr] + if !ok { + (*bp)[addr] = &backoffAddr{ tries: 1, until: time.Now().Add(BackoffBase), } return } - backoffTime := BackoffBase + BackoffCoef*time.Duration(bp.tries*bp.tries) + backoffTime := BackoffBase + BackoffCoef*time.Duration(ba.tries*ba.tries) if backoffTime > BackoffMax { backoffTime = BackoffMax } - bp.until = time.Now().Add(backoffTime) - bp.tries++ + ba.until = time.Now().Add(backoffTime) + ba.tries++ } // Clear removes a backoff record. Clients should call this after a @@ -210,12 +227,6 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { return conn, nil } - // if this peer has been backed off, lets get out of here - if s.backf.Backoff(p) { - log.Event(ctx, "swarmDialBackoff", p) - return nil, ErrDialBackoff - } - // apply the DialPeer timeout ctx, cancel := context.WithTimeout(ctx, network.GetDialPeerTimeout(ctx)) defer cancel() @@ -268,10 +279,6 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { log.Debugf("ignoring dial error because we have a connection: %s", err) return conn, nil } - if err != context.Canceled { - log.Event(ctx, "swarmDialBackoffAdd", logdial) - s.backf.AddBackoff(p) // let others know to backoff - } // ok, we failed. return nil, err @@ -318,10 +325,18 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return nil, &DialError{Peer: p, Cause: ErrNoGoodAddresses} } goodAddrsChan := make(chan ma.Multiaddr, len(goodAddrs)) + nonBackoff := false for _, a := range goodAddrs { - goodAddrsChan <- a + // skip addresses in back-off + if !s.backf.Backoff(p, a) { + nonBackoff = true + goodAddrsChan <- a + } } close(goodAddrsChan) + if !nonBackoff { + return nil, ErrDialBackoff + } ///////// // try to get a connection to any addr @@ -402,6 +417,10 @@ dialLoop: active-- if resp.Err != nil { // Errors are normal, lots of dials will fail + if resp.Err != context.Canceled { + s.backf.AddBackoff(p, resp.Addr) + } + log.Infof("got error on dial: %s", resp.Err) err.recordErr(resp.Addr, resp.Err) } else if resp.Conn != nil { @@ -429,6 +448,10 @@ dialLoop: active-- if resp.Err != nil { // Errors are normal, lots of dials will fail + if resp.Err != context.Canceled { + s.backf.AddBackoff(p, resp.Addr) + } + log.Infof("got error on dial: %s", resp.Err) err.recordErr(resp.Addr, resp.Err) } else if resp.Conn != nil { From ac37d0dd17212a9c17adaaa28076471f64561210 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 1 Apr 2020 15:19:38 -0700 Subject: [PATCH 158/259] add background cleanup task --- p2p/net/swarm/swarm.go | 1 + p2p/net/swarm/swarm_dial.go | 57 ++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index fe4e1ee978..198d88a72b 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -114,6 +114,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc s.limiter = newDialLimiter(s.dialAddr) s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) + s.backf.init(s.ctx) // Set teardown after setting the context/process so we don't start the // teardown process early. diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 40821644bd..9c2886fcb3 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -97,7 +97,7 @@ const DefaultPerPeerRateLimit = 8 // * It's thread-safe. // * It's *not* safe to move this type after using. type DialBackoff struct { - entries map[peer.ID]*backoffPeer + entries map[peer.ID]backoffPeer lock sync.RWMutex } @@ -108,9 +108,23 @@ type backoffAddr struct { until time.Time } -func (db *DialBackoff) init() { +func (db *DialBackoff) init(ctx context.Context) { if db.entries == nil { - db.entries = make(map[peer.ID]*backoffPeer) + db.entries = make(map[peer.ID]backoffPeer) + } + go db.background(ctx) +} + +func (db *DialBackoff) background(ctx context.Context) { + ticker := time.NewTicker(BackoffMax) + for { + select { + case <-ctx.Done(): + ticker.Stop() + return + case <-ticker.C: + db.cleanup() + } } } @@ -119,10 +133,9 @@ func (db *DialBackoff) init() { func (db *DialBackoff) Backoff(p peer.ID, addr ma.Multiaddr) (backoff bool) { db.lock.Lock() defer db.lock.Unlock() - db.init() bp, found := db.entries[p] if found && bp != nil { - ap, found := (*bp)[addr] + ap, found := bp[addr] // TODO: cleanup out of date entries. if found && time.Now().Before(ap.until) { return true @@ -154,21 +167,18 @@ var BackoffMax = time.Minute * 5 func (db *DialBackoff) AddBackoff(p peer.ID, addr ma.Multiaddr) { db.lock.Lock() defer db.lock.Unlock() - db.init() bp, ok := db.entries[p] if !ok { - bp := backoffPeer(make(map[ma.Multiaddr]*backoffAddr)) - db.entries[p] = &bp - bp[addr] = &backoffAddr{ + db.entries[p] = backoffPeer(make(map[ma.Multiaddr]*backoffAddr)) + db.entries[p][addr] = &backoffAddr{ tries: 1, until: time.Now().Add(BackoffBase), } return } - // todo: cleanup out of date entries. - ba, ok := (*bp)[addr] + ba, ok := bp[addr] if !ok { - (*bp)[addr] = &backoffAddr{ + bp[addr] = &backoffAddr{ tries: 1, until: time.Now().Add(BackoffBase), } @@ -188,10 +198,31 @@ func (db *DialBackoff) AddBackoff(p peer.ID, addr ma.Multiaddr) { func (db *DialBackoff) Clear(p peer.ID) { db.lock.Lock() defer db.lock.Unlock() - db.init() delete(db.entries, p) } +func (db *DialBackoff) cleanup() { + db.lock.Lock() + defer db.lock.Unlock() + now := time.Now() + deletePeers := []peer.ID{} + for p, e := range db.entries { + good := false + for _, backoff := range e { + if now.Before(backoff.until) { + good = true + break + } + } + if !good { + deletePeers = append(deletePeers, p) + } + } + for _, p := range deletePeers { + delete(db.entries, p) + } +} + // DialPeer connects to a peer. // // The idea is that the client of Swarm does not need to know what network From 390b8ca72567672433d8c1065b6da57bdfb5bca1 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 2 Apr 2020 09:15:44 -0700 Subject: [PATCH 159/259] simplify data structure --- p2p/net/swarm/swarm_dial.go | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 9c2886fcb3..75c9ad26ba 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -97,12 +97,10 @@ const DefaultPerPeerRateLimit = 8 // * It's thread-safe. // * It's *not* safe to move this type after using. type DialBackoff struct { - entries map[peer.ID]backoffPeer + entries map[peer.ID]map[string]*backoffAddr lock sync.RWMutex } -type backoffPeer map[ma.Multiaddr]*backoffAddr - type backoffAddr struct { tries int until time.Time @@ -110,17 +108,17 @@ type backoffAddr struct { func (db *DialBackoff) init(ctx context.Context) { if db.entries == nil { - db.entries = make(map[peer.ID]backoffPeer) + db.entries = make(map[peer.ID]map[string]*backoffAddr) } go db.background(ctx) } func (db *DialBackoff) background(ctx context.Context) { ticker := time.NewTicker(BackoffMax) + defer ticker.Stop() for { select { case <-ctx.Done(): - ticker.Stop() return case <-ticker.C: db.cleanup() @@ -134,9 +132,8 @@ func (db *DialBackoff) Backoff(p peer.ID, addr ma.Multiaddr) (backoff bool) { db.lock.Lock() defer db.lock.Unlock() bp, found := db.entries[p] - if found && bp != nil { - ap, found := bp[addr] - // TODO: cleanup out of date entries. + if found { + ap, found := bp[string(addr.Bytes())] if found && time.Now().Before(ap.until) { return true } @@ -165,20 +162,21 @@ var BackoffMax = time.Minute * 5 // // Where PriorBackoffs is the number of previous backoffs. func (db *DialBackoff) AddBackoff(p peer.ID, addr ma.Multiaddr) { + saddr := string(addr.Bytes()) db.lock.Lock() defer db.lock.Unlock() bp, ok := db.entries[p] if !ok { - db.entries[p] = backoffPeer(make(map[ma.Multiaddr]*backoffAddr)) - db.entries[p][addr] = &backoffAddr{ + db.entries[p] = make(map[string]*backoffAddr) + db.entries[p][saddr] = &backoffAddr{ tries: 1, until: time.Now().Add(BackoffBase), } return } - ba, ok := bp[addr] + ba, ok := bp[saddr] if !ok { - bp[addr] = &backoffAddr{ + bp[saddr] = &backoffAddr{ tries: 1, until: time.Now().Add(BackoffBase), } @@ -205,7 +203,6 @@ func (db *DialBackoff) cleanup() { db.lock.Lock() defer db.lock.Unlock() now := time.Now() - deletePeers := []peer.ID{} for p, e := range db.entries { good := false for _, backoff := range e { @@ -215,12 +212,9 @@ func (db *DialBackoff) cleanup() { } } if !good { - deletePeers = append(deletePeers, p) + delete(db.entries, p) } } - for _, p := range deletePeers { - delete(db.entries, p) - } } // DialPeer connects to a peer. From 035e5dcada370ef5d01ea77e81d32afbac52d895 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 2 Apr 2020 11:36:55 -0700 Subject: [PATCH 160/259] chore: slightly simplify backoff logic --- p2p/net/swarm/swarm_dial.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 75c9ad26ba..970cf0105e 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -131,15 +131,9 @@ func (db *DialBackoff) background(ctx context.Context) { func (db *DialBackoff) Backoff(p peer.ID, addr ma.Multiaddr) (backoff bool) { db.lock.Lock() defer db.lock.Unlock() - bp, found := db.entries[p] - if found { - ap, found := bp[string(addr.Bytes())] - if found && time.Now().Before(ap.until) { - return true - } - } - return false + ap, found := db.entries[p][string(addr.Bytes())] + return found && time.Now().Before(ap.until) } // BackoffBase is the base amount of time to backoff (default: 5s). @@ -167,12 +161,8 @@ func (db *DialBackoff) AddBackoff(p peer.ID, addr ma.Multiaddr) { defer db.lock.Unlock() bp, ok := db.entries[p] if !ok { - db.entries[p] = make(map[string]*backoffAddr) - db.entries[p][saddr] = &backoffAddr{ - tries: 1, - until: time.Now().Add(BackoffBase), - } - return + bp = make(map[string]*backoffAddr, 1) + db.entries[p] = bp } ba, ok := bp[saddr] if !ok { From 240d2fe66fc18a25673e10cefebda89253bf5721 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 2 Apr 2020 11:53:58 -0700 Subject: [PATCH 161/259] don't expire backoffs until 2x backoff period --- p2p/net/swarm/swarm_dial.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 970cf0105e..cae4110c01 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -196,7 +196,11 @@ func (db *DialBackoff) cleanup() { for p, e := range db.entries { good := false for _, backoff := range e { - if now.Before(backoff.until) { + backoffTime := BackoffBase + BackoffCoef*time.Duration(backoff.tries*backoff.tries) + if backoffTime > BackoffMax { + backoffTime = BackoffMax + } + if now.Before(backoff.until.Add(backoffTime)) { good = true break } From 9b8dbb79824f7ec75c8c10e79ec552caa4c36565 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 22 Apr 2020 17:50:50 -0700 Subject: [PATCH 162/259] fix: avoid calling AddChild after the process may shutdown. --- p2p/net/swarm/testing/testing.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 5e39602426..10de2ace09 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -73,7 +73,9 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter()) - s.Process().AddChild(goprocess.WithTeardown(ps.Close)) + // Call AddChildNoWait because we can't call AddChild after the process + // may have been closed (e.g., if the context was canceled). + s.Process().AddChildNoWait(goprocess.WithTeardown(ps.Close)) tcpTransport := tcp.NewTCPTransport(GenUpgrader(s)) tcpTransport.DisableReuseport = cfg.disableReuseport From dc499b7e0737ce09bb031a44eab86bdc1cf73eb7 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Fri, 15 May 2020 18:56:21 +0530 Subject: [PATCH 163/259] implement connection gating support: intercept peer, address dials, upgraded conns (#201) --- p2p/net/swarm/addrs.go | 4 +- p2p/net/swarm/swarm.go | 87 ++++++++++-------- p2p/net/swarm/swarm_dial.go | 17 +++- p2p/net/swarm/swarm_listen.go | 1 + p2p/net/swarm/swarm_test.go | 146 ++++++++++++++++++++----------- p2p/net/swarm/testing/testing.go | 87 ++++++++++++++++-- 6 files changed, 242 insertions(+), 100 deletions(-) diff --git a/p2p/net/swarm/addrs.go b/p2p/net/swarm/addrs.go index ed510f2626..17c65ae7b2 100644 --- a/p2p/net/swarm/addrs.go +++ b/p2p/net/swarm/addrs.go @@ -1,12 +1,12 @@ package swarm import ( - mafilter "github.com/libp2p/go-maddr-filter" + ma "github.com/multiformats/go-multiaddr" mamask "github.com/whyrusleeping/multiaddr-filter" ) // http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml -var lowTimeoutFilters = mafilter.NewFilters() +var lowTimeoutFilters = ma.NewFilters() func init() { for _, p := range []string{ diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 198d88a72b..f5c0209c80 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -19,9 +20,7 @@ import ( "github.com/jbenet/goprocess" goprocessctx "github.com/jbenet/goprocess/context" - filter "github.com/libp2p/go-maddr-filter" ma "github.com/multiformats/go-multiaddr" - mafilter "github.com/whyrusleeping/multiaddr-filter" ) // DialTimeoutLocal is the maximum duration a Dial to local network address @@ -87,22 +86,24 @@ type Swarm struct { dsync *DialSync backf DialBackoff limiter *dialLimiter - - // filters for addresses that shouldnt be dialed (or accepted) - Filters *filter.Filters + gater connmgr.ConnectionGater proc goprocess.Process ctx context.Context bwc metrics.Reporter } -// NewSwarm constructs a Swarm -func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter) *Swarm { +// NewSwarm constructs a Swarm. +// +// NOTE: go-libp2p will be moving to dependency injection soon. The variadic +// `extra` interface{} parameter facilitates the future migration. Supported +// elements are: +// - connmgr.ConnectionGater +func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter, extra ...interface{}) *Swarm { s := &Swarm{ - local: local, - peers: peers, - bwc: bwc, - Filters: filter.NewFilters(), + local: local, + peers: peers, + bwc: bwc, } s.conns.m = make(map[peer.ID][]*Conn) @@ -110,6 +111,13 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc s.transports.m = make(map[int]transport.Transport) s.notifs.m = make(map[network.Notifiee]struct{}) + for _, i := range extra { + switch v := i.(type) { + case connmgr.ConnectionGater: + s.gater = v + } + } + s.dsync = NewDialSync(s.doDial) s.limiter = newDialLimiter(s.dialAddr) s.proc = goprocessctx.WithContext(ctx) @@ -168,33 +176,46 @@ func (s *Swarm) teardown() error { return nil } -// AddAddrFilter adds a multiaddr filter to the set of filters the swarm will use to determine which -// addresses not to dial to. -func (s *Swarm) AddAddrFilter(f string) error { - m, err := mafilter.NewMask(f) - if err != nil { - return err - } - - s.Filters.AddDialFilter(m) - return nil -} - // Process returns the Process of the swarm func (s *Swarm) Process() goprocess.Process { return s.proc } func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, error) { - // The underlying transport (or the dialer) *should* filter it's own - // connections but we should double check anyways. - raddr := tc.RemoteMultiaddr() - if s.Filters.AddrBlocked(raddr) { - tc.Close() - return nil, ErrAddrFiltered + var ( + p = tc.RemotePeer() + addr = tc.RemoteMultiaddr() + ) + + if s.gater != nil { + if allow := s.gater.InterceptAddrDial(p, addr); !allow { + err := tc.Close() + if err != nil { + log.Warnf("failed to close connection with peer %s and addr %s; err: %s", p.Pretty(), addr, err) + } + return nil, ErrAddrFiltered + } } - p := tc.RemotePeer() + stat := network.Stat{Direction: dir} + c := &Conn{ + conn: tc, + swarm: s, + stat: stat, + } + + // we ONLY check upgraded connections here so we can send them a Disconnect message. + // If we do this in the Upgrader, we will not be able to do this. + if s.gater != nil { + if allow, _ := s.gater.InterceptUpgraded(c); !allow { + // TODO Send disconnect with reason here + err := tc.Close() + if err != nil { + log.Warnf("failed to close connection with peer %s and addr %s; err: %s", p.Pretty(), addr, err) + } + return nil, ErrGaterDisallowedConnection + } + } // Add the public key. if pk := tc.RemotePublicKey(); pk != nil { @@ -214,12 +235,6 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, } // Wrap and register the connection. - stat := network.Stat{Direction: dir} - c := &Conn{ - conn: tc, - swarm: s, - stat: stat, - } c.streams.m = make(map[*Stream]struct{}) s.conns.m[p] = append(s.conns.m[p], c) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index cae4110c01..f35f2b6372 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -50,6 +50,10 @@ var ( // ErrNoGoodAddresses is returned when we find addresses for a peer but // can't use any of them. ErrNoGoodAddresses = errors.New("no good addresses") + + // ErrGaterDisallowedConnection is returned when the gater prevents us from + // forming a connection with a peer. + ErrGaterDisallowedConnection = errors.New("gater disallows connection to peer") ) // DialAttempts governs how many times a goroutine will try to dial a given peer. @@ -218,6 +222,11 @@ func (db *DialBackoff) cleanup() { // This allows us to use various transport protocols, do NAT traversal/relay, // etc. to achieve connection. func (s *Swarm) DialPeer(ctx context.Context, p peer.ID) (network.Conn, error) { + if s.gater != nil && !s.gater.InterceptPeerDial(p) { + log.Debugf("gater disallowed outbound connection to peer %s", p.Pretty()) + return nil, &DialError{Peer: p, Cause: ErrGaterDisallowedConnection} + } + return s.dialPeer(ctx, p) } @@ -339,7 +348,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { if len(peerAddrs) == 0 { return nil, &DialError{Peer: p, Cause: ErrNoAddresses} } - goodAddrs := s.filterKnownUndialables(peerAddrs) + goodAddrs := s.filterKnownUndialables(p, peerAddrs) if len(goodAddrs) == 0 { return nil, &DialError{Peer: p, Cause: ErrNoGoodAddresses} } @@ -393,7 +402,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { // IPv6 link-local addresses, addresses without a dial-capable transport, // and addresses that we know to be our own. // This is an optimization to avoid wasting time on dials that we know are going to fail. -func (s *Swarm) filterKnownUndialables(addrs []ma.Multiaddr) []ma.Multiaddr { +func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) []ma.Multiaddr { lisAddrs, _ := s.InterfaceListenAddresses() var ourAddrs []ma.Multiaddr for _, addr := range lisAddrs { @@ -409,7 +418,9 @@ func (s *Swarm) filterKnownUndialables(addrs []ma.Multiaddr) []ma.Multiaddr { s.canDial, // TODO: Consider allowing link-local addresses addrutil.AddrOverNonLocalIP, - addrutil.FilterNeg(s.Filters.AddrBlocked), + func(addr ma.Multiaddr) bool { + return s.gater == nil || s.gater.InterceptAddrDial(p, addr) + }, ) } diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 09d411dfd8..ab5c42a345 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -89,6 +89,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { } return } + log.Debugf("swarm listener accepted connection: %s", c) s.refs.Add(1) go func() { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index b155373f67..4750b6a64d 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -5,20 +5,21 @@ import ( "context" "fmt" "io" - "net" "sync" "testing" "time" - logging "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p-core/control" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" - ma "github.com/multiformats/go-multiaddr" - . "github.com/libp2p/go-libp2p-swarm" . "github.com/libp2p/go-libp2p-swarm/testing" + + logging "github.com/ipfs/go-log" + ma "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/require" ) var log = logging.Logger("swarm_test") @@ -280,60 +281,105 @@ func TestConnHandler(t *testing.T) { } } -func TestAddrBlocking(t *testing.T) { +func TestConnectionGating(t *testing.T) { ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) - - swarms[0].SetConnHandler(func(conn network.Conn) { - t.Errorf("no connections should happen! -- %s", conn) - }) - - _, block, err := net.ParseCIDR("127.0.0.1/8") - if err != nil { - t.Fatal(err) - } - - swarms[1].Filters.AddDialFilter(block) - - swarms[1].Peerstore().AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], peerstore.PermanentAddrTTL) - _, err = swarms[1].DialPeer(ctx, swarms[0].LocalPeer()) - if err == nil { - t.Fatal("dial should have failed") - } - - swarms[0].Peerstore().AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], peerstore.PermanentAddrTTL) - _, err = swarms[0].DialPeer(ctx, swarms[1].LocalPeer()) - if err == nil { - t.Fatal("dial should have failed") + tcs := map[string]struct { + p1Gater func(gater *MockConnectionGater) *MockConnectionGater + p2Gater func(gater *MockConnectionGater) *MockConnectionGater + + p1ConnectednessToP2 network.Connectedness + p2ConnectednessToP1 network.Connectedness + isP1OutboundErr bool + }{ + "no gating": { + p1ConnectednessToP2: network.Connected, + p2ConnectednessToP1: network.Connected, + isP1OutboundErr: false, + }, + "p1 gates outbound peer dial": { + p1Gater: func(c *MockConnectionGater) *MockConnectionGater { + c.PeerDial = func(p peer.ID) bool { return false } + return c + }, + p1ConnectednessToP2: network.NotConnected, + p2ConnectednessToP1: network.NotConnected, + isP1OutboundErr: true, + }, + "p1 gates outbound addr dialing": { + p1Gater: func(c *MockConnectionGater) *MockConnectionGater { + c.Dial = func(p peer.ID, addr ma.Multiaddr) bool { return false } + return c + }, + p1ConnectednessToP2: network.NotConnected, + p2ConnectednessToP1: network.NotConnected, + isP1OutboundErr: true, + }, + "p2 gates inbound peer dial before securing": { + p2Gater: func(c *MockConnectionGater) *MockConnectionGater { + c.Accept = func(c network.ConnMultiaddrs) bool { return false } + return c + }, + p1ConnectednessToP2: network.NotConnected, + p2ConnectednessToP1: network.NotConnected, + isP1OutboundErr: true, + }, + "p2 gates inbound peer dial before multiplexing": { + p1Gater: func(c *MockConnectionGater) *MockConnectionGater { + c.Secured = func(network.Direction, peer.ID, network.ConnMultiaddrs) bool { return false } + return c + }, + p1ConnectednessToP2: network.NotConnected, + p2ConnectednessToP1: network.NotConnected, + isP1OutboundErr: true, + }, + "p2 gates inbound peer dial after upgrading": { + p1Gater: func(c *MockConnectionGater) *MockConnectionGater { + c.Upgraded = func(c network.Conn) (bool, control.DisconnectReason) { return false, 0 } + return c + }, + p1ConnectednessToP2: network.NotConnected, + p2ConnectednessToP1: network.NotConnected, + isP1OutboundErr: true, + }, + "p2 gates outbound dials": { + p2Gater: func(c *MockConnectionGater) *MockConnectionGater { + c.PeerDial = func(p peer.ID) bool { return false } + return c + }, + p1ConnectednessToP2: network.Connected, + p2ConnectednessToP1: network.Connected, + isP1OutboundErr: false, + }, } -} -func TestFilterBounds(t *testing.T) { - ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + for n, tc := range tcs { + t.Run(n, func(t *testing.T) { + p1Gater := DefaultMockConnectionGater() + p2Gater := DefaultMockConnectionGater() + if tc.p1Gater != nil { + p1Gater = tc.p1Gater(p1Gater) + } + if tc.p2Gater != nil { + p2Gater = tc.p2Gater(p2Gater) + } - conns := make(chan struct{}, 8) - swarms[0].SetConnHandler(func(conn network.Conn) { - conns <- struct{}{} - }) + sw1 := GenSwarm(t, ctx, OptConnGater(p1Gater)) + sw2 := GenSwarm(t, ctx, OptConnGater(p2Gater)) - // Address that we wont be dialing from - _, block, err := net.ParseCIDR("192.0.0.1/8") - if err != nil { - t.Fatal(err) - } + p1 := sw1.LocalPeer() + p2 := sw2.LocalPeer() + sw1.Peerstore().AddAddr(p2, sw2.ListenAddresses()[0], peerstore.PermanentAddrTTL) + // 1 -> 2 + _, err := sw1.DialPeer(ctx, p2) - // set filter on both sides, shouldnt matter - swarms[1].Filters.AddDialFilter(block) - swarms[0].Filters.AddDialFilter(block) + require.Equal(t, tc.isP1OutboundErr, err != nil, n) + require.Equal(t, tc.p1ConnectednessToP2, sw1.Connectedness(p2), n) - connectSwarms(t, ctx, swarms) + require.Eventually(t, func() bool { + return tc.p2ConnectednessToP1 == sw2.Connectedness(p1) + }, 2*time.Second, 100*time.Millisecond, n) + }) - select { - case <-time.After(time.Second): - t.Fatal("should have gotten connection") - case <-conns: - t.Log("got connect") } } diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 10de2ace09..0d252c3a1c 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -4,26 +4,31 @@ import ( "context" "testing" + "github.com/libp2p/go-libp2p-core/connmgr" + "github.com/libp2p/go-libp2p-core/control" "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" - "github.com/libp2p/go-libp2p-testing/net" - "github.com/libp2p/go-tcp-transport" - goprocess "github.com/jbenet/goprocess" csms "github.com/libp2p/go-conn-security-multistream" pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" secio "github.com/libp2p/go-libp2p-secio" + swarm "github.com/libp2p/go-libp2p-swarm" + "github.com/libp2p/go-libp2p-testing/net" tptu "github.com/libp2p/go-libp2p-transport-upgrader" yamux "github.com/libp2p/go-libp2p-yamux" msmux "github.com/libp2p/go-stream-muxer-multistream" + "github.com/libp2p/go-tcp-transport" - swarm "github.com/libp2p/go-libp2p-swarm" + goprocess "github.com/jbenet/goprocess" + ma "github.com/multiformats/go-multiaddr" ) type config struct { disableReuseport bool dialOnly bool + connectionGater connmgr.ConnectionGater } // Option is an option that can be passed when constructing a test swarm. @@ -39,6 +44,13 @@ var OptDialOnly Option = func(_ *testing.T, c *config) { c.dialOnly = true } +// OptConnGater configures the given connection gater on the test +func OptConnGater(cg connmgr.ConnectionGater) Option { + return func(_ *testing.T, c *config) { + c.connectionGater = cg + } +} + // GenUpgrader creates a new connection upgrader for use with this swarm. func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { id := n.LocalPeer() @@ -53,9 +65,8 @@ func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { stMuxer.AddTransport("/yamux/1.0.0", yamux.DefaultTransport) return &tptu.Upgrader{ - Secure: secMuxer, - Muxer: stMuxer, - Filters: n.Filters, + Secure: secMuxer, + Muxer: stMuxer, } } @@ -72,12 +83,16 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { ps := pstoremem.NewPeerstore() ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) - s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter()) + + s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter(), cfg.connectionGater) + // Call AddChildNoWait because we can't call AddChild after the process // may have been closed (e.g., if the context was canceled). s.Process().AddChildNoWait(goprocess.WithTeardown(ps.Close)) - tcpTransport := tcp.NewTCPTransport(GenUpgrader(s)) + upgrader := GenUpgrader(s) + upgrader.ConnGater = cfg.connectionGater + tcpTransport := tcp.NewTCPTransport(upgrader) tcpTransport.DisableReuseport = cfg.disableReuseport if err := s.AddTransport(tcpTransport); err != nil { @@ -101,3 +116,57 @@ func DivulgeAddresses(a, b network.Network) { addrs := a.Peerstore().Addrs(id) b.Peerstore().AddAddrs(id, addrs, peerstore.PermanentAddrTTL) } + +// MockConnectionGater is a mock connection gater to be used by the tests. +type MockConnectionGater struct { + Dial func(p peer.ID, addr ma.Multiaddr) bool + PeerDial func(p peer.ID) bool + Accept func(c network.ConnMultiaddrs) bool + Secured func(network.Direction, peer.ID, network.ConnMultiaddrs) bool + Upgraded func(c network.Conn) (bool, control.DisconnectReason) +} + +func DefaultMockConnectionGater() *MockConnectionGater { + m := &MockConnectionGater{} + m.Dial = func(p peer.ID, addr ma.Multiaddr) bool { + return true + } + + m.PeerDial = func(p peer.ID) bool { + return true + } + + m.Accept = func(c network.ConnMultiaddrs) bool { + return true + } + + m.Secured = func(network.Direction, peer.ID, network.ConnMultiaddrs) bool { + return true + } + + m.Upgraded = func(c network.Conn) (bool, control.DisconnectReason) { + return true, 0 + } + + return m +} + +func (m *MockConnectionGater) InterceptAddrDial(p peer.ID, addr ma.Multiaddr) (allow bool) { + return m.Dial(p, addr) +} + +func (m *MockConnectionGater) InterceptPeerDial(p peer.ID) (allow bool) { + return m.PeerDial(p) +} + +func (m *MockConnectionGater) InterceptAccept(c network.ConnMultiaddrs) (allow bool) { + return m.Accept(c) +} + +func (m *MockConnectionGater) InterceptSecured(d network.Direction, p peer.ID, c network.ConnMultiaddrs) (allow bool) { + return m.Secured(d, p, c) +} + +func (m *MockConnectionGater) InterceptUpgraded(tc network.Conn) (allow bool, reason control.DisconnectReason) { + return m.Upgraded(tc) +} From 499b4d8467f7ed673a80e14b3b18ab88f110e2c2 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Wed, 20 May 2020 14:34:11 +0530 Subject: [PATCH 164/259] Rank Dial addresses (#212) * Rank dial addresss. --- p2p/net/swarm/limiter.go | 32 ++++++---- p2p/net/swarm/limiter_test.go | 45 ++++++++++++-- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_dial.go | 107 ++++++++++++++++++++++++++++------ p2p/net/swarm/swarm_test.go | 54 +++++++++++++++++ 5 files changed, 205 insertions(+), 35 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 6808dd71a6..3e20976b32 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -10,7 +10,6 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" - addrutil "github.com/libp2p/go-addr-util" ma "github.com/multiformats/go-multiaddr" ) @@ -43,9 +42,10 @@ func (dj *dialJob) dialTimeout() time.Duration { type dialLimiter struct { lk sync.Mutex - fdConsuming int - fdLimit int - waitingOnFd []*dialJob + isFdConsumingFnc isFdConsumingFnc + fdConsuming int + fdLimit int + waitingOnFd []*dialJob dialFunc dialfunc @@ -55,19 +55,21 @@ type dialLimiter struct { } type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (transport.CapableConn, error) +type isFdConsumingFnc func(ma.Multiaddr) bool -func newDialLimiter(df dialfunc) *dialLimiter { +func newDialLimiter(df dialfunc, fdFnc isFdConsumingFnc) *dialLimiter { fd := ConcurrentFdDials if env := os.Getenv("LIBP2P_SWARM_FD_LIMIT"); env != "" { if n, err := strconv.ParseInt(env, 10, 32); err == nil { fd = int(n) } } - return newDialLimiterWithParams(df, fd, DefaultPerPeerRateLimit) + return newDialLimiterWithParams(fdFnc, df, fd, DefaultPerPeerRateLimit) } -func newDialLimiterWithParams(df dialfunc, fdLimit, perPeerLimit int) *dialLimiter { +func newDialLimiterWithParams(isFdConsumingFnc isFdConsumingFnc, df dialfunc, fdLimit, perPeerLimit int) *dialLimiter { return &dialLimiter{ + isFdConsumingFnc: isFdConsumingFnc, fdLimit: fdLimit, perPeerLimit: perPeerLimit, waitingOnPeerLimit: make(map[peer.ID][]*dialJob), @@ -140,16 +142,26 @@ func (dl *dialLimiter) freePeerToken(dj *dialJob) { func (dl *dialLimiter) finishedDial(dj *dialJob) { dl.lk.Lock() defer dl.lk.Unlock() - - if addrutil.IsFDCostlyTransport(dj.addr) { + if dl.shouldConsumeFd(dj.addr) { dl.freeFDToken() } dl.freePeerToken(dj) } +func (dl *dialLimiter) shouldConsumeFd(addr ma.Multiaddr) bool { + // we don't consume FD's for relay addresses for now as they will be consumed when the Relay Transport + // actually dials the Relay server. That dial call will also pass through this limiter with + // the address of the relay server i.e. non-relay address. + _, err := addr.ValueForProtocol(ma.P_CIRCUIT) + + isRelay := err == nil + + return !isRelay && dl.isFdConsumingFnc(addr) +} + func (dl *dialLimiter) addCheckFdLimit(dj *dialJob) { - if addrutil.IsFDCostlyTransport(dj.addr) { + if dl.shouldConsumeFd(dj.addr) { if dl.fdConsuming >= dl.fdLimit { log.Debugf("[limiter] blocked dial waiting on FD token; peer: %s; addr: %s; consuming: %d; "+ "limit: %d; waiting: %d", dj.peer, dj.addr, dl.fdConsuming, dl.fdLimit, len(dl.waitingOnFd)) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 7373ddbb3e..367e099fd9 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -10,11 +10,26 @@ import ( "time" "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/test" "github.com/libp2p/go-libp2p-core/transport" + ma "github.com/multiformats/go-multiaddr" mafmt "github.com/multiformats/go-multiaddr-fmt" ) +var isFdConsuming = func(addr ma.Multiaddr) bool { + res := false + + ma.ForEach(addr, func(c ma.Component) bool { + if c.Protocol().Code == ma.P_TCP { + res = true + return false + } + return true + }) + return res +} + func mustAddr(t *testing.T, s string) ma.Multiaddr { a, err := ma.NewMultiaddr(s) if err != nil { @@ -61,6 +76,11 @@ func hangDialFunc(hang chan struct{}) dialfunc { return transport.CapableConn(nil), nil } + _, err := a.ValueForProtocol(ma.P_CIRCUIT) + if err == nil { + return transport.CapableConn(nil), nil + } + if tcpPortOver(a, 10) { return transport.CapableConn(nil), nil } @@ -74,7 +94,7 @@ func TestLimiterBasicDials(t *testing.T) { hang := make(chan struct{}) defer close(hang) - l := newDialLimiterWithParams(hangDialFunc(hang), ConcurrentFdDials, 4) + l := newDialLimiterWithParams(isFdConsuming, hangDialFunc(hang), ConcurrentFdDials, 4) bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} good := addrWithPort(t, 20) @@ -123,7 +143,7 @@ func TestLimiterBasicDials(t *testing.T) { func TestFDLimiting(t *testing.T) { hang := make(chan struct{}) defer close(hang) - l := newDialLimiterWithParams(hangDialFunc(hang), 16, 5) + l := newDialLimiterWithParams(isFdConsuming, hangDialFunc(hang), 16, 5) bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} pids := []peer.ID{"testpeer1", "testpeer2", "testpeer3", "testpeer4"} @@ -168,6 +188,21 @@ func TestFDLimiting(t *testing.T) { case <-time.After(time.Second * 5): t.Fatal("timeout waiting for utp addr success") } + + // A relay address with tcp transport will complete because we do not consume fds for dials + // with relay addresses as the fd will be consumed when we actually dial the relay server. + pid6 := test.RandPeerIDFatal(t) + relayAddr := mustAddr(t, fmt.Sprintf("/ip4/127.0.0.1/tcp/20/p2p-circuit/p2p/%s", pid6)) + l.AddDialJob(&dialJob{ctx: ctx, peer: pid6, addr: relayAddr, resp: resch}) + + select { + case res := <-resch: + if res.Err != nil { + t.Fatal("should have gotten successful response") + } + case <-time.After(time.Second * 5): + t.Fatal("timeout waiting for relay addr success") + } } func TestTokenRedistribution(t *testing.T) { @@ -184,7 +219,7 @@ func TestTokenRedistribution(t *testing.T) { <-ch return nil, fmt.Errorf("test bad dial") } - l := newDialLimiterWithParams(df, 8, 4) + l := newDialLimiterWithParams(isFdConsuming, df, 8, 4) bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} pids := []peer.ID{"testpeer1", "testpeer2"} @@ -277,7 +312,7 @@ func TestStressLimiter(t *testing.T) { return nil, fmt.Errorf("test bad dial") } - l := newDialLimiterWithParams(df, 20, 5) + l := newDialLimiterWithParams(isFdConsuming, df, 20, 5) var bads []ma.Multiaddr for i := 0; i < 100; i++ { @@ -337,7 +372,7 @@ func TestFDLimitUnderflow(t *testing.T) { return nil, fmt.Errorf("df timed out") } - l := newDialLimiterWithParams(df, 20, 3) + l := newDialLimiterWithParams(isFdConsuming, df, 20, 3) var addrs []ma.Multiaddr for i := 0; i <= 1000; i++ { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index f5c0209c80..85f8b2f81c 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -119,7 +119,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc } s.dsync = NewDialSync(s.doDial) - s.limiter = newDialLimiter(s.dialAddr) + s.limiter = newDialLimiter(s.dialAddr, s.IsFdConsumingAddr) s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) s.backf.init(s.ctx) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index f35f2b6372..68b1045c78 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -10,11 +10,13 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" + + addrutil "github.com/libp2p/go-addr-util" lgbl "github.com/libp2p/go-libp2p-loggables" logging "github.com/ipfs/go-log" - addrutil "github.com/libp2p/go-addr-util" ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" ) // Diagram of dial sync: @@ -337,13 +339,6 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { } ////// - /* - This slice-to-chan code is temporary, the peerstore can currently provide - a channel as an interface for receiving addresses, but more thought - needs to be put into the execution. For now, this allows us to use - the improved rate limiter, while maintaining the outward behaviour - that we previously had (halting a dial when we run out of addrs) - */ peerAddrs := s.peers.Addrs(p) if len(peerAddrs) == 0 { return nil, &DialError{Peer: p, Cause: ErrNoAddresses} @@ -352,23 +347,60 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { if len(goodAddrs) == 0 { return nil, &DialError{Peer: p, Cause: ErrNoGoodAddresses} } - goodAddrsChan := make(chan ma.Multiaddr, len(goodAddrs)) - nonBackoff := false + + /////// Check backoff andnRank addresses + var nonBackoff bool for _, a := range goodAddrs { // skip addresses in back-off if !s.backf.Backoff(p, a) { nonBackoff = true - goodAddrsChan <- a } } - close(goodAddrsChan) if !nonBackoff { return nil, ErrDialBackoff } - ///////// - // try to get a connection to any addr - connC, dialErr := s.dialAddrs(ctx, p, goodAddrsChan) + // ranks addresses in descending order of preference for dialing + // Private UDP > Public UDP > Private TCP > Public TCP > UDP Relay server > TCP Relay server + rankAddrsFnc := func(addrs []ma.Multiaddr) []ma.Multiaddr { + var localUdpAddrs []ma.Multiaddr // private udp + var relayUdpAddrs []ma.Multiaddr // relay udp + var othersUdp []ma.Multiaddr // public udp + + var localFdAddrs []ma.Multiaddr // private fd consuming + var relayFdAddrs []ma.Multiaddr // relay fd consuming + var othersFd []ma.Multiaddr // public fd consuming + + for _, a := range addrs { + if _, err := a.ValueForProtocol(ma.P_CIRCUIT); err == nil { + if s.IsFdConsumingAddr(a) { + relayFdAddrs = append(relayFdAddrs, a) + continue + } + relayUdpAddrs = append(relayUdpAddrs, a) + } else if manet.IsPrivateAddr(a) { + if s.IsFdConsumingAddr(a) { + localFdAddrs = append(localFdAddrs, a) + continue + } + localUdpAddrs = append(localUdpAddrs, a) + } else { + if s.IsFdConsumingAddr(a) { + othersFd = append(othersFd, a) + continue + } + othersUdp = append(othersUdp, a) + } + } + + relays := append(relayUdpAddrs, relayFdAddrs...) + fds := append(localFdAddrs, othersFd...) + + return append(append(append(localUdpAddrs, othersUdp...), fds...), relays...) + } + + connC, dialErr := s.dialAddrs(ctx, p, rankAddrsFnc(goodAddrs)) + if dialErr != nil { logdial["error"] = dialErr.Cause.Error() switch dialErr.Cause { @@ -424,7 +456,23 @@ func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) []ma.Mul ) } -func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.Multiaddr) (transport.CapableConn, *DialError) { +func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs []ma.Multiaddr) (transport.CapableConn, *DialError) { + /* + This slice-to-chan code is temporary, the peerstore can currently provide + a channel as an interface for receiving addresses, but more thought + needs to be put into the execution. For now, this allows us to use + the improved rate limiter, while maintaining the outward behaviour + that we previously had (halting a dial when we run out of addrs) + */ + var remoteAddrChan chan ma.Multiaddr + if len(remoteAddrs) > 0 { + remoteAddrChan = make(chan ma.Multiaddr, len(remoteAddrs)) + for i := range remoteAddrs { + remoteAddrChan <- remoteAddrs[i] + } + close(remoteAddrChan) + } + log.Debugf("%s swarm dialing %s", s.local, p) ctx, cancel := context.WithCancel(ctx) @@ -438,7 +486,7 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma. var active int dialLoop: - for remoteAddrs != nil || active > 0 { + for remoteAddrChan != nil || active > 0 { // Check for context cancellations and/or responses first. select { case <-ctx.Done(): @@ -464,9 +512,9 @@ dialLoop: // Now, attempt to dial. select { - case addr, ok := <-remoteAddrs: + case addr, ok := <-remoteAddrChan: if !ok { - remoteAddrs = nil + remoteAddrChan = nil continue } @@ -540,3 +588,24 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (tra // success! we got one! return connC, nil } + +// TODO We should have a `IsFdConsuming() bool` method on the `Transport` interface in go-libp2p-core/transport. +// This function checks if any of the transport protocols in the address requires a file descriptor. +// For now: +// A Non-circuit address which has the TCP/UNIX protocol is deemed FD consuming. +// For a circuit-relay address, we look at the address of the relay server/proxy +// and use the same logic as above to decide. +func (s *Swarm) IsFdConsumingAddr(addr ma.Multiaddr) bool { + first, _ := ma.SplitFunc(addr, func(c ma.Component) bool { + return c.Protocol().Code == ma.P_CIRCUIT + }) + + // for safety + if first == nil { + return true + } + + _, err1 := first.ValueForProtocol(ma.P_TCP) + _, err2 := first.ValueForProtocol(ma.P_UNIX) + return err1 == nil || err2 == nil +} diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 4750b6a64d..aa18904acc 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -13,7 +13,10 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/test" + circuit "github.com/libp2p/go-libp2p-circuit" + qc "github.com/libp2p/go-libp2p-quic-transport" . "github.com/libp2p/go-libp2p-swarm" . "github.com/libp2p/go-libp2p-swarm/testing" @@ -383,6 +386,57 @@ func TestConnectionGating(t *testing.T) { } } +func TestIsFdConsuming(t *testing.T) { + tcs := map[string]struct { + addr string + isFdConsuming bool + }{ + "tcp": { + addr: "/ip4/127.0.0.1/tcp/20", + isFdConsuming: true, + }, + "quic": { + addr: "/ip4/127.0.0.1/udp/0/quic", + isFdConsuming: false, + }, + "addr-without-registered-transport": { + addr: "/ip4/127.0.0.1/tcp/20/ws", + isFdConsuming: true, + }, + "relay-tcp": { + addr: fmt.Sprintf("/ip4/127.0.0.1/tcp/20/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), + isFdConsuming: true, + }, + "relay-quic": { + addr: fmt.Sprintf("/ip4/127.0.0.1/udp/20/quic/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), + isFdConsuming: false, + }, + "relay-without-serveraddr": { + addr: fmt.Sprintf("/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), + isFdConsuming: true, + }, + "relay-without-registered-transport-server": { + addr: fmt.Sprintf("/ip4/127.0.0.1/tcp/20/ws/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), + isFdConsuming: true, + }, + } + + ctx := context.Background() + sw := GenSwarm(t, ctx) + sk := sw.Peerstore().PrivKey(sw.LocalPeer()) + require.NotNil(t, sk) + qtpt, err := qc.NewTransport(sk, nil, nil) + require.NoError(t, err) + require.NoError(t, sw.AddTransport(qtpt)) + require.NoError(t, sw.AddTransport(&circuit.RelayTransport{})) + + for name := range tcs { + maddr, err := ma.NewMultiaddr(tcs[name].addr) + require.NoError(t, err, name) + require.Equal(t, tcs[name].isFdConsuming, sw.IsFdConsumingAddr(maddr), name) + } +} + func TestNoDial(t *testing.T) { ctx := context.Background() swarms := makeSwarms(ctx, t, 2) From fba72557b3b8ac387f80aed9bdfff4d6f0e7eef0 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Tue, 2 Jun 2020 15:45:59 +0530 Subject: [PATCH 165/259] configure private key for test swarm --- p2p/net/swarm/testing/testing.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 0d252c3a1c..6ba62a587c 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -6,6 +6,7 @@ import ( "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/control" + "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -29,6 +30,7 @@ type config struct { disableReuseport bool dialOnly bool connectionGater connmgr.ConnectionGater + sk crypto.PrivKey } // Option is an option that can be passed when constructing a test swarm. @@ -51,6 +53,13 @@ func OptConnGater(cg connmgr.ConnectionGater) Option { } } +// OptPeerPrivateKey configures the peer private key which is then used to derive the public key and peer ID. +func OptPeerPrivateKey(sk crypto.PrivKey) Option { + return func(_ *testing.T, c *config) { + c.sk = sk + } +} + // GenUpgrader creates a new connection upgrader for use with this swarm. func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { id := n.LocalPeer() @@ -78,12 +87,24 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { o(t, &cfg) } - p := tnet.RandPeerNetParamsOrFatal(t) + var p tnet.PeerNetParams + if cfg.sk == nil { + p = tnet.RandPeerNetParamsOrFatal(t) + } else { + pk := cfg.sk.GetPublic() + id, err := peer.IDFromPublicKey(pk) + if err != nil { + t.Fatal(err) + } + p.PrivKey = cfg.sk + p.PubKey = pk + p.ID = id + p.Addr = tnet.ZeroLocalTCPAddress + } ps := pstoremem.NewPeerstore() ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) - s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter(), cfg.connectionGater) // Call AddChildNoWait because we can't call AddChild after the process From ac5726ba2f3ec10df8069e59796844ddcb6de8e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 5 Jun 2020 12:44:40 +0100 Subject: [PATCH 166/259] `ID()` method on connections and streams + record opening time (#224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aarsh Shah Co-authored-by: Raúl Kripalani --- p2p/net/swarm/swarm.go | 8 ++++++-- p2p/net/swarm/swarm_conn.go | 15 ++++++++++++++- p2p/net/swarm/swarm_stream.go | 7 +++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 85f8b2f81c..db3f4f1cf3 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -54,6 +54,9 @@ type Swarm struct { local peer.ID peers peerstore.Peerstore + nextConnID uint32 // guarded by atomic + nextStreamID uint32 // guarded by atomic + conns struct { sync.RWMutex m map[peer.ID][]*Conn @@ -197,11 +200,13 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, } } - stat := network.Stat{Direction: dir} + // Wrap and register the connection. + stat := network.Stat{Direction: dir, Opened: time.Now()} c := &Conn{ conn: tc, swarm: s, stat: stat, + id: atomic.AddUint32(&s.nextConnID, 1), } // we ONLY check upgraded connections here so we can send them a Disconnect message. @@ -234,7 +239,6 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, return nil, ErrSwarmClosed } - // Wrap and register the connection. c.streams.m = make(map[*Stream]struct{}) s.conns.m[p] = append(s.conns.m[p], c) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index c09957c130..946936cfc6 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "sync" + "sync/atomic" + "time" ic "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/mux" @@ -22,6 +24,7 @@ var ErrConnClosed = errors.New("connection closed") // Conn is the connection type used by swarm. In general, you won't use this // type directly. type Conn struct { + id uint32 conn transport.CapableConn swarm *Swarm @@ -38,6 +41,11 @@ type Conn struct { stat network.Stat } +func (c *Conn) ID() string { + // format: - + return fmt.Sprintf("%s-%d", c.RemotePeer().Pretty()[0:10], c.id) +} + // Close closes this connection. // // Note: This method won't wait for the close notifications to finish as that @@ -169,6 +177,7 @@ func (c *Conn) Stat() network.Stat { // NewStream returns a new Stream from this connection func (c *Conn) NewStream() (network.Stream, error) { ts, err := c.conn.OpenStream() + if err != nil { return nil, err } @@ -185,11 +194,15 @@ func (c *Conn) addStream(ts mux.MuxedStream, dir network.Direction) (*Stream, er } // Wrap and register the stream. - stat := network.Stat{Direction: dir} + stat := network.Stat{ + Direction: dir, + Opened: time.Now(), + } s := &Stream{ stream: ts, conn: c, stat: stat, + id: atomic.AddUint32(&c.swarm.nextStreamID, 1), } c.streams.m[s] = struct{}{} diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 9dded2a9d7..803ba4db63 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -28,6 +28,8 @@ var _ network.Stream = &Stream{} // Stream is the stream type used by swarm. In general, you won't use this type // directly. type Stream struct { + id uint32 + stream mux.MuxedStream conn *Conn @@ -43,6 +45,11 @@ type Stream struct { stat network.Stat } +func (s *Stream) ID() string { + // format: -- + return fmt.Sprintf("%s-%d", s.conn.ID(), s.id) +} + func (s *Stream) String() string { return fmt.Sprintf( " %s (%s)>", From 4bd32948e10c36b6ef5a734a566839f4f22aaa71 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 24 Jun 2020 10:38:00 -0700 Subject: [PATCH 167/259] slim down dependencies Removes secio, quic, and circuit dependencies. We were using them for testing but we don't actually need them. --- p2p/net/swarm/swarm_test.go | 6 ------ p2p/net/swarm/testing/testing.go | 7 ++----- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index aa18904acc..81e1412f11 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -15,8 +15,6 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/test" - circuit "github.com/libp2p/go-libp2p-circuit" - qc "github.com/libp2p/go-libp2p-quic-transport" . "github.com/libp2p/go-libp2p-swarm" . "github.com/libp2p/go-libp2p-swarm/testing" @@ -425,10 +423,6 @@ func TestIsFdConsuming(t *testing.T) { sw := GenSwarm(t, ctx) sk := sw.Peerstore().PrivKey(sw.LocalPeer()) require.NotNil(t, sk) - qtpt, err := qc.NewTransport(sk, nil, nil) - require.NoError(t, err) - require.NoError(t, sw.AddTransport(qtpt)) - require.NoError(t, sw.AddTransport(&circuit.RelayTransport{})) for name := range tcs { maddr, err := ma.NewMultiaddr(tcs[name].addr) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 6ba62a587c..83d9fcd12e 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -11,10 +11,10 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/sec/insecure" csms "github.com/libp2p/go-conn-security-multistream" pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" - secio "github.com/libp2p/go-libp2p-secio" swarm "github.com/libp2p/go-libp2p-swarm" "github.com/libp2p/go-libp2p-testing/net" tptu "github.com/libp2p/go-libp2p-transport-upgrader" @@ -65,10 +65,7 @@ func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { id := n.LocalPeer() pk := n.Peerstore().PrivKey(id) secMuxer := new(csms.SSMuxer) - secMuxer.AddTransport(secio.ID, &secio.Transport{ - LocalID: id, - PrivateKey: pk, - }) + secMuxer.AddTransport(insecure.ID, insecure.NewWithIdentity(id, pk)) stMuxer := msmux.NewBlankTransport() stMuxer.AddTransport("/yamux/1.0.0", yamux.DefaultTransport) From 720a4d3fe9ccf625ee3e54e15b2e4036e45e98f6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 1 Sep 2020 22:07:30 -0700 Subject: [PATCH 168/259] feat: update to latest go-libp2p-core interfaces This commit switches over to the new Close/CloseWrite/CloseRead interface changes in go-libp2p-core 0.7.0. This change also removes a now obsolete hack to implicitly clean up streams after reading an EOF; calling Close or Reset is now required. See: https://github.com/libp2p/go-libp2p-core/releases/tag/v0.7.0 --- p2p/net/swarm/dial_test.go | 2 +- p2p/net/swarm/swarm_conn.go | 4 +- p2p/net/swarm/swarm_dial.go | 2 +- p2p/net/swarm/swarm_stream.go | 72 +++++++++++------------------------ 4 files changed, 27 insertions(+), 53 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 49c21fd6d3..14b2b93396 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -18,7 +18,7 @@ import ( "github.com/libp2p/go-libp2p-testing/ci" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" . "github.com/libp2p/go-libp2p-swarm" ) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 946936cfc6..bd531f273f 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -87,10 +87,12 @@ func (c *Conn) doClose() { }() } -func (c *Conn) removeStream(s *Stream) { +func (c *Conn) removeStream(s *Stream) bool { c.streams.Lock() + _, has := c.streams.m[s] delete(c.streams.m, s) c.streams.Unlock() + return has } // listens for new streams. diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 68b1045c78..4dbb1f2dd1 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -16,7 +16,7 @@ import ( logging "github.com/ipfs/go-log" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" ) // Diagram of dial sync: diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 803ba4db63..74ac62cb05 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -2,7 +2,6 @@ package swarm import ( "fmt" - "io" "sync" "sync/atomic" "time" @@ -12,16 +11,6 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" ) -type streamState int - -const ( - streamOpen streamState = iota - streamCloseRead - streamCloseWrite - streamCloseBoth - streamReset -) - // Validate Stream conforms to the go-libp2p-net Stream interface var _ network.Stream = &Stream{} @@ -33,11 +22,6 @@ type Stream struct { stream mux.MuxedStream conn *Conn - state struct { - sync.Mutex - v streamState - } - notifyLk sync.Mutex protocol atomic.Value @@ -74,19 +58,6 @@ func (s *Stream) Read(p []byte) (int, error) { s.conn.swarm.bwc.LogRecvMessage(int64(n)) s.conn.swarm.bwc.LogRecvMessageStream(int64(n), s.Protocol(), s.Conn().RemotePeer()) } - // If we observe an EOF, this stream is now closed for reading. - // If we're already closed for writing, this stream is now fully closed. - if err == io.EOF { - s.state.Lock() - switch s.state.v { - case streamCloseWrite: - s.state.v = streamCloseBoth - s.remove() - case streamOpen: - s.state.v = streamCloseRead - } - s.state.Unlock() - } return n, err } @@ -101,38 +72,39 @@ func (s *Stream) Write(p []byte) (int, error) { return n, err } -// Close closes the stream, indicating this side is finished -// with the stream. +// Close closes the stream, closing both ends and freeing all associated +// resources. func (s *Stream) Close() error { err := s.stream.Close() - - s.state.Lock() - switch s.state.v { - case streamCloseRead: - s.state.v = streamCloseBoth - s.remove() - case streamOpen: - s.state.v = streamCloseWrite - } - s.state.Unlock() + s.remove() return err } -// Reset resets the stream, closing both ends. +// Reset resets the stream, signaling an error on both ends and freeing all +// associated resources. func (s *Stream) Reset() error { err := s.stream.Reset() - s.state.Lock() - switch s.state.v { - case streamOpen, streamCloseRead, streamCloseWrite: - s.state.v = streamReset - s.remove() - } - s.state.Unlock() + s.remove() return err } +// Close closes the stream for writing, flushing all data and sending an EOF. +// This function does not free resources, call Close or Reset when done with the +// stream. +func (s *Stream) CloseWrite() error { + return s.stream.CloseWrite() +} + +// Close closes the stream for reading. This function does not free resources, +// call Close or Reset when done with the stream. +func (s *Stream) CloseRead() error { + return s.stream.CloseRead() +} + func (s *Stream) remove() { - s.conn.removeStream(s) + if !s.conn.removeStream(s) { + return + } // We *must* do this in a goroutine. This can be called during a // an open notification and will block until that notification is done. From 0526dce9adc951072b52e51d99b67261b0139ff3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 2 Sep 2020 14:52:56 -0700 Subject: [PATCH 169/259] fix: handle case where swarm closes before stream When we close a connection, we set the "stream" set to nil to avoid opening new stream. Unfortunately, this meant we wouldn't decrement the reference count on the swarm. --- p2p/net/swarm/swarm_conn.go | 4 +--- p2p/net/swarm/swarm_stream.go | 10 +++++----- p2p/net/swarm/swarm_test.go | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index bd531f273f..946936cfc6 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -87,12 +87,10 @@ func (c *Conn) doClose() { }() } -func (c *Conn) removeStream(s *Stream) bool { +func (c *Conn) removeStream(s *Stream) { c.streams.Lock() - _, has := c.streams.m[s] delete(c.streams.m, s) c.streams.Unlock() - return has } // listens for new streams. diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 74ac62cb05..c09b712a52 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -22,6 +22,8 @@ type Stream struct { stream mux.MuxedStream conn *Conn + closeOnce sync.Once + notifyLk sync.Mutex protocol atomic.Value @@ -76,7 +78,7 @@ func (s *Stream) Write(p []byte) (int, error) { // resources. func (s *Stream) Close() error { err := s.stream.Close() - s.remove() + s.closeOnce.Do(s.remove) return err } @@ -84,7 +86,7 @@ func (s *Stream) Close() error { // associated resources. func (s *Stream) Reset() error { err := s.stream.Reset() - s.remove() + s.closeOnce.Do(s.remove) return err } @@ -102,9 +104,7 @@ func (s *Stream) CloseRead() error { } func (s *Stream) remove() { - if !s.conn.removeStream(s) { - return - } + s.conn.removeStream(s) // We *must* do this in a goroutine. This can be called during a // an open notification and will block until that notification is done. diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 81e1412f11..bd2c844fca 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -440,3 +440,20 @@ func TestNoDial(t *testing.T) { t.Fatal("should have failed with ErrNoConn") } } + +func TestCloseWithOpenStreams(t *testing.T) { + ctx := context.Background() + swarms := makeSwarms(ctx, t, 2) + connectSwarms(t, ctx, swarms) + + s, err := swarms[0].NewStream(ctx, swarms[1].LocalPeer()) + if err != nil { + t.Fatal(err) + } + defer s.Close() + // close swarm before stream. + err = swarms[0].Close() + if err != nil { + t.Fatal(err) + } +} From 1b9128f18e4e6256464601504efe0c3c18a03b99 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 17 Dec 2020 16:03:06 +0700 Subject: [PATCH 170/259] use a context for OpenStream and NewStream --- p2p/net/swarm/dial_test.go | 4 ++-- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_conn.go | 5 +++-- p2p/net/swarm/swarm_notif_test.go | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 14b2b93396..6462d7e11f 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -49,7 +49,7 @@ func TestBasicDialPeer(t *testing.T) { t.Fatal(err) } - s, err := c.NewStream() + s, err := c.NewStream(ctx) if err != nil { t.Fatal(err) } @@ -74,7 +74,7 @@ func TestDialWithNoListeners(t *testing.T) { t.Fatal(err) } - s, err := c.NewStream() + s, err := c.NewStream(ctx) if err != nil { t.Fatal(err) } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index db3f4f1cf3..8f0e18e57d 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -351,7 +351,7 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (network.Stream, error return nil, err } } - s, err := c.NewStream() + s, err := c.NewStream(ctx) if err != nil { if c.conn.IsClosed() { continue diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 946936cfc6..c6af3da93d 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -1,6 +1,7 @@ package swarm import ( + "context" "errors" "fmt" "sync" @@ -175,8 +176,8 @@ func (c *Conn) Stat() network.Stat { } // NewStream returns a new Stream from this connection -func (c *Conn) NewStream() (network.Stream, error) { - ts, err := c.conn.OpenStream() +func (c *Conn) NewStream(ctx context.Context) (network.Stream, error) { + ts, err := c.conn.OpenStream(ctx) if err != nil { return nil, err diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 60cf461f5f..8537363a19 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -145,7 +145,7 @@ func TestNotifications(t *testing.T) { for _, c := range s.Conns() { _, n2, _ := complement(c) - st1, err := c.NewStream() + st1, err := c.NewStream(context.Background()) if err != nil { t.Error(err) } else { From 0432da356d815b7301eb026d5b5cae3913d079be Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Fri, 29 Jan 2021 00:23:57 +0530 Subject: [PATCH 171/259] Enable QUIC in Test Swarm (#235) * add quic to swarm testing * update go --- p2p/net/swarm/testing/testing.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 83d9fcd12e..eb1e1e42b6 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -12,6 +12,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/sec/insecure" + quic "github.com/libp2p/go-libp2p-quic-transport" csms "github.com/libp2p/go-conn-security-multistream" pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" @@ -113,14 +114,26 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { tcpTransport := tcp.NewTCPTransport(upgrader) tcpTransport.DisableReuseport = cfg.disableReuseport + quicTransport, err := quic.NewTransport(p.PrivKey, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := s.AddTransport(tcpTransport); err != nil { t.Fatal(err) } + if err := s.AddTransport(quicTransport); err != nil { + t.Fatal(err) + } + if !cfg.dialOnly { if err := s.Listen(p.Addr); err != nil { t.Fatal(err) } + if err := s.Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/quic")); err != nil { + t.Fatal(err) + } s.Peerstore().AddAddrs(p.ID, s.ListenAddresses(), peerstore.PermanentAddrTTL) } From 96218e227b8d3db2e9e9ee4cfbdfa50bb2b0f70e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 8 Feb 2021 17:06:32 +0800 Subject: [PATCH 172/259] prevent dialing addresses that we're listening on It's impossible to run two nodes on the same IP:port, so we know for sure that any dial to an address that we're listening on will fail (as the peer IDs won't match). In practice, this will be most useful for preventing dials to localhost for nodes that are listening on the default port. --- p2p/net/swarm/swarm_dial.go | 2 +- p2p/net/swarm/swarm_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 4dbb1f2dd1..30a9ab06cb 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -440,7 +440,7 @@ func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) []ma.Mul for _, addr := range lisAddrs { protos := addr.Protocols() // we're only sure about filtering out /ip4 and /ip6 addresses, so far - if len(protos) == 2 && (protos[0].Code == ma.P_IP4 || protos[0].Code == ma.P_IP6) { + if protos[0].Code == ma.P_IP4 || protos[0].Code == ma.P_IP6 { ourAddrs = append(ourAddrs, addr) } } diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index bd2c844fca..9b1e9c42d7 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -3,8 +3,10 @@ package swarm_test import ( "bytes" "context" + "errors" "fmt" "io" + "strings" "sync" "testing" "time" @@ -20,6 +22,7 @@ import ( logging "github.com/ipfs/go-log" ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" "github.com/stretchr/testify/require" ) @@ -457,3 +460,31 @@ func TestCloseWithOpenStreams(t *testing.T) { t.Fatal(err) } } + +func TestPreventDialListenAddr(t *testing.T) { + s := GenSwarm(t, context.Background(), OptDialOnly) + if err := s.Listen(ma.StringCast("/ip4/0.0.0.0/udp/0/quic")); err != nil { + t.Fatal(err) + } + addrs, err := s.InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + var addr ma.Multiaddr + for _, a := range addrs { + _, s, err := manet.DialArgs(a) + if err != nil { + t.Fatal(err) + } + if strings.Split(s, ":")[0] == "127.0.0.1" { + addr = a + break + } + } + remote := peer.ID("foobar") + s.Peerstore().AddAddr(remote, addr, time.Hour) + _, err = s.DialPeer(context.Background(), remote) + if !errors.Is(err, ErrNoGoodAddresses) { + t.Fatal("expected dial to fail: %w", err) + } +} From 6e0d187aff84be00bf3306141d2ab09b94aaad3d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 9 Feb 2021 11:12:34 +0800 Subject: [PATCH 173/259] only listen on localhost in tests --- p2p/net/swarm/dial_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 6462d7e11f..9fc5df4189 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -153,7 +153,7 @@ func TestSimultDials(t *testing.T) { func newSilentPeer(t *testing.T) (peer.ID, ma.Multiaddr, net.Listener) { dst := testutil.RandPeerIDFatal(t) - lst, err := net.Listen("tcp4", ":0") + lst, err := net.Listen("tcp4", "localhost:0") if err != nil { t.Fatal(err) } From 8e9ae255c1a9aef4e0a613d0a2e7080ce7562f3d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 12 Feb 2021 16:23:10 +0800 Subject: [PATCH 174/259] avoid assigning a function to a variable --- p2p/net/swarm/swarm_dial.go | 81 ++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 30a9ab06cb..65f123a08f 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -321,6 +321,45 @@ func (s *Swarm) canDial(addr ma.Multiaddr) bool { return t != nil && t.CanDial(addr) } +// ranks addresses in descending order of preference for dialing +// Private UDP > Public UDP > Private TCP > Public TCP > UDP Relay server > TCP Relay server +func (s *Swarm) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { + var localUdpAddrs []ma.Multiaddr // private udp + var relayUdpAddrs []ma.Multiaddr // relay udp + var othersUdp []ma.Multiaddr // public udp + + var localFdAddrs []ma.Multiaddr // private fd consuming + var relayFdAddrs []ma.Multiaddr // relay fd consuming + var othersFd []ma.Multiaddr // public fd consuming + + for _, a := range addrs { + if _, err := a.ValueForProtocol(ma.P_CIRCUIT); err == nil { + if s.IsFdConsumingAddr(a) { + relayFdAddrs = append(relayFdAddrs, a) + continue + } + relayUdpAddrs = append(relayUdpAddrs, a) + } else if manet.IsPrivateAddr(a) { + if s.IsFdConsumingAddr(a) { + localFdAddrs = append(localFdAddrs, a) + continue + } + localUdpAddrs = append(localUdpAddrs, a) + } else { + if s.IsFdConsumingAddr(a) { + othersFd = append(othersFd, a) + continue + } + othersUdp = append(othersUdp, a) + } + } + + relays := append(relayUdpAddrs, relayFdAddrs...) + fds := append(localFdAddrs, othersFd...) + + return append(append(append(localUdpAddrs, othersUdp...), fds...), relays...) +} + // dial is the actual swarm's dial logic, gated by Dial. func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) @@ -360,47 +399,7 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return nil, ErrDialBackoff } - // ranks addresses in descending order of preference for dialing - // Private UDP > Public UDP > Private TCP > Public TCP > UDP Relay server > TCP Relay server - rankAddrsFnc := func(addrs []ma.Multiaddr) []ma.Multiaddr { - var localUdpAddrs []ma.Multiaddr // private udp - var relayUdpAddrs []ma.Multiaddr // relay udp - var othersUdp []ma.Multiaddr // public udp - - var localFdAddrs []ma.Multiaddr // private fd consuming - var relayFdAddrs []ma.Multiaddr // relay fd consuming - var othersFd []ma.Multiaddr // public fd consuming - - for _, a := range addrs { - if _, err := a.ValueForProtocol(ma.P_CIRCUIT); err == nil { - if s.IsFdConsumingAddr(a) { - relayFdAddrs = append(relayFdAddrs, a) - continue - } - relayUdpAddrs = append(relayUdpAddrs, a) - } else if manet.IsPrivateAddr(a) { - if s.IsFdConsumingAddr(a) { - localFdAddrs = append(localFdAddrs, a) - continue - } - localUdpAddrs = append(localUdpAddrs, a) - } else { - if s.IsFdConsumingAddr(a) { - othersFd = append(othersFd, a) - continue - } - othersUdp = append(othersUdp, a) - } - } - - relays := append(relayUdpAddrs, relayFdAddrs...) - fds := append(localFdAddrs, othersFd...) - - return append(append(append(localUdpAddrs, othersUdp...), fds...), relays...) - } - - connC, dialErr := s.dialAddrs(ctx, p, rankAddrsFnc(goodAddrs)) - + connC, dialErr := s.dialAddrs(ctx, p, s.rankAddrs(goodAddrs)) if dialErr != nil { logdial["error"] = dialErr.Cause.Error() switch dialErr.Cause { From 0528e4482231cdd6e9bd0f072510a34b68e24ac1 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 4 Feb 2021 00:18:04 +0200 Subject: [PATCH 175/259] treat transient conns as opt-in for opening new streams --- p2p/net/swarm/swarm.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 8f0e18e57d..b36946b38a 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -351,6 +351,13 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (network.Stream, error return nil, err } } + + if c.Stat().Transient { + if !network.GetUseTransient(ctx) { + return nil, network.ErrTransientConn + } + } + s, err := c.NewStream(ctx) if err != nil { if c.conn.IsClosed() { From 9d3ae86fb3b2b4c76a99ecabc8f777cced57ca5e Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 4 Feb 2021 13:34:29 +0200 Subject: [PATCH 176/259] initialize conn stat with underlying conn stat if available --- p2p/net/swarm/swarm.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index b36946b38a..d297f355cf 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -200,8 +200,16 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, } } + // create the Stat object, initializing with the underlying connection Stat if available + var stat network.Stat + cs, ok := tc.(network.ConnStat) + if ok { + stat = cs.Stat() + } + stat.Direction = dir + stat.Opened = time.Now() + // Wrap and register the connection. - stat := network.Stat{Direction: dir, Opened: time.Now()} c := &Conn{ conn: tc, swarm: s, From 8fa595c3c6486b2a572a590226c69602339a9d37 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 4 Feb 2021 13:42:17 +0200 Subject: [PATCH 177/259] more idiomatic code --- p2p/net/swarm/swarm.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index d297f355cf..b75e24b351 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -202,8 +202,7 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, // create the Stat object, initializing with the underlying connection Stat if available var stat network.Stat - cs, ok := tc.(network.ConnStat) - if ok { + if cs, ok := tc.(network.ConnStat); ok { stat = cs.Stat() } stat.Direction = dir From 6b0e958728ed9516ce1097ba19f0c9af24e6ccd1 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 17 Feb 2021 11:47:47 +0200 Subject: [PATCH 178/259] fix GetTransient usage for interface change --- p2p/net/swarm/swarm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index b75e24b351..272662a059 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -360,7 +360,7 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (network.Stream, error } if c.Stat().Transient { - if !network.GetUseTransient(ctx) { + if useTransient, _ := network.GetUseTransient(ctx); !useTransient { return nil, network.ErrTransientConn } } From d97c24c74bf01d91fbfb783f3bb00f7a0c20cf46 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Thu, 18 Feb 2021 14:34:00 +0530 Subject: [PATCH 179/259] Support for Hole punching (#233) * support for forced direct connections. --- p2p/net/swarm/dial_sync.go | 6 ++--- p2p/net/swarm/swarm.go | 22 +++++++++++---- p2p/net/swarm/swarm_dial.go | 54 +++++++++++++++++++++++++++---------- 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 4c1230f5fc..edb6c89821 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -96,13 +96,13 @@ func (ad *activeDial) start(ctx context.Context) { ad.cancel() } -func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { +func (ds *DialSync) getActiveDial(ctx context.Context, p peer.ID) *activeDial { ds.dialsLk.Lock() defer ds.dialsLk.Unlock() actd, ok := ds.dials[p] if !ok { - adctx, cancel := context.WithCancel(context.Background()) + adctx, cancel := context.WithCancel(ctx) actd = &activeDial{ id: p, cancel: cancel, @@ -123,7 +123,7 @@ func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { // DialLock initiates a dial to the given peer if there are none in progress // then waits for the dial to that peer to complete. func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { - return ds.getActiveDial(p).wait(ctx) + return ds.getActiveDial(ctx, p).wait(ctx) } // CancelDial cancels all in-progress dials to the given peer. diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 272662a059..68f7603fd7 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -341,6 +341,7 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (network.Stream, error // a non-closed connection. dials := 0 for { + // will prefer direct connections over relayed connections for opening streams c := s.bestConnToPeer(p) if c == nil { if nodial, _ := network.GetNoDial(ctx); nodial { @@ -392,9 +393,10 @@ func (s *Swarm) ConnsToPeer(p peer.ID) []network.Conn { // bestConnToPeer returns the best connection to peer. func (s *Swarm) bestConnToPeer(p peer.ID) *Conn { - // Selects the best connection we have to the peer. - // TODO: Prefer some transports over others. Currently, we just select - // the newest non-closed connection with the most streams. + + // TODO: Prefer some transports over others. + // For now, prefers direct connections over Relayed connections. + // For tie-breaking, select the newest non-closed connection with the most streams. s.conns.RLock() defer s.conns.RUnlock() @@ -409,15 +411,25 @@ func (s *Swarm) bestConnToPeer(p peer.ID) *Conn { cLen := len(c.streams.m) c.streams.Unlock() - if cLen >= bestLen { + // We will never prefer a Relayed connection over a direct connection. + if isDirectConn(best) && !isDirectConn(c) { + continue + } + + // 1. Always prefer a direct connection over a relayed connection. + // 2. If both conns are direct or relayed, pick the one with as many or more streams. + if (!isDirectConn(best) && isDirectConn(c)) || (cLen >= bestLen) { best = c bestLen = cLen } - } return best } +func isDirectConn(c *Conn) bool { + return c != nil && !c.conn.Transport().Proxy() +} + // Connectedness returns our "connectedness" state with the given peer. // // To check if we have an open connection, use `s.Connectedness(p) == diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 65f123a08f..052da67fe5 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -251,9 +251,14 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { defer log.EventBegin(ctx, "swarmDialAttemptSync", p).Done() - // check if we already have an open connection first conn := s.bestConnToPeer(p) - if conn != nil { + forceDirect, _ := network.GetForceDirectDial(ctx) + if forceDirect { + if isDirectConn(conn) { + return conn, nil + } + } else if conn != nil { + // check if we already have an open connection first return conn, nil } @@ -287,8 +292,13 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { // Short circuit. // By the time we take the dial lock, we may already *have* a connection // to the peer. + forceDirect, _ := network.GetForceDirectDial(ctx) c := s.bestConnToPeer(p) - if c != nil { + if forceDirect { + if isDirectConn(c) { + return c, nil + } + } else if c != nil { return c, nil } @@ -301,12 +311,17 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { conn, err := s.dial(ctx, p) if err != nil { conn = s.bestConnToPeer(p) - if conn != nil { + if forceDirect { + if isDirectConn(conn) { + log.Debugf("ignoring dial error because we already have a direct connection: %s", err) + return conn, nil + } + } else if conn != nil { // Hm? What error? // Could have canceled the dial because we received a // connection or some other random reason. // Just ignore the error and return the connection. - log.Debugf("ignoring dial error because we have a connection: %s", err) + log.Debugf("ignoring dial error because we already have a connection: %s", err) return conn, nil } @@ -321,6 +336,11 @@ func (s *Swarm) canDial(addr ma.Multiaddr) bool { return t != nil && t.CanDial(addr) } +func (s *Swarm) nonProxyAddr(addr ma.Multiaddr) bool { + t := s.TransportForDialing(addr) + return !t.Proxy() +} + // ranks addresses in descending order of preference for dialing // Private UDP > Public UDP > Private TCP > Public TCP > UDP Relay server > TCP Relay server func (s *Swarm) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { @@ -362,6 +382,7 @@ func (s *Swarm) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { // dial is the actual swarm's dial logic, gated by Dial. func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { + forceDirect, _ := network.GetForceDirectDial(ctx) var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) if p == s.local { log.Event(ctx, "swarmDialDoDialSelf", logdial) @@ -383,20 +404,25 @@ func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { return nil, &DialError{Peer: p, Cause: ErrNoAddresses} } goodAddrs := s.filterKnownUndialables(p, peerAddrs) + if forceDirect { + goodAddrs = addrutil.FilterAddrs(goodAddrs, s.nonProxyAddr) + } if len(goodAddrs) == 0 { return nil, &DialError{Peer: p, Cause: ErrNoGoodAddresses} } - /////// Check backoff andnRank addresses - var nonBackoff bool - for _, a := range goodAddrs { - // skip addresses in back-off - if !s.backf.Backoff(p, a) { - nonBackoff = true + if !forceDirect { + /////// Check backoff andnRank addresses + var nonBackoff bool + for _, a := range goodAddrs { + // skip addresses in back-off + if !s.backf.Backoff(p, a) { + nonBackoff = true + } + } + if !nonBackoff { + return nil, ErrDialBackoff } - } - if !nonBackoff { - return nil, ErrDialBackoff } connC, dialErr := s.dialAddrs(ctx, p, s.rankAddrs(goodAddrs)) From 953af40e2e67c0fa7f96fbfbf9837c7e7686cc85 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Fri, 19 Feb 2021 17:47:39 +0530 Subject: [PATCH 180/259] fix swarm transient conn (#241) --- p2p/net/swarm/swarm.go | 6 ------ p2p/net/swarm/swarm_conn.go | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 68f7603fd7..25f4590178 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -360,12 +360,6 @@ func (s *Swarm) NewStream(ctx context.Context, p peer.ID) (network.Stream, error } } - if c.Stat().Transient { - if useTransient, _ := network.GetUseTransient(ctx); !useTransient { - return nil, network.ErrTransientConn - } - } - s, err := c.NewStream(ctx) if err != nil { if c.conn.IsClosed() { diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index c6af3da93d..74c0b522ba 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -177,6 +177,12 @@ func (c *Conn) Stat() network.Stat { // NewStream returns a new Stream from this connection func (c *Conn) NewStream(ctx context.Context) (network.Stream, error) { + if c.Stat().Transient { + if useTransient, _ := network.GetUseTransient(ctx); !useTransient { + return nil, network.ErrTransientConn + } + } + ts, err := c.conn.OpenStream(ctx) if err != nil { From 757fb56c538a8f6f8ea7246487467217fa455e17 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 19 Mar 2021 12:23:44 -0700 Subject: [PATCH 181/259] fix: prefer non-transient connections Given two relay connections, prefer the one that's non-transient. Otherwise, a transient connection could prevent us from opening streams even if we have a second non-transient connection through a better relay. --- p2p/net/swarm/swarm.go | 47 +++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 25f4590178..7a241b4b69 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -385,6 +385,38 @@ func (s *Swarm) ConnsToPeer(p peer.ID) []network.Conn { return output } +func isBetterConn(a, b *Conn) bool { + // If one is transient and not the other, prefer the non-transient connection. + aTransient := a.Stat().Transient + bTransient := b.Stat().Transient + if aTransient != bTransient { + return !aTransient + } + + // If one is direct and not the other, prefer the direct connection. + aDirect := isDirectConn(a) + bDirect := isDirectConn(b) + if aDirect != bDirect { + return aDirect + } + + // Otherwise, prefer the connection with more open streams. + a.streams.Lock() + aLen := len(a.streams.m) + a.streams.Unlock() + + b.streams.Lock() + bLen := len(b.streams.m) + b.streams.Unlock() + + if aLen != bLen { + return aLen > bLen + } + + // finally, pick the last connection. + return true +} + // bestConnToPeer returns the best connection to peer. func (s *Swarm) bestConnToPeer(p peer.ID) *Conn { @@ -395,26 +427,13 @@ func (s *Swarm) bestConnToPeer(p peer.ID) *Conn { defer s.conns.RUnlock() var best *Conn - bestLen := 0 for _, c := range s.conns.m[p] { if c.conn.IsClosed() { // We *will* garbage collect this soon anyways. continue } - c.streams.Lock() - cLen := len(c.streams.m) - c.streams.Unlock() - - // We will never prefer a Relayed connection over a direct connection. - if isDirectConn(best) && !isDirectConn(c) { - continue - } - - // 1. Always prefer a direct connection over a relayed connection. - // 2. If both conns are direct or relayed, pick the one with as many or more streams. - if (!isDirectConn(best) && isDirectConn(c)) || (cLen >= bestLen) { + if best == nil || isBetterConn(c, best) { best = c - bestLen = cLen } } return best From 1e011b1e8460a329a28e2c695942439877d64387 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 19 Mar 2021 12:27:38 -0700 Subject: [PATCH 182/259] fix: DRY direct connect logic --- p2p/net/swarm/swarm.go | 11 +++++++++++ p2p/net/swarm/swarm_dial.go | 29 +++++++---------------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 7a241b4b69..3b3cf83228 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -439,6 +439,17 @@ func (s *Swarm) bestConnToPeer(p peer.ID) *Conn { return best } +func (s *Swarm) bestAcceptableConnToPeer(ctx context.Context, p peer.ID) *Conn { + conn := s.bestConnToPeer(p) + if conn != nil { + forceDirect, _ := network.GetForceDirectDial(ctx) + if !forceDirect || isDirectConn(conn) { + return conn + } + } + return nil +} + func isDirectConn(c *Conn) bool { return c != nil && !c.conn.Transport().Proxy() } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 052da67fe5..ccf33c2e4c 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -251,14 +251,9 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { defer log.EventBegin(ctx, "swarmDialAttemptSync", p).Done() - conn := s.bestConnToPeer(p) - forceDirect, _ := network.GetForceDirectDial(ctx) - if forceDirect { - if isDirectConn(conn) { - return conn, nil - } - } else if conn != nil { - // check if we already have an open connection first + // check if we already have an open (usable) connection first + conn := s.bestAcceptableConnToPeer(ctx, p) + if conn != nil { return conn, nil } @@ -292,13 +287,8 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { // Short circuit. // By the time we take the dial lock, we may already *have* a connection // to the peer. - forceDirect, _ := network.GetForceDirectDial(ctx) - c := s.bestConnToPeer(p) - if forceDirect { - if isDirectConn(c) { - return c, nil - } - } else if c != nil { + c := s.bestAcceptableConnToPeer(ctx, p) + if c != nil { return c, nil } @@ -310,13 +300,8 @@ func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { conn, err := s.dial(ctx, p) if err != nil { - conn = s.bestConnToPeer(p) - if forceDirect { - if isDirectConn(conn) { - log.Debugf("ignoring dial error because we already have a direct connection: %s", err) - return conn, nil - } - } else if conn != nil { + conn := s.bestAcceptableConnToPeer(ctx, p) + if conn != nil { // Hm? What error? // Could have canceled the dial because we received a // connection or some other random reason. From 1b4be471ed9ff90b5a2e0efa29638c1bd475c1c9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 19 Mar 2021 12:36:04 -0700 Subject: [PATCH 183/259] fix: do not use the request context when dialing Otherwise, canceling one dial request will cancel all "joined" dial requests. --- p2p/net/swarm/dial_sync.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index edb6c89821..50f3a69821 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -96,13 +96,17 @@ func (ad *activeDial) start(ctx context.Context) { ad.cancel() } -func (ds *DialSync) getActiveDial(ctx context.Context, p peer.ID) *activeDial { +func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { ds.dialsLk.Lock() defer ds.dialsLk.Unlock() actd, ok := ds.dials[p] if !ok { - adctx, cancel := context.WithCancel(ctx) + // This code intentionally uses the background context. Otherwise, if the first call + // to Dial is canceled, subsequent dial calls will also be canceled. + // XXX: this also breaks direct connection logic. We will need to pipe the + // information through some other way. + adctx, cancel := context.WithCancel(context.Background()) actd = &activeDial{ id: p, cancel: cancel, @@ -123,7 +127,7 @@ func (ds *DialSync) getActiveDial(ctx context.Context, p peer.ID) *activeDial { // DialLock initiates a dial to the given peer if there are none in progress // then waits for the dial to that peer to complete. func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { - return ds.getActiveDial(ctx, p).wait(ctx) + return ds.getActiveDial(p).wait(ctx) } // CancelDial cancels all in-progress dials to the given peer. From 4ace0858ff195370b46cf52a84fad39810873885 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 28 Aug 2020 11:13:44 -0700 Subject: [PATCH 184/259] feat: close transports that implement io.Closer This way, transports with shared resources (e.g., reused sockets) can clean them up. fixes https://github.com/libp2p/go-libp2p/issues/999 --- p2p/net/swarm/swarm.go | 22 +++++++++++++++++++ p2p/net/swarm/swarm_listen.go | 12 ++++++++++- p2p/net/swarm/swarm_transport.go | 13 +++++++++-- p2p/net/swarm/transport_test.go | 37 ++++++++++++++++++++++++++++++-- 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 25f4590178..7dd957f230 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "strings" "sync" "sync/atomic" @@ -176,6 +177,27 @@ func (s *Swarm) teardown() error { // Wait for everything to finish. s.refs.Wait() + // Now close out any transports (if necessary). Do this after closing + // all connections/listeners. + s.transports.Lock() + transports := s.transports.m + s.transports.m = nil + s.transports.Unlock() + + var wg sync.WaitGroup + for _, t := range transports { + if closer, ok := t.(io.Closer); ok { + wg.Add(1) + go func(c io.Closer) { + defer wg.Done() + if err := closer.Close(); err != nil { + log.Errorf("error when closing down transport %T: %s", c, err) + } + }(closer) + } + } + wg.Wait() + return nil } diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index ab5c42a345..5bd1015d99 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -40,7 +40,17 @@ func (s *Swarm) Listen(addrs ...ma.Multiaddr) error { func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { tpt := s.TransportForListening(a) if tpt == nil { - return ErrNoTransport + // TransportForListening will return nil if either: + // 1. No transport has been registered. + // 2. We're closed (so we've nulled out the transport map. + // + // Distinguish between these two cases to avoid confusing users. + select { + case <-s.proc.Closing(): + return ErrSwarmClosed + default: + return ErrNoTransport + } } list, err := tpt.Listen(a) diff --git a/p2p/net/swarm/swarm_transport.go b/p2p/net/swarm/swarm_transport.go index 307bfe641f..21728ac3b5 100644 --- a/p2p/net/swarm/swarm_transport.go +++ b/p2p/net/swarm/swarm_transport.go @@ -20,7 +20,10 @@ func (s *Swarm) TransportForDialing(a ma.Multiaddr) transport.Transport { s.transports.RLock() defer s.transports.RUnlock() if len(s.transports.m) == 0 { - log.Error("you have no transports configured") + // make sure we're not just shutting down. + if s.transports.m != nil { + log.Error("you have no transports configured") + } return nil } @@ -48,7 +51,10 @@ func (s *Swarm) TransportForListening(a ma.Multiaddr) transport.Transport { s.transports.RLock() defer s.transports.RUnlock() if len(s.transports.m) == 0 { - log.Error("you have no transports configured") + // make sure we're not just shutting down. + if s.transports.m != nil { + log.Error("you have no transports configured") + } return nil } @@ -77,6 +83,9 @@ func (s *Swarm) AddTransport(t transport.Transport) error { s.transports.Lock() defer s.transports.Unlock() + if s.transports.m == nil { + return ErrSwarmClosed + } var registered []string for _, p := range protocols { if _, ok := s.transports.m[p]; ok { diff --git a/p2p/net/swarm/transport_test.go b/p2p/net/swarm/transport_test.go index f6090a6ef8..82225840b9 100644 --- a/p2p/net/swarm/transport_test.go +++ b/p2p/net/swarm/transport_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + swarm "github.com/libp2p/go-libp2p-swarm" swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-libp2p-core/peer" @@ -14,6 +15,7 @@ import ( type dummyTransport struct { protocols []int proxy bool + closed bool } func (dt *dummyTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.CapableConn, error) { @@ -35,13 +37,44 @@ func (dt *dummyTransport) Proxy() bool { func (dt *dummyTransport) Protocols() []int { return dt.protocols } +func (dt *dummyTransport) Close() error { + dt.closed = true + return nil +} func TestUselessTransport(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - swarm := swarmt.GenSwarm(t, ctx) - err := swarm.AddTransport(new(dummyTransport)) + s := swarmt.GenSwarm(t, ctx) + err := s.AddTransport(new(dummyTransport)) if err == nil { t.Fatal("adding a transport that supports no protocols should have failed") } } + +func TestTransportClose(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := swarmt.GenSwarm(t, ctx) + tpt := &dummyTransport{protocols: []int{1}} + if err := s.AddTransport(tpt); err != nil { + t.Fatal(err) + } + _ = s.Close() + if !tpt.closed { + t.Fatal("expected transport to be closed") + } + +} + +func TestTransportAfterClose(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := swarmt.GenSwarm(t, ctx) + s.Close() + + tpt := &dummyTransport{protocols: []int{1}} + if err := s.AddTransport(tpt); err != swarm.ErrSwarmClosed { + t.Fatal("expected swarm closed error, got: ", err) + } +} From 32635431a660a209abce5183f639a97218031464 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 29 Mar 2021 11:21:12 -0700 Subject: [PATCH 185/259] fix: use 64bit stream/conn IDs Given 1k requests per second (assuming one stream per request), we can easily loop around the stream ID after less than 2 months. 32bits is plenty (usually) for connection-scoped stream IDs because individual connections don't usually last that long, but isn't enough for a _global_ stream ID. Given that there's no reason for these to be 32bit IDs, let's just make them 64bits. --- p2p/net/swarm/swarm.go | 8 ++++---- p2p/net/swarm/swarm_conn.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 25f4590178..91e8bd4b1c 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -47,6 +47,9 @@ var ErrDialTimeout = errors.New("dial timed out") // communication. The Chan sends/receives Messages, which note the // destination or source Peer. type Swarm struct { + nextConnID uint64 // guarded by atomic + nextStreamID uint64 // guarded by atomic + // Close refcount. This allows us to fully wait for the swarm to be torn // down before continuing. refs sync.WaitGroup @@ -54,9 +57,6 @@ type Swarm struct { local peer.ID peers peerstore.Peerstore - nextConnID uint32 // guarded by atomic - nextStreamID uint32 // guarded by atomic - conns struct { sync.RWMutex m map[peer.ID][]*Conn @@ -213,7 +213,7 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, conn: tc, swarm: s, stat: stat, - id: atomic.AddUint32(&s.nextConnID, 1), + id: atomic.AddUint64(&s.nextConnID, 1), } // we ONLY check upgraded connections here so we can send them a Disconnect message. diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 74c0b522ba..3d5f8f3735 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -25,7 +25,7 @@ var ErrConnClosed = errors.New("connection closed") // Conn is the connection type used by swarm. In general, you won't use this // type directly. type Conn struct { - id uint32 + id uint64 conn transport.CapableConn swarm *Swarm @@ -209,7 +209,7 @@ func (c *Conn) addStream(ts mux.MuxedStream, dir network.Direction) (*Stream, er stream: ts, conn: c, stat: stat, - id: atomic.AddUint32(&c.swarm.nextStreamID, 1), + id: atomic.AddUint64(&c.swarm.nextStreamID, 1), } c.streams.m[s] = struct{}{} diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index c09b712a52..a5f0738ad6 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -17,7 +17,7 @@ var _ network.Stream = &Stream{} // Stream is the stream type used by swarm. In general, you won't use this type // directly. type Stream struct { - id uint32 + id uint64 stream mux.MuxedStream conn *Conn From 28671c3153f1d18edc4b23aad780540f88a83dc8 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 30 Mar 2021 17:14:50 +0300 Subject: [PATCH 186/259] implement dial worker for synchronizing simultaneous dials --- p2p/net/swarm/dial_sync.go | 114 ++++----- p2p/net/swarm/swarm.go | 8 +- p2p/net/swarm/swarm_dial.go | 493 ++++++++++++++++++++++-------------- 3 files changed, 349 insertions(+), 266 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 50f3a69821..2efdf067b6 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -5,6 +5,7 @@ import ( "errors" "sync" + "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" ) @@ -12,88 +13,74 @@ import ( var errDialCanceled = errors.New("dial was aborted internally, likely due to https://git.io/Je2wW") // DialFunc is the type of function expected by DialSync. -type DialFunc func(context.Context, peer.ID) (*Conn, error) +type DialWorkerFunc func(context.Context, peer.ID, <-chan dialRequest) // NewDialSync constructs a new DialSync -func NewDialSync(dfn DialFunc) *DialSync { +func NewDialSync(worker DialWorkerFunc) *DialSync { return &DialSync{ - dials: make(map[peer.ID]*activeDial), - dialFunc: dfn, + dials: make(map[peer.ID]*activeDial), + dialWorker: worker, } } // DialSync is a dial synchronization helper that ensures that at most one dial // to any given peer is active at any given time. type DialSync struct { - dials map[peer.ID]*activeDial - dialsLk sync.Mutex - dialFunc DialFunc + dials map[peer.ID]*activeDial + dialsLk sync.Mutex + dialWorker DialWorkerFunc } type activeDial struct { - id peer.ID - refCnt int - refCntLk sync.Mutex - cancel func() + id peer.ID + refCnt int - err error - conn *Conn - waitch chan struct{} + ctx context.Context + cancel func() - ds *DialSync -} + reqch chan dialRequest -func (ad *activeDial) wait(ctx context.Context) (*Conn, error) { - defer ad.decref() - select { - case <-ad.waitch: - return ad.conn, ad.err - case <-ctx.Done(): - return nil, ctx.Err() - } + ds *DialSync } func (ad *activeDial) incref() { - ad.refCntLk.Lock() - defer ad.refCntLk.Unlock() ad.refCnt++ } func (ad *activeDial) decref() { - ad.refCntLk.Lock() + ad.ds.dialsLk.Lock() ad.refCnt-- - maybeZero := (ad.refCnt <= 0) - ad.refCntLk.Unlock() - - // make sure to always take locks in correct order. - if maybeZero { - ad.ds.dialsLk.Lock() - ad.refCntLk.Lock() - // check again after lock swap drop to make sure nobody else called incref - // in between locks - if ad.refCnt <= 0 { - ad.cancel() - delete(ad.ds.dials, ad.id) - } - ad.refCntLk.Unlock() - ad.ds.dialsLk.Unlock() + if ad.refCnt == 0 { + ad.cancel() + close(ad.reqch) + delete(ad.ds.dials, ad.id) } + ad.ds.dialsLk.Unlock() } -func (ad *activeDial) start(ctx context.Context) { - ad.conn, ad.err = ad.ds.dialFunc(ctx, ad.id) - - // This isn't the user's context so we should fix the error. - switch ad.err { - case context.Canceled: - // The dial was canceled with `CancelDial`. - ad.err = errDialCanceled - case context.DeadlineExceeded: - // We hit an internal timeout, not a context timeout. - ad.err = ErrDialTimeout +func (ad *activeDial) dial(ctx context.Context, p peer.ID) (*Conn, error) { + dialCtx := ad.ctx + + if forceDirect, reason := network.GetForceDirectDial(ctx); forceDirect { + dialCtx = network.WithForceDirectDial(dialCtx, reason) + } + if simConnect, reason := network.GetSimultaneousConnect(ctx); simConnect { + dialCtx = network.WithSimultaneousConnect(dialCtx, reason) + } + + resch := make(chan dialResponse, 1) + select { + case ad.reqch <- dialRequest{ctx: dialCtx, resch: resch}: + case <-ctx.Done(): + return nil, ctx.Err() + } + + select { + case res := <-resch: + return res.conn, res.err + case <-ctx.Done(): + return nil, ctx.Err() } - close(ad.waitch) - ad.cancel() } func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { @@ -109,13 +96,14 @@ func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { adctx, cancel := context.WithCancel(context.Background()) actd = &activeDial{ id: p, + ctx: adctx, cancel: cancel, - waitch: make(chan struct{}), + reqch: make(chan dialRequest), ds: ds, } ds.dials[p] = actd - go actd.start(adctx) + go ds.dialWorker(adctx, p, actd.reqch) } // increase ref count before dropping dialsLk @@ -127,14 +115,8 @@ func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { // DialLock initiates a dial to the given peer if there are none in progress // then waits for the dial to that peer to complete. func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { - return ds.getActiveDial(p).wait(ctx) -} + ad := ds.getActiveDial(p) + defer ad.decref() -// CancelDial cancels all in-progress dials to the given peer. -func (ds *DialSync) CancelDial(p peer.ID) { - ds.dialsLk.Lock() - defer ds.dialsLk.Unlock() - if ad, ok := ds.dials[p]; ok { - ad.cancel() - } + return ad.dial(ctx, p) } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 3b3cf83228..c57c563c25 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -121,7 +121,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc } } - s.dsync = NewDialSync(s.doDial) + s.dsync = NewDialSync(s.dialWorker) s.limiter = newDialLimiter(s.dialAddr, s.IsFdConsumingAddr) s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) @@ -259,12 +259,6 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, c.notifyLk.Lock() s.conns.Unlock() - // We have a connection now. Cancel all other in-progress dials. - // This should be fast, no reason to wait till later. - if dir == network.DirOutbound { - s.dsync.CancelDial(p) - } - s.notifyAll(func(f network.Notifiee) { f.Connected(s, c) }) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index ccf33c2e4c..cc739791dc 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -14,7 +14,6 @@ import ( addrutil "github.com/libp2p/go-addr-util" lgbl "github.com/libp2p/go-libp2p-loggables" - logging "github.com/ipfs/go-log" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) @@ -58,6 +57,12 @@ var ( ErrGaterDisallowedConnection = errors.New("gater disallows connection to peer") ) +var ( + DelayDialPrivateAddr = 5 * time.Millisecond + DelayDialPublicAddr = 50 * time.Millisecond + DelayDialRelayAddr = 100 * time.Millisecond +) + // DialAttempts governs how many times a goroutine will try to dial a given peer. // Note: this is down to one, as we have _too many dials_ atm. To add back in, // add loop back in Dial(.) @@ -281,39 +286,306 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { return nil, err } -// doDial is an ugly shim method to retain all the logging and backoff logic -// of the old dialsync code -func (s *Swarm) doDial(ctx context.Context, p peer.ID) (*Conn, error) { - // Short circuit. - // By the time we take the dial lock, we may already *have* a connection - // to the peer. - c := s.bestAcceptableConnToPeer(ctx, p) - if c != nil { - return c, nil +/////////////////////////////////////////////////////////////////////////////////// +// lo and behold, The Dialer +// TODO explain how all this works +////////////////////////////////////////////////////////////////////////////////// +type dialRequest struct { + ctx context.Context + resch chan dialResponse +} + +type dialResponse struct { + conn *Conn + err error +} + +type dialComplete struct { + addr ma.Multiaddr + conn *Conn + err error +} + +// dialWorker is an active dial goroutine that synchronizes and executes concurrent dials +func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan dialRequest) { + if p == s.local { + for { + select { + case req, ok := <-reqch: + if !ok { + return + } + + req.resch <- dialResponse{err: ErrDialToSelf} + } + } } - logdial := lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + s.dialWorkerLoop(ctx, p, reqch) +} - // ok, we have been charged to dial! let's do it. - // if it succeeds, dial will add the conn to the swarm itself. - defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done() +func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan dialRequest) { + defer s.limiter.clearAllPeerDials(p) - conn, err := s.dial(ctx, p) - if err != nil { - conn := s.bestAcceptableConnToPeer(ctx, p) - if conn != nil { - // Hm? What error? - // Could have canceled the dial because we received a - // connection or some other random reason. - // Just ignore the error and return the connection. - log.Debugf("ignoring dial error because we already have a connection: %s", err) - return conn, nil + type pendRequest struct { + req dialRequest // the original request + err *DialError // dial error accumulator + addrs map[ma.Multiaddr]struct{} // pending addr dials + } + + type addrDial struct { + ctx context.Context + conn *Conn + err error + requests []int + } + + reqno := 0 + requests := make(map[int]*pendRequest) + pending := make(map[ma.Multiaddr]*addrDial) + + var triggerDial <-chan time.Time + var nextDial []ma.Multiaddr + active := 0 + done := false + + resch := make(chan dialComplete) + +loop: + for { + select { + case req, ok := <-reqch: + if !ok { + // request channel has been closed, wait for pending dials to complete + if active > 0 { + done = true + reqch = nil + triggerDial = nil + continue loop + } + + // no active dials, we are done + return + } + + c := s.bestAcceptableConnToPeer(req.ctx, p) + if c != nil { + req.resch <- dialResponse{conn: c} + continue loop + } + + addrs, err := s.addrsForDial(req.ctx, p) + if err != nil { + req.resch <- dialResponse{err: err} + continue loop + } + + // at this point, len(addrs) > 0 or else it would be error from addrsForDial + // ranke them to process in order + addrs = s.rankAddrs(addrs) + + // create the pending request object + pr := &pendRequest{ + req: req, + err: &DialError{Peer: p}, + addrs: make(map[ma.Multiaddr]struct{}), + } + for _, a := range addrs { + pr.addrs[a] = struct{}{} + } + + // check if any of the addrs has been successfully dialed and accumulate + // errors from complete dials while collecting new addrs to dial/join + var todial []ma.Multiaddr + var tojoin []*addrDial + + for _, a := range addrs { + ad, ok := pending[a] + if !ok { + todial = append(todial, a) + continue + } + + if ad.conn != nil { + // dial to this addr was successful, complete the request + req.resch <- dialResponse{conn: ad.conn} + continue loop + } + + if ad.err != nil { + // dial to this addr errored, accumulate the error + pr.err.recordErr(a, ad.err) + delete(pr.addrs, a) + } + + // dial is still pending, add to the join list + tojoin = append(tojoin, ad) + } + + if len(todial) == 0 && len(tojoin) == 0 { + // all request applicable addrs have been dialed, we must have errored + req.resch <- dialResponse{err: pr.err} + continue loop + } + + // the request has some pending or new dials, track it and schedule new dials + reqno++ + requests[reqno] = pr + + for _, ad := range tojoin { + ad.requests = append(ad.requests, reqno) + } + + if len(todial) > 0 { + for _, a := range todial { + pending[a] = &addrDial{ctx: req.ctx, requests: []int{reqno}} + } + + nextDial = append(nextDial, todial...) + nextDial = s.rankAddrs(nextDial) + + if triggerDial == nil { + trigger := make(chan time.Time) + close(trigger) + triggerDial = trigger + } + } + + case <-triggerDial: + if len(nextDial) == 0 { + triggerDial = nil + continue loop + } + + next := nextDial[0] + nextDial = nextDial[1:] + + // spawn the next dial + ad := pending[next] + go s.dialNextAddr(ad.ctx, p, next, resch) + active++ + + // select an appropriate delay for the next dial trigger + delay := s.delayForNextDial(next) + triggerDial = time.After(delay) + + case res := <-resch: + active-- + + if done && active == 0 { + return + } + + ad := pending[res.addr] + ad.conn = res.conn + ad.err = res.err + + dialRequests := ad.requests + ad.requests = nil + + if res.conn != nil { + // we got a connection, dispatch to still pending requests + for _, reqno := range dialRequests { + pr, ok := requests[reqno] + if !ok { + // it has already dispatched a connection + continue + } + + pr.req.resch <- dialResponse{conn: res.conn} + delete(requests, reqno) + } + + continue loop + } + + // it must be an error, accumulate it and dispatch dial error if the request has tried all addrs + for _, reqno := range dialRequests { + pr, ok := requests[reqno] + if !ok { + // has already been dispatched + continue + } + + // accumulate the error + pr.err.recordErr(res.addr, res.err) + + delete(pr.addrs, res.addr) + if len(pr.addrs) == 0 { + // all addrs have erred, dispatch dial error + pr.req.resch <- dialResponse{err: pr.err} + delete(requests, reqno) + } + } } + } +} - // ok, we failed. - return nil, err +func (s *Swarm) addrsForDial(ctx context.Context, p peer.ID) ([]ma.Multiaddr, error) { + peerAddrs := s.peers.Addrs(p) + if len(peerAddrs) == 0 { + return nil, ErrNoAddresses + } + + goodAddrs := s.filterKnownUndialables(p, peerAddrs) + if forceDirect, _ := network.GetForceDirectDial(ctx); forceDirect { + goodAddrs = addrutil.FilterAddrs(goodAddrs, s.nonProxyAddr) } - return conn, nil + + if len(goodAddrs) == 0 { + return nil, ErrNoGoodAddresses + } + + return goodAddrs, nil +} + +func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, resch chan dialComplete) { + // check the dial backoff + if forceDirect, _ := network.GetForceDirectDial(ctx); !forceDirect { + if s.backf.Backoff(p, addr) { + resch <- dialComplete{addr: addr, err: ErrDialBackoff} + return + } + } + + // start the dial + dresch := make(chan dialResult) + s.limitedDial(ctx, p, addr, dresch) + select { + case res := <-dresch: + if res.Err != nil { + if res.Err != context.Canceled { + s.backf.AddBackoff(p, addr) + } + + resch <- dialComplete{addr: addr, err: res.Err} + return + } + + conn, err := s.addConn(res.Conn, network.DirOutbound) + if err != nil { + res.Conn.Close() + resch <- dialComplete{addr: addr, err: err} + return + } + + resch <- dialComplete{addr: addr, conn: conn} + + case <-ctx.Done(): + resch <- dialComplete{addr: addr, err: ctx.Err()} + } +} + +func (s *Swarm) delayForNextDial(addr ma.Multiaddr) time.Duration { + if _, err := addr.ValueForProtocol(ma.P_CIRCUIT); err == nil { + return DelayDialRelayAddr + } + + if manet.IsPrivateAddr(addr) { + return DelayDialPrivateAddr + } + + return DelayDialPublicAddr } func (s *Swarm) canDial(addr ma.Multiaddr) bool { @@ -365,80 +637,6 @@ func (s *Swarm) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { return append(append(append(localUdpAddrs, othersUdp...), fds...), relays...) } -// dial is the actual swarm's dial logic, gated by Dial. -func (s *Swarm) dial(ctx context.Context, p peer.ID) (*Conn, error) { - forceDirect, _ := network.GetForceDirectDial(ctx) - var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) - if p == s.local { - log.Event(ctx, "swarmDialDoDialSelf", logdial) - return nil, ErrDialToSelf - } - defer log.EventBegin(ctx, "swarmDialDo", logdial).Done() - logdial["dial"] = "failure" // start off with failure. set to "success" at the end. - - sk := s.peers.PrivKey(s.local) - logdial["encrypted"] = sk != nil // log whether this will be an encrypted dial or not. - if sk == nil { - // fine for sk to be nil, just log. - log.Debug("Dial not given PrivateKey, so WILL NOT SECURE conn.") - } - - ////// - peerAddrs := s.peers.Addrs(p) - if len(peerAddrs) == 0 { - return nil, &DialError{Peer: p, Cause: ErrNoAddresses} - } - goodAddrs := s.filterKnownUndialables(p, peerAddrs) - if forceDirect { - goodAddrs = addrutil.FilterAddrs(goodAddrs, s.nonProxyAddr) - } - if len(goodAddrs) == 0 { - return nil, &DialError{Peer: p, Cause: ErrNoGoodAddresses} - } - - if !forceDirect { - /////// Check backoff andnRank addresses - var nonBackoff bool - for _, a := range goodAddrs { - // skip addresses in back-off - if !s.backf.Backoff(p, a) { - nonBackoff = true - } - } - if !nonBackoff { - return nil, ErrDialBackoff - } - } - - connC, dialErr := s.dialAddrs(ctx, p, s.rankAddrs(goodAddrs)) - if dialErr != nil { - logdial["error"] = dialErr.Cause.Error() - switch dialErr.Cause { - case context.Canceled, context.DeadlineExceeded: - // Always prefer the context errors as we rely on being - // able to check them. - // - // Removing this will BREAK backoff (causing us to - // backoff when canceling dials). - return nil, dialErr.Cause - } - return nil, dialErr - } - logdial["conn"] = logging.Metadata{ - "localAddr": connC.LocalMultiaddr(), - "remoteAddr": connC.RemoteMultiaddr(), - } - swarmC, err := s.addConn(connC, network.DirOutbound) - if err != nil { - logdial["error"] = err.Error() - connC.Close() // close the connection. didn't work out :( - return nil, &DialError{Peer: p, Cause: err} - } - - logdial["dial"] = "success" - return swarmC, nil -} - // filterKnownUndialables takes a list of multiaddrs, and removes those // that we definitely don't want to dial: addresses configured to be blocked, // IPv6 link-local addresses, addresses without a dial-capable transport, @@ -466,98 +664,6 @@ func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) []ma.Mul ) } -func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs []ma.Multiaddr) (transport.CapableConn, *DialError) { - /* - This slice-to-chan code is temporary, the peerstore can currently provide - a channel as an interface for receiving addresses, but more thought - needs to be put into the execution. For now, this allows us to use - the improved rate limiter, while maintaining the outward behaviour - that we previously had (halting a dial when we run out of addrs) - */ - var remoteAddrChan chan ma.Multiaddr - if len(remoteAddrs) > 0 { - remoteAddrChan = make(chan ma.Multiaddr, len(remoteAddrs)) - for i := range remoteAddrs { - remoteAddrChan <- remoteAddrs[i] - } - close(remoteAddrChan) - } - - log.Debugf("%s swarm dialing %s", s.local, p) - - ctx, cancel := context.WithCancel(ctx) - defer cancel() // cancel work when we exit func - - // use a single response type instead of errs and conns, reduces complexity *a ton* - respch := make(chan dialResult) - err := &DialError{Peer: p} - - defer s.limiter.clearAllPeerDials(p) - - var active int -dialLoop: - for remoteAddrChan != nil || active > 0 { - // Check for context cancellations and/or responses first. - select { - case <-ctx.Done(): - break dialLoop - case resp := <-respch: - active-- - if resp.Err != nil { - // Errors are normal, lots of dials will fail - if resp.Err != context.Canceled { - s.backf.AddBackoff(p, resp.Addr) - } - - log.Infof("got error on dial: %s", resp.Err) - err.recordErr(resp.Addr, resp.Err) - } else if resp.Conn != nil { - return resp.Conn, nil - } - - // We got a result, try again from the top. - continue - default: - } - - // Now, attempt to dial. - select { - case addr, ok := <-remoteAddrChan: - if !ok { - remoteAddrChan = nil - continue - } - - s.limitedDial(ctx, p, addr, respch) - active++ - case <-ctx.Done(): - break dialLoop - case resp := <-respch: - active-- - if resp.Err != nil { - // Errors are normal, lots of dials will fail - if resp.Err != context.Canceled { - s.backf.AddBackoff(p, resp.Addr) - } - - log.Infof("got error on dial: %s", resp.Err) - err.recordErr(resp.Addr, resp.Err) - } else if resp.Conn != nil { - return resp.Conn, nil - } - } - } - - if ctxErr := ctx.Err(); ctxErr != nil { - err.Cause = ctxErr - } else if len(err.DialErrors) == 0 { - err.Cause = network.ErrNoRemoteAddrs - } else { - err.Cause = ErrAllDialsFailed - } - return nil, err -} - // limitedDial will start a dial to the given peer when // it is able, respecting the various different types of rate // limiting that occur without using extra goroutines per addr @@ -570,6 +676,7 @@ func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp }) } +// dialAddr is the actual dial for an addr, indirectly invoked through the limiter func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.CapableConn, error) { // Just to double check. Costs nothing. if s.local == p { From 30d51370fc298dac8f9f6cb77d319c5e4caf5e05 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 30 Mar 2021 17:46:01 +0300 Subject: [PATCH 187/259] adjust next dial delays --- p2p/net/swarm/swarm_dial.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index cc739791dc..428845f344 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -59,8 +59,8 @@ var ( var ( DelayDialPrivateAddr = 5 * time.Millisecond - DelayDialPublicAddr = 50 * time.Millisecond - DelayDialRelayAddr = 100 * time.Millisecond + DelayDialPublicAddr = 25 * time.Millisecond + DelayDialRelayAddr = 50 * time.Millisecond ) // DialAttempts governs how many times a goroutine will try to dial a given peer. From 1efe95aff0fff1b2dc776e951f69ae0714ea52b0 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 30 Mar 2021 20:38:48 +0300 Subject: [PATCH 188/259] fix dial_sync tests --- p2p/net/swarm/dial_sync.go | 12 +++--- p2p/net/swarm/dial_sync_test.go | 70 +++++++++++++++++++++++++-------- p2p/net/swarm/swarm_dial.go | 38 +++++++++--------- 3 files changed, 79 insertions(+), 41 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 2efdf067b6..54a21067b7 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -13,7 +13,7 @@ import ( var errDialCanceled = errors.New("dial was aborted internally, likely due to https://git.io/Je2wW") // DialFunc is the type of function expected by DialSync. -type DialWorkerFunc func(context.Context, peer.ID, <-chan dialRequest) +type DialWorkerFunc func(context.Context, peer.ID, <-chan DialRequest) // NewDialSync constructs a new DialSync func NewDialSync(worker DialWorkerFunc) *DialSync { @@ -38,7 +38,7 @@ type activeDial struct { ctx context.Context cancel func() - reqch chan dialRequest + reqch chan DialRequest ds *DialSync } @@ -68,16 +68,16 @@ func (ad *activeDial) dial(ctx context.Context, p peer.ID) (*Conn, error) { dialCtx = network.WithSimultaneousConnect(dialCtx, reason) } - resch := make(chan dialResponse, 1) + resch := make(chan DialResponse, 1) select { - case ad.reqch <- dialRequest{ctx: dialCtx, resch: resch}: + case ad.reqch <- DialRequest{Ctx: dialCtx, Resch: resch}: case <-ctx.Done(): return nil, ctx.Err() } select { case res := <-resch: - return res.conn, res.err + return res.Conn, res.Err case <-ctx.Done(): return nil, ctx.Err() } @@ -98,7 +98,7 @@ func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { id: p, ctx: adctx, cancel: cancel, - reqch: make(chan dialRequest), + reqch: make(chan DialRequest), ds: ds, } ds.dials[p] = actd diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 485d1a3171..ef7458a554 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -12,19 +12,33 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -func getMockDialFunc() (DialFunc, func(), context.Context, <-chan struct{}) { +func getMockDialFunc() (DialWorkerFunc, func(), context.Context, <-chan struct{}) { dfcalls := make(chan struct{}, 512) // buffer it large enough that we won't care dialctx, cancel := context.WithCancel(context.Background()) ch := make(chan struct{}) - f := func(ctx context.Context, p peer.ID) (*Conn, error) { + f := func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { dfcalls <- struct{}{} - defer cancel() - select { - case <-ch: - return new(Conn), nil - case <-ctx.Done(): - return nil, ctx.Err() - } + go func() { + defer cancel() + for { + select { + case req, ok := <-reqch: + if !ok { + return + } + + select { + case <-ch: + req.Resch <- DialResponse{Conn: new(Conn)} + case <-ctx.Done(): + req.Resch <- DialResponse{Err: ctx.Err()} + return + } + case <-ctx.Done(): + return + } + } + }() } o := new(sync.Once) @@ -174,12 +188,25 @@ func TestDialSyncAllCancel(t *testing.T) { func TestFailFirst(t *testing.T) { var count int - f := func(ctx context.Context, p peer.ID) (*Conn, error) { - if count > 0 { - return new(Conn), nil + f := func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { + for { + select { + case req, ok := <-reqch: + if !ok { + return + } + + if count > 0 { + req.Resch <- DialResponse{Conn: new(Conn)} + } else { + req.Resch <- DialResponse{Err: fmt.Errorf("gophers ate the modem")} + } + count++ + + case <-ctx.Done(): + return + } } - count++ - return nil, fmt.Errorf("gophers ate the modem") } ds := NewDialSync(f) @@ -205,8 +232,19 @@ func TestFailFirst(t *testing.T) { } func TestStressActiveDial(t *testing.T) { - ds := NewDialSync(func(ctx context.Context, p peer.ID) (*Conn, error) { - return nil, nil + ds := NewDialSync(func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { + for { + select { + case req, ok := <-reqch: + if !ok { + return + } + + req.Resch <- DialResponse{} + case <-ctx.Done(): + return + } + } }) wg := sync.WaitGroup{} diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 428845f344..ae4c87f1ad 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -290,14 +290,14 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { // lo and behold, The Dialer // TODO explain how all this works ////////////////////////////////////////////////////////////////////////////////// -type dialRequest struct { - ctx context.Context - resch chan dialResponse +type DialRequest struct { + Ctx context.Context + Resch chan DialResponse } -type dialResponse struct { - conn *Conn - err error +type DialResponse struct { + Conn *Conn + Err error } type dialComplete struct { @@ -307,7 +307,7 @@ type dialComplete struct { } // dialWorker is an active dial goroutine that synchronizes and executes concurrent dials -func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan dialRequest) { +func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { if p == s.local { for { select { @@ -316,7 +316,7 @@ func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan dialRequ return } - req.resch <- dialResponse{err: ErrDialToSelf} + req.Resch <- DialResponse{Err: ErrDialToSelf} } } } @@ -324,11 +324,11 @@ func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan dialRequ s.dialWorkerLoop(ctx, p, reqch) } -func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan dialRequest) { +func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { defer s.limiter.clearAllPeerDials(p) type pendRequest struct { - req dialRequest // the original request + req DialRequest // the original request err *DialError // dial error accumulator addrs map[ma.Multiaddr]struct{} // pending addr dials } @@ -368,15 +368,15 @@ loop: return } - c := s.bestAcceptableConnToPeer(req.ctx, p) + c := s.bestAcceptableConnToPeer(req.Ctx, p) if c != nil { - req.resch <- dialResponse{conn: c} + req.Resch <- DialResponse{Conn: c} continue loop } - addrs, err := s.addrsForDial(req.ctx, p) + addrs, err := s.addrsForDial(req.Ctx, p) if err != nil { - req.resch <- dialResponse{err: err} + req.Resch <- DialResponse{Err: err} continue loop } @@ -408,7 +408,7 @@ loop: if ad.conn != nil { // dial to this addr was successful, complete the request - req.resch <- dialResponse{conn: ad.conn} + req.Resch <- DialResponse{Conn: ad.conn} continue loop } @@ -424,7 +424,7 @@ loop: if len(todial) == 0 && len(tojoin) == 0 { // all request applicable addrs have been dialed, we must have errored - req.resch <- dialResponse{err: pr.err} + req.Resch <- DialResponse{Err: pr.err} continue loop } @@ -438,7 +438,7 @@ loop: if len(todial) > 0 { for _, a := range todial { - pending[a] = &addrDial{ctx: req.ctx, requests: []int{reqno}} + pending[a] = &addrDial{ctx: req.Ctx, requests: []int{reqno}} } nextDial = append(nextDial, todial...) @@ -492,7 +492,7 @@ loop: continue } - pr.req.resch <- dialResponse{conn: res.conn} + pr.req.Resch <- DialResponse{Conn: res.conn} delete(requests, reqno) } @@ -513,7 +513,7 @@ loop: delete(pr.addrs, res.addr) if len(pr.addrs) == 0 { // all addrs have erred, dispatch dial error - pr.req.resch <- dialResponse{err: pr.err} + pr.req.Resch <- DialResponse{Err: pr.err} delete(requests, reqno) } } From b69afe8dd9eb1d813b8b1cfac227de017641b520 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 30 Mar 2021 21:35:48 +0300 Subject: [PATCH 189/259] clear address dial when they fail because of backoff makes TestDialBackoff happy --- p2p/net/swarm/swarm_dial.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index ae4c87f1ad..9b10db165b 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -517,6 +517,11 @@ loop: delete(requests, reqno) } } + + // if it was a backoff, clear the address dial so that it doesn't inhibit new dial requests + if res.err == ErrDialBackoff { + delete(pending, res.addr) + } } } } From 790eddaaa647ba3c91f3729d4af3d5916a119803 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 12:27:51 +0300 Subject: [PATCH 190/259] nuke incref, it's useless --- p2p/net/swarm/dial_sync.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 54a21067b7..e5f25478cb 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -43,10 +43,6 @@ type activeDial struct { ds *DialSync } -func (ad *activeDial) incref() { - ad.refCnt++ -} - func (ad *activeDial) decref() { ad.ds.dialsLk.Lock() ad.refCnt-- @@ -107,7 +103,7 @@ func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { } // increase ref count before dropping dialsLk - actd.incref() + actd.refCnt++ return actd } From df78bf89f4194eec30d3bfdb140729659146ae64 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 12:34:54 +0300 Subject: [PATCH 191/259] make dialWorker return an error for self dials and responsible for spawning the loop --- p2p/net/swarm/dial_sync.go | 19 ++++++---- p2p/net/swarm/dial_sync_test.go | 61 ++++++++++++++++++--------------- p2p/net/swarm/swarm_dial.go | 16 +++------ 3 files changed, 51 insertions(+), 45 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index e5f25478cb..32196cb2d5 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -13,7 +13,7 @@ import ( var errDialCanceled = errors.New("dial was aborted internally, likely due to https://git.io/Je2wW") // DialFunc is the type of function expected by DialSync. -type DialWorkerFunc func(context.Context, peer.ID, <-chan DialRequest) +type DialWorkerFunc func(context.Context, peer.ID, <-chan DialRequest) error // NewDialSync constructs a new DialSync func NewDialSync(worker DialWorkerFunc) *DialSync { @@ -79,7 +79,7 @@ func (ad *activeDial) dial(ctx context.Context, p peer.ID) (*Conn, error) { } } -func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { +func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { ds.dialsLk.Lock() defer ds.dialsLk.Unlock() @@ -99,20 +99,27 @@ func (ds *DialSync) getActiveDial(p peer.ID) *activeDial { } ds.dials[p] = actd - go ds.dialWorker(adctx, p, actd.reqch) + err := ds.dialWorker(adctx, p, actd.reqch) + if err != nil { + cancel() + return nil, err + } } // increase ref count before dropping dialsLk actd.refCnt++ - return actd + return actd, nil } // DialLock initiates a dial to the given peer if there are none in progress // then waits for the dial to that peer to complete. func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { - ad := ds.getActiveDial(p) - defer ad.decref() + ad, err := ds.getActiveDial(p) + if err != nil { + return nil, err + } + defer ad.decref() return ad.dial(ctx, p) } diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index ef7458a554..f1a9f8a539 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -16,7 +16,7 @@ func getMockDialFunc() (DialWorkerFunc, func(), context.Context, <-chan struct{} dfcalls := make(chan struct{}, 512) // buffer it large enough that we won't care dialctx, cancel := context.WithCancel(context.Background()) ch := make(chan struct{}) - f := func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { + f := func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { dfcalls <- struct{}{} go func() { defer cancel() @@ -39,6 +39,7 @@ func getMockDialFunc() (DialWorkerFunc, func(), context.Context, <-chan struct{} } } }() + return nil } o := new(sync.Once) @@ -188,25 +189,28 @@ func TestDialSyncAllCancel(t *testing.T) { func TestFailFirst(t *testing.T) { var count int - f := func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { - for { - select { - case req, ok := <-reqch: - if !ok { - return - } + f := func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { + go func() { + for { + select { + case req, ok := <-reqch: + if !ok { + return + } - if count > 0 { - req.Resch <- DialResponse{Conn: new(Conn)} - } else { - req.Resch <- DialResponse{Err: fmt.Errorf("gophers ate the modem")} - } - count++ + if count > 0 { + req.Resch <- DialResponse{Conn: new(Conn)} + } else { + req.Resch <- DialResponse{Err: fmt.Errorf("gophers ate the modem")} + } + count++ - case <-ctx.Done(): - return + case <-ctx.Done(): + return + } } - } + }() + return nil } ds := NewDialSync(f) @@ -232,19 +236,22 @@ func TestFailFirst(t *testing.T) { } func TestStressActiveDial(t *testing.T) { - ds := NewDialSync(func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { - for { - select { - case req, ok := <-reqch: - if !ok { + ds := NewDialSync(func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { + go func() { + for { + select { + case req, ok := <-reqch: + if !ok { + return + } + + req.Resch <- DialResponse{} + case <-ctx.Done(): return } - - req.Resch <- DialResponse{} - case <-ctx.Done(): - return } - } + }() + return nil }) wg := sync.WaitGroup{} diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 9b10db165b..77cb8a6bc7 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -307,21 +307,13 @@ type dialComplete struct { } // dialWorker is an active dial goroutine that synchronizes and executes concurrent dials -func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { +func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { if p == s.local { - for { - select { - case req, ok := <-reqch: - if !ok { - return - } - - req.Resch <- DialResponse{Err: ErrDialToSelf} - } - } + return ErrDialToSelf } - s.dialWorkerLoop(ctx, p, reqch) + go s.dialWorkerLoop(ctx, p, reqch) + return nil } func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { From 88df3095026e362e8eb89b90dd2fa0317d9722a4 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 13:58:16 +0300 Subject: [PATCH 192/259] don't use a goroutine for the actual dial --- p2p/net/swarm/swarm_dial.go | 138 +++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 66 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 77cb8a6bc7..8f1e2c3cb8 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -290,6 +290,7 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { // lo and behold, The Dialer // TODO explain how all this works ////////////////////////////////////////////////////////////////////////////////// + type DialRequest struct { Ctx context.Context Resch chan DialResponse @@ -300,12 +301,6 @@ type DialResponse struct { Err error } -type dialComplete struct { - addr ma.Multiaddr - conn *Conn - err error -} - // dialWorker is an active dial goroutine that synchronizes and executes concurrent dials func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { if p == s.local { @@ -326,6 +321,7 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial } type addrDial struct { + addr ma.Multiaddr ctx context.Context conn *Conn err error @@ -336,12 +332,40 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial requests := make(map[int]*pendRequest) pending := make(map[ma.Multiaddr]*addrDial) + dispatchError := func(ad *addrDial, err error) { + ad.err = err + for _, reqno := range ad.requests { + pr, ok := requests[reqno] + if !ok { + // has already been dispatched + continue + } + + // accumulate the error + pr.err.recordErr(ad.addr, err) + + delete(pr.addrs, ad.addr) + if len(pr.addrs) == 0 { + // all addrs have erred, dispatch dial error + pr.req.Resch <- DialResponse{Err: pr.err} + delete(requests, reqno) + } + } + + ad.requests = nil + + // if it was a backoff, clear the address dial so that it doesn't inhibit new dial requests + if err == ErrDialBackoff { + delete(pending, ad.addr) + } + } + var triggerDial <-chan time.Time var nextDial []ma.Multiaddr active := 0 done := false - resch := make(chan dialComplete) + resch := make(chan dialResult) loop: for { @@ -408,6 +432,7 @@ loop: // dial to this addr errored, accumulate the error pr.err.recordErr(a, ad.err) delete(pr.addrs, a) + continue } // dial is still pending, add to the join list @@ -430,7 +455,7 @@ loop: if len(todial) > 0 { for _, a := range todial { - pending[a] = &addrDial{ctx: req.Ctx, requests: []int{reqno}} + pending[a] = &addrDial{addr: a, ctx: req.Ctx, requests: []int{reqno}} } nextDial = append(nextDial, todial...) @@ -454,7 +479,12 @@ loop: // spawn the next dial ad := pending[next] - go s.dialNextAddr(ad.ctx, p, next, resch) + err := s.dialNextAddr(ad.ctx, p, next, resch) + if err != nil { + dispatchError(ad, err) + continue loop + } + active++ // select an appropriate delay for the next dial trigger @@ -465,55 +495,54 @@ loop: active-- if done && active == 0 { + if res.Conn != nil { + // we got an actual connection, but the dial has been cancelled + // Should we close it? I think not, we should just add it to the swarm + _, err := s.addConn(res.Conn, network.DirOutbound) + if err != nil { + // well duh, now we have to close it + res.Conn.Close() + } + } return } - ad := pending[res.addr] - ad.conn = res.conn - ad.err = res.err + ad := pending[res.Addr] - dialRequests := ad.requests - ad.requests = nil + if res.Conn != nil { + // we got a connection, add it to the swarm + conn, err := s.addConn(res.Conn, network.DirOutbound) + if err != nil { + // oops no, we failed to add it to the swarm + res.Conn.Close() + dispatchError(ad, err) + continue loop + } - if res.conn != nil { - // we got a connection, dispatch to still pending requests - for _, reqno := range dialRequests { + // dispatch to still pending requests + for _, reqno := range ad.requests { pr, ok := requests[reqno] if !ok { // it has already dispatched a connection continue } - pr.req.Resch <- DialResponse{Conn: res.conn} + pr.req.Resch <- DialResponse{Conn: conn} delete(requests, reqno) } + ad.conn = conn + ad.requests = nil + continue loop } - // it must be an error, accumulate it and dispatch dial error if the request has tried all addrs - for _, reqno := range dialRequests { - pr, ok := requests[reqno] - if !ok { - // has already been dispatched - continue - } - - // accumulate the error - pr.err.recordErr(res.addr, res.err) - - delete(pr.addrs, res.addr) - if len(pr.addrs) == 0 { - // all addrs have erred, dispatch dial error - pr.req.Resch <- DialResponse{Err: pr.err} - delete(requests, reqno) - } + // it must be an error -- add backoff if applicable and dispatch + if res.Err != context.Canceled { + s.backf.AddBackoff(p, res.Addr) } - // if it was a backoff, clear the address dial so that it doesn't inhibit new dial requests - if res.err == ErrDialBackoff { - delete(pending, res.addr) - } + dispatchError(ad, res.Err) } } } @@ -536,41 +565,18 @@ func (s *Swarm) addrsForDial(ctx context.Context, p peer.ID) ([]ma.Multiaddr, er return goodAddrs, nil } -func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, resch chan dialComplete) { +func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, resch chan dialResult) error { // check the dial backoff if forceDirect, _ := network.GetForceDirectDial(ctx); !forceDirect { if s.backf.Backoff(p, addr) { - resch <- dialComplete{addr: addr, err: ErrDialBackoff} - return + return ErrDialBackoff } } // start the dial - dresch := make(chan dialResult) - s.limitedDial(ctx, p, addr, dresch) - select { - case res := <-dresch: - if res.Err != nil { - if res.Err != context.Canceled { - s.backf.AddBackoff(p, addr) - } - - resch <- dialComplete{addr: addr, err: res.Err} - return - } + s.limitedDial(ctx, p, addr, resch) - conn, err := s.addConn(res.Conn, network.DirOutbound) - if err != nil { - res.Conn.Close() - resch <- dialComplete{addr: addr, err: err} - return - } - - resch <- dialComplete{addr: addr, conn: conn} - - case <-ctx.Done(): - resch <- dialComplete{addr: addr, err: ctx.Err()} - } + return nil } func (s *Swarm) delayForNextDial(addr ma.Multiaddr) time.Duration { From 0cf03ba71aad3a5b63780e3b178aadf40538f95b Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 16:12:45 +0300 Subject: [PATCH 193/259] batch dials together, rework address ranking --- p2p/net/swarm/swarm_dial.go | 163 +++++++++++++++++++++++------------- 1 file changed, 107 insertions(+), 56 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 8f1e2c3cb8..3d8ec3df33 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -58,9 +58,9 @@ var ( ) var ( - DelayDialPrivateAddr = 5 * time.Millisecond - DelayDialPublicAddr = 25 * time.Millisecond - DelayDialRelayAddr = 50 * time.Millisecond + delayDialPrivateAddr = 5 * time.Millisecond + delayDialPublicAddr = 25 * time.Millisecond + delayDialRelayAddr = 50 * time.Millisecond ) // DialAttempts governs how many times a goroutine will try to dial a given peer. @@ -361,6 +361,9 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial } var triggerDial <-chan time.Time + triggerNow := make(chan time.Time) + close(triggerNow) + var nextDial []ma.Multiaddr active := 0 done := false @@ -461,34 +464,46 @@ loop: nextDial = append(nextDial, todial...) nextDial = s.rankAddrs(nextDial) - if triggerDial == nil { - trigger := make(chan time.Time) - close(trigger) - triggerDial = trigger - } + // trigger a new dial now to account for the new addrs we added + triggerDial = triggerNow } case <-triggerDial: - if len(nextDial) == 0 { - triggerDial = nil - continue loop - } + // we dial batches of addresses together, logically belonging to the same batch + // after a batch of addresses has been dialed, we add a delay before initiating the next batch + dialed := false + last := 0 + next := 0 + for i, addr := range nextDial { + if dialed && !s.sameAddrBatch(nextDial[last], addr) { + break + } - next := nextDial[0] - nextDial = nextDial[1:] + next = i + 1 - // spawn the next dial - ad := pending[next] - err := s.dialNextAddr(ad.ctx, p, next, resch) - if err != nil { - dispatchError(ad, err) - continue loop + // spawn the dial + ad := pending[addr] + err := s.dialNextAddr(ad.ctx, p, addr, resch) + if err != nil { + dispatchError(ad, err) + continue + } + + dialed = true + last = i + active++ } - active++ + lastDial := nextDial[last] + nextDial = nextDial[next:] + if !dialed || len(nextDial) == 0 { + // we didn't dial anything because of backoff or we don't have any more addresses + triggerDial = nil + continue loop + } - // select an appropriate delay for the next dial trigger - delay := s.delayForNextDial(next) + // select an appropriate delay for the next dial batch + delay := s.delayForNextDial(lastDial) triggerDial = time.After(delay) case res := <-resch: @@ -516,6 +531,9 @@ loop: // oops no, we failed to add it to the swarm res.Conn.Close() dispatchError(ad, err) + if active == 0 && len(nextDial) > 0 { + triggerDial = triggerNow + } continue loop } @@ -543,6 +561,9 @@ loop: } dispatchError(ad, res.Err) + if active == 0 && len(nextDial) > 0 { + triggerDial = triggerNow + } } } } @@ -579,16 +600,37 @@ func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, return nil } +func (s *Swarm) sameAddrBatch(a, b ma.Multiaddr) bool { + // is it a relay addr? + if s.IsRelayAddr(a) { + return s.IsRelayAddr(b) + } + + // is it an expensive addr? + if s.IsExpensiveAddr(a) { + return s.IsExpensiveAddr(b) + } + + // is it a public addr? + if !manet.IsPrivateAddr(a) { + return !manet.IsPrivateAddr(b) && + s.IsFdConsumingAddr(a) == s.IsFdConsumingAddr(b) + } + + // it's a private addr + return manet.IsPrivateAddr(b) +} + func (s *Swarm) delayForNextDial(addr ma.Multiaddr) time.Duration { if _, err := addr.ValueForProtocol(ma.P_CIRCUIT); err == nil { - return DelayDialRelayAddr + return delayDialRelayAddr } if manet.IsPrivateAddr(addr) { - return DelayDialPrivateAddr + return delayDialPrivateAddr } - return DelayDialPublicAddr + return delayDialPublicAddr } func (s *Swarm) canDial(addr ma.Multiaddr) bool { @@ -601,43 +643,41 @@ func (s *Swarm) nonProxyAddr(addr ma.Multiaddr) bool { return !t.Proxy() } -// ranks addresses in descending order of preference for dialing -// Private UDP > Public UDP > Private TCP > Public TCP > UDP Relay server > TCP Relay server +// ranks addresses in descending order of preference for dialing, with the following rules: +// NonRelay > Relay +// NonWS > WS +// Private > Public +// UDP > TCP func (s *Swarm) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { - var localUdpAddrs []ma.Multiaddr // private udp - var relayUdpAddrs []ma.Multiaddr // relay udp - var othersUdp []ma.Multiaddr // public udp + addrTier := func(a ma.Multiaddr) (tier int) { + if s.IsRelayAddr(a) { + tier |= 0b1000 + } + if s.IsExpensiveAddr(a) { + tier |= 0b0100 + } + if !manet.IsPrivateAddr(a) { + tier |= 0b0010 + } + if s.IsFdConsumingAddr(a) { + tier |= 0b0001 + } - var localFdAddrs []ma.Multiaddr // private fd consuming - var relayFdAddrs []ma.Multiaddr // relay fd consuming - var othersFd []ma.Multiaddr // public fd consuming + return tier + } + tiers := make([][]ma.Multiaddr, 16) for _, a := range addrs { - if _, err := a.ValueForProtocol(ma.P_CIRCUIT); err == nil { - if s.IsFdConsumingAddr(a) { - relayFdAddrs = append(relayFdAddrs, a) - continue - } - relayUdpAddrs = append(relayUdpAddrs, a) - } else if manet.IsPrivateAddr(a) { - if s.IsFdConsumingAddr(a) { - localFdAddrs = append(localFdAddrs, a) - continue - } - localUdpAddrs = append(localUdpAddrs, a) - } else { - if s.IsFdConsumingAddr(a) { - othersFd = append(othersFd, a) - continue - } - othersUdp = append(othersUdp, a) - } + tier := addrTier(a) + tiers[tier] = append(tiers[tier], a) } - relays := append(relayUdpAddrs, relayFdAddrs...) - fds := append(localFdAddrs, othersFd...) + result := make([]ma.Multiaddr, 0, len(addrs)) + for _, tier := range tiers { + result = append(result, tier...) + } - return append(append(append(localUdpAddrs, othersUdp...), fds...), relays...) + return result } // filterKnownUndialables takes a list of multiaddrs, and removes those @@ -729,3 +769,14 @@ func (s *Swarm) IsFdConsumingAddr(addr ma.Multiaddr) bool { _, err2 := first.ValueForProtocol(ma.P_UNIX) return err1 == nil || err2 == nil } + +func (s *Swarm) IsExpensiveAddr(addr ma.Multiaddr) bool { + _, err1 := addr.ValueForProtocol(ma.P_WS) + _, err2 := addr.ValueForProtocol(ma.P_WSS) + return err1 == nil || err2 == nil +} + +func (s *Swarm) IsRelayAddr(addr ma.Multiaddr) bool { + _, err := addr.ValueForProtocol(ma.P_CIRCUIT) + return err == nil +} From 3707a55a7e0b9f50912ad37cadc9556e63be0339 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 16:28:10 +0300 Subject: [PATCH 194/259] tune down batch dial delays --- p2p/net/swarm/swarm_dial.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 3d8ec3df33..2b7f0b03e9 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -58,9 +58,9 @@ var ( ) var ( - delayDialPrivateAddr = 5 * time.Millisecond - delayDialPublicAddr = 25 * time.Millisecond - delayDialRelayAddr = 50 * time.Millisecond + delayDialPrivateAddr = 1 * time.Millisecond + delayDialPublicAddr = 5 * time.Millisecond + delayDialRelayAddr = 10 * time.Millisecond ) // DialAttempts governs how many times a goroutine will try to dial a given peer. From 0bd0fcded706eed3345f92092951ff96dd385554 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 16:42:04 +0300 Subject: [PATCH 195/259] use a timer instead of time.After --- p2p/net/swarm/swarm_dial.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 2b7f0b03e9..de16ba3fbb 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -361,9 +361,16 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial } var triggerDial <-chan time.Time + var triggerTimer *time.Timer triggerNow := make(chan time.Time) close(triggerNow) + defer func() { + if triggerTimer != nil { + triggerTimer.Stop() + } + }() + var nextDial []ma.Multiaddr active := 0 done := false @@ -504,7 +511,15 @@ loop: // select an appropriate delay for the next dial batch delay := s.delayForNextDial(lastDial) - triggerDial = time.After(delay) + if triggerTimer == nil { + triggerTimer = time.NewTimer(delay) + } else { + if !triggerTimer.Stop() && triggerDial != triggerTimer.C { + <-triggerTimer.C + } + triggerTimer.Reset(delay) + } + triggerDial = triggerTimer.C case res := <-resch: active-- From 5539ac088539fe462dee49e0597db88407b83293 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 17:46:14 +0300 Subject: [PATCH 196/259] kill dial jump delays --- p2p/net/swarm/swarm_dial.go | 43 ++----------------------------------- 1 file changed, 2 insertions(+), 41 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index de16ba3fbb..1fe37a1fdd 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -57,12 +57,6 @@ var ( ErrGaterDisallowedConnection = errors.New("gater disallows connection to peer") ) -var ( - delayDialPrivateAddr = 1 * time.Millisecond - delayDialPublicAddr = 5 * time.Millisecond - delayDialRelayAddr = 10 * time.Millisecond -) - // DialAttempts governs how many times a goroutine will try to dial a given peer. // Note: this is down to one, as we have _too many dials_ atm. To add back in, // add loop back in Dial(.) @@ -360,17 +354,10 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial } } - var triggerDial <-chan time.Time - var triggerTimer *time.Timer - triggerNow := make(chan time.Time) + var triggerDial <-chan struct{} + triggerNow := make(chan struct{}) close(triggerNow) - defer func() { - if triggerTimer != nil { - triggerTimer.Stop() - } - }() - var nextDial []ma.Multiaddr active := 0 done := false @@ -501,26 +488,12 @@ loop: active++ } - lastDial := nextDial[last] nextDial = nextDial[next:] if !dialed || len(nextDial) == 0 { // we didn't dial anything because of backoff or we don't have any more addresses triggerDial = nil - continue loop } - // select an appropriate delay for the next dial batch - delay := s.delayForNextDial(lastDial) - if triggerTimer == nil { - triggerTimer = time.NewTimer(delay) - } else { - if !triggerTimer.Stop() && triggerDial != triggerTimer.C { - <-triggerTimer.C - } - triggerTimer.Reset(delay) - } - triggerDial = triggerTimer.C - case res := <-resch: active-- @@ -636,18 +609,6 @@ func (s *Swarm) sameAddrBatch(a, b ma.Multiaddr) bool { return manet.IsPrivateAddr(b) } -func (s *Swarm) delayForNextDial(addr ma.Multiaddr) time.Duration { - if _, err := addr.ValueForProtocol(ma.P_CIRCUIT); err == nil { - return delayDialRelayAddr - } - - if manet.IsPrivateAddr(addr) { - return delayDialPrivateAddr - } - - return delayDialPublicAddr -} - func (s *Swarm) canDial(addr ma.Multiaddr) bool { t := s.TransportForDialing(addr) return t != nil && t.CanDial(addr) From 91c366808709be2424c2f5de7caf9622d88b744a Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 18:10:43 +0300 Subject: [PATCH 197/259] add TestDialExistingConnection --- p2p/net/swarm/dial_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 9fc5df4189..c8b8bbe9b1 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -524,3 +524,29 @@ func TestDialPeerFailed(t *testing.T) { t.Errorf("expected %d errors, got %d", expectedErrorsCount, len(dialErr.DialErrors)) } } + +func TestDialExistingConnection(t *testing.T) { + ctx := context.Background() + + swarms := makeSwarms(ctx, t, 2) + defer closeSwarms(swarms) + s1 := swarms[0] + s2 := swarms[1] + + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) + + c1, err := s1.DialPeer(ctx, s2.LocalPeer()) + if err != nil { + t.Fatal(err) + } + + c2, err := s1.DialPeer(ctx, s2.LocalPeer()) + if err != nil { + t.Fatal(err) + } + + if c1 != c2 { + t.Fatal("expecting the same connection from both dials") + } + +} From ad3f67685ce2dd4127011bd9ca0f008fdd761e73 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 18:18:22 +0300 Subject: [PATCH 198/259] do a last ditch check for acceptable connections before dispatching a dial error --- p2p/net/swarm/swarm_dial.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 1fe37a1fdd..2d3255b4a6 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -341,7 +341,14 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial delete(pr.addrs, ad.addr) if len(pr.addrs) == 0 { // all addrs have erred, dispatch dial error - pr.req.Resch <- DialResponse{Err: pr.err} + // but first do a last one check in case an acceptable connection has landed from + // a simultaneous dial that started later and added new acceptable addrs + c := s.bestAcceptableConnToPeer(pr.req.Ctx, p) + if c != nil { + pr.req.Resch <- DialResponse{Conn: c} + } else { + pr.req.Resch <- DialResponse{Err: pr.err} + } delete(requests, reqno) } } From c20de2016f79ef68d02835d4dfcc5500ca1c18f4 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 18:30:30 +0300 Subject: [PATCH 199/259] merge dial contexts where possible --- p2p/net/swarm/swarm_dial.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 2d3255b4a6..513ce1c617 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -320,6 +320,7 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial conn *Conn err error requests []int + dialed bool } reqno := 0 @@ -454,6 +455,9 @@ loop: requests[reqno] = pr for _, ad := range tojoin { + if !ad.dialed { + ad.ctx = s.mergeDialContexts(ad.ctx, req.Ctx) + } ad.requests = append(ad.requests, reqno) } @@ -490,6 +494,7 @@ loop: continue } + ad.dialed = true dialed = true last = i active++ @@ -581,6 +586,18 @@ func (s *Swarm) addrsForDial(ctx context.Context, p peer.ID) ([]ma.Multiaddr, er return goodAddrs, nil } +func (s *Swarm) mergeDialContexts(a, b context.Context) context.Context { + dialCtx := a + + if simConnect, reason := network.GetSimultaneousConnect(b); simConnect { + if simConnect, _ := network.GetSimultaneousConnect(a); !simConnect { + dialCtx = network.WithSimultaneousConnect(dialCtx, reason) + } + } + + return dialCtx +} + func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, resch chan dialResult) error { // check the dial backoff if forceDirect, _ := network.GetForceDirectDial(ctx); !forceDirect { From d2bc4f43119f9ec074b5f906bb51617fbc62e8db Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 18:55:52 +0300 Subject: [PATCH 200/259] add TestDialSimultaneousJoin test --- p2p/net/swarm/dial_test.go | 101 +++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index c8b8bbe9b1..86390d6254 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -9,6 +9,7 @@ import ( addrutil "github.com/libp2p/go-addr-util" + "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/transport" @@ -548,5 +549,105 @@ func TestDialExistingConnection(t *testing.T) { if c1 != c2 { t.Fatal("expecting the same connection from both dials") } +} + +func newSilentListener(t *testing.T) ([]ma.Multiaddr, net.Listener) { + lst, err := net.Listen("tcp4", "localhost:0") + if err != nil { + t.Fatal(err) + } + addr, err := manet.FromNetAddr(lst.Addr()) + if err != nil { + t.Fatal(err) + } + addrs := []ma.Multiaddr{addr} + addrs, err = addrutil.ResolveUnspecifiedAddresses(addrs, nil) + if err != nil { + t.Fatal(err) + } + return addrs, lst + +} + +func TestDialSimultaneousJoin(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + swarms := makeSwarms(ctx, t, 2) + s1 := swarms[0] + s2 := swarms[1] + defer s1.Close() + defer s2.Close() + + s2silentAddrs, s2silentListener := newSilentListener(t) + go acceptAndHang(s2silentListener) + + connch := make(chan network.Conn, 512) + + // start a dial to s2 through the silent addr + go func() { + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2silentAddrs, peerstore.PermanentAddrTTL) + + c, err := s1.DialPeer(ctx, s2.LocalPeer()) + if err != nil { + t.Fatal(err) + } + + t.Logf("first dial succedded; conn: %+v", c) + + connch <- c + }() + + // wait a bit for the dial to take hold + time.Sleep(100 * time.Millisecond) + + // start a second dial to s2 that uses the real s2 addrs + go func() { + s2addrs, err := s2.InterfaceListenAddresses() + if err != nil { + t.Fatal(err) + } + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2addrs[:1], peerstore.PermanentAddrTTL) + + c, err := s1.DialPeer(ctx, s2.LocalPeer()) + if err != nil { + t.Fatal(err) + } + t.Logf("second dial succedded; conn: %+v", c) + + connch <- c + }() + + // wait for the second dial to finish + c2 := <-connch + + // start a third dial to s2, this should get the existing connection from the successful dial + go func() { + c, err := s1.DialPeer(ctx, s2.LocalPeer()) + if err != nil { + t.Fatal(err) + } + + t.Logf("third dial succedded; conn: %+v", c) + + connch <- c + }() + + c3 := <-connch + + if c2 != c3 { + t.Fatal("expected c2 and c3 to be the same") + } + + // next, the first dial to s2, using the silent addr should timeout; at this point the dial + // will error but the last chance check will see the existing connection and return it + select { + case c1 := <-connch: + if c1 != c2 { + t.Fatal("expected c1 and c2 to be the same") + } + case <-time.After(2 * transport.DialTimeout): + t.Fatal("no connection from first dial") + } } From 167d64587bce771cede74d9df0d07b64253fc349 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 20:04:42 +0300 Subject: [PATCH 201/259] don't add backoff if we have successfully connected for consistency with the old dialer behaviour. --- p2p/net/swarm/swarm_dial.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 513ce1c617..1a5cb1208a 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -368,7 +368,8 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial var nextDial []ma.Multiaddr active := 0 - done := false + done := false // true when the request channel has been closed + connected := false // true when a connection has been successfully established resch := make(chan dialResult) @@ -509,6 +510,10 @@ loop: case res := <-resch: active-- + if res.Conn != nil { + connected = true + } + if done && active == 0 { if res.Conn != nil { // we got an actual connection, but the dial has been cancelled @@ -556,7 +561,9 @@ loop: } // it must be an error -- add backoff if applicable and dispatch - if res.Err != context.Canceled { + if res.Err != context.Canceled && !connected { + // we only add backoff if there has not been a successful connection + // for consistency with the old dialer behavior. s.backf.AddBackoff(p, res.Addr) } From 80b33ec0b4a910f3584d7f6cf08d828ab9cf8973 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 20:14:52 +0300 Subject: [PATCH 202/259] fix TestConnectednessCorrect we might get more connections because simultaneous dials can succeed and we have both TCP and QUIC addrs by default --- p2p/net/swarm/swarm_net_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 2ba64edb96..64121bb1b4 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -57,8 +57,8 @@ func TestConnectednessCorrect(t *testing.T) { t.Fatal("expected net 0 to have two peers") } - if len(nets[2].Conns()) != 2 { - t.Fatal("expected net 2 to have two conns") + if len(nets[2].Peers()) != 2 { + t.Fatal("expected net 2 to have two peers") } if len(nets[1].ConnsToPeer(nets[3].LocalPeer())) != 0 { From 0af34e70f0a5010157f329a2f501baa2527035c4 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 23:04:42 +0300 Subject: [PATCH 203/259] don't store the active dial if it errors while starting the worker --- p2p/net/swarm/dial_sync.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 32196cb2d5..ae3578a53a 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -97,13 +97,14 @@ func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { reqch: make(chan DialRequest), ds: ds, } - ds.dials[p] = actd err := ds.dialWorker(adctx, p, actd.reqch) if err != nil { cancel() return nil, err } + + ds.dials[p] = actd } // increase ref count before dropping dialsLk From 302743444250277bf1c4100c794dbb49e4306aa8 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 31 Mar 2021 23:04:49 +0300 Subject: [PATCH 204/259] add TestSelfDial --- p2p/net/swarm/dial_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 86390d6254..26bf47cac1 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -651,3 +651,23 @@ func TestDialSimultaneousJoin(t *testing.T) { t.Fatal("no connection from first dial") } } + +func TestDialSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + swarms := makeSwarms(ctx, t, 2) + s1 := swarms[0] + defer s1.Close() + + _, err := s1.DialPeer(ctx, s1.LocalPeer()) + if err != ErrDialToSelf { + t.Fatal("expected error from self dial") + } + + // do it twice to make sure we get a new active dial object that fails again + _, err = s1.DialPeer(ctx, s1.LocalPeer()) + if err != ErrDialToSelf { + t.Fatal("expected error from self dial") + } +} From 7db6ec9ea181fd49d18ff7c1ff9d9cf5fc179a8a Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 1 Apr 2021 12:13:03 +0300 Subject: [PATCH 205/259] make DialRequest and DialResponse private --- p2p/net/swarm/dial_sync.go | 14 +++++------ p2p/net/swarm/dial_sync_test.go | 20 +++++++--------- p2p/net/swarm/swarm_dial.go | 42 ++++++++++++++++----------------- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index ae3578a53a..24781dd19d 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -12,8 +12,8 @@ import ( // TODO: change this text when we fix the bug var errDialCanceled = errors.New("dial was aborted internally, likely due to https://git.io/Je2wW") -// DialFunc is the type of function expected by DialSync. -type DialWorkerFunc func(context.Context, peer.ID, <-chan DialRequest) error +// DialWorerFunc is used by DialSync to spawn a new dial worker +type DialWorkerFunc func(context.Context, peer.ID, <-chan dialRequest) error // NewDialSync constructs a new DialSync func NewDialSync(worker DialWorkerFunc) *DialSync { @@ -38,7 +38,7 @@ type activeDial struct { ctx context.Context cancel func() - reqch chan DialRequest + reqch chan dialRequest ds *DialSync } @@ -64,16 +64,16 @@ func (ad *activeDial) dial(ctx context.Context, p peer.ID) (*Conn, error) { dialCtx = network.WithSimultaneousConnect(dialCtx, reason) } - resch := make(chan DialResponse, 1) + resch := make(chan dialResponse, 1) select { - case ad.reqch <- DialRequest{Ctx: dialCtx, Resch: resch}: + case ad.reqch <- dialRequest{ctx: dialCtx, resch: resch}: case <-ctx.Done(): return nil, ctx.Err() } select { case res := <-resch: - return res.Conn, res.Err + return res.conn, res.err case <-ctx.Done(): return nil, ctx.Err() } @@ -94,7 +94,7 @@ func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { id: p, ctx: adctx, cancel: cancel, - reqch: make(chan DialRequest), + reqch: make(chan dialRequest), ds: ds, } diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index f1a9f8a539..e414dd529e 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -1,4 +1,4 @@ -package swarm_test +package swarm import ( "context" @@ -7,8 +7,6 @@ import ( "testing" "time" - . "github.com/libp2p/go-libp2p-swarm" - "github.com/libp2p/go-libp2p-core/peer" ) @@ -16,7 +14,7 @@ func getMockDialFunc() (DialWorkerFunc, func(), context.Context, <-chan struct{} dfcalls := make(chan struct{}, 512) // buffer it large enough that we won't care dialctx, cancel := context.WithCancel(context.Background()) ch := make(chan struct{}) - f := func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { + f := func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { dfcalls <- struct{}{} go func() { defer cancel() @@ -29,9 +27,9 @@ func getMockDialFunc() (DialWorkerFunc, func(), context.Context, <-chan struct{} select { case <-ch: - req.Resch <- DialResponse{Conn: new(Conn)} + req.resch <- dialResponse{conn: new(Conn)} case <-ctx.Done(): - req.Resch <- DialResponse{Err: ctx.Err()} + req.resch <- dialResponse{err: ctx.Err()} return } case <-ctx.Done(): @@ -189,7 +187,7 @@ func TestDialSyncAllCancel(t *testing.T) { func TestFailFirst(t *testing.T) { var count int - f := func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { + f := func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { go func() { for { select { @@ -199,9 +197,9 @@ func TestFailFirst(t *testing.T) { } if count > 0 { - req.Resch <- DialResponse{Conn: new(Conn)} + req.resch <- dialResponse{conn: new(Conn)} } else { - req.Resch <- DialResponse{Err: fmt.Errorf("gophers ate the modem")} + req.resch <- dialResponse{err: fmt.Errorf("gophers ate the modem")} } count++ @@ -236,7 +234,7 @@ func TestFailFirst(t *testing.T) { } func TestStressActiveDial(t *testing.T) { - ds := NewDialSync(func(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { + ds := NewDialSync(func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { go func() { for { select { @@ -245,7 +243,7 @@ func TestStressActiveDial(t *testing.T) { return } - req.Resch <- DialResponse{} + req.resch <- dialResponse{} case <-ctx.Done(): return } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 1a5cb1208a..ab95a46127 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -285,18 +285,18 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { // TODO explain how all this works ////////////////////////////////////////////////////////////////////////////////// -type DialRequest struct { - Ctx context.Context - Resch chan DialResponse +type dialRequest struct { + ctx context.Context + resch chan dialResponse } -type DialResponse struct { - Conn *Conn - Err error +type dialResponse struct { + conn *Conn + err error } // dialWorker is an active dial goroutine that synchronizes and executes concurrent dials -func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan DialRequest) error { +func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { if p == s.local { return ErrDialToSelf } @@ -305,11 +305,11 @@ func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan DialRequ return nil } -func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan DialRequest) { +func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan dialRequest) { defer s.limiter.clearAllPeerDials(p) type pendRequest struct { - req DialRequest // the original request + req dialRequest // the original request err *DialError // dial error accumulator addrs map[ma.Multiaddr]struct{} // pending addr dials } @@ -344,11 +344,11 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan Dial // all addrs have erred, dispatch dial error // but first do a last one check in case an acceptable connection has landed from // a simultaneous dial that started later and added new acceptable addrs - c := s.bestAcceptableConnToPeer(pr.req.Ctx, p) + c := s.bestAcceptableConnToPeer(pr.req.ctx, p) if c != nil { - pr.req.Resch <- DialResponse{Conn: c} + pr.req.resch <- dialResponse{conn: c} } else { - pr.req.Resch <- DialResponse{Err: pr.err} + pr.req.resch <- dialResponse{err: pr.err} } delete(requests, reqno) } @@ -390,15 +390,15 @@ loop: return } - c := s.bestAcceptableConnToPeer(req.Ctx, p) + c := s.bestAcceptableConnToPeer(req.ctx, p) if c != nil { - req.Resch <- DialResponse{Conn: c} + req.resch <- dialResponse{conn: c} continue loop } - addrs, err := s.addrsForDial(req.Ctx, p) + addrs, err := s.addrsForDial(req.ctx, p) if err != nil { - req.Resch <- DialResponse{Err: err} + req.resch <- dialResponse{err: err} continue loop } @@ -430,7 +430,7 @@ loop: if ad.conn != nil { // dial to this addr was successful, complete the request - req.Resch <- DialResponse{Conn: ad.conn} + req.resch <- dialResponse{conn: ad.conn} continue loop } @@ -447,7 +447,7 @@ loop: if len(todial) == 0 && len(tojoin) == 0 { // all request applicable addrs have been dialed, we must have errored - req.Resch <- DialResponse{Err: pr.err} + req.resch <- dialResponse{err: pr.err} continue loop } @@ -457,14 +457,14 @@ loop: for _, ad := range tojoin { if !ad.dialed { - ad.ctx = s.mergeDialContexts(ad.ctx, req.Ctx) + ad.ctx = s.mergeDialContexts(ad.ctx, req.ctx) } ad.requests = append(ad.requests, reqno) } if len(todial) > 0 { for _, a := range todial { - pending[a] = &addrDial{addr: a, ctx: req.Ctx, requests: []int{reqno}} + pending[a] = &addrDial{addr: a, ctx: req.ctx, requests: []int{reqno}} } nextDial = append(nextDial, todial...) @@ -550,7 +550,7 @@ loop: continue } - pr.req.Resch <- DialResponse{Conn: conn} + pr.req.resch <- dialResponse{conn: conn} delete(requests, reqno) } From c55e86c4d210bc2802d6694d1da9e255cc3906e9 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 1 Apr 2021 12:16:06 +0300 Subject: [PATCH 206/259] add comment about the necessity of removing the address tracking when a dial backoff occurs --- p2p/net/swarm/swarm_dial.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index ab95a46127..29e3a8d70c 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -356,7 +356,12 @@ func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan dial ad.requests = nil - // if it was a backoff, clear the address dial so that it doesn't inhibit new dial requests + // if it was a backoff, clear the address dial so that it doesn't inhibit new dial requests. + // this is necessary to support active listen scenarios, where a new dial comes in while + // another dial is in progress, and needs to do a direct connection without inhibitions from + // dial backoff. + // it is also necessary to preserve consisent behaviour with the old dialer -- TestDialBackoff + // regresses without this. if err == ErrDialBackoff { delete(pending, ad.addr) } From ace5e258c941df093bcad4ed576b53d55e383c70 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 1 Apr 2021 12:20:25 +0300 Subject: [PATCH 207/259] remove dial batching --- p2p/net/swarm/swarm_dial.go | 47 +++---------------------------------- 1 file changed, 3 insertions(+), 44 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 29e3a8d70c..c656df5a26 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -480,37 +480,17 @@ loop: } case <-triggerDial: - // we dial batches of addresses together, logically belonging to the same batch - // after a batch of addresses has been dialed, we add a delay before initiating the next batch - dialed := false - last := 0 - next := 0 - for i, addr := range nextDial { - if dialed && !s.sameAddrBatch(nextDial[last], addr) { - break - } - - next = i + 1 - + for _, addr := range nextDial { // spawn the dial ad := pending[addr] err := s.dialNextAddr(ad.ctx, p, addr, resch) if err != nil { dispatchError(ad, err) - continue } - - ad.dialed = true - dialed = true - last = i - active++ } - nextDial = nextDial[next:] - if !dialed || len(nextDial) == 0 { - // we didn't dial anything because of backoff or we don't have any more addresses - triggerDial = nil - } + nextDial = nil + triggerDial = nil case res := <-resch: active-- @@ -624,27 +604,6 @@ func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, return nil } -func (s *Swarm) sameAddrBatch(a, b ma.Multiaddr) bool { - // is it a relay addr? - if s.IsRelayAddr(a) { - return s.IsRelayAddr(b) - } - - // is it an expensive addr? - if s.IsExpensiveAddr(a) { - return s.IsExpensiveAddr(b) - } - - // is it a public addr? - if !manet.IsPrivateAddr(a) { - return !manet.IsPrivateAddr(b) && - s.IsFdConsumingAddr(a) == s.IsFdConsumingAddr(b) - } - - // it's a private addr - return manet.IsPrivateAddr(b) -} - func (s *Swarm) canDial(addr ma.Multiaddr) bool { t := s.TransportForDialing(addr) return t != nil && t.CanDial(addr) From c2e44d4b3d6b116577d08d6d813b25c9c6579225 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 1 Apr 2021 12:41:58 +0300 Subject: [PATCH 208/259] add new TestDialSelf so that we exercise the dialWorker dial to self error path --- p2p/net/swarm/dial_sync_test.go | 23 +++++++++++++++++++++++ p2p/net/swarm/dial_test.go | 8 +------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index e414dd529e..e5a7da69eb 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -270,3 +270,26 @@ func TestStressActiveDial(t *testing.T) { wg.Wait() } + +func TestDialSelf(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + self := peer.ID("ABC") + s := NewSwarm(ctx, self, nil, nil) + defer s.Close() + + ds := NewDialSync(s.dialWorker) + + // this should fail + _, err := ds.DialLock(ctx, self) + if err != ErrDialToSelf { + t.Fatal("expected error from self dial") + } + + // do it twice to make sure we get a new active dial object that fails again + _, err = ds.DialLock(ctx, self) + if err != ErrDialToSelf { + t.Fatal("expected error from self dial") + } +} diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 26bf47cac1..2a966a4662 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -652,7 +652,7 @@ func TestDialSimultaneousJoin(t *testing.T) { } } -func TestDialSelf(t *testing.T) { +func TestDialSelf2(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -664,10 +664,4 @@ func TestDialSelf(t *testing.T) { if err != ErrDialToSelf { t.Fatal("expected error from self dial") } - - // do it twice to make sure we get a new active dial object that fails again - _, err = s1.DialPeer(ctx, s1.LocalPeer()) - if err != ErrDialToSelf { - t.Fatal("expected error from self dial") - } } From c7e4304bd428954e742b622582d0fa9bee7c6e46 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 1 Apr 2021 15:36:55 +0300 Subject: [PATCH 209/259] make DialWorkerFunc, NewDialSync private they work with private data types, so there is no point in having them public --- p2p/net/swarm/dial_sync.go | 8 ++++---- p2p/net/swarm/dial_sync_test.go | 18 ++++++++---------- p2p/net/swarm/swarm.go | 2 +- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 24781dd19d..3179016661 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -13,10 +13,10 @@ import ( var errDialCanceled = errors.New("dial was aborted internally, likely due to https://git.io/Je2wW") // DialWorerFunc is used by DialSync to spawn a new dial worker -type DialWorkerFunc func(context.Context, peer.ID, <-chan dialRequest) error +type dialWorkerFunc func(context.Context, peer.ID, <-chan dialRequest) error -// NewDialSync constructs a new DialSync -func NewDialSync(worker DialWorkerFunc) *DialSync { +// newDialSync constructs a new DialSync +func newDialSync(worker dialWorkerFunc) *DialSync { return &DialSync{ dials: make(map[peer.ID]*activeDial), dialWorker: worker, @@ -28,7 +28,7 @@ func NewDialSync(worker DialWorkerFunc) *DialSync { type DialSync struct { dials map[peer.ID]*activeDial dialsLk sync.Mutex - dialWorker DialWorkerFunc + dialWorker dialWorkerFunc } type activeDial struct { diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index e5a7da69eb..59ace9ae67 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -10,7 +10,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -func getMockDialFunc() (DialWorkerFunc, func(), context.Context, <-chan struct{}) { +func getMockDialFunc() (dialWorkerFunc, func(), context.Context, <-chan struct{}) { dfcalls := make(chan struct{}, 512) // buffer it large enough that we won't care dialctx, cancel := context.WithCancel(context.Background()) ch := make(chan struct{}) @@ -48,7 +48,7 @@ func getMockDialFunc() (DialWorkerFunc, func(), context.Context, <-chan struct{} func TestBasicDialSync(t *testing.T) { df, done, _, callsch := getMockDialFunc() - dsync := NewDialSync(df) + dsync := newDialSync(df) p := peer.ID("testpeer") @@ -86,7 +86,7 @@ func TestBasicDialSync(t *testing.T) { func TestDialSyncCancel(t *testing.T) { df, done, _, dcall := getMockDialFunc() - dsync := NewDialSync(df) + dsync := newDialSync(df) p := peer.ID("testpeer") @@ -137,7 +137,7 @@ func TestDialSyncCancel(t *testing.T) { func TestDialSyncAllCancel(t *testing.T) { df, done, dctx, _ := getMockDialFunc() - dsync := NewDialSync(df) + dsync := newDialSync(df) p := peer.ID("testpeer") @@ -211,7 +211,7 @@ func TestFailFirst(t *testing.T) { return nil } - ds := NewDialSync(f) + ds := newDialSync(f) p := peer.ID("testing") @@ -234,7 +234,7 @@ func TestFailFirst(t *testing.T) { } func TestStressActiveDial(t *testing.T) { - ds := NewDialSync(func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { + ds := newDialSync(func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { go func() { for { select { @@ -279,16 +279,14 @@ func TestDialSelf(t *testing.T) { s := NewSwarm(ctx, self, nil, nil) defer s.Close() - ds := NewDialSync(s.dialWorker) - // this should fail - _, err := ds.DialLock(ctx, self) + _, err := s.dsync.DialLock(ctx, self) if err != ErrDialToSelf { t.Fatal("expected error from self dial") } // do it twice to make sure we get a new active dial object that fails again - _, err = ds.DialLock(ctx, self) + _, err = s.dsync.DialLock(ctx, self) if err != ErrDialToSelf { t.Fatal("expected error from self dial") } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c57c563c25..95b164d13c 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -121,7 +121,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc } } - s.dsync = NewDialSync(s.dialWorker) + s.dsync = newDialSync(s.dialWorker) s.limiter = newDialLimiter(s.dialAddr, s.IsFdConsumingAddr) s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) From bc47c2f657d3b983179a59a76702717e405e91de Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 1 Apr 2021 23:11:25 +0300 Subject: [PATCH 210/259] rename dialWorker to startDialWorker --- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_dial.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 95b164d13c..1abe12273e 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -121,7 +121,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc } } - s.dsync = newDialSync(s.dialWorker) + s.dsync = newDialSync(s.startDialWorker) s.limiter = newDialLimiter(s.dialAddr, s.IsFdConsumingAddr) s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index c656df5a26..d60c9f39bb 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -295,8 +295,8 @@ type dialResponse struct { err error } -// dialWorker is an active dial goroutine that synchronizes and executes concurrent dials -func (s *Swarm) dialWorker(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { +// startDialWorker starts an active dial goroutine that synchronizes and executes concurrent dials +func (s *Swarm) startDialWorker(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { if p == s.local { return ErrDialToSelf } From 4f223e98dfc9861e3b449ef17ee7a0eda4c7b349 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 1 Apr 2021 23:14:59 +0300 Subject: [PATCH 211/259] make addr utility funcs standalone and not exported --- p2p/net/swarm/swarm_dial.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index d60c9f39bb..3cc3275ef8 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -621,10 +621,10 @@ func (s *Swarm) nonProxyAddr(addr ma.Multiaddr) bool { // UDP > TCP func (s *Swarm) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { addrTier := func(a ma.Multiaddr) (tier int) { - if s.IsRelayAddr(a) { + if isRelayAddr(a) { tier |= 0b1000 } - if s.IsExpensiveAddr(a) { + if isExpensiveAddr(a) { tier |= 0b0100 } if !manet.IsPrivateAddr(a) { @@ -741,13 +741,13 @@ func (s *Swarm) IsFdConsumingAddr(addr ma.Multiaddr) bool { return err1 == nil || err2 == nil } -func (s *Swarm) IsExpensiveAddr(addr ma.Multiaddr) bool { +func isExpensiveAddr(addr ma.Multiaddr) bool { _, err1 := addr.ValueForProtocol(ma.P_WS) _, err2 := addr.ValueForProtocol(ma.P_WSS) return err1 == nil || err2 == nil } -func (s *Swarm) IsRelayAddr(addr ma.Multiaddr) bool { +func isRelayAddr(addr ma.Multiaddr) bool { _, err := addr.ValueForProtocol(ma.P_CIRCUIT) return err == nil } From 78037d70cb2c5d1205bc07f4a3efa1c6e7d6a7fe Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 1 Apr 2021 23:31:29 +0300 Subject: [PATCH 212/259] make IsFdConsumingAddr a standalone utility func --- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_dial.go | 4 +-- p2p/net/swarm/swarm_test.go | 48 --------------------------------- p2p/net/swarm/util_test.go | 53 +++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 51 deletions(-) create mode 100644 p2p/net/swarm/util_test.go diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 1abe12273e..00c43f7a6f 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -122,7 +122,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc } s.dsync = newDialSync(s.startDialWorker) - s.limiter = newDialLimiter(s.dialAddr, s.IsFdConsumingAddr) + s.limiter = newDialLimiter(s.dialAddr, isFdConsumingAddr) s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) s.backf.init(s.ctx) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 3cc3275ef8..14129257be 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -630,7 +630,7 @@ func (s *Swarm) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { if !manet.IsPrivateAddr(a) { tier |= 0b0010 } - if s.IsFdConsumingAddr(a) { + if isFdConsumingAddr(a) { tier |= 0b0001 } @@ -726,7 +726,7 @@ func (s *Swarm) dialAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr) (tra // A Non-circuit address which has the TCP/UNIX protocol is deemed FD consuming. // For a circuit-relay address, we look at the address of the relay server/proxy // and use the same logic as above to decide. -func (s *Swarm) IsFdConsumingAddr(addr ma.Multiaddr) bool { +func isFdConsumingAddr(addr ma.Multiaddr) bool { first, _ := ma.SplitFunc(addr, func(c ma.Component) bool { return c.Protocol().Code == ma.P_CIRCUIT }) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 9b1e9c42d7..4e9801ad08 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -15,7 +15,6 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" - "github.com/libp2p/go-libp2p-core/test" . "github.com/libp2p/go-libp2p-swarm" . "github.com/libp2p/go-libp2p-swarm/testing" @@ -387,53 +386,6 @@ func TestConnectionGating(t *testing.T) { } } -func TestIsFdConsuming(t *testing.T) { - tcs := map[string]struct { - addr string - isFdConsuming bool - }{ - "tcp": { - addr: "/ip4/127.0.0.1/tcp/20", - isFdConsuming: true, - }, - "quic": { - addr: "/ip4/127.0.0.1/udp/0/quic", - isFdConsuming: false, - }, - "addr-without-registered-transport": { - addr: "/ip4/127.0.0.1/tcp/20/ws", - isFdConsuming: true, - }, - "relay-tcp": { - addr: fmt.Sprintf("/ip4/127.0.0.1/tcp/20/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), - isFdConsuming: true, - }, - "relay-quic": { - addr: fmt.Sprintf("/ip4/127.0.0.1/udp/20/quic/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), - isFdConsuming: false, - }, - "relay-without-serveraddr": { - addr: fmt.Sprintf("/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), - isFdConsuming: true, - }, - "relay-without-registered-transport-server": { - addr: fmt.Sprintf("/ip4/127.0.0.1/tcp/20/ws/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), - isFdConsuming: true, - }, - } - - ctx := context.Background() - sw := GenSwarm(t, ctx) - sk := sw.Peerstore().PrivKey(sw.LocalPeer()) - require.NotNil(t, sk) - - for name := range tcs { - maddr, err := ma.NewMultiaddr(tcs[name].addr) - require.NoError(t, err, name) - require.Equal(t, tcs[name].isFdConsuming, sw.IsFdConsumingAddr(maddr), name) - } -} - func TestNoDial(t *testing.T) { ctx := context.Background() swarms := makeSwarms(ctx, t, 2) diff --git a/p2p/net/swarm/util_test.go b/p2p/net/swarm/util_test.go new file mode 100644 index 0000000000..11124adb27 --- /dev/null +++ b/p2p/net/swarm/util_test.go @@ -0,0 +1,53 @@ +package swarm + +import ( + "fmt" + "testing" + + "github.com/libp2p/go-libp2p-core/test" + ma "github.com/multiformats/go-multiaddr" + + "github.com/stretchr/testify/require" +) + +func TestIsFdConsuming(t *testing.T) { + tcs := map[string]struct { + addr string + isFdConsuming bool + }{ + "tcp": { + addr: "/ip4/127.0.0.1/tcp/20", + isFdConsuming: true, + }, + "quic": { + addr: "/ip4/127.0.0.1/udp/0/quic", + isFdConsuming: false, + }, + "addr-without-registered-transport": { + addr: "/ip4/127.0.0.1/tcp/20/ws", + isFdConsuming: true, + }, + "relay-tcp": { + addr: fmt.Sprintf("/ip4/127.0.0.1/tcp/20/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), + isFdConsuming: true, + }, + "relay-quic": { + addr: fmt.Sprintf("/ip4/127.0.0.1/udp/20/quic/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), + isFdConsuming: false, + }, + "relay-without-serveraddr": { + addr: fmt.Sprintf("/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), + isFdConsuming: true, + }, + "relay-without-registered-transport-server": { + addr: fmt.Sprintf("/ip4/127.0.0.1/tcp/20/ws/p2p-circuit/p2p/%s", test.RandPeerIDFatal(t)), + isFdConsuming: true, + }, + } + + for name := range tcs { + maddr, err := ma.NewMultiaddr(tcs[name].addr) + require.NoError(t, err, name) + require.Equal(t, tcs[name].isFdConsuming, isFdConsumingAddr(maddr), name) + } +} From fbba08216e7b996a74e28fce626f603eab480e80 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 14 Apr 2021 19:19:58 -0700 Subject: [PATCH 213/259] fix go vet --- p2p/net/swarm/limiter_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 367e099fd9..5999ff10a6 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -2,6 +2,7 @@ package swarm import ( "context" + "errors" "fmt" "math/rand" "strconv" @@ -379,8 +380,12 @@ func TestFDLimitUnderflow(t *testing.T) { addrs = append(addrs, addrWithPort(t, i)) } + wg := sync.WaitGroup{} + wg.Add(1000) + errs := make(chan error, 1000) for i := 0; i < 1000; i++ { go func(id peer.ID, i int) { + defer wg.Done() ctx, cancel := context.WithCancel(context.Background()) resp := make(chan dialResult) @@ -406,12 +411,19 @@ func TestFDLimitUnderflow(t *testing.T) { if res.Err != nil { return } - t.Fatal("got dial res, shouldn't") + errs <- errors.New("got dial res, but shouldn't") } }(peer.ID(fmt.Sprintf("testpeer%d", i%20)), i) } - time.Sleep(time.Second * 3) + go func() { + wg.Wait() + close(errs) + }() + + for err := range errs { + t.Fatal(err) + } l.lk.Lock() fdConsuming := l.fdConsuming From 8dc70f2421de38865b6d86cf40125ff30151abc9 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 14 Apr 2021 21:34:52 -0700 Subject: [PATCH 214/259] more tests after merge --- p2p/net/swarm/dial_test.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 2a966a4662..1bad4d9f3f 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -583,6 +583,7 @@ func TestDialSimultaneousJoin(t *testing.T) { go acceptAndHang(s2silentListener) connch := make(chan network.Conn, 512) + errs := make(chan error, 2) // start a dial to s2 through the silent addr go func() { @@ -590,12 +591,15 @@ func TestDialSimultaneousJoin(t *testing.T) { c, err := s1.DialPeer(ctx, s2.LocalPeer()) if err != nil { - t.Fatal(err) + errs <- err + connch <- nil + return } t.Logf("first dial succedded; conn: %+v", c) connch <- c + errs <- nil }() // wait a bit for the dial to take hold @@ -605,18 +609,22 @@ func TestDialSimultaneousJoin(t *testing.T) { go func() { s2addrs, err := s2.InterfaceListenAddresses() if err != nil { - t.Fatal(err) + errs <- err + return } s1.Peerstore().AddAddrs(s2.LocalPeer(), s2addrs[:1], peerstore.PermanentAddrTTL) c, err := s1.DialPeer(ctx, s2.LocalPeer()) if err != nil { - t.Fatal(err) + errs <- err + connch <- nil + return } t.Logf("second dial succedded; conn: %+v", c) connch <- c + errs <- nil }() // wait for the second dial to finish @@ -626,16 +634,27 @@ func TestDialSimultaneousJoin(t *testing.T) { go func() { c, err := s1.DialPeer(ctx, s2.LocalPeer()) if err != nil { - t.Fatal(err) + errs <- err + connch <- nil + return } t.Logf("third dial succedded; conn: %+v", c) connch <- c + errs <- nil }() c3 := <-connch + // raise any errors from the two + for i := 0; i < 3; i++ { + err := <-errs + if err != nil { + t.Fatal(err) + } + } + if c2 != c3 { t.Fatal("expected c2 and c3 to be the same") } From b00734a0b3e6c525543384fa13838689209a8ed9 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 14 Apr 2021 21:36:09 -0700 Subject: [PATCH 215/259] typeo --- p2p/net/swarm/dial_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 1bad4d9f3f..70eeb9f51d 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -647,7 +647,7 @@ func TestDialSimultaneousJoin(t *testing.T) { c3 := <-connch - // raise any errors from the two + // raise any errors from the previous goroutines for i := 0; i < 3; i++ { err := <-errs if err != nil { From 6cc76d55e57952d64d981fa9fffe83c00cebe225 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 14 Apr 2021 19:40:39 -0700 Subject: [PATCH 216/259] fix staticcheck --- p2p/net/swarm/addrs.go | 3 ++- p2p/net/swarm/dial_sync.go | 4 ---- p2p/net/swarm/dial_test.go | 5 +++-- p2p/net/swarm/peers_test.go | 4 ++-- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_dial.go | 8 +------- p2p/net/swarm/swarm_listen.go | 4 ++-- p2p/net/swarm/swarm_test.go | 3 +-- 8 files changed, 12 insertions(+), 21 deletions(-) diff --git a/p2p/net/swarm/addrs.go b/p2p/net/swarm/addrs.go index 17c65ae7b2..b9068720c5 100644 --- a/p2p/net/swarm/addrs.go +++ b/p2p/net/swarm/addrs.go @@ -1,6 +1,7 @@ package swarm import ( + filter "github.com/libp2p/go-maddr-filter" ma "github.com/multiformats/go-multiaddr" mamask "github.com/whyrusleeping/multiaddr-filter" ) @@ -30,6 +31,6 @@ func init() { if err != nil { panic("error in lowTimeoutFilters init: " + err.Error()) } - lowTimeoutFilters.AddDialFilter(f) + lowTimeoutFilters.AddFilter(*f, filter.ActionDeny) } } diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 3179016661..e334ef5a25 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -2,16 +2,12 @@ package swarm import ( "context" - "errors" "sync" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" ) -// TODO: change this text when we fix the bug -var errDialCanceled = errors.New("dial was aborted internally, likely due to https://git.io/Je2wW") - // DialWorerFunc is used by DialSync to spawn a new dial worker type dialWorkerFunc func(context.Context, peer.ID, <-chan dialRequest) error diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 70eeb9f51d..6258d0eda1 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -437,9 +437,10 @@ func TestDialBackoffClears(t *testing.T) { s1.Peerstore().AddAddr(s2.LocalPeer(), s2bad, peerstore.PermanentAddrTTL) before := time.Now() - if c, err := s1.DialPeer(ctx, s2.LocalPeer()); err == nil { - t.Fatal("dialing to broken addr worked...", err) + c, err := s1.DialPeer(ctx, s2.LocalPeer()) + if err == nil { defer c.Close() + t.Fatal("dialing to broken addr worked...", err) } else { t.Log("correctly got error:", err) } diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index fd1984d072..8e82bf5bde 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -33,8 +33,8 @@ func TestPeers(t *testing.T) { // t.Log(s.swarm.Dump()) } - s1GotConn := make(chan struct{}, 0) - s2GotConn := make(chan struct{}, 0) + s1GotConn := make(chan struct{}) + s2GotConn := make(chan struct{}) s1.SetConnHandler(func(c network.Conn) { s1GotConn <- struct{}{} }) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index d42020a89d..de7a9350b6 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -512,7 +512,7 @@ func (s *Swarm) ClosePeer(p peer.ID) error { } var errs []string - for _ = range conns { + for range conns { err := <-errCh if err != nil { errs = append(errs, err.Error()) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 14129257be..68d8cfecb8 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -12,8 +12,6 @@ import ( "github.com/libp2p/go-libp2p-core/transport" addrutil "github.com/libp2p/go-addr-util" - lgbl "github.com/libp2p/go-libp2p-loggables" - ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) @@ -236,20 +234,16 @@ func (s *Swarm) DialPeer(ctx context.Context, p peer.ID) (network.Conn, error) { // It is gated by the swarm's dial synchronization systems: dialsync and // dialbackoff. func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { - log.Debugf("[%s] swarm dialing peer [%s]", s.local, p) - var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) + log.Debugw("dialing peer", "from", s.local, "to", p) err := p.Validate() if err != nil { return nil, err } if p == s.local { - log.Event(ctx, "swarmDialSelf", logdial) return nil, ErrDialToSelf } - defer log.EventBegin(ctx, "swarmDialAttemptSync", p).Done() - // check if we already have an open (usable) connection first conn := s.bestAcceptableConnToPeer(ctx, p) if conn != nil { diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index 5bd1015d99..c064ae851e 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -24,7 +24,7 @@ func (s *Swarm) Listen(addrs ...ma.Multiaddr) error { for i, e := range errs { if e != nil { - log.Warningf("listen on %s failed: %s", addrs[i], errs[i]) + log.Warnw("listening failed", "on", addrs[i], "error", errs[i]) } } @@ -111,7 +111,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { // ignore. return default: - log.Warningf("add conn %s failed: ", err) + log.Warnw("adding connection failed", "to", a, "error", err) return } }() diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 4e9801ad08..6aad60006b 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -169,8 +169,6 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { count := 0 countShouldBe := MsgNum * (len(swarms) - 1) for stream := range streamChan { // one per peer - defer stream.Close() - // get peer on the other side p := stream.Conn().RemotePeer() @@ -196,6 +194,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { got[p] = msgCount count += msgCount + stream.Close() } if count != countShouldBe { From 98d7eef9507afee19e12a1c13946a0addda125df Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 20 Apr 2021 11:36:20 -0700 Subject: [PATCH 217/259] fix: avoid returning typed nils --- p2p/net/swarm/swarm_dial.go | 7 ++++++- p2p/net/swarm/swarm_test.go | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 68d8cfecb8..e130c18301 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -226,7 +226,12 @@ func (s *Swarm) DialPeer(ctx context.Context, p peer.ID) (network.Conn, error) { return nil, &DialError{Peer: p, Cause: ErrGaterDisallowedConnection} } - return s.dialPeer(ctx, p) + // Avoid typed nil issues. + c, err := s.dialPeer(ctx, p) + if err != nil { + return nil, err + } + return c, nil } // internal dial method that returns an unwrapped conn diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 6aad60006b..97ea90762d 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -412,6 +412,19 @@ func TestCloseWithOpenStreams(t *testing.T) { } } +func TestTypedNilConn(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := GenSwarm(t, ctx) + defer s.Close() + + // We can't dial ourselves. + c, err := s.DialPeer(ctx, s.LocalPeer()) + require.Error(t, err) + // If we fail to dial, the connection should be nil. + require.True(t, c == nil) +} + func TestPreventDialListenAddr(t *testing.T) { s := GenSwarm(t, context.Background(), OptDialOnly) if err := s.Listen(ma.StringCast("/ip4/0.0.0.0/udp/0/quic")); err != nil { From 67f91a0be9dc913ae9faa6a495cc8ddec6a2dabd Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 23 Apr 2021 10:25:57 +0700 Subject: [PATCH 218/259] run connection gating tests on both TCP and QUIC Also disables one test that doesn't work with QUIC, as we always complete the handshake before gating there. --- p2p/net/swarm/swarm_test.go | 57 ++++++++++++++++++------------ p2p/net/swarm/testing/testing.go | 60 ++++++++++++++++++++------------ 2 files changed, 72 insertions(+), 45 deletions(-) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 97ea90762d..739afe9182 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -292,6 +292,7 @@ func TestConnectionGating(t *testing.T) { p1ConnectednessToP2 network.Connectedness p2ConnectednessToP1 network.Connectedness isP1OutboundErr bool + disableOnQUIC bool }{ "no gating": { p1ConnectednessToP2: network.Connected, @@ -324,6 +325,8 @@ func TestConnectionGating(t *testing.T) { p1ConnectednessToP2: network.NotConnected, p2ConnectednessToP1: network.NotConnected, isP1OutboundErr: true, + // QUIC gates the connection after completion of the handshake + disableOnQUIC: true, }, "p2 gates inbound peer dial before multiplexing": { p1Gater: func(c *MockConnectionGater) *MockConnectionGater { @@ -355,33 +358,43 @@ func TestConnectionGating(t *testing.T) { } for n, tc := range tcs { - t.Run(n, func(t *testing.T) { - p1Gater := DefaultMockConnectionGater() - p2Gater := DefaultMockConnectionGater() - if tc.p1Gater != nil { - p1Gater = tc.p1Gater(p1Gater) - } - if tc.p2Gater != nil { - p2Gater = tc.p2Gater(p2Gater) + for _, useQuic := range []bool{false, true} { + trString := "TCP" + optTransport := OptDisableQUIC + if useQuic { + if tc.disableOnQUIC { + continue + } + trString = "QUIC" + optTransport = OptDisableTCP } + t.Run(fmt.Sprintf("%s %s", n, trString), func(t *testing.T) { + p1Gater := DefaultMockConnectionGater() + p2Gater := DefaultMockConnectionGater() + if tc.p1Gater != nil { + p1Gater = tc.p1Gater(p1Gater) + } + if tc.p2Gater != nil { + p2Gater = tc.p2Gater(p2Gater) + } - sw1 := GenSwarm(t, ctx, OptConnGater(p1Gater)) - sw2 := GenSwarm(t, ctx, OptConnGater(p2Gater)) - - p1 := sw1.LocalPeer() - p2 := sw2.LocalPeer() - sw1.Peerstore().AddAddr(p2, sw2.ListenAddresses()[0], peerstore.PermanentAddrTTL) - // 1 -> 2 - _, err := sw1.DialPeer(ctx, p2) + sw1 := GenSwarm(t, ctx, OptConnGater(p1Gater), optTransport) + sw2 := GenSwarm(t, ctx, OptConnGater(p2Gater), optTransport) - require.Equal(t, tc.isP1OutboundErr, err != nil, n) - require.Equal(t, tc.p1ConnectednessToP2, sw1.Connectedness(p2), n) + p1 := sw1.LocalPeer() + p2 := sw2.LocalPeer() + sw1.Peerstore().AddAddr(p2, sw2.ListenAddresses()[0], peerstore.PermanentAddrTTL) + // 1 -> 2 + _, err := sw1.DialPeer(ctx, p2) - require.Eventually(t, func() bool { - return tc.p2ConnectednessToP1 == sw2.Connectedness(p1) - }, 2*time.Second, 100*time.Millisecond, n) - }) + require.Equal(t, tc.isP1OutboundErr, err != nil, n) + require.Equal(t, tc.p1ConnectednessToP2, sw1.Connectedness(p2), n) + require.Eventually(t, func() bool { + return tc.p2ConnectednessToP1 == sw2.Connectedness(p1) + }, 2*time.Second, 100*time.Millisecond, n) + }) + } } } diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index eb1e1e42b6..313e3d771e 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -4,6 +4,7 @@ import ( "context" "testing" + csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/control" "github.com/libp2p/go-libp2p-core/crypto" @@ -12,10 +13,8 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/sec/insecure" + "github.com/libp2p/go-libp2p-peerstore/pstoremem" quic "github.com/libp2p/go-libp2p-quic-transport" - - csms "github.com/libp2p/go-conn-security-multistream" - pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" swarm "github.com/libp2p/go-libp2p-swarm" "github.com/libp2p/go-libp2p-testing/net" tptu "github.com/libp2p/go-libp2p-transport-upgrader" @@ -23,13 +22,15 @@ import ( msmux "github.com/libp2p/go-stream-muxer-multistream" "github.com/libp2p/go-tcp-transport" - goprocess "github.com/jbenet/goprocess" + "github.com/jbenet/goprocess" ma "github.com/multiformats/go-multiaddr" ) type config struct { disableReuseport bool dialOnly bool + disableTCP bool + disableQUIC bool connectionGater connmgr.ConnectionGater sk crypto.PrivKey } @@ -47,6 +48,16 @@ var OptDialOnly Option = func(_ *testing.T, c *config) { c.dialOnly = true } +// OptDisableTCP disables TCP. +var OptDisableTCP Option = func(_ *testing.T, c *config) { + c.disableTCP = true +} + +// OptDisableQUIC disables QUIC. +var OptDisableQUIC Option = func(_ *testing.T, c *config) { + c.disableQUIC = true +} + // OptConnGater configures the given connection gater on the test func OptConnGater(cg connmgr.ConnectionGater) Option { return func(_ *testing.T, c *config) { @@ -111,33 +122,36 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { upgrader := GenUpgrader(s) upgrader.ConnGater = cfg.connectionGater - tcpTransport := tcp.NewTCPTransport(upgrader) - tcpTransport.DisableReuseport = cfg.disableReuseport - - quicTransport, err := quic.NewTransport(p.PrivKey, nil, nil) - if err != nil { - t.Fatal(err) - } - - if err := s.AddTransport(tcpTransport); err != nil { - t.Fatal(err) - } - if err := s.AddTransport(quicTransport); err != nil { - t.Fatal(err) + if !cfg.disableTCP { + tcpTransport := tcp.NewTCPTransport(upgrader) + tcpTransport.DisableReuseport = cfg.disableReuseport + if err := s.AddTransport(tcpTransport); err != nil { + t.Fatal(err) + } + if !cfg.dialOnly { + if err := s.Listen(p.Addr); err != nil { + t.Fatal(err) + } + } } - - if !cfg.dialOnly { - if err := s.Listen(p.Addr); err != nil { + if !cfg.disableQUIC { + quicTransport, err := quic.NewTransport(p.PrivKey, nil, cfg.connectionGater) + if err != nil { t.Fatal(err) } - if err := s.Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/quic")); err != nil { + if err := s.AddTransport(quicTransport); err != nil { t.Fatal(err) } - + if !cfg.dialOnly { + if err := s.Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/quic")); err != nil { + t.Fatal(err) + } + } + } + if !cfg.dialOnly { s.Peerstore().AddAddrs(p.ID, s.ListenAddresses(), peerstore.PermanentAddrTTL) } - return s } From 168f8abb883e7c19f76665b755ab69a96bfcb838 Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Fri, 30 Apr 2021 18:29:52 -0400 Subject: [PATCH 219/259] sync: update CI config files (#248) --- p2p/net/swarm/swarm_net_test.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 64121bb1b4..05984f6b02 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -3,6 +3,7 @@ package swarm_test import ( "context" "fmt" + "io/ioutil" "testing" "time" @@ -108,6 +109,8 @@ func TestNetworkOpenStream(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + testString := "hello ipfs" + nets := make([]network.Network, 4) for i := 0; i < 4; i++ { nets[i] = GenSwarm(t, ctx) @@ -129,13 +132,12 @@ func TestNetworkOpenStream(t *testing.T) { defer close(done) defer s.Close() - buf := make([]byte, 10) - _, err := s.Read(buf) + buf, err := ioutil.ReadAll(s) if err != nil { t.Error(err) return } - if string(buf) != "hello ipfs" { + if string(buf) != testString { t.Error("got wrong message") } }) @@ -145,18 +147,20 @@ func TestNetworkOpenStream(t *testing.T) { t.Fatal(err) } - streams := nets[0].ConnsToPeer(nets[1].LocalPeer())[0].GetStreams() - if err != nil { - t.Fatal(err) + numStreams := 0 + for _, conn := range nets[0].ConnsToPeer(nets[1].LocalPeer()) { + numStreams += len(conn.GetStreams()) } - if len(streams) != 1 { + if numStreams != 1 { t.Fatal("should only have one stream there") } - _, err = s.Write([]byte("hello ipfs")) + n, err := s.Write([]byte(testString)) if err != nil { t.Fatal(err) + } else if n != len(testString) { + t.Errorf("expected to write %d bytes, wrote %d", len(testString), n) } err = s.Close() From 048c5ddc49b286b55b3271516bc402e619537ba4 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 18 May 2021 11:29:57 -0700 Subject: [PATCH 220/259] remove incorrect call to InterceptAddrDial addConn is called both when we add a dialed and an accepted connection to the swarm. InterceptAddrDial is only supposed to intercept outgoing connections though. When dialing, we already call InterceptAddrDial when we compose the list of dialable addresses. --- p2p/net/swarm/swarm.go | 10 ---------- p2p/net/swarm/swarm_test.go | 9 +++++++++ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index de7a9350b6..e975548fe1 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -212,16 +212,6 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, addr = tc.RemoteMultiaddr() ) - if s.gater != nil { - if allow := s.gater.InterceptAddrDial(p, addr); !allow { - err := tc.Close() - if err != nil { - log.Warnf("failed to close connection with peer %s and addr %s; err: %s", p.Pretty(), addr, err) - } - return nil, ErrAddrFiltered - } - } - // create the Stat object, initializing with the underlying connection Stat if available var stat network.Stat if cs, ok := tc.(network.ConnStat); ok { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 739afe9182..a94281b1f5 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -317,6 +317,15 @@ func TestConnectionGating(t *testing.T) { p2ConnectednessToP1: network.NotConnected, isP1OutboundErr: true, }, + "p2 accepts inbound peer dial if outgoing dial is gated": { + p2Gater: func(c *MockConnectionGater) *MockConnectionGater { + c.Dial = func(peer.ID, ma.Multiaddr) bool { return false } + return c + }, + p1ConnectednessToP2: network.Connected, + p2ConnectednessToP1: network.Connected, + isP1OutboundErr: false, + }, "p2 gates inbound peer dial before securing": { p2Gater: func(c *MockConnectionGater) *MockConnectionGater { c.Accept = func(c network.ConnMultiaddrs) bool { return false } From d965b92478ef42e5cb5c25b3fc0476cd78ebed2f Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 19 May 2021 15:25:45 -0700 Subject: [PATCH 221/259] speed up the TestFDLimitUnderflow test We're setting a limit of 20 fds, so we don't need to test 1000 dials. --- p2p/net/swarm/limiter_test.go | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 5999ff10a6..1aefffec6d 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -358,22 +358,16 @@ func TestStressLimiter(t *testing.T) { } func TestFDLimitUnderflow(t *testing.T) { - df := func(ctx context.Context, p peer.ID, a ma.Multiaddr) (transport.CapableConn, error) { - timeout := make(chan bool, 1) - go func() { - time.Sleep(time.Second * 5) - timeout <- true - }() - + df := func(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.CapableConn, error) { select { case <-ctx.Done(): - case <-timeout: + case <-time.After(5 * time.Second): } - return nil, fmt.Errorf("df timed out") } - l := newDialLimiterWithParams(isFdConsuming, df, 20, 3) + const fdLimit = 20 + l := newDialLimiterWithParams(isFdConsuming, df, fdLimit, 3) var addrs []ma.Multiaddr for i := 0; i <= 1000; i++ { @@ -381,12 +375,14 @@ func TestFDLimitUnderflow(t *testing.T) { } wg := sync.WaitGroup{} - wg.Add(1000) - errs := make(chan error, 1000) - for i := 0; i < 1000; i++ { + const num = 3 * fdLimit + wg.Add(num) + errs := make(chan error, num) + for i := 0; i < num; i++ { go func(id peer.ID, i int) { defer wg.Done() ctx, cancel := context.WithCancel(context.Background()) + defer cancel() resp := make(chan dialResult) l.AddDialJob(&dialJob{ @@ -396,17 +392,6 @@ func TestFDLimitUnderflow(t *testing.T) { resp: resp, }) - //cancel first 60 after 1s, next 60 after 2s - if i > 60 { - time.Sleep(time.Second * 1) - } - if i < 120 { - time.Sleep(time.Second * 1) - cancel() - return - } - defer cancel() - for res := range resp { if res.Err != nil { return From b6f93bdc536e3363808fbceb6bec35cc03415cd6 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 27 Jul 2021 12:38:27 +0200 Subject: [PATCH 222/259] wait longer for connection establishment in TestNotifications --- p2p/net/swarm/swarm_notif_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 8537363a19..3383617218 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -39,7 +39,7 @@ func TestNotifications(t *testing.T) { } }() - timeout := 5 * time.Second + const timeout = 5 * time.Second // signup notifs for i, swarm := range swarms { @@ -50,7 +50,7 @@ func TestNotifications(t *testing.T) { connectSwarms(t, ctx, swarms) - <-time.After(time.Millisecond) + time.Sleep(50 * time.Millisecond) // should've gotten 5 by now. // test everyone got the correct connection opened calls From 258e1e698b5b7b4effb15ba8a6ac98cf26a948a6 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 22 Aug 2021 10:16:39 +0100 Subject: [PATCH 223/259] fix race condition in TestFailFirst --- p2p/net/swarm/dial_sync_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 59ace9ae67..71c03b0da6 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "sync/atomic" "testing" "time" @@ -186,7 +187,7 @@ func TestDialSyncAllCancel(t *testing.T) { } func TestFailFirst(t *testing.T) { - var count int + var count int32 f := func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { go func() { for { @@ -196,12 +197,12 @@ func TestFailFirst(t *testing.T) { return } - if count > 0 { + if atomic.LoadInt32(&count) > 0 { req.resch <- dialResponse{conn: new(Conn)} } else { req.resch <- dialResponse{err: fmt.Errorf("gophers ate the modem")} } - count++ + atomic.AddInt32(&count, 1) case <-ctx.Done(): return From 8252673e1d25625be93dd3b26c6add079e4bd369 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 22 Aug 2021 10:06:02 +0100 Subject: [PATCH 224/259] remove unused context in Swarm.dialWorkerLoop --- p2p/net/swarm/dial_sync.go | 5 +- p2p/net/swarm/dial_sync_test.go | 94 ++++++++++----------------------- p2p/net/swarm/swarm_dial.go | 6 +-- 3 files changed, 34 insertions(+), 71 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index e334ef5a25..03ba9cc089 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -9,7 +9,7 @@ import ( ) // DialWorerFunc is used by DialSync to spawn a new dial worker -type dialWorkerFunc func(context.Context, peer.ID, <-chan dialRequest) error +type dialWorkerFunc func(peer.ID, <-chan dialRequest) error // newDialSync constructs a new DialSync func newDialSync(worker dialWorkerFunc) *DialSync { @@ -94,8 +94,7 @@ func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { ds: ds, } - err := ds.dialWorker(adctx, p, actd.reqch) - if err != nil { + if err := ds.dialWorker(p, actd.reqch); err != nil { cancel() return nil, err } diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 71c03b0da6..441c8b7de9 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -15,58 +15,37 @@ func getMockDialFunc() (dialWorkerFunc, func(), context.Context, <-chan struct{} dfcalls := make(chan struct{}, 512) // buffer it large enough that we won't care dialctx, cancel := context.WithCancel(context.Background()) ch := make(chan struct{}) - f := func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { + f := func(p peer.ID, reqch <-chan dialRequest) error { + defer cancel() dfcalls <- struct{}{} go func() { - defer cancel() - for { - select { - case req, ok := <-reqch: - if !ok { - return - } - - select { - case <-ch: - req.resch <- dialResponse{conn: new(Conn)} - case <-ctx.Done(): - req.resch <- dialResponse{err: ctx.Err()} - return - } - case <-ctx.Done(): - return - } + for req := range reqch { + <-ch + req.resch <- dialResponse{conn: new(Conn)} } }() return nil } - o := new(sync.Once) - - return f, func() { o.Do(func() { close(ch) }) }, dialctx, dfcalls + var once sync.Once + return f, func() { once.Do(func() { close(ch) }) }, dialctx, dfcalls } func TestBasicDialSync(t *testing.T) { df, done, _, callsch := getMockDialFunc() - dsync := newDialSync(df) - p := peer.ID("testpeer") - ctx := context.Background() - - finished := make(chan struct{}) + finished := make(chan struct{}, 2) go func() { - _, err := dsync.DialLock(ctx, p) - if err != nil { + if _, err := dsync.DialLock(context.Background(), p); err != nil { t.Error(err) } finished <- struct{}{} }() go func() { - _, err := dsync.DialLock(ctx, p) - if err != nil { + if _, err := dsync.DialLock(context.Background(), p); err != nil { t.Error(err) } finished <- struct{}{} @@ -139,15 +118,12 @@ func TestDialSyncAllCancel(t *testing.T) { df, done, dctx, _ := getMockDialFunc() dsync := newDialSync(df) - p := peer.ID("testpeer") - - ctx1, cancel1 := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) finished := make(chan struct{}) go func() { - _, err := dsync.DialLock(ctx1, p) - if err != ctx1.Err() { + if _, err := dsync.DialLock(ctx, p); err != ctx.Err() { t.Error("should have gotten context error") } finished <- struct{}{} @@ -155,14 +131,13 @@ func TestDialSyncAllCancel(t *testing.T) { // Add a second dialwait in so two actors are waiting on the same dial go func() { - _, err := dsync.DialLock(ctx1, p) - if err != ctx1.Err() { + if _, err := dsync.DialLock(ctx, p); err != ctx.Err() { t.Error("should have gotten context error") } finished <- struct{}{} }() - cancel1() + cancel() for i := 0; i < 2; i++ { select { case <-finished: @@ -180,33 +155,27 @@ func TestDialSyncAllCancel(t *testing.T) { // should be able to successfully dial that peer again done() - _, err := dsync.DialLock(context.Background(), p) - if err != nil { + if _, err := dsync.DialLock(context.Background(), p); err != nil { t.Fatal(err) } } func TestFailFirst(t *testing.T) { var count int32 - f := func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { + f := func(p peer.ID, reqch <-chan dialRequest) error { go func() { for { - select { - case req, ok := <-reqch: - if !ok { - return - } - - if atomic.LoadInt32(&count) > 0 { - req.resch <- dialResponse{conn: new(Conn)} - } else { - req.resch <- dialResponse{err: fmt.Errorf("gophers ate the modem")} - } - atomic.AddInt32(&count, 1) - - case <-ctx.Done(): + req, ok := <-reqch + if !ok { return } + + if atomic.LoadInt32(&count) > 0 { + req.resch <- dialResponse{conn: new(Conn)} + } else { + req.resch <- dialResponse{err: fmt.Errorf("gophers ate the modem")} + } + atomic.AddInt32(&count, 1) } }() return nil @@ -235,19 +204,14 @@ func TestFailFirst(t *testing.T) { } func TestStressActiveDial(t *testing.T) { - ds := newDialSync(func(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { + ds := newDialSync(func(p peer.ID, reqch <-chan dialRequest) error { go func() { for { - select { - case req, ok := <-reqch: - if !ok { - return - } - - req.resch <- dialResponse{} - case <-ctx.Done(): + req, ok := <-reqch + if !ok { return } + req.resch <- dialResponse{} } }() return nil diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index e130c18301..83a0468f9f 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -295,16 +295,16 @@ type dialResponse struct { } // startDialWorker starts an active dial goroutine that synchronizes and executes concurrent dials -func (s *Swarm) startDialWorker(ctx context.Context, p peer.ID, reqch <-chan dialRequest) error { +func (s *Swarm) startDialWorker(p peer.ID, reqch <-chan dialRequest) error { if p == s.local { return ErrDialToSelf } - go s.dialWorkerLoop(ctx, p, reqch) + go s.dialWorkerLoop(p, reqch) return nil } -func (s *Swarm) dialWorkerLoop(ctx context.Context, p peer.ID, reqch <-chan dialRequest) { +func (s *Swarm) dialWorkerLoop(p peer.ID, reqch <-chan dialRequest) { defer s.limiter.clearAllPeerDials(p) type pendRequest struct { From 89af06a0d20e6932ae2e72661a7591d74b22ddcc Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 25 Aug 2021 11:52:13 +0100 Subject: [PATCH 225/259] simplify limiter by removing the injected isFdConsumingFnc --- p2p/net/swarm/limiter.go | 17 +++++++---------- p2p/net/swarm/limiter_test.go | 23 +++++------------------ p2p/net/swarm/swarm.go | 2 +- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 3e20976b32..5cd8cfa29e 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -42,10 +42,9 @@ func (dj *dialJob) dialTimeout() time.Duration { type dialLimiter struct { lk sync.Mutex - isFdConsumingFnc isFdConsumingFnc - fdConsuming int - fdLimit int - waitingOnFd []*dialJob + fdConsuming int + fdLimit int + waitingOnFd []*dialJob dialFunc dialfunc @@ -55,21 +54,19 @@ type dialLimiter struct { } type dialfunc func(context.Context, peer.ID, ma.Multiaddr) (transport.CapableConn, error) -type isFdConsumingFnc func(ma.Multiaddr) bool -func newDialLimiter(df dialfunc, fdFnc isFdConsumingFnc) *dialLimiter { +func newDialLimiter(df dialfunc) *dialLimiter { fd := ConcurrentFdDials if env := os.Getenv("LIBP2P_SWARM_FD_LIMIT"); env != "" { if n, err := strconv.ParseInt(env, 10, 32); err == nil { fd = int(n) } } - return newDialLimiterWithParams(fdFnc, df, fd, DefaultPerPeerRateLimit) + return newDialLimiterWithParams(df, fd, DefaultPerPeerRateLimit) } -func newDialLimiterWithParams(isFdConsumingFnc isFdConsumingFnc, df dialfunc, fdLimit, perPeerLimit int) *dialLimiter { +func newDialLimiterWithParams(df dialfunc, fdLimit, perPeerLimit int) *dialLimiter { return &dialLimiter{ - isFdConsumingFnc: isFdConsumingFnc, fdLimit: fdLimit, perPeerLimit: perPeerLimit, waitingOnPeerLimit: make(map[peer.ID][]*dialJob), @@ -157,7 +154,7 @@ func (dl *dialLimiter) shouldConsumeFd(addr ma.Multiaddr) bool { isRelay := err == nil - return !isRelay && dl.isFdConsumingFnc(addr) + return !isRelay && isFdConsumingAddr(addr) } func (dl *dialLimiter) addCheckFdLimit(dj *dialJob) { diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 1aefffec6d..4a592b8651 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -18,19 +18,6 @@ import ( mafmt "github.com/multiformats/go-multiaddr-fmt" ) -var isFdConsuming = func(addr ma.Multiaddr) bool { - res := false - - ma.ForEach(addr, func(c ma.Component) bool { - if c.Protocol().Code == ma.P_TCP { - res = true - return false - } - return true - }) - return res -} - func mustAddr(t *testing.T, s string) ma.Multiaddr { a, err := ma.NewMultiaddr(s) if err != nil { @@ -95,7 +82,7 @@ func TestLimiterBasicDials(t *testing.T) { hang := make(chan struct{}) defer close(hang) - l := newDialLimiterWithParams(isFdConsuming, hangDialFunc(hang), ConcurrentFdDials, 4) + l := newDialLimiterWithParams(hangDialFunc(hang), ConcurrentFdDials, 4) bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} good := addrWithPort(t, 20) @@ -144,7 +131,7 @@ func TestLimiterBasicDials(t *testing.T) { func TestFDLimiting(t *testing.T) { hang := make(chan struct{}) defer close(hang) - l := newDialLimiterWithParams(isFdConsuming, hangDialFunc(hang), 16, 5) + l := newDialLimiterWithParams(hangDialFunc(hang), 16, 5) bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} pids := []peer.ID{"testpeer1", "testpeer2", "testpeer3", "testpeer4"} @@ -220,7 +207,7 @@ func TestTokenRedistribution(t *testing.T) { <-ch return nil, fmt.Errorf("test bad dial") } - l := newDialLimiterWithParams(isFdConsuming, df, 8, 4) + l := newDialLimiterWithParams(df, 8, 4) bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} pids := []peer.ID{"testpeer1", "testpeer2"} @@ -313,7 +300,7 @@ func TestStressLimiter(t *testing.T) { return nil, fmt.Errorf("test bad dial") } - l := newDialLimiterWithParams(isFdConsuming, df, 20, 5) + l := newDialLimiterWithParams(df, 20, 5) var bads []ma.Multiaddr for i := 0; i < 100; i++ { @@ -367,7 +354,7 @@ func TestFDLimitUnderflow(t *testing.T) { } const fdLimit = 20 - l := newDialLimiterWithParams(isFdConsuming, df, fdLimit, 3) + l := newDialLimiterWithParams(df, fdLimit, 3) var addrs []ma.Multiaddr for i := 0; i <= 1000; i++ { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index e975548fe1..35eb415681 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -123,7 +123,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc } s.dsync = newDialSync(s.startDialWorker) - s.limiter = newDialLimiter(s.dialAddr, isFdConsumingAddr) + s.limiter = newDialLimiter(s.dialAddr) s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) s.backf.init(s.ctx) From fc08179d1c6ce1a8de3a7f683b3b3976de085ddf Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 27 Aug 2021 11:54:25 +0100 Subject: [PATCH 226/259] add a test case for the testing package --- p2p/net/swarm/testing/testing.go | 3 +-- p2p/net/swarm/testing/testing_test.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 p2p/net/swarm/testing/testing_test.go diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 313e3d771e..ba51776993 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -16,7 +16,7 @@ import ( "github.com/libp2p/go-libp2p-peerstore/pstoremem" quic "github.com/libp2p/go-libp2p-quic-transport" swarm "github.com/libp2p/go-libp2p-swarm" - "github.com/libp2p/go-libp2p-testing/net" + tnet "github.com/libp2p/go-libp2p-testing/net" tptu "github.com/libp2p/go-libp2p-transport-upgrader" yamux "github.com/libp2p/go-libp2p-yamux" msmux "github.com/libp2p/go-stream-muxer-multistream" @@ -86,7 +86,6 @@ func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { Secure: secMuxer, Muxer: stMuxer, } - } // GenSwarm generates a new test swarm. diff --git a/p2p/net/swarm/testing/testing_test.go b/p2p/net/swarm/testing/testing_test.go new file mode 100644 index 0000000000..a80cca177c --- /dev/null +++ b/p2p/net/swarm/testing/testing_test.go @@ -0,0 +1,14 @@ +package testing + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGenSwarm(t *testing.T) { + swarm := GenSwarm(t, context.Background()) + require.NoError(t, swarm.Close()) + GenUpgrader(swarm) +} From e5b2a35de8aeed11a9a13e751d0a0b5111cd7cca Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 23 Aug 2021 17:58:36 +0100 Subject: [PATCH 227/259] rename DialSync.DialLock to Dial --- p2p/net/swarm/dial_sync.go | 4 ++-- p2p/net/swarm/dial_sync_test.go | 24 ++++++++++++------------ p2p/net/swarm/swarm_dial.go | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 03ba9cc089..83167f6359 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -108,9 +108,9 @@ func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { return actd, nil } -// DialLock initiates a dial to the given peer if there are none in progress +// Dial initiates a dial to the given peer if there are none in progress // then waits for the dial to that peer to complete. -func (ds *DialSync) DialLock(ctx context.Context, p peer.ID) (*Conn, error) { +func (ds *DialSync) Dial(ctx context.Context, p peer.ID) (*Conn, error) { ad, err := ds.getActiveDial(p) if err != nil { return nil, err diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 441c8b7de9..47aa68f7ab 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -38,14 +38,14 @@ func TestBasicDialSync(t *testing.T) { finished := make(chan struct{}, 2) go func() { - if _, err := dsync.DialLock(context.Background(), p); err != nil { + if _, err := dsync.Dial(context.Background(), p); err != nil { t.Error(err) } finished <- struct{}{} }() go func() { - if _, err := dsync.DialLock(context.Background(), p); err != nil { + if _, err := dsync.Dial(context.Background(), p); err != nil { t.Error(err) } finished <- struct{}{} @@ -74,7 +74,7 @@ func TestDialSyncCancel(t *testing.T) { finished := make(chan struct{}) go func() { - _, err := dsync.DialLock(ctx1, p) + _, err := dsync.Dial(ctx1, p) if err != ctx1.Err() { t.Error("should have gotten context error") } @@ -90,7 +90,7 @@ func TestDialSyncCancel(t *testing.T) { // Add a second dialwait in so two actors are waiting on the same dial go func() { - _, err := dsync.DialLock(context.Background(), p) + _, err := dsync.Dial(context.Background(), p) if err != nil { t.Error(err) } @@ -123,7 +123,7 @@ func TestDialSyncAllCancel(t *testing.T) { finished := make(chan struct{}) go func() { - if _, err := dsync.DialLock(ctx, p); err != ctx.Err() { + if _, err := dsync.Dial(ctx, p); err != ctx.Err() { t.Error("should have gotten context error") } finished <- struct{}{} @@ -131,7 +131,7 @@ func TestDialSyncAllCancel(t *testing.T) { // Add a second dialwait in so two actors are waiting on the same dial go func() { - if _, err := dsync.DialLock(ctx, p); err != ctx.Err() { + if _, err := dsync.Dial(ctx, p); err != ctx.Err() { t.Error("should have gotten context error") } finished <- struct{}{} @@ -155,7 +155,7 @@ func TestDialSyncAllCancel(t *testing.T) { // should be able to successfully dial that peer again done() - if _, err := dsync.DialLock(context.Background(), p); err != nil { + if _, err := dsync.Dial(context.Background(), p); err != nil { t.Fatal(err) } } @@ -188,12 +188,12 @@ func TestFailFirst(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - _, err := ds.DialLock(ctx, p) + _, err := ds.Dial(ctx, p) if err == nil { t.Fatal("expected gophers to have eaten the modem") } - c, err := ds.DialLock(ctx, p) + c, err := ds.Dial(ctx, p) if err != nil { t.Fatal(err) } @@ -223,7 +223,7 @@ func TestStressActiveDial(t *testing.T) { makeDials := func() { for i := 0; i < 10000; i++ { - ds.DialLock(context.Background(), pid) + ds.Dial(context.Background(), pid) } wg.Done() } @@ -245,13 +245,13 @@ func TestDialSelf(t *testing.T) { defer s.Close() // this should fail - _, err := s.dsync.DialLock(ctx, self) + _, err := s.dsync.Dial(ctx, self) if err != ErrDialToSelf { t.Fatal("expected error from self dial") } // do it twice to make sure we get a new active dial object that fails again - _, err = s.dsync.DialLock(ctx, self) + _, err = s.dsync.Dial(ctx, self) if err != ErrDialToSelf { t.Fatal("expected error from self dial") } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 83a0468f9f..9c9e48cc7e 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -259,7 +259,7 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { ctx, cancel := context.WithTimeout(ctx, network.GetDialPeerTimeout(ctx)) defer cancel() - conn, err = s.dsync.DialLock(ctx, p) + conn, err = s.dsync.Dial(ctx, p) if err == nil { return conn, nil } From 9adcf8f5ea8ef467a8a08f2b99df8e11794120b4 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 23 Aug 2021 18:00:01 +0100 Subject: [PATCH 228/259] only check for self dials once --- p2p/net/swarm/dial_sync.go | 11 +++-------- p2p/net/swarm/dial_sync_test.go | 35 ++++----------------------------- p2p/net/swarm/dial_test.go | 2 +- p2p/net/swarm/swarm_dial.go | 9 ++------- 4 files changed, 10 insertions(+), 47 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 83167f6359..2271221d5f 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -8,8 +8,8 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -// DialWorerFunc is used by DialSync to spawn a new dial worker -type dialWorkerFunc func(peer.ID, <-chan dialRequest) error +// dialWorkerFunc is used by DialSync to spawn a new dial worker +type dialWorkerFunc func(peer.ID, <-chan dialRequest) // newDialSync constructs a new DialSync func newDialSync(worker dialWorkerFunc) *DialSync { @@ -93,12 +93,7 @@ func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { reqch: make(chan dialRequest), ds: ds, } - - if err := ds.dialWorker(p, actd.reqch); err != nil { - cancel() - return nil, err - } - + ds.dialWorker(p, actd.reqch) ds.dials[p] = actd } diff --git a/p2p/net/swarm/dial_sync_test.go b/p2p/net/swarm/dial_sync_test.go index 47aa68f7ab..0d9c6ca413 100644 --- a/p2p/net/swarm/dial_sync_test.go +++ b/p2p/net/swarm/dial_sync_test.go @@ -15,7 +15,7 @@ func getMockDialFunc() (dialWorkerFunc, func(), context.Context, <-chan struct{} dfcalls := make(chan struct{}, 512) // buffer it large enough that we won't care dialctx, cancel := context.WithCancel(context.Background()) ch := make(chan struct{}) - f := func(p peer.ID, reqch <-chan dialRequest) error { + f := func(p peer.ID, reqch <-chan dialRequest) { defer cancel() dfcalls <- struct{}{} go func() { @@ -24,7 +24,6 @@ func getMockDialFunc() (dialWorkerFunc, func(), context.Context, <-chan struct{} req.resch <- dialResponse{conn: new(Conn)} } }() - return nil } var once sync.Once @@ -162,7 +161,7 @@ func TestDialSyncAllCancel(t *testing.T) { func TestFailFirst(t *testing.T) { var count int32 - f := func(p peer.ID, reqch <-chan dialRequest) error { + f := func(p peer.ID, reqch <-chan dialRequest) { go func() { for { req, ok := <-reqch @@ -178,18 +177,15 @@ func TestFailFirst(t *testing.T) { atomic.AddInt32(&count, 1) } }() - return nil } ds := newDialSync(f) - p := peer.ID("testing") ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - _, err := ds.Dial(ctx, p) - if err == nil { + if _, err := ds.Dial(ctx, p); err == nil { t.Fatal("expected gophers to have eaten the modem") } @@ -197,14 +193,13 @@ func TestFailFirst(t *testing.T) { if err != nil { t.Fatal(err) } - if c == nil { t.Fatal("should have gotten a 'real' conn back") } } func TestStressActiveDial(t *testing.T) { - ds := newDialSync(func(p peer.ID, reqch <-chan dialRequest) error { + ds := newDialSync(func(p peer.ID, reqch <-chan dialRequest) { go func() { for { req, ok := <-reqch @@ -214,7 +209,6 @@ func TestStressActiveDial(t *testing.T) { req.resch <- dialResponse{} } }() - return nil }) wg := sync.WaitGroup{} @@ -235,24 +229,3 @@ func TestStressActiveDial(t *testing.T) { wg.Wait() } - -func TestDialSelf(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - self := peer.ID("ABC") - s := NewSwarm(ctx, self, nil, nil) - defer s.Close() - - // this should fail - _, err := s.dsync.Dial(ctx, self) - if err != ErrDialToSelf { - t.Fatal("expected error from self dial") - } - - // do it twice to make sure we get a new active dial object that fails again - _, err = s.dsync.Dial(ctx, self) - if err != ErrDialToSelf { - t.Fatal("expected error from self dial") - } -} diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 6258d0eda1..3bbc5e2b41 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -672,7 +672,7 @@ func TestDialSimultaneousJoin(t *testing.T) { } } -func TestDialSelf2(t *testing.T) { +func TestDialSelf(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 9c9e48cc7e..9f76c975e3 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -294,14 +294,9 @@ type dialResponse struct { err error } -// startDialWorker starts an active dial goroutine that synchronizes and executes concurrent dials -func (s *Swarm) startDialWorker(p peer.ID, reqch <-chan dialRequest) error { - if p == s.local { - return ErrDialToSelf - } - +// startDialWorker starts an active dial goroutine that synchronizes and executes concurrent dials to a single peer +func (s *Swarm) startDialWorker(p peer.ID, reqch <-chan dialRequest) { go s.dialWorkerLoop(p, reqch) - return nil } func (s *Swarm) dialWorkerLoop(p peer.ID, reqch <-chan dialRequest) { From 2f389ce389b8cfce05298e4751090b47fa8c130a Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 24 Aug 2021 12:03:46 +0100 Subject: [PATCH 229/259] simplify starting of the dialWorkerLoop --- p2p/net/swarm/dial_sync.go | 2 +- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_dial.go | 6 +----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 2271221d5f..6b3f2afeee 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -93,7 +93,7 @@ func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { reqch: make(chan dialRequest), ds: ds, } - ds.dialWorker(p, actd.reqch) + go ds.dialWorker(p, actd.reqch) ds.dials[p] = actd } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 35eb415681..c34497f74e 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -122,7 +122,7 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc } } - s.dsync = newDialSync(s.startDialWorker) + s.dsync = newDialSync(s.dialWorkerLoop) s.limiter = newDialLimiter(s.dialAddr) s.proc = goprocessctx.WithContext(ctx) s.ctx = goprocessctx.OnClosingContext(s.proc) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 9f76c975e3..21b5b84ecb 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -294,11 +294,7 @@ type dialResponse struct { err error } -// startDialWorker starts an active dial goroutine that synchronizes and executes concurrent dials to a single peer -func (s *Swarm) startDialWorker(p peer.ID, reqch <-chan dialRequest) { - go s.dialWorkerLoop(p, reqch) -} - +// dialWorkerLoop synchronizes and executes concurrent dials to a single peer func (s *Swarm) dialWorkerLoop(p peer.ID, reqch <-chan dialRequest) { defer s.limiter.clearAllPeerDials(p) From 76a1e92bd12eee2fb0287b12c6d7cda6246fa057 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 5 Sep 2021 16:17:03 +0100 Subject: [PATCH 230/259] read and use the direction from the simultaneous connect context --- p2p/net/swarm/dial_sync.go | 4 ++-- p2p/net/swarm/swarm_dial.go | 18 +++++------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 03ba9cc089..ed7766711c 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -56,8 +56,8 @@ func (ad *activeDial) dial(ctx context.Context, p peer.ID) (*Conn, error) { if forceDirect, reason := network.GetForceDirectDial(ctx); forceDirect { dialCtx = network.WithForceDirectDial(dialCtx, reason) } - if simConnect, reason := network.GetSimultaneousConnect(ctx); simConnect { - dialCtx = network.WithSimultaneousConnect(dialCtx, reason) + if simConnect, isClient, reason := network.GetSimultaneousConnect(ctx); simConnect { + dialCtx = network.WithSimultaneousConnect(dialCtx, isClient, reason) } resch := make(chan dialResponse, 1) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 83a0468f9f..bbd5c6a0a2 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -461,7 +461,11 @@ loop: for _, ad := range tojoin { if !ad.dialed { - ad.ctx = s.mergeDialContexts(ad.ctx, req.ctx) + if simConnect, isClient, reason := network.GetSimultaneousConnect(req.ctx); simConnect { + if simConnect, _, _ := network.GetSimultaneousConnect(ad.ctx); !simConnect { + ad.ctx = network.WithSimultaneousConnect(ad.ctx, isClient, reason) + } + } } ad.requests = append(ad.requests, reqno) } @@ -577,18 +581,6 @@ func (s *Swarm) addrsForDial(ctx context.Context, p peer.ID) ([]ma.Multiaddr, er return goodAddrs, nil } -func (s *Swarm) mergeDialContexts(a, b context.Context) context.Context { - dialCtx := a - - if simConnect, reason := network.GetSimultaneousConnect(b); simConnect { - if simConnect, _ := network.GetSimultaneousConnect(a); !simConnect { - dialCtx = network.WithSimultaneousConnect(dialCtx, reason) - } - } - - return dialCtx -} - func (s *Swarm) dialNextAddr(ctx context.Context, p peer.ID, addr ma.Multiaddr, resch chan dialResult) error { // check the dial backoff if forceDirect, _ := network.GetForceDirectDial(ctx); !forceDirect { From 0e0111c6f5fe5d3bf3c91295b39c0ea340d24c14 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 23 Aug 2021 17:38:57 +0100 Subject: [PATCH 231/259] simplify the DialSync code It's easier to reason about this code if activeDial doesn't contain a pointer back to DialSync (which already has a map of activeDials). It also allows us to remove the memory footprint of the activeDial struct, so this should be (slightly) more efficient. --- p2p/net/swarm/dial_sync.go | 45 +++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 6b3f2afeee..325abdf5f4 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -22,35 +22,26 @@ func newDialSync(worker dialWorkerFunc) *DialSync { // DialSync is a dial synchronization helper that ensures that at most one dial // to any given peer is active at any given time. type DialSync struct { + mutex sync.Mutex dials map[peer.ID]*activeDial - dialsLk sync.Mutex dialWorker dialWorkerFunc } type activeDial struct { - id peer.ID refCnt int ctx context.Context cancel func() reqch chan dialRequest - - ds *DialSync } -func (ad *activeDial) decref() { - ad.ds.dialsLk.Lock() - ad.refCnt-- - if ad.refCnt == 0 { - ad.cancel() - close(ad.reqch) - delete(ad.ds.dials, ad.id) - } - ad.ds.dialsLk.Unlock() +func (ad *activeDial) close() { + ad.cancel() + close(ad.reqch) } -func (ad *activeDial) dial(ctx context.Context, p peer.ID) (*Conn, error) { +func (ad *activeDial) dial(ctx context.Context) (*Conn, error) { dialCtx := ad.ctx if forceDirect, reason := network.GetForceDirectDial(ctx); forceDirect { @@ -76,8 +67,8 @@ func (ad *activeDial) dial(ctx context.Context, p peer.ID) (*Conn, error) { } func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { - ds.dialsLk.Lock() - defer ds.dialsLk.Unlock() + ds.mutex.Lock() + defer ds.mutex.Unlock() actd, ok := ds.dials[p] if !ok { @@ -85,21 +76,17 @@ func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { // to Dial is canceled, subsequent dial calls will also be canceled. // XXX: this also breaks direct connection logic. We will need to pipe the // information through some other way. - adctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) actd = &activeDial{ - id: p, - ctx: adctx, + ctx: ctx, cancel: cancel, reqch: make(chan dialRequest), - ds: ds, } go ds.dialWorker(p, actd.reqch) ds.dials[p] = actd } - - // increase ref count before dropping dialsLk + // increase ref count before dropping mutex actd.refCnt++ - return actd, nil } @@ -111,6 +98,14 @@ func (ds *DialSync) Dial(ctx context.Context, p peer.ID) (*Conn, error) { return nil, err } - defer ad.decref() - return ad.dial(ctx, p) + defer func() { + ds.mutex.Lock() + defer ds.mutex.Unlock() + ad.refCnt-- + if ad.refCnt == 0 { + ad.close() + delete(ds.dials, p) + } + }() + return ad.dial(ctx) } From 782897ea41f376916c9f7fab6e1e344597f8bc94 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 5 Sep 2021 19:00:36 +0100 Subject: [PATCH 232/259] stop using goprocess to control teardown --- p2p/net/swarm/dial_test.go | 86 ++++++++++++--------------- p2p/net/swarm/peers_test.go | 8 +-- p2p/net/swarm/simul_test.go | 11 ++-- p2p/net/swarm/swarm.go | 49 ++++----------- p2p/net/swarm/swarm_addr_test.go | 32 +++------- p2p/net/swarm/swarm_listen.go | 2 +- p2p/net/swarm/swarm_net_test.go | 35 +++-------- p2p/net/swarm/swarm_notif_test.go | 14 ++--- p2p/net/swarm/swarm_test.go | 61 +++++++------------ p2p/net/swarm/testing/testing.go | 25 +++++--- p2p/net/swarm/testing/testing_test.go | 3 +- p2p/net/swarm/transport_test.go | 29 ++++----- 12 files changed, 137 insertions(+), 218 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 3bbc5e2b41..de41e7f740 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -7,28 +7,33 @@ import ( "testing" "time" - addrutil "github.com/libp2p/go-addr-util" + . "github.com/libp2p/go-libp2p-swarm" + addrutil "github.com/libp2p/go-addr-util" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" - "github.com/libp2p/go-libp2p-core/transport" - testutil "github.com/libp2p/go-libp2p-core/test" + "github.com/libp2p/go-libp2p-core/transport" swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-libp2p-testing/ci" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" - . "github.com/libp2p/go-libp2p-swarm" + "github.com/stretchr/testify/require" ) func init() { transport.DialTimeout = time.Second } -func closeSwarms(swarms []*Swarm) { +type swarmWithBackoff interface { + network.Network + Backoff() *DialBackoff +} + +func closeSwarms(swarms []network.Network) { for _, s := range swarms { s.Close() } @@ -36,50 +41,37 @@ func closeSwarms(swarms []*Swarm) { func TestBasicDialPeer(t *testing.T) { t.Parallel() - ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) defer closeSwarms(swarms) s1 := swarms[0] s2 := swarms[1] s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) - c, err := s1.DialPeer(ctx, s2.LocalPeer()) - if err != nil { - t.Fatal(err) - } - - s, err := c.NewStream(ctx) - if err != nil { - t.Fatal(err) - } + c, err := s1.DialPeer(context.Background(), s2.LocalPeer()) + require.NoError(t, err) + s, err := c.NewStream(context.Background()) + require.NoError(t, err) s.Close() } func TestDialWithNoListeners(t *testing.T) { t.Parallel() - ctx := context.Background() - s1 := makeDialOnlySwarm(ctx, t) - - swarms := makeSwarms(ctx, t, 1) + s1 := makeDialOnlySwarm(t) + swarms := makeSwarms(t, 1) defer closeSwarms(swarms) s2 := swarms[0] s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) - c, err := s1.DialPeer(ctx, s2.LocalPeer()) - if err != nil { - t.Fatal(err) - } - - s, err := c.NewStream(ctx) - if err != nil { - t.Fatal(err) - } + c, err := s1.DialPeer(context.Background(), s2.LocalPeer()) + require.NoError(t, err) + s, err := c.NewStream(context.Background()) + require.NoError(t, err) s.Close() } @@ -104,12 +96,12 @@ func TestSimultDials(t *testing.T) { t.Parallel() ctx := context.Background() - swarms := makeSwarms(ctx, t, 2, swarmt.OptDisableReuseport) + swarms := makeSwarms(t, 2, swarmt.OptDisableReuseport) // connect everyone { var wg sync.WaitGroup - connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + connect := func(s network.Network, dst peer.ID, addr ma.Multiaddr) { // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) s.Peerstore().AddAddr(dst, addr, peerstore.TempAddrTTL) @@ -175,7 +167,7 @@ func TestDialWait(t *testing.T) { t.Parallel() ctx := context.Background() - swarms := makeSwarms(ctx, t, 1) + swarms := makeSwarms(t, 1) s1 := swarms[0] defer s1.Close() @@ -201,7 +193,7 @@ func TestDialWait(t *testing.T) { t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) } - if !s1.Backoff().Backoff(s2p, s2addr) { + if !s1.(swarmWithBackoff).Backoff().Backoff(s2p, s2addr) { t.Error("s2 should now be on backoff") } } @@ -215,7 +207,7 @@ func TestDialBackoff(t *testing.T) { t.Parallel() ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) s1 := swarms[0] s2 := swarms[1] defer s1.Close() @@ -338,10 +330,10 @@ func TestDialBackoff(t *testing.T) { } // check backoff state - if s1.Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { + if s1.(swarmWithBackoff).Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { t.Error("s2 should not be on backoff") } - if !s1.Backoff().Backoff(s3p, s3addr) { + if !s1.(swarmWithBackoff).Backoff().Backoff(s3p, s3addr) { t.Error("s3 should be on backoff") } @@ -408,10 +400,10 @@ func TestDialBackoff(t *testing.T) { } // check backoff state (the same) - if s1.Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { + if s1.(swarmWithBackoff).Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { t.Error("s2 should not be on backoff") } - if !s1.Backoff().Backoff(s3p, s3addr) { + if !s1.(swarmWithBackoff).Backoff().Backoff(s3p, s3addr) { t.Error("s3 should be on backoff") } } @@ -422,7 +414,7 @@ func TestDialBackoffClears(t *testing.T) { t.Parallel() ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) s1 := swarms[0] s2 := swarms[1] defer s1.Close() @@ -453,7 +445,7 @@ func TestDialBackoffClears(t *testing.T) { t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) } - if !s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { + if !s1.(swarmWithBackoff).Backoff().Backoff(s2.LocalPeer(), s2bad) { t.Error("s2 should now be on backoff") } else { t.Log("correctly added to backoff") @@ -480,7 +472,7 @@ func TestDialBackoffClears(t *testing.T) { t.Log("correctly connected") } - if s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { + if s1.(swarmWithBackoff).Backoff().Backoff(s2.LocalPeer(), s2bad) { t.Error("s2 should no longer be on backoff") } else { t.Log("correctly cleared backoff") @@ -491,7 +483,7 @@ func TestDialPeerFailed(t *testing.T) { t.Parallel() ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) defer closeSwarms(swarms) testedSwarm, targetSwarm := swarms[0], swarms[1] @@ -530,7 +522,7 @@ func TestDialPeerFailed(t *testing.T) { func TestDialExistingConnection(t *testing.T) { ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) defer closeSwarms(swarms) s1 := swarms[0] s2 := swarms[1] @@ -574,7 +566,7 @@ func TestDialSimultaneousJoin(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) s1 := swarms[0] s2 := swarms[1] defer s1.Close() @@ -676,12 +668,10 @@ func TestDialSelf(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) s1 := swarms[0] defer s1.Close() _, err := s1.DialPeer(ctx, s1.LocalPeer()) - if err != ErrDialToSelf { - t.Fatal("expected error from self dial") - } + require.ErrorIs(t, err, ErrDialToSelf, "expected error from self dial") } diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 8e82bf5bde..908abe91a4 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -9,17 +9,15 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" ma "github.com/multiformats/go-multiaddr" - - . "github.com/libp2p/go-libp2p-swarm" ) func TestPeers(t *testing.T) { ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) s1 := swarms[0] s2 := swarms[1] - connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + connect := func(s network.Network, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) // t.Logf("connections from %s", s.LocalPeer()) @@ -55,7 +53,7 @@ func TestPeers(t *testing.T) { log.Infof("%s swarm routing table: %s", s.LocalPeer(), s.Peers()) } - test := func(s *Swarm) { + test := func(s network.Network) { expect := 1 actual := len(s.Peers()) if actual != expect { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 0373e37dfb..326c4e21b6 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,32 +7,29 @@ import ( "testing" "time" + "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" ma "github.com/multiformats/go-multiaddr" - . "github.com/libp2p/go-libp2p-swarm" swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-libp2p-testing/ci" ) func TestSimultOpen(t *testing.T) { - t.Parallel() - - ctx := context.Background() - swarms := makeSwarms(ctx, t, 2, swarmt.OptDisableReuseport) + swarms := makeSwarms(t, 2, swarmt.OptDisableReuseport) // connect everyone { var wg sync.WaitGroup - connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + connect := func(s network.Network, dst peer.ID, addr ma.Multiaddr) { defer wg.Done() // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) - if _, err := s.DialPeer(ctx, dst); err != nil { + if _, err := s.DialPeer(context.Background(), dst); err != nil { t.Error("error swarm dialing to peer", err) } } diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c34497f74e..0f4e3c75bd 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -18,8 +18,6 @@ import ( "github.com/libp2p/go-libp2p-core/transport" logging "github.com/ipfs/go-log" - "github.com/jbenet/goprocess" - goprocessctx "github.com/jbenet/goprocess/context" ma "github.com/multiformats/go-multiaddr" ) @@ -92,9 +90,10 @@ type Swarm struct { limiter *dialLimiter gater connmgr.ConnectionGater - proc goprocess.Process - ctx context.Context - bwc metrics.Reporter + ctx context.Context // is canceled when Close is called + ctxCancel context.CancelFunc + + bwc metrics.Reporter } // NewSwarm constructs a Swarm. @@ -103,11 +102,14 @@ type Swarm struct { // `extra` interface{} parameter facilitates the future migration. Supported // elements are: // - connmgr.ConnectionGater -func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter, extra ...interface{}) *Swarm { +func NewSwarm(local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter, extra ...interface{}) *Swarm { + ctx, cancel := context.WithCancel(context.Background()) s := &Swarm{ - local: local, - peers: peers, - bwc: bwc, + local: local, + peers: peers, + bwc: bwc, + ctx: ctx, + ctxCancel: cancel, } s.conns.m = make(map[peer.ID][]*Conn) @@ -124,23 +126,11 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc s.dsync = newDialSync(s.dialWorkerLoop) s.limiter = newDialLimiter(s.dialAddr) - s.proc = goprocessctx.WithContext(ctx) - s.ctx = goprocessctx.OnClosingContext(s.proc) s.backf.init(s.ctx) - - // Set teardown after setting the context/process so we don't start the - // teardown process early. - s.proc.SetTeardown(s.teardown) - return s } -func (s *Swarm) teardown() error { - // Wait for the context to be canceled. - // This allows other parts of the swarm to detect that we're shutting - // down. - <-s.ctx.Done() - +func (s *Swarm) Close() error { // Prevents new connections and/or listeners from being added to the swarm. s.listeners.Lock() @@ -201,11 +191,6 @@ func (s *Swarm) teardown() error { return nil } -// Process returns the Process of the swarm -func (s *Swarm) Process() goprocess.Process { - return s.proc -} - func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, error) { var ( p = tc.RemotePeer() @@ -293,16 +278,6 @@ func (s *Swarm) Peerstore() peerstore.Peerstore { return s.peers } -// Context returns the context of the swarm -func (s *Swarm) Context() context.Context { - return s.ctx -} - -// Close stops the Swarm. -func (s *Swarm) Close() error { - return s.proc.Close() -} - // TODO: We probably don't need the conn handlers. // SetConnHandler assigns the handler for new connections. diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index baeac46203..21ecafa2ad 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -6,6 +6,7 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/test" + "github.com/stretchr/testify/require" ma "github.com/multiformats/go-multiaddr" @@ -13,7 +14,6 @@ import ( ) func TestDialBadAddrs(t *testing.T) { - m := func(s string) ma.Multiaddr { maddr, err := ma.NewMultiaddr(s) if err != nil { @@ -22,13 +22,12 @@ func TestDialBadAddrs(t *testing.T) { return maddr } - ctx := context.Background() - s := makeSwarms(ctx, t, 1)[0] + s := makeSwarms(t, 1)[0] test := func(a ma.Multiaddr) { p := test.RandPeerIDFatal(t) s.Peerstore().AddAddr(p, a, peerstore.PermanentAddrTTL) - if _, err := s.DialPeer(ctx, p); err == nil { + if _, err := s.DialPeer(context.Background(), p); err == nil { t.Errorf("swarm should not dial: %s", p) } } @@ -39,19 +38,13 @@ func TestDialBadAddrs(t *testing.T) { } func TestAddrRace(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := makeSwarms(ctx, t, 1)[0] + s := makeSwarms(t, 1)[0] defer s.Close() a1, err := s.InterfaceListenAddresses() - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) a2, err := s.InterfaceListenAddresses() - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if len(a1) > 0 && len(a2) > 0 && &a1[0] == &a2[0] { t.Fatal("got the exact same address set twice; this could lead to data races") @@ -59,15 +52,8 @@ func TestAddrRace(t *testing.T) { } func TestAddressesWithoutListening(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := swarmt.GenSwarm(t, ctx, swarmt.OptDialOnly) - + s := swarmt.GenSwarm(t, swarmt.OptDialOnly) a1, err := s.InterfaceListenAddresses() - if err != nil { - t.Fatal(err) - } - if len(a1) != 0 { - t.Fatalf("expected to be listening on no addresses, was listening on %d", len(a1)) - } + require.NoError(t, err) + require.Empty(t, a1, "expected to be listening on no addresses") } diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go index c064ae851e..ca54280c08 100644 --- a/p2p/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -46,7 +46,7 @@ func (s *Swarm) AddListenAddr(a ma.Multiaddr) error { // // Distinguish between these two cases to avoid confusing users. select { - case <-s.proc.Closing(): + case <-s.ctx.Done(): return ErrSwarmClosed default: return ErrNoTransport diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 05984f6b02..1f1d0454db 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/libp2p/go-libp2p-core/network" . "github.com/libp2p/go-libp2p-swarm/testing" @@ -15,19 +17,16 @@ import ( // TestConnectednessCorrect starts a few networks, connects a few // and tests Connectedness value is correct. func TestConnectednessCorrect(t *testing.T) { - - ctx := context.Background() - nets := make([]network.Network, 4) for i := 0; i < 4; i++ { - nets[i] = GenSwarm(t, ctx) + nets[i] = GenSwarm(t) } // connect 0-1, 0-2, 0-3, 1-2, 2-3 dial := func(a, b network.Network) { DivulgeAddresses(b, a) - if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { + if _, err := a.DialPeer(context.Background(), b.LocalPeer()); err != nil { t.Fatalf("Failed to dial: %s", err) } } @@ -54,33 +53,17 @@ func TestConnectednessCorrect(t *testing.T) { expectConnectedness(t, nets[0], nets[2], network.NotConnected) expectConnectedness(t, nets[1], nets[3], network.NotConnected) - if len(nets[0].Peers()) != 2 { - t.Fatal("expected net 0 to have two peers") - } - - if len(nets[2].Peers()) != 2 { - t.Fatal("expected net 2 to have two peers") - } - - if len(nets[1].ConnsToPeer(nets[3].LocalPeer())) != 0 { - t.Fatal("net 1 should have no connections to net 3") - } - - if err := nets[2].ClosePeer(nets[1].LocalPeer()); err != nil { - t.Fatal(err) - } + require.Len(t, nets[0].Peers(), 2, "expected net 0 to have two peers") + require.Len(t, nets[2].Peers(), 2, "expected net 2 to have two peers") + require.NotZerof(t, nets[1].ConnsToPeer(nets[3].LocalPeer()), "net 1 should have no connections to net 3") + require.NoError(t, nets[2].ClosePeer(nets[1].LocalPeer())) time.Sleep(time.Millisecond * 50) - expectConnectedness(t, nets[2], nets[1], network.NotConnected) for _, n := range nets { n.Close() } - - for _, n := range nets { - <-n.Process().Closed() - } } func expectConnectedness(t *testing.T, a, b network.Network, expected network.Connectedness) { @@ -113,7 +96,7 @@ func TestNetworkOpenStream(t *testing.T) { nets := make([]network.Network, 4) for i := 0; i < 4; i++ { - nets[i] = GenSwarm(t, ctx) + nets[i] = GenSwarm(t) } dial := func(a, b network.Network) { diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 3383617218..157405cba1 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -18,8 +20,7 @@ func TestNotifications(t *testing.T) { notifiees := make([]*netNotifiee, swarmSize) - ctx := context.Background() - swarms := makeSwarms(ctx, t, swarmSize) + swarms := makeSwarms(t, swarmSize) defer func() { for i, s := range swarms { select { @@ -27,10 +28,7 @@ func TestNotifications(t *testing.T) { t.Error("should not have been closed") default: } - err := s.Close() - if err != nil { - t.Error(err) - } + require.NoError(t, s.Close()) select { case <-notifiees[i].listenClose: default: @@ -48,7 +46,7 @@ func TestNotifications(t *testing.T) { notifiees[i] = n } - connectSwarms(t, ctx, swarms) + connectSwarms(t, context.Background(), swarms) time.Sleep(50 * time.Millisecond) // should've gotten 5 by now. @@ -96,7 +94,7 @@ func TestNotifications(t *testing.T) { } } - complement := func(c network.Conn) (*Swarm, *netNotifiee, *Conn) { + complement := func(c network.Conn) (network.Network, *netNotifiee, *Conn) { for i, s := range swarms { for _, c2 := range s.Conns() { if c.LocalMultiaddr().Equal(c2.RemoteMultiaddr()) && diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index a94281b1f5..e6cb0ba307 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -58,29 +58,25 @@ func EchoStreamHandler(stream network.Stream) { }() } -func makeDialOnlySwarm(ctx context.Context, t *testing.T) *Swarm { - swarm := GenSwarm(t, ctx, OptDialOnly) +func makeDialOnlySwarm(t *testing.T) network.Network { + swarm := GenSwarm(t, OptDialOnly) swarm.SetStreamHandler(EchoStreamHandler) - return swarm } -func makeSwarms(ctx context.Context, t *testing.T, num int, opts ...Option) []*Swarm { - swarms := make([]*Swarm, 0, num) - +func makeSwarms(t *testing.T, num int, opts ...Option) []network.Network { + swarms := make([]network.Network, 0, num) for i := 0; i < num; i++ { - swarm := GenSwarm(t, ctx, opts...) + swarm := GenSwarm(t, opts...) swarm.SetStreamHandler(EchoStreamHandler) swarms = append(swarms, swarm) } - return swarms } -func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { - +func connectSwarms(t *testing.T, ctx context.Context, swarms []network.Network) { var wg sync.WaitGroup - connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + connect := func(s network.Network, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) if _, err := s.DialPeer(ctx, dst); err != nil { @@ -104,13 +100,10 @@ func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { } func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { - // t.Skip("skipping for another test") - - ctx := context.Background() - swarms := makeSwarms(ctx, t, SwarmNum, OptDisableReuseport) + swarms := makeSwarms(t, SwarmNum, OptDisableReuseport) // connect everyone - connectSwarms(t, ctx, swarms) + connectSwarms(t, context.Background(), swarms) // ping/pong for _, s1 := range swarms { @@ -118,7 +111,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { log.Debugf("%s ping pong round", s1.LocalPeer()) log.Debugf("-------------------------------------------------------") - _, cancel := context.WithCancel(ctx) + _, cancel := context.WithCancel(context.Background()) got := map[peer.ID]int{} errChan := make(chan error, MsgNum*len(swarms)) streamChan := make(chan network.Stream, MsgNum) @@ -132,7 +125,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { defer wg.Done() // first, one stream per peer (nice) - stream, err := s1.NewStream(ctx, p) + stream, err := s1.NewStream(context.Background(), p) if err != nil { errChan <- err return @@ -253,7 +246,7 @@ func TestConnHandler(t *testing.T) { t.Parallel() ctx := context.Background() - swarms := makeSwarms(ctx, t, 5) + swarms := makeSwarms(t, 5) gotconn := make(chan struct{}, 10) swarms[0].SetConnHandler(func(conn network.Conn) { @@ -387,8 +380,8 @@ func TestConnectionGating(t *testing.T) { p2Gater = tc.p2Gater(p2Gater) } - sw1 := GenSwarm(t, ctx, OptConnGater(p1Gater), optTransport) - sw2 := GenSwarm(t, ctx, OptConnGater(p2Gater), optTransport) + sw1 := GenSwarm(t, OptConnGater(p1Gater), optTransport) + sw2 := GenSwarm(t, OptConnGater(p2Gater), optTransport) p1 := sw1.LocalPeer() p2 := sw2.LocalPeer() @@ -408,10 +401,9 @@ func TestConnectionGating(t *testing.T) { } func TestNoDial(t *testing.T) { - ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) - _, err := swarms[0].NewStream(network.WithNoDial(ctx, "swarm test"), swarms[1].LocalPeer()) + _, err := swarms[0].NewStream(network.WithNoDial(context.Background(), "swarm test"), swarms[1].LocalPeer()) if err != network.ErrNoConn { t.Fatal("should have failed with ErrNoConn") } @@ -419,36 +411,29 @@ func TestNoDial(t *testing.T) { func TestCloseWithOpenStreams(t *testing.T) { ctx := context.Background() - swarms := makeSwarms(ctx, t, 2) + swarms := makeSwarms(t, 2) connectSwarms(t, ctx, swarms) s, err := swarms[0].NewStream(ctx, swarms[1].LocalPeer()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer s.Close() // close swarm before stream. - err = swarms[0].Close() - if err != nil { - t.Fatal(err) - } + require.NoError(t, swarms[0].Close()) } func TestTypedNilConn(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := GenSwarm(t, ctx) + s := GenSwarm(t) defer s.Close() // We can't dial ourselves. - c, err := s.DialPeer(ctx, s.LocalPeer()) + c, err := s.DialPeer(context.Background(), s.LocalPeer()) require.Error(t, err) // If we fail to dial, the connection should be nil. - require.True(t, c == nil) + require.Nil(t, c) } func TestPreventDialListenAddr(t *testing.T) { - s := GenSwarm(t, context.Background(), OptDialOnly) + s := GenSwarm(t, OptDialOnly) if err := s.Listen(ma.StringCast("/ip4/0.0.0.0/udp/0/quic")); err != nil { t.Fatal(err) } diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index ba51776993..d6091354b5 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -1,7 +1,6 @@ package testing import ( - "context" "testing" csms "github.com/libp2p/go-conn-security-multistream" @@ -22,7 +21,6 @@ import ( msmux "github.com/libp2p/go-stream-muxer-multistream" "github.com/libp2p/go-tcp-transport" - "github.com/jbenet/goprocess" ma "github.com/multiformats/go-multiaddr" ) @@ -73,7 +71,7 @@ func OptPeerPrivateKey(sk crypto.PrivKey) Option { } // GenUpgrader creates a new connection upgrader for use with this swarm. -func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { +func GenUpgrader(n network.Network) *tptu.Upgrader { id := n.LocalPeer() pk := n.Peerstore().PrivKey(id) secMuxer := new(csms.SSMuxer) @@ -88,8 +86,18 @@ func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { } } +type mSwarm struct { + *swarm.Swarm + ps peerstore.Peerstore +} + +func (s *mSwarm) Close() error { + s.ps.Close() + return s.Swarm.Close() +} + // GenSwarm generates a new test swarm. -func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { +func GenSwarm(t *testing.T, opts ...Option) network.Network { var cfg config for _, o := range opts { o(t, &cfg) @@ -113,11 +121,10 @@ func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm { ps := pstoremem.NewPeerstore() ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) - s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter(), cfg.connectionGater) - - // Call AddChildNoWait because we can't call AddChild after the process - // may have been closed (e.g., if the context was canceled). - s.Process().AddChildNoWait(goprocess.WithTeardown(ps.Close)) + s := &mSwarm{ + Swarm: swarm.NewSwarm(p.ID, ps, metrics.NewBandwidthCounter(), cfg.connectionGater), + ps: ps, + } upgrader := GenUpgrader(s) upgrader.ConnGater = cfg.connectionGater diff --git a/p2p/net/swarm/testing/testing_test.go b/p2p/net/swarm/testing/testing_test.go index a80cca177c..60cd278719 100644 --- a/p2p/net/swarm/testing/testing_test.go +++ b/p2p/net/swarm/testing/testing_test.go @@ -1,14 +1,13 @@ package testing import ( - "context" "testing" "github.com/stretchr/testify/require" ) func TestGenSwarm(t *testing.T) { - swarm := GenSwarm(t, context.Background()) + swarm := GenSwarm(t) require.NoError(t, swarm.Close()) GenUpgrader(swarm) } diff --git a/p2p/net/swarm/transport_test.go b/p2p/net/swarm/transport_test.go index 82225840b9..5272602650 100644 --- a/p2p/net/swarm/transport_test.go +++ b/p2p/net/swarm/transport_test.go @@ -7,9 +7,13 @@ import ( swarm "github.com/libp2p/go-libp2p-swarm" swarmt "github.com/libp2p/go-libp2p-swarm/testing" + "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" + ma "github.com/multiformats/go-multiaddr" + + "github.com/stretchr/testify/require" ) type dummyTransport struct { @@ -42,24 +46,23 @@ func (dt *dummyTransport) Close() error { return nil } +type swarmWithTransport interface { + network.Network + AddTransport(transport.Transport) error +} + func TestUselessTransport(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := swarmt.GenSwarm(t, ctx) - err := s.AddTransport(new(dummyTransport)) + s := swarmt.GenSwarm(t) + err := s.(swarmWithTransport).AddTransport(new(dummyTransport)) if err == nil { t.Fatal("adding a transport that supports no protocols should have failed") } } func TestTransportClose(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := swarmt.GenSwarm(t, ctx) + s := swarmt.GenSwarm(t) tpt := &dummyTransport{protocols: []int{1}} - if err := s.AddTransport(tpt); err != nil { - t.Fatal(err) - } + require.NoError(t, s.(swarmWithTransport).AddTransport(tpt)) _ = s.Close() if !tpt.closed { t.Fatal("expected transport to be closed") @@ -68,13 +71,11 @@ func TestTransportClose(t *testing.T) { } func TestTransportAfterClose(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := swarmt.GenSwarm(t, ctx) + s := swarmt.GenSwarm(t) s.Close() tpt := &dummyTransport{protocols: []int{1}} - if err := s.AddTransport(tpt); err != swarm.ErrSwarmClosed { + if err := s.(swarmWithTransport).AddTransport(tpt); err != swarm.ErrSwarmClosed { t.Fatal("expected swarm closed error, got: ", err) } } From a872d26b7c566429e744fb3130879454f64ee3a6 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 7 Sep 2021 10:42:50 +0100 Subject: [PATCH 233/259] use testing.Cleanup to shut down peerstore and revert most test changes --- p2p/net/swarm/dial_test.go | 23 +++++++++-------------- p2p/net/swarm/peers_test.go | 6 ++++-- p2p/net/swarm/simul_test.go | 4 ++-- p2p/net/swarm/swarm_notif_test.go | 2 +- p2p/net/swarm/swarm_test.go | 10 +++++----- p2p/net/swarm/testing/testing.go | 21 +++++---------------- p2p/net/swarm/transport_test.go | 16 +++------------- 7 files changed, 29 insertions(+), 53 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index de41e7f740..5fe4bdc223 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -28,12 +28,7 @@ func init() { transport.DialTimeout = time.Second } -type swarmWithBackoff interface { - network.Network - Backoff() *DialBackoff -} - -func closeSwarms(swarms []network.Network) { +func closeSwarms(swarms []*Swarm) { for _, s := range swarms { s.Close() } @@ -101,7 +96,7 @@ func TestSimultDials(t *testing.T) { // connect everyone { var wg sync.WaitGroup - connect := func(s network.Network, dst peer.ID, addr ma.Multiaddr) { + connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) s.Peerstore().AddAddr(dst, addr, peerstore.TempAddrTTL) @@ -193,7 +188,7 @@ func TestDialWait(t *testing.T) { t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) } - if !s1.(swarmWithBackoff).Backoff().Backoff(s2p, s2addr) { + if !s1.Backoff().Backoff(s2p, s2addr) { t.Error("s2 should now be on backoff") } } @@ -330,10 +325,10 @@ func TestDialBackoff(t *testing.T) { } // check backoff state - if s1.(swarmWithBackoff).Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { + if s1.Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { t.Error("s2 should not be on backoff") } - if !s1.(swarmWithBackoff).Backoff().Backoff(s3p, s3addr) { + if !s1.Backoff().Backoff(s3p, s3addr) { t.Error("s3 should be on backoff") } @@ -400,10 +395,10 @@ func TestDialBackoff(t *testing.T) { } // check backoff state (the same) - if s1.(swarmWithBackoff).Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { + if s1.Backoff().Backoff(s2.LocalPeer(), s2addrs[0]) { t.Error("s2 should not be on backoff") } - if !s1.(swarmWithBackoff).Backoff().Backoff(s3p, s3addr) { + if !s1.Backoff().Backoff(s3p, s3addr) { t.Error("s3 should be on backoff") } } @@ -445,7 +440,7 @@ func TestDialBackoffClears(t *testing.T) { t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) } - if !s1.(swarmWithBackoff).Backoff().Backoff(s2.LocalPeer(), s2bad) { + if !s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { t.Error("s2 should now be on backoff") } else { t.Log("correctly added to backoff") @@ -472,7 +467,7 @@ func TestDialBackoffClears(t *testing.T) { t.Log("correctly connected") } - if s1.(swarmWithBackoff).Backoff().Backoff(s2.LocalPeer(), s2bad) { + if s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { t.Error("s2 should no longer be on backoff") } else { t.Log("correctly cleared backoff") diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 908abe91a4..3145d862bb 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -9,6 +9,8 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" ma "github.com/multiformats/go-multiaddr" + + . "github.com/libp2p/go-libp2p-swarm" ) func TestPeers(t *testing.T) { @@ -17,7 +19,7 @@ func TestPeers(t *testing.T) { s1 := swarms[0] s2 := swarms[1] - connect := func(s network.Network, dst peer.ID, addr ma.Multiaddr) { + connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) // t.Logf("connections from %s", s.LocalPeer()) @@ -53,7 +55,7 @@ func TestPeers(t *testing.T) { log.Infof("%s swarm routing table: %s", s.LocalPeer(), s.Peers()) } - test := func(s network.Network) { + test := func(s *Swarm) { expect := 1 actual := len(s.Peers()) if actual != expect { diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 326c4e21b6..aa4eb59095 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,12 +7,12 @@ import ( "testing" "time" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" ma "github.com/multiformats/go-multiaddr" + . "github.com/libp2p/go-libp2p-swarm" swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-libp2p-testing/ci" ) @@ -24,7 +24,7 @@ func TestSimultOpen(t *testing.T) { // connect everyone { var wg sync.WaitGroup - connect := func(s network.Network, dst peer.ID, addr ma.Multiaddr) { + connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { defer wg.Done() // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index 157405cba1..c0c6f82db8 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -94,7 +94,7 @@ func TestNotifications(t *testing.T) { } } - complement := func(c network.Conn) (network.Network, *netNotifiee, *Conn) { + complement := func(c network.Conn) (*Swarm, *netNotifiee, *Conn) { for i, s := range swarms { for _, c2 := range s.Conns() { if c.LocalMultiaddr().Equal(c2.RemoteMultiaddr()) && diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index e6cb0ba307..9f799f663e 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -58,14 +58,14 @@ func EchoStreamHandler(stream network.Stream) { }() } -func makeDialOnlySwarm(t *testing.T) network.Network { +func makeDialOnlySwarm(t *testing.T) *Swarm { swarm := GenSwarm(t, OptDialOnly) swarm.SetStreamHandler(EchoStreamHandler) return swarm } -func makeSwarms(t *testing.T, num int, opts ...Option) []network.Network { - swarms := make([]network.Network, 0, num) +func makeSwarms(t *testing.T, num int, opts ...Option) []*Swarm { + swarms := make([]*Swarm, 0, num) for i := 0; i < num; i++ { swarm := GenSwarm(t, opts...) swarm.SetStreamHandler(EchoStreamHandler) @@ -74,9 +74,9 @@ func makeSwarms(t *testing.T, num int, opts ...Option) []network.Network { return swarms } -func connectSwarms(t *testing.T, ctx context.Context, swarms []network.Network) { +func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { var wg sync.WaitGroup - connect := func(s network.Network, dst peer.ID, addr ma.Multiaddr) { + connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) if _, err := s.DialPeer(ctx, dst); err != nil { diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index d6091354b5..201b4f0f1a 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -71,7 +71,7 @@ func OptPeerPrivateKey(sk crypto.PrivKey) Option { } // GenUpgrader creates a new connection upgrader for use with this swarm. -func GenUpgrader(n network.Network) *tptu.Upgrader { +func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { id := n.LocalPeer() pk := n.Peerstore().PrivKey(id) secMuxer := new(csms.SSMuxer) @@ -86,18 +86,8 @@ func GenUpgrader(n network.Network) *tptu.Upgrader { } } -type mSwarm struct { - *swarm.Swarm - ps peerstore.Peerstore -} - -func (s *mSwarm) Close() error { - s.ps.Close() - return s.Swarm.Close() -} - // GenSwarm generates a new test swarm. -func GenSwarm(t *testing.T, opts ...Option) network.Network { +func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { var cfg config for _, o := range opts { o(t, &cfg) @@ -121,10 +111,9 @@ func GenSwarm(t *testing.T, opts ...Option) network.Network { ps := pstoremem.NewPeerstore() ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) - s := &mSwarm{ - Swarm: swarm.NewSwarm(p.ID, ps, metrics.NewBandwidthCounter(), cfg.connectionGater), - ps: ps, - } + t.Cleanup(func() { ps.Close() }) + + s := swarm.NewSwarm(p.ID, ps, metrics.NewBandwidthCounter(), cfg.connectionGater) upgrader := GenUpgrader(s) upgrader.ConnGater = cfg.connectionGater diff --git a/p2p/net/swarm/transport_test.go b/p2p/net/swarm/transport_test.go index 5272602650..6d5913cf58 100644 --- a/p2p/net/swarm/transport_test.go +++ b/p2p/net/swarm/transport_test.go @@ -7,7 +7,6 @@ import ( swarm "github.com/libp2p/go-libp2p-swarm" swarmt "github.com/libp2p/go-libp2p-swarm/testing" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" @@ -46,28 +45,19 @@ func (dt *dummyTransport) Close() error { return nil } -type swarmWithTransport interface { - network.Network - AddTransport(transport.Transport) error -} - func TestUselessTransport(t *testing.T) { s := swarmt.GenSwarm(t) - err := s.(swarmWithTransport).AddTransport(new(dummyTransport)) - if err == nil { - t.Fatal("adding a transport that supports no protocols should have failed") - } + require.Error(t, s.AddTransport(new(dummyTransport)), "adding a transport that supports no protocols should have failed") } func TestTransportClose(t *testing.T) { s := swarmt.GenSwarm(t) tpt := &dummyTransport{protocols: []int{1}} - require.NoError(t, s.(swarmWithTransport).AddTransport(tpt)) + require.NoError(t, s.AddTransport(tpt)) _ = s.Close() if !tpt.closed { t.Fatal("expected transport to be closed") } - } func TestTransportAfterClose(t *testing.T) { @@ -75,7 +65,7 @@ func TestTransportAfterClose(t *testing.T) { s.Close() tpt := &dummyTransport{protocols: []int{1}} - if err := s.(swarmWithTransport).AddTransport(tpt); err != swarm.ErrSwarmClosed { + if err := s.AddTransport(tpt); err != swarm.ErrSwarmClosed { t.Fatal("expected swarm closed error, got: ", err) } } From 0537306605873e8e18bc658045a6022cc20d5beb Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 7 Sep 2021 10:56:55 +0100 Subject: [PATCH 234/259] cancel the ctx when closing, use a sync.Once to only close once --- p2p/net/swarm/swarm.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 0f4e3c75bd..43f7a36f90 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -90,6 +90,7 @@ type Swarm struct { limiter *dialLimiter gater connmgr.ConnectionGater + closeOnce sync.Once ctx context.Context // is canceled when Close is called ctxCancel context.CancelFunc @@ -131,8 +132,14 @@ func NewSwarm(local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter, ex } func (s *Swarm) Close() error { - // Prevents new connections and/or listeners from being added to the swarm. + s.closeOnce.Do(s.close) + return nil +} + +func (s *Swarm) close() { + s.ctxCancel() + // Prevents new connections and/or listeners from being added to the swarm. s.listeners.Lock() listeners := s.listeners.m s.listeners.m = nil @@ -187,8 +194,6 @@ func (s *Swarm) Close() error { } } wg.Wait() - - return nil } func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, error) { From 21c448606039639f1d9c81197dbc455e0c6558b1 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 7 Sep 2021 11:26:57 +0100 Subject: [PATCH 235/259] use functional options to configure the swarm --- p2p/net/swarm/swarm.go | 23 ++++++++++++----------- p2p/net/swarm/swarm_test.go | 14 +++++++------- p2p/net/swarm/testing/testing.go | 6 +++++- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 43f7a36f90..e9768ab1b4 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -41,6 +41,15 @@ var ErrAddrFiltered = errors.New("address filtered") // ErrDialTimeout is returned when one a dial times out due to the global timeout var ErrDialTimeout = errors.New("dial timed out") +type Option func(*Swarm) + +// WithConnectionGater sets a connection gater +func WithConnectionGater(gater connmgr.ConnectionGater) Option { + return func(s *Swarm) { + s.gater = gater + } +} + // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the @@ -98,12 +107,7 @@ type Swarm struct { } // NewSwarm constructs a Swarm. -// -// NOTE: go-libp2p will be moving to dependency injection soon. The variadic -// `extra` interface{} parameter facilitates the future migration. Supported -// elements are: -// - connmgr.ConnectionGater -func NewSwarm(local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter, extra ...interface{}) *Swarm { +func NewSwarm(local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter, opts ...Option) *Swarm { ctx, cancel := context.WithCancel(context.Background()) s := &Swarm{ local: local, @@ -118,11 +122,8 @@ func NewSwarm(local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter, ex s.transports.m = make(map[int]transport.Transport) s.notifs.m = make(map[network.Notifiee]struct{}) - for _, i := range extra { - switch v := i.(type) { - case connmgr.ConnectionGater: - s.gater = v - } + for _, opt := range opts { + opt(s) } s.dsync = newDialSync(s.dialWorkerLoop) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 9f799f663e..07551fd0b3 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -16,7 +16,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" - . "github.com/libp2p/go-libp2p-swarm" + swarm "github.com/libp2p/go-libp2p-swarm" . "github.com/libp2p/go-libp2p-swarm/testing" logging "github.com/ipfs/go-log" @@ -58,14 +58,14 @@ func EchoStreamHandler(stream network.Stream) { }() } -func makeDialOnlySwarm(t *testing.T) *Swarm { +func makeDialOnlySwarm(t *testing.T) *swarm.Swarm { swarm := GenSwarm(t, OptDialOnly) swarm.SetStreamHandler(EchoStreamHandler) return swarm } -func makeSwarms(t *testing.T, num int, opts ...Option) []*Swarm { - swarms := make([]*Swarm, 0, num) +func makeSwarms(t *testing.T, num int, opts ...Option) []*swarm.Swarm { + swarms := make([]*swarm.Swarm, 0, num) for i := 0; i < num; i++ { swarm := GenSwarm(t, opts...) swarm.SetStreamHandler(EchoStreamHandler) @@ -74,9 +74,9 @@ func makeSwarms(t *testing.T, num int, opts ...Option) []*Swarm { return swarms } -func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { +func connectSwarms(t *testing.T, ctx context.Context, swarms []*swarm.Swarm) { var wg sync.WaitGroup - connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { + connect := func(s *swarm.Swarm, dst peer.ID, addr ma.Multiaddr) { // TODO: make a DialAddr func. s.Peerstore().AddAddr(dst, addr, peerstore.PermanentAddrTTL) if _, err := s.DialPeer(ctx, dst); err != nil { @@ -455,7 +455,7 @@ func TestPreventDialListenAddr(t *testing.T) { remote := peer.ID("foobar") s.Peerstore().AddAddr(remote, addr, time.Hour) _, err = s.DialPeer(context.Background(), remote) - if !errors.Is(err, ErrNoGoodAddresses) { + if !errors.Is(err, swarm.ErrNoGoodAddresses) { t.Fatal("expected dial to fail: %w", err) } } diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 201b4f0f1a..ff84252b86 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -113,7 +113,11 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { ps.AddPrivKey(p.ID, p.PrivKey) t.Cleanup(func() { ps.Close() }) - s := swarm.NewSwarm(p.ID, ps, metrics.NewBandwidthCounter(), cfg.connectionGater) + var swarmOpts []swarm.Option + if cfg.connectionGater != nil { + swarmOpts = append(swarmOpts, swarm.WithConnectionGater(cfg.connectionGater)) + } + s := swarm.NewSwarm(p.ID, ps, metrics.NewBandwidthCounter(), swarmOpts...) upgrader := GenUpgrader(s) upgrader.ConnGater = cfg.connectionGater From c2439b03ed6e0c5ab679f8800d877249e514fd47 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 7 Sep 2021 13:38:10 +0100 Subject: [PATCH 236/259] convert the metrics reporter to an option --- p2p/net/swarm/swarm.go | 10 ++++++++-- p2p/net/swarm/testing/testing.go | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index e9768ab1b4..a55c8d9c73 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -50,6 +50,13 @@ func WithConnectionGater(gater connmgr.ConnectionGater) Option { } } +// WithMetrics sets a metrics reporter +func WithMetrics(reporter metrics.Reporter) Option { + return func(s *Swarm) { + s.bwc = reporter + } +} + // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the @@ -107,12 +114,11 @@ type Swarm struct { } // NewSwarm constructs a Swarm. -func NewSwarm(local peer.ID, peers peerstore.Peerstore, bwc metrics.Reporter, opts ...Option) *Swarm { +func NewSwarm(local peer.ID, peers peerstore.Peerstore, opts ...Option) *Swarm { ctx, cancel := context.WithCancel(context.Background()) s := &Swarm{ local: local, peers: peers, - bwc: bwc, ctx: ctx, ctxCancel: cancel, } diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index ff84252b86..b0d4c218ef 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -113,11 +113,11 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { ps.AddPrivKey(p.ID, p.PrivKey) t.Cleanup(func() { ps.Close() }) - var swarmOpts []swarm.Option + swarmOpts := []swarm.Option{swarm.WithMetrics(metrics.NewBandwidthCounter())} if cfg.connectionGater != nil { swarmOpts = append(swarmOpts, swarm.WithConnectionGater(cfg.connectionGater)) } - s := swarm.NewSwarm(p.ID, ps, metrics.NewBandwidthCounter(), swarmOpts...) + s := swarm.NewSwarm(p.ID, ps, swarmOpts...) upgrader := GenUpgrader(s) upgrader.ConnGater = cfg.connectionGater From 96eb4c7e1c560badbbfc56e6392fe94a24fca54d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 8 Sep 2021 17:58:30 +0100 Subject: [PATCH 237/259] add an error return value to the constructor --- p2p/net/swarm/swarm.go | 16 ++++++++++------ p2p/net/swarm/testing/testing.go | 5 ++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index a55c8d9c73..c5c3bf2850 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -41,19 +41,21 @@ var ErrAddrFiltered = errors.New("address filtered") // ErrDialTimeout is returned when one a dial times out due to the global timeout var ErrDialTimeout = errors.New("dial timed out") -type Option func(*Swarm) +type Option func(*Swarm) error // WithConnectionGater sets a connection gater func WithConnectionGater(gater connmgr.ConnectionGater) Option { - return func(s *Swarm) { + return func(s *Swarm) error { s.gater = gater + return nil } } // WithMetrics sets a metrics reporter func WithMetrics(reporter metrics.Reporter) Option { - return func(s *Swarm) { + return func(s *Swarm) error { s.bwc = reporter + return nil } } @@ -114,7 +116,7 @@ type Swarm struct { } // NewSwarm constructs a Swarm. -func NewSwarm(local peer.ID, peers peerstore.Peerstore, opts ...Option) *Swarm { +func NewSwarm(local peer.ID, peers peerstore.Peerstore, opts ...Option) (*Swarm, error) { ctx, cancel := context.WithCancel(context.Background()) s := &Swarm{ local: local, @@ -129,13 +131,15 @@ func NewSwarm(local peer.ID, peers peerstore.Peerstore, opts ...Option) *Swarm { s.notifs.m = make(map[network.Notifiee]struct{}) for _, opt := range opts { - opt(s) + if err := opt(s); err != nil { + return nil, err + } } s.dsync = newDialSync(s.dialWorkerLoop) s.limiter = newDialLimiter(s.dialAddr) s.backf.init(s.ctx) - return s + return s, nil } func (s *Swarm) Close() error { diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index b0d4c218ef..0e4974f41d 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -3,6 +3,8 @@ package testing import ( "testing" + "github.com/stretchr/testify/require" + csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/control" @@ -117,7 +119,8 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { if cfg.connectionGater != nil { swarmOpts = append(swarmOpts, swarm.WithConnectionGater(cfg.connectionGater)) } - s := swarm.NewSwarm(p.ID, ps, swarmOpts...) + s, err := swarm.NewSwarm(p.ID, ps, swarmOpts...) + require.NoError(t, err) upgrader := GenUpgrader(s) upgrader.ConnGater = cfg.connectionGater From 0af4415ada749183a9a4f48d20cd8233a071ea61 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 8 Sep 2021 18:05:42 +0100 Subject: [PATCH 238/259] unexport the DialSync --- p2p/net/swarm/dial_sync.go | 16 ++++++++-------- p2p/net/swarm/swarm.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index e135d9396e..73d8af6b1f 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -8,20 +8,20 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -// dialWorkerFunc is used by DialSync to spawn a new dial worker +// dialWorkerFunc is used by dialSync to spawn a new dial worker type dialWorkerFunc func(peer.ID, <-chan dialRequest) -// newDialSync constructs a new DialSync -func newDialSync(worker dialWorkerFunc) *DialSync { - return &DialSync{ +// newDialSync constructs a new dialSync +func newDialSync(worker dialWorkerFunc) *dialSync { + return &dialSync{ dials: make(map[peer.ID]*activeDial), dialWorker: worker, } } -// DialSync is a dial synchronization helper that ensures that at most one dial +// dialSync is a dial synchronization helper that ensures that at most one dial // to any given peer is active at any given time. -type DialSync struct { +type dialSync struct { mutex sync.Mutex dials map[peer.ID]*activeDial dialWorker dialWorkerFunc @@ -66,7 +66,7 @@ func (ad *activeDial) dial(ctx context.Context) (*Conn, error) { } } -func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { +func (ds *dialSync) getActiveDial(p peer.ID) (*activeDial, error) { ds.mutex.Lock() defer ds.mutex.Unlock() @@ -92,7 +92,7 @@ func (ds *DialSync) getActiveDial(p peer.ID) (*activeDial, error) { // Dial initiates a dial to the given peer if there are none in progress // then waits for the dial to that peer to complete. -func (ds *DialSync) Dial(ctx context.Context, p peer.ID) (*Conn, error) { +func (ds *dialSync) Dial(ctx context.Context, p peer.ID) (*Conn, error) { ad, err := ds.getActiveDial(p) if err != nil { return nil, err diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index a55c8d9c73..17e281e0d2 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -101,7 +101,7 @@ type Swarm struct { streamh atomic.Value // dialing helpers - dsync *DialSync + dsync *dialSync backf DialBackoff limiter *dialLimiter gater connmgr.ConnectionGater From 6d9ce0e3cf5ee45da4f4541534697c0d1033507c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 21 Sep 2021 22:50:10 +0100 Subject: [PATCH 239/259] remove the ConnHandler --- p2p/net/swarm/peers_test.go | 17 +++++------------ p2p/net/swarm/swarm.go | 25 +------------------------ p2p/net/swarm/swarm_test.go | 35 ----------------------------------- 3 files changed, 6 insertions(+), 71 deletions(-) diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index 3145d862bb..d7c19b4f15 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -3,14 +3,16 @@ package swarm_test import ( "context" "testing" + "time" + + . "github.com/libp2p/go-libp2p-swarm" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" ma "github.com/multiformats/go-multiaddr" - . "github.com/libp2p/go-libp2p-swarm" + "github.com/stretchr/testify/require" ) func TestPeers(t *testing.T) { @@ -33,17 +35,8 @@ func TestPeers(t *testing.T) { // t.Log(s.swarm.Dump()) } - s1GotConn := make(chan struct{}) - s2GotConn := make(chan struct{}) - s1.SetConnHandler(func(c network.Conn) { - s1GotConn <- struct{}{} - }) - s2.SetConnHandler(func(c network.Conn) { - s2GotConn <- struct{}{} - }) - connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) - <-s2GotConn // have to wait here so the other side catches up. + require.Eventually(t, func() bool { return len(s2.Peers()) > 0 }, 3*time.Second, 50*time.Millisecond) connect(s2, s1.LocalPeer(), s1.ListenAddresses()[0]) for i := 0; i < 100; i++ { diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 73edeb8b33..cce11a7b25 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -98,8 +98,7 @@ type Swarm struct { m map[int]transport.Transport } - // new connection and stream handlers - connh atomic.Value + // stream handlers streamh atomic.Value // dialing helpers @@ -278,14 +277,6 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, c.notifyLk.Unlock() c.start() - - // TODO: Get rid of this. We use it for identify but that happen much - // earlier (really, inside the transport and, if not then, during the - // notifications). - if h := s.ConnHandler(); h != nil { - go h(c) - } - return c, nil } @@ -294,20 +285,6 @@ func (s *Swarm) Peerstore() peerstore.Peerstore { return s.peers } -// TODO: We probably don't need the conn handlers. - -// SetConnHandler assigns the handler for new connections. -// You will rarely use this. See SetStreamHandler -func (s *Swarm) SetConnHandler(handler network.ConnHandler) { - s.connh.Store(handler) -} - -// ConnHandler gets the handler for new connections. -func (s *Swarm) ConnHandler() network.ConnHandler { - handler, _ := s.connh.Load().(network.ConnHandler) - return handler -} - // SetStreamHandler assigns the handler for new streams. func (s *Swarm) SetStreamHandler(handler network.StreamHandler) { s.streamh.Store(handler) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 07551fd0b3..8400e3a865 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -241,41 +241,6 @@ func TestBasicSwarm(t *testing.T) { SubtestSwarm(t, swarms, msgs) } -func TestConnHandler(t *testing.T) { - // t.Skip("skipping for another test") - t.Parallel() - - ctx := context.Background() - swarms := makeSwarms(t, 5) - - gotconn := make(chan struct{}, 10) - swarms[0].SetConnHandler(func(conn network.Conn) { - gotconn <- struct{}{} - }) - - connectSwarms(t, ctx, swarms) - - <-time.After(time.Millisecond) - // should've gotten 5 by now. - - swarms[0].SetConnHandler(nil) - - expect := 4 - for i := 0; i < expect; i++ { - select { - case <-time.After(time.Second): - t.Fatal("failed to get connections") - case <-gotconn: - } - } - - select { - case <-gotconn: - t.Fatalf("should have connected to %d swarms, got an extra.", expect) - default: - } -} - func TestConnectionGating(t *testing.T) { ctx := context.Background() tcs := map[string]struct { From c3cd06d61c4a25270451c16c74add89a1030fc19 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 29 Sep 2021 15:56:41 +0100 Subject: [PATCH 240/259] update go-tcp-transport to v0.4.0 --- p2p/net/swarm/testing/testing.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 0e4974f41d..737da71515 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -126,8 +126,12 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { upgrader.ConnGater = cfg.connectionGater if !cfg.disableTCP { - tcpTransport := tcp.NewTCPTransport(upgrader) - tcpTransport.DisableReuseport = cfg.disableReuseport + var tcpOpts []tcp.Option + if cfg.disableReuseport { + tcpOpts = append(tcpOpts, tcp.DisableReuseport()) + } + tcpTransport, err := tcp.NewTCPTransport(upgrader, tcpOpts...) + require.NoError(t, err) if err := s.AddTransport(tcpTransport); err != nil { t.Fatal(err) } From 8707662663c88551330af5a70ceaee419cfcd410 Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Tue, 9 Nov 2021 10:44:28 -0500 Subject: [PATCH 241/259] feat: plumb contexts through from peerstore (#290) --- p2p/net/swarm/testing/testing.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 737da71515..e2ffabe28b 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -110,7 +110,8 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { p.Addr = tnet.ZeroLocalTCPAddress } - ps := pstoremem.NewPeerstore() + ps, err := pstoremem.NewPeerstore() + require.NoError(t, err) ps.AddPubKey(p.ID, p.PubKey) ps.AddPrivKey(p.ID, p.PrivKey) t.Cleanup(func() { ps.Close() }) From 212b671494e067c312482359bd983827e050769e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 7 Dec 2021 11:19:13 +0400 Subject: [PATCH 242/259] chore: update go-log to v2 (#294) --- p2p/net/swarm/swarm.go | 3 +-- p2p/net/swarm/swarm_test.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index cce11a7b25..e5077ac0c3 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -17,8 +17,7 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/transport" - logging "github.com/ipfs/go-log" - + logging "github.com/ipfs/go-log/v2" ma "github.com/multiformats/go-multiaddr" ) diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 8400e3a865..043e5735ad 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -19,7 +19,7 @@ import ( swarm "github.com/libp2p/go-libp2p-swarm" . "github.com/libp2p/go-libp2p-swarm/testing" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/stretchr/testify/require" From f417a8d5ced60b16a88ae7c0b63b3b21f81dbff1 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 10 Dec 2021 15:28:54 +0400 Subject: [PATCH 243/259] count the number of streams on a connection for the stats --- p2p/net/swarm/swarm.go | 2 +- p2p/net/swarm/swarm_conn.go | 19 ++++++++------- p2p/net/swarm/swarm_net_test.go | 4 ++-- p2p/net/swarm/swarm_stream.go | 4 ++-- p2p/net/swarm/swarm_test.go | 41 +++++++++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 13 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index e5077ac0c3..b1f51a351a 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -212,7 +212,7 @@ func (s *Swarm) addConn(tc transport.CapableConn, dir network.Direction) (*Conn, ) // create the Stat object, initializing with the underlying connection Stat if available - var stat network.Stat + var stat network.ConnStats if cs, ok := tc.(network.ConnStat); ok { stat = cs.Stat() } diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 3d5f8f3735..388e9e76e4 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -39,7 +39,7 @@ type Conn struct { m map[*Stream]struct{} } - stat network.Stat + stat network.ConnStats } func (c *Conn) ID() string { @@ -90,6 +90,7 @@ func (c *Conn) doClose() { func (c *Conn) removeStream(s *Stream) { c.streams.Lock() + c.stat.NumStreams-- delete(c.streams.m, s) c.streams.Unlock() } @@ -171,7 +172,9 @@ func (c *Conn) RemotePublicKey() ic.PubKey { } // Stat returns metadata pertaining to this connection -func (c *Conn) Stat() network.Stat { +func (c *Conn) Stat() network.ConnStats { + c.streams.Lock() + defer c.streams.Unlock() return c.stat } @@ -201,16 +204,16 @@ func (c *Conn) addStream(ts mux.MuxedStream, dir network.Direction) (*Stream, er } // Wrap and register the stream. - stat := network.Stat{ - Direction: dir, - Opened: time.Now(), - } s := &Stream{ stream: ts, conn: c, - stat: stat, - id: atomic.AddUint64(&c.swarm.nextStreamID, 1), + stat: network.Stats{ + Direction: dir, + Opened: time.Now(), + }, + id: atomic.AddUint64(&c.swarm.nextStreamID, 1), } + c.stat.NumStreams++ c.streams.m[s] = struct{}{} // Released once the stream disconnect notifications have finished diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 1f1d0454db..3096574d7a 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -130,9 +130,9 @@ func TestNetworkOpenStream(t *testing.T) { t.Fatal(err) } - numStreams := 0 + var numStreams int for _, conn := range nets[0].ConnsToPeer(nets[1].LocalPeer()) { - numStreams += len(conn.GetStreams()) + numStreams += conn.Stat().NumStreams } if numStreams != 1 { diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index a5f0738ad6..b0063b1b21 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -28,7 +28,7 @@ type Stream struct { protocol atomic.Value - stat network.Stat + stat network.Stats } func (s *Stream) ID() string { @@ -151,6 +151,6 @@ func (s *Stream) SetWriteDeadline(t time.Time) error { } // Stat returns metadata information for this stream. -func (s *Stream) Stat() network.Stat { +func (s *Stream) Stat() network.Stats { return s.stat } diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 043e5735ad..0dac49d1c7 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -424,3 +424,44 @@ func TestPreventDialListenAddr(t *testing.T) { t.Fatal("expected dial to fail: %w", err) } } + +func TestStreamCount(t *testing.T) { + s1 := GenSwarm(t) + s2 := GenSwarm(t) + connectSwarms(t, context.Background(), []*swarm.Swarm{s2, s1}) + + countStreams := func() (n int) { + var num int + for _, c := range s1.ConnsToPeer(s2.LocalPeer()) { + n += c.Stat().NumStreams + num += len(c.GetStreams()) + } + require.Equal(t, n, num, "inconsistent stream count") + return + } + + streams := make(chan network.Stream, 20) + streamAccepted := make(chan struct{}, 1) + s1.SetStreamHandler(func(str network.Stream) { + streams <- str + streamAccepted <- struct{}{} + }) + + for i := 0; i < 10; i++ { + str, err := s2.NewStream(context.Background(), s1.LocalPeer()) + require.NoError(t, err) + str.Write([]byte("foobar")) + <-streamAccepted + } + require.Eventually(t, func() bool { return len(streams) == 10 }, 5*time.Second, 10*time.Millisecond) + require.Equal(t, countStreams(), 10) + (<-streams).Reset() + (<-streams).Close() + require.Equal(t, countStreams(), 8) + + str, err := s1.NewStream(context.Background(), s2.LocalPeer()) + require.NoError(t, err) + require.Equal(t, countStreams(), 9) + str.Close() + require.Equal(t, countStreams(), 8) +} From a32993f6ee423e33c3cf487e5d4350bbd53c371d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 14 Dec 2021 16:46:32 +0400 Subject: [PATCH 244/259] remove addrutil.SubtractFilter --- p2p/net/swarm/swarm_dial.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 80d6958a78..cccf80c13d 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -279,10 +279,10 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { return nil, err } -/////////////////////////////////////////////////////////////////////////////////// +// ///////////////////////////////////////////////////////////////////////////////// // lo and behold, The Dialer // TODO explain how all this works -////////////////////////////////////////////////////////////////////////////////// +// //////////////////////////////////////////////////////////////////////////////// type dialRequest struct { ctx context.Context @@ -650,7 +650,14 @@ func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) []ma.Mul } return addrutil.FilterAddrs(addrs, - addrutil.SubtractFilter(ourAddrs...), + func(addr ma.Multiaddr) bool { + for _, a := range ourAddrs { + if a.Equal(addr) { + return false + } + } + return true + }, s.canDial, // TODO: Consider allowing link-local addresses addrutil.AddrOverNonLocalIP, From f545ea34aa7ee00d9c15a8e8fc269f5b1f2153c4 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 19 Dec 2021 19:39:16 +0400 Subject: [PATCH 245/259] add constructor options for timeout, stop using transport.DialTimeout --- p2p/net/swarm/dial_test.go | 52 ++++++++++++++++---------------- p2p/net/swarm/limiter.go | 20 ++++-------- p2p/net/swarm/swarm.go | 41 +++++++++++++++++++------ p2p/net/swarm/swarm_dial.go | 17 +++++++---- p2p/net/swarm/testing/testing.go | 11 +++++++ 5 files changed, 86 insertions(+), 55 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 5fe4bdc223..a3448640f5 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -14,7 +14,6 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" testutil "github.com/libp2p/go-libp2p-core/test" - "github.com/libp2p/go-libp2p-core/transport" swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-libp2p-testing/ci" @@ -24,10 +23,6 @@ import ( "github.com/stretchr/testify/require" ) -func init() { - transport.DialTimeout = time.Second -} - func closeSwarms(swarms []*Swarm) { for _, s := range swarms { s.Close() @@ -161,8 +156,9 @@ func newSilentPeer(t *testing.T) (peer.ID, ma.Multiaddr, net.Listener) { func TestDialWait(t *testing.T) { t.Parallel() - ctx := context.Background() - swarms := makeSwarms(t, 1) + const dialTimeout = 250 * time.Millisecond + + swarms := makeSwarms(t, 1, swarmt.DialTimeout(dialTimeout)) s1 := swarms[0] defer s1.Close() @@ -173,7 +169,7 @@ func TestDialWait(t *testing.T) { s1.Peerstore().AddAddr(s2p, s2addr, peerstore.PermanentAddrTTL) before := time.Now() - if c, err := s1.DialPeer(ctx, s2p); err == nil { + if c, err := s1.DialPeer(context.Background(), s2p); err == nil { defer c.Close() t.Fatal("error swarm dialing to unknown peer worked...", err) } else { @@ -181,11 +177,11 @@ func TestDialWait(t *testing.T) { } duration := time.Since(before) - if duration < transport.DialTimeout*DialAttempts { - t.Error("< transport.DialTimeout * DialAttempts not being respected", duration, transport.DialTimeout*DialAttempts) + if duration < dialTimeout*DialAttempts { + t.Error("< dialTimeout * DialAttempts not being respected", duration, dialTimeout*DialAttempts) } - if duration > 2*transport.DialTimeout*DialAttempts { - t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) + if duration > 2*dialTimeout*DialAttempts { + t.Error("> 2*dialTimeout * DialAttempts not being respected", duration, 2*dialTimeout*DialAttempts) } if !s1.Backoff().Backoff(s2p, s2addr) { @@ -194,15 +190,16 @@ func TestDialWait(t *testing.T) { } func TestDialBackoff(t *testing.T) { - // t.Skip("skipping for another test") if ci.IsRunning() { t.Skip("travis will never have fun with this test") } t.Parallel() + const dialTimeout = 250 * time.Millisecond + ctx := context.Background() - swarms := makeSwarms(t, 2) + swarms := makeSwarms(t, 2, swarmt.DialTimeout(dialTimeout)) s1 := swarms[0] s2 := swarms[1] defer s1.Close() @@ -269,8 +266,8 @@ func TestDialBackoff(t *testing.T) { s3done := dialOfflineNode(s3p, N) // when all dials should be done by: - dialTimeout1x := time.After(transport.DialTimeout) - dialTimeout10Ax := time.After(transport.DialTimeout * 2 * 10) // DialAttempts * 10) + dialTimeout1x := time.After(dialTimeout) + dialTimeout10Ax := time.After(dialTimeout * 2 * 10) // DialAttempts * 10) // 2) all dials should hang select { @@ -352,8 +349,8 @@ func TestDialBackoff(t *testing.T) { s3done := dialOfflineNode(s3p, N) // when all dials should be done by: - dialTimeout1x := time.After(transport.DialTimeout) - dialTimeout10Ax := time.After(transport.DialTimeout * 2 * 10) // DialAttempts * 10) + dialTimeout1x := time.After(dialTimeout) + dialTimeout10Ax := time.After(dialTimeout * 2 * 10) // DialAttempts * 10) // 7) s3 dials should all return immediately (except 1) for i := 0; i < N-1; i++ { @@ -405,11 +402,12 @@ func TestDialBackoff(t *testing.T) { } func TestDialBackoffClears(t *testing.T) { - // t.Skip("skipping for another test") t.Parallel() + const dialTimeout = 250 * time.Millisecond + ctx := context.Background() - swarms := makeSwarms(t, 2) + swarms := makeSwarms(t, 2, swarmt.DialTimeout(dialTimeout)) s1 := swarms[0] s2 := swarms[1] defer s1.Close() @@ -433,11 +431,11 @@ func TestDialBackoffClears(t *testing.T) { } duration := time.Since(before) - if duration < transport.DialTimeout*DialAttempts { - t.Error("< transport.DialTimeout * DialAttempts not being respected", duration, transport.DialTimeout*DialAttempts) + if duration < dialTimeout*DialAttempts { + t.Error("< dialTimeout * DialAttempts not being respected", duration, dialTimeout*DialAttempts) } - if duration > 2*transport.DialTimeout*DialAttempts { - t.Error("> 2*transport.DialTimeout * DialAttempts not being respected", duration, 2*transport.DialTimeout*DialAttempts) + if duration > 2*dialTimeout*DialAttempts { + t.Error("> 2*dialTimeout * DialAttempts not being respected", duration, 2*dialTimeout*DialAttempts) } if !s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { @@ -561,7 +559,9 @@ func TestDialSimultaneousJoin(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - swarms := makeSwarms(t, 2) + const dialTimeout = 250 * time.Millisecond + + swarms := makeSwarms(t, 2, swarmt.DialTimeout(dialTimeout)) s1 := swarms[0] s2 := swarms[1] defer s1.Close() @@ -654,7 +654,7 @@ func TestDialSimultaneousJoin(t *testing.T) { if c1 != c2 { t.Fatal("expected c1 and c2 to be the same") } - case <-time.After(2 * transport.DialTimeout): + case <-time.After(2 * dialTimeout): t.Fatal("no connection from first dial") } } diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index 5cd8cfa29e..ac72279bfe 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -20,25 +20,17 @@ type dialResult struct { } type dialJob struct { - addr ma.Multiaddr - peer peer.ID - ctx context.Context - resp chan dialResult + addr ma.Multiaddr + peer peer.ID + ctx context.Context + resp chan dialResult + timeout time.Duration } func (dj *dialJob) cancelled() bool { return dj.ctx.Err() != nil } -func (dj *dialJob) dialTimeout() time.Duration { - timeout := transport.DialTimeout - if lowTimeoutFilters.AddrBlocked(dj.addr) { - timeout = DialTimeoutLocal - } - - return timeout -} - type dialLimiter struct { lk sync.Mutex @@ -221,7 +213,7 @@ func (dl *dialLimiter) executeDial(j *dialJob) { return } - dctx, cancel := context.WithTimeout(j.ctx, j.dialTimeout()) + dctx, cancel := context.WithTimeout(j.ctx, j.timeout) defer cancel() con, err := dl.dialFunc(dctx, j.peer, j.addr) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index b1f51a351a..4761790769 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -21,11 +21,15 @@ import ( ma "github.com/multiformats/go-multiaddr" ) -// DialTimeoutLocal is the maximum duration a Dial to local network address -// is allowed to take. -// This includes the time between dialing the raw network connection, -// protocol selection as well the handshake, if applicable. -var DialTimeoutLocal = 5 * time.Second +const ( + defaultDialTimeout = 15 * time.Second + + // defaultDialTimeoutLocal is the maximum duration a Dial to local network address + // is allowed to take. + // This includes the time between dialing the raw network connection, + // protocol selection as well the handshake, if applicable. + defaultDialTimeoutLocal = 5 * time.Second +) var log = logging.Logger("swarm2") @@ -58,6 +62,20 @@ func WithMetrics(reporter metrics.Reporter) Option { } } +func WithDialTimeout(t time.Duration) Option { + return func(s *Swarm) error { + s.dialTimeout = t + return nil + } +} + +func WithDialTimeoutLocal(t time.Duration) Option { + return func(s *Swarm) error { + s.dialTimeoutLocal = t + return nil + } +} + // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the @@ -73,6 +91,9 @@ type Swarm struct { local peer.ID peers peerstore.Peerstore + dialTimeout time.Duration + dialTimeoutLocal time.Duration + conns struct { sync.RWMutex m map[peer.ID][]*Conn @@ -117,10 +138,12 @@ type Swarm struct { func NewSwarm(local peer.ID, peers peerstore.Peerstore, opts ...Option) (*Swarm, error) { ctx, cancel := context.WithCancel(context.Background()) s := &Swarm{ - local: local, - peers: peers, - ctx: ctx, - ctxCancel: cancel, + local: local, + peers: peers, + ctx: ctx, + ctxCancel: cancel, + dialTimeout: defaultDialTimeout, + dialTimeoutLocal: defaultDialTimeoutLocal, } s.conns.m = make(map[peer.ID][]*Conn) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index 80d6958a78..df5f0d947b 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -279,10 +279,10 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { return nil, err } -/////////////////////////////////////////////////////////////////////////////////// +// ///////////////////////////////////////////////////////////////////////////////// // lo and behold, The Dialer // TODO explain how all this works -////////////////////////////////////////////////////////////////////////////////// +// //////////////////////////////////////////////////////////////////////////////// type dialRequest struct { ctx context.Context @@ -664,11 +664,16 @@ func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) []ma.Mul // it is able, respecting the various different types of rate // limiting that occur without using extra goroutines per addr func (s *Swarm) limitedDial(ctx context.Context, p peer.ID, a ma.Multiaddr, resp chan dialResult) { + timeout := s.dialTimeout + if lowTimeoutFilters.AddrBlocked(a) && s.dialTimeoutLocal < s.dialTimeout { + timeout = s.dialTimeoutLocal + } s.limiter.AddDialJob(&dialJob{ - addr: a, - peer: p, - resp: resp, - ctx: ctx, + addr: a, + peer: p, + resp: resp, + ctx: ctx, + timeout: timeout, }) } diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index e2ffabe28b..0ed3273591 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -2,6 +2,7 @@ package testing import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -31,6 +32,7 @@ type config struct { dialOnly bool disableTCP bool disableQUIC bool + dialTimeout time.Duration connectionGater connmgr.ConnectionGater sk crypto.PrivKey } @@ -72,6 +74,12 @@ func OptPeerPrivateKey(sk crypto.PrivKey) Option { } } +func DialTimeout(t time.Duration) Option { + return func(_ *testing.T, c *config) { + c.dialTimeout = t + } +} + // GenUpgrader creates a new connection upgrader for use with this swarm. func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { id := n.LocalPeer() @@ -120,6 +128,9 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { if cfg.connectionGater != nil { swarmOpts = append(swarmOpts, swarm.WithConnectionGater(cfg.connectionGater)) } + if cfg.dialTimeout != 0 { + swarmOpts = append(swarmOpts, swarm.WithDialTimeout(cfg.dialTimeout)) + } s, err := swarm.NewSwarm(p.ID, ps, swarmOpts...) require.NoError(t, err) From d12b3851b9b784ffba8165740665d8190230e275 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 19 Dec 2021 17:33:14 +0400 Subject: [PATCH 246/259] speed up the dial tests --- p2p/net/swarm/dial_test.go | 127 +++++++++------------------------- p2p/net/swarm/limiter_test.go | 43 ++++++------ p2p/net/swarm/simul_test.go | 4 +- p2p/net/swarm/swarm_test.go | 18 +---- 4 files changed, 59 insertions(+), 133 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index a3448640f5..fe8b32c2fe 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -82,11 +82,11 @@ func acceptAndHang(l net.Listener) { } func TestSimultDials(t *testing.T) { - // t.Skip("skipping for another test") t.Parallel() ctx := context.Background() swarms := makeSwarms(t, 2, swarmt.OptDisableReuseport) + defer closeSwarms(swarms) // connect everyone { @@ -128,10 +128,6 @@ func TestSimultDials(t *testing.T) { if c10l > 2 { t.Error("1->0 has", c10l) } - - for _, s := range swarms { - s.Close() - } } func newSilentPeer(t *testing.T) (peer.ID, ma.Multiaddr, net.Listener) { @@ -154,8 +150,6 @@ func newSilentPeer(t *testing.T) (peer.ID, ma.Multiaddr, net.Listener) { } func TestDialWait(t *testing.T) { - t.Parallel() - const dialTimeout = 250 * time.Millisecond swarms := makeSwarms(t, 1, swarmt.DialTimeout(dialTimeout)) @@ -193,22 +187,18 @@ func TestDialBackoff(t *testing.T) { if ci.IsRunning() { t.Skip("travis will never have fun with this test") } - t.Parallel() - const dialTimeout = 250 * time.Millisecond + const dialTimeout = 100 * time.Millisecond ctx := context.Background() swarms := makeSwarms(t, 2, swarmt.DialTimeout(dialTimeout)) + defer closeSwarms(swarms) s1 := swarms[0] s2 := swarms[1] - defer s1.Close() - defer s2.Close() s2addrs, err := s2.InterfaceListenAddresses() - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) s1.Peerstore().AddAddrs(s2.LocalPeer(), s2addrs, peerstore.PermanentAddrTTL) // dial to a non-existent peer. @@ -405,13 +395,10 @@ func TestDialBackoffClears(t *testing.T) { t.Parallel() const dialTimeout = 250 * time.Millisecond - - ctx := context.Background() swarms := makeSwarms(t, 2, swarmt.DialTimeout(dialTimeout)) + defer closeSwarms(swarms) s1 := swarms[0] s2 := swarms[1] - defer s1.Close() - defer s2.Close() // use another address first, that accept and hang on conns _, s2bad, s2l := newSilentPeer(t) @@ -422,13 +409,8 @@ func TestDialBackoffClears(t *testing.T) { s1.Peerstore().AddAddr(s2.LocalPeer(), s2bad, peerstore.PermanentAddrTTL) before := time.Now() - c, err := s1.DialPeer(ctx, s2.LocalPeer()) - if err == nil { - defer c.Close() - t.Fatal("dialing to broken addr worked...", err) - } else { - t.Log("correctly got error:", err) - } + _, err := s1.DialPeer(context.Background(), s2.LocalPeer()) + require.Error(t, err, "dialing to broken addr worked...") duration := time.Since(before) if duration < dialTimeout*DialAttempts { @@ -437,65 +419,38 @@ func TestDialBackoffClears(t *testing.T) { if duration > 2*dialTimeout*DialAttempts { t.Error("> 2*dialTimeout * DialAttempts not being respected", duration, 2*dialTimeout*DialAttempts) } - - if !s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { - t.Error("s2 should now be on backoff") - } else { - t.Log("correctly added to backoff") - } + require.True(t, s1.Backoff().Backoff(s2.LocalPeer(), s2bad), "s2 should now be on backoff") // phase 2 -- add the working address. dial should succeed. - ifaceAddrs1, err := swarms[1].InterfaceListenAddresses() - if err != nil { - t.Fatal(err) - } + ifaceAddrs1, err := s2.InterfaceListenAddresses() + require.NoError(t, err) s1.Peerstore().AddAddrs(s2.LocalPeer(), ifaceAddrs1, peerstore.PermanentAddrTTL) - if c, err := s1.DialPeer(ctx, s2.LocalPeer()); err == nil { - c.Close() - t.Log("backoffs are per address, not peer") - } - - time.Sleep(BackoffBase) - - if c, err := s1.DialPeer(ctx, s2.LocalPeer()); err != nil { - t.Fatal(err) - } else { - c.Close() - t.Log("correctly connected") - } - - if s1.Backoff().Backoff(s2.LocalPeer(), s2bad) { - t.Error("s2 should no longer be on backoff") - } else { - t.Log("correctly cleared backoff") - } + // backoffs are per address, not peer + c, err := s1.DialPeer(context.Background(), s2.LocalPeer()) + require.NoError(t, err) + defer c.Close() + require.False(t, s1.Backoff().Backoff(s2.LocalPeer(), s2bad), "s2 should no longer be on backoff") } func TestDialPeerFailed(t *testing.T) { t.Parallel() - ctx := context.Background() - swarms := makeSwarms(t, 2) + swarms := makeSwarms(t, 2, swarmt.DialTimeout(100*time.Millisecond)) defer closeSwarms(swarms) testedSwarm, targetSwarm := swarms[0], swarms[1] - expectedErrorsCount := 5 + const expectedErrorsCount = 5 for i := 0; i < expectedErrorsCount; i++ { _, silentPeerAddress, silentPeerListener := newSilentPeer(t) go acceptAndHang(silentPeerListener) defer silentPeerListener.Close() - testedSwarm.Peerstore().AddAddr( - targetSwarm.LocalPeer(), - silentPeerAddress, - peerstore.PermanentAddrTTL) + testedSwarm.Peerstore().AddAddr(targetSwarm.LocalPeer(), silentPeerAddress, peerstore.PermanentAddrTTL) } - _, err := testedSwarm.DialPeer(ctx, targetSwarm.LocalPeer()) - if err == nil { - t.Fatal(err) - } + _, err := testedSwarm.DialPeer(context.Background(), targetSwarm.LocalPeer()) + require.Error(t, err) // dial_test.go:508: correctly get a combined error: failed to dial PEER: all dials failed // * [/ip4/127.0.0.1/tcp/46485] failed to negotiate security protocol: context deadline exceeded @@ -513,8 +468,6 @@ func TestDialPeerFailed(t *testing.T) { } func TestDialExistingConnection(t *testing.T) { - ctx := context.Background() - swarms := makeSwarms(t, 2) defer closeSwarms(swarms) s1 := swarms[0] @@ -522,19 +475,13 @@ func TestDialExistingConnection(t *testing.T) { s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) - c1, err := s1.DialPeer(ctx, s2.LocalPeer()) - if err != nil { - t.Fatal(err) - } + c1, err := s1.DialPeer(context.Background(), s2.LocalPeer()) + require.NoError(t, err) - c2, err := s1.DialPeer(ctx, s2.LocalPeer()) - if err != nil { - t.Fatal(err) - } + c2, err := s1.DialPeer(context.Background(), s2.LocalPeer()) + require.NoError(t, err) - if c1 != c2 { - t.Fatal("expecting the same connection from both dials") - } + require.Equal(t, c1, c2, "expecting the same connection from both dials") } func newSilentListener(t *testing.T) ([]ma.Multiaddr, net.Listener) { @@ -556,16 +503,12 @@ func newSilentListener(t *testing.T) ([]ma.Multiaddr, net.Listener) { } func TestDialSimultaneousJoin(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - const dialTimeout = 250 * time.Millisecond swarms := makeSwarms(t, 2, swarmt.DialTimeout(dialTimeout)) + defer closeSwarms(swarms) s1 := swarms[0] s2 := swarms[1] - defer s1.Close() - defer s2.Close() s2silentAddrs, s2silentListener := newSilentListener(t) go acceptAndHang(s2silentListener) @@ -577,7 +520,7 @@ func TestDialSimultaneousJoin(t *testing.T) { go func() { s1.Peerstore().AddAddrs(s2.LocalPeer(), s2silentAddrs, peerstore.PermanentAddrTTL) - c, err := s1.DialPeer(ctx, s2.LocalPeer()) + c, err := s1.DialPeer(context.Background(), s2.LocalPeer()) if err != nil { errs <- err connch <- nil @@ -602,7 +545,7 @@ func TestDialSimultaneousJoin(t *testing.T) { } s1.Peerstore().AddAddrs(s2.LocalPeer(), s2addrs[:1], peerstore.PermanentAddrTTL) - c, err := s1.DialPeer(ctx, s2.LocalPeer()) + c, err := s1.DialPeer(context.Background(), s2.LocalPeer()) if err != nil { errs <- err connch <- nil @@ -620,7 +563,7 @@ func TestDialSimultaneousJoin(t *testing.T) { // start a third dial to s2, this should get the existing connection from the successful dial go func() { - c, err := s1.DialPeer(ctx, s2.LocalPeer()) + c, err := s1.DialPeer(context.Background(), s2.LocalPeer()) if err != nil { errs <- err connch <- nil @@ -637,10 +580,7 @@ func TestDialSimultaneousJoin(t *testing.T) { // raise any errors from the previous goroutines for i := 0; i < 3; i++ { - err := <-errs - if err != nil { - t.Fatal(err) - } + require.NoError(t, <-errs) } if c2 != c3 { @@ -660,13 +600,12 @@ func TestDialSimultaneousJoin(t *testing.T) { } func TestDialSelf(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + t.Parallel() swarms := makeSwarms(t, 2) + defer closeSwarms(swarms) s1 := swarms[0] - defer s1.Close() - _, err := s1.DialPeer(ctx, s1.LocalPeer()) + _, err := s1.DialPeer(context.Background(), s1.LocalPeer()) require.ErrorIs(t, err, ErrDialToSelf, "expected error from self dial") } diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 4a592b8651..cd48586143 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -18,16 +18,14 @@ import ( mafmt "github.com/multiformats/go-multiaddr-fmt" ) -func mustAddr(t *testing.T, s string) ma.Multiaddr { - a, err := ma.NewMultiaddr(s) - if err != nil { - t.Fatal(err) - } - return a +func setDialTimeout(t time.Duration) (reset func()) { + orig := transport.DialTimeout + transport.DialTimeout = t + return func() { transport.DialTimeout = orig } } -func addrWithPort(t *testing.T, p int) ma.Multiaddr { - return mustAddr(t, fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p)) +func addrWithPort(p int) ma.Multiaddr { + return ma.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p)) } // in these tests I use addresses with tcp ports over a certain number to @@ -84,8 +82,8 @@ func TestLimiterBasicDials(t *testing.T) { l := newDialLimiterWithParams(hangDialFunc(hang), ConcurrentFdDials, 4) - bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} - good := addrWithPort(t, 20) + bads := []ma.Multiaddr{addrWithPort(1), addrWithPort(2), addrWithPort(3), addrWithPort(4)} + good := addrWithPort(20) resch := make(chan dialResult) pid := peer.ID("testpeer") @@ -133,9 +131,9 @@ func TestFDLimiting(t *testing.T) { defer close(hang) l := newDialLimiterWithParams(hangDialFunc(hang), 16, 5) - bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} + bads := []ma.Multiaddr{addrWithPort(1), addrWithPort(2), addrWithPort(3), addrWithPort(4)} pids := []peer.ID{"testpeer1", "testpeer2", "testpeer3", "testpeer4"} - goodTCP := addrWithPort(t, 20) + goodTCP := addrWithPort(20) ctx := context.Background() resch := make(chan dialResult) @@ -163,7 +161,7 @@ func TestFDLimiting(t *testing.T) { } pid5 := peer.ID("testpeer5") - utpaddr := mustAddr(t, "/ip4/127.0.0.1/udp/7777/utp") + utpaddr := ma.StringCast("/ip4/127.0.0.1/udp/7777/utp") // This should complete immediately since utp addresses arent blocked by fd rate limiting l.AddDialJob(&dialJob{ctx: ctx, peer: pid5, addr: utpaddr, resp: resch}) @@ -180,7 +178,7 @@ func TestFDLimiting(t *testing.T) { // A relay address with tcp transport will complete because we do not consume fds for dials // with relay addresses as the fd will be consumed when we actually dial the relay server. pid6 := test.RandPeerIDFatal(t) - relayAddr := mustAddr(t, fmt.Sprintf("/ip4/127.0.0.1/tcp/20/p2p-circuit/p2p/%s", pid6)) + relayAddr := ma.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/20/p2p-circuit/p2p/%s", pid6)) l.AddDialJob(&dialJob{ctx: ctx, peer: pid6, addr: relayAddr, resp: resch}) select { @@ -209,7 +207,7 @@ func TestTokenRedistribution(t *testing.T) { } l := newDialLimiterWithParams(df, 8, 4) - bads := []ma.Multiaddr{addrWithPort(t, 1), addrWithPort(t, 2), addrWithPort(t, 3), addrWithPort(t, 4)} + bads := []ma.Multiaddr{addrWithPort(1), addrWithPort(2), addrWithPort(3), addrWithPort(4)} pids := []peer.ID{"testpeer1", "testpeer2"} ctx := context.Background() @@ -224,13 +222,11 @@ func TestTokenRedistribution(t *testing.T) { tryDialAddrs(ctx, l, pid, bads, resch) } - good := mustAddr(t, "/ip4/127.0.0.1/tcp/1001") - // add a good dial job for peer 1 l.AddDialJob(&dialJob{ ctx: ctx, peer: pids[1], - addr: good, + addr: ma.StringCast("/ip4/127.0.0.1/tcp/1001"), resp: resch, }) @@ -263,7 +259,7 @@ func TestTokenRedistribution(t *testing.T) { l.AddDialJob(&dialJob{ ctx: ctx, peer: pids[0], - addr: addrWithPort(t, 7), + addr: addrWithPort(7), resp: resch, }) @@ -304,10 +300,10 @@ func TestStressLimiter(t *testing.T) { var bads []ma.Multiaddr for i := 0; i < 100; i++ { - bads = append(bads, addrWithPort(t, i)) + bads = append(bads, addrWithPort(i)) } - addresses := append(bads, addrWithPort(t, 2000)) + addresses := append(bads, addrWithPort(2000)) success := make(chan struct{}) for i := 0; i < 20; i++ { @@ -345,6 +341,9 @@ func TestStressLimiter(t *testing.T) { } func TestFDLimitUnderflow(t *testing.T) { + reset := setDialTimeout(250 * time.Millisecond) + defer reset() + df := func(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.CapableConn, error) { select { case <-ctx.Done(): @@ -358,7 +357,7 @@ func TestFDLimitUnderflow(t *testing.T) { var addrs []ma.Multiaddr for i := 0; i <= 1000; i++ { - addrs = append(addrs, addrWithPort(t, i)) + addrs = append(addrs, addrWithPort(i)) } wg := sync.WaitGroup{} diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index aa4eb59095..0e7d6da705 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -56,7 +56,7 @@ func TestSimultOpenMany(t *testing.T) { addrs = 10 rounds = 5 } - SubtestSwarm(t, addrs, rounds) + subtestSwarm(t, addrs, rounds) } func TestSimultOpenFewStress(t *testing.T) { @@ -72,7 +72,7 @@ func TestSimultOpenFewStress(t *testing.T) { // rounds := 100 for i := 0; i < rounds; i++ { - SubtestSwarm(t, swarms, msgs) + subtestSwarm(t, swarms, msgs) <-time.After(10 * time.Millisecond) } } diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 0dac49d1c7..0c5c341db2 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -99,7 +99,7 @@ func connectSwarms(t *testing.T, ctx context.Context, swarms []*swarm.Swarm) { } } -func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { +func subtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { swarms := makeSwarms(t, SwarmNum, OptDisableReuseport) // connect everyone @@ -216,29 +216,17 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { cancel() <-time.After(10 * time.Millisecond) } - - for _, s := range swarms { - s.Close() - } } func TestSwarm(t *testing.T) { - // t.Skip("skipping for another test") t.Parallel() - - // msgs := 1000 - msgs := 100 - swarms := 5 - SubtestSwarm(t, swarms, msgs) + subtestSwarm(t, 5, 100) } func TestBasicSwarm(t *testing.T) { // t.Skip("skipping for another test") t.Parallel() - - msgs := 1 - swarms := 2 - SubtestSwarm(t, swarms, msgs) + subtestSwarm(t, 2, 1) } func TestConnectionGating(t *testing.T) { From 15bb90236f669326e441869eaee956851ff5840a Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 20 Dec 2021 11:23:32 +0400 Subject: [PATCH 247/259] stop using the deprecated libp2p/go-maddr-filter --- p2p/net/swarm/addrs.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/p2p/net/swarm/addrs.go b/p2p/net/swarm/addrs.go index b9068720c5..09f8df766c 100644 --- a/p2p/net/swarm/addrs.go +++ b/p2p/net/swarm/addrs.go @@ -1,7 +1,6 @@ package swarm import ( - filter "github.com/libp2p/go-maddr-filter" ma "github.com/multiformats/go-multiaddr" mamask "github.com/whyrusleeping/multiaddr-filter" ) @@ -31,6 +30,6 @@ func init() { if err != nil { panic("error in lowTimeoutFilters init: " + err.Error()) } - lowTimeoutFilters.AddFilter(*f, filter.ActionDeny) + lowTimeoutFilters.AddFilter(*f, ma.ActionDeny) } } From 6299c02c158db41c05dcdd0332e354ecc146538e Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 21 Dec 2021 11:13:02 +0200 Subject: [PATCH 248/259] increment active dial counter --- p2p/net/swarm/swarm_dial.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index df5f0d947b..59457cc338 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -480,6 +480,8 @@ loop: err := s.dialNextAddr(ad.ctx, p, addr, resch) if err != nil { dispatchError(ad, err) + } else { + active++ } } From c35965b8696cfb1c04a6aebae40cb88d95f149c3 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 30 Dec 2021 04:45:24 -0800 Subject: [PATCH 249/259] stop using transport.DialTimeout in tests (#307) --- p2p/net/swarm/limiter_test.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index cd48586143..87f9e930d2 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -18,12 +18,6 @@ import ( mafmt "github.com/multiformats/go-multiaddr-fmt" ) -func setDialTimeout(t time.Duration) (reset func()) { - orig := transport.DialTimeout - transport.DialTimeout = t - return func() { transport.DialTimeout = orig } -} - func addrWithPort(p int) ma.Multiaddr { return ma.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p)) } @@ -341,9 +335,6 @@ func TestStressLimiter(t *testing.T) { } func TestFDLimitUnderflow(t *testing.T) { - reset := setDialTimeout(250 * time.Millisecond) - defer reset() - df := func(ctx context.Context, p peer.ID, addr ma.Multiaddr) (transport.CapableConn, error) { select { case <-ctx.Done(): From a2a7e6bb42ac65a99ae61300dadd8f02e1c1fd4e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 14 Dec 2021 17:08:46 +0400 Subject: [PATCH 250/259] remove dependency on go-addr-util --- p2p/net/swarm/dial_test.go | 7 ++----- p2p/net/swarm/swarm_addr.go | 6 +++--- p2p/net/swarm/swarm_dial.go | 7 +++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 5fe4bdc223..ae5d281c06 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -9,7 +9,6 @@ import ( . "github.com/libp2p/go-libp2p-swarm" - addrutil "github.com/libp2p/go-addr-util" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" @@ -149,8 +148,7 @@ func newSilentPeer(t *testing.T) (peer.ID, ma.Multiaddr, net.Listener) { if err != nil { t.Fatal(err) } - addrs := []ma.Multiaddr{addr} - addrs, err = addrutil.ResolveUnspecifiedAddresses(addrs, nil) + addrs, err := manet.ResolveUnspecifiedAddresses([]ma.Multiaddr{addr}, nil) if err != nil { t.Fatal(err) } @@ -548,8 +546,7 @@ func newSilentListener(t *testing.T) ([]ma.Multiaddr, net.Listener) { if err != nil { t.Fatal(err) } - addrs := []ma.Multiaddr{addr} - addrs, err = addrutil.ResolveUnspecifiedAddresses(addrs, nil) + addrs, err := manet.ResolveUnspecifiedAddresses([]ma.Multiaddr{addr}, nil) if err != nil { t.Fatal(err) } diff --git a/p2p/net/swarm/swarm_addr.go b/p2p/net/swarm/swarm_addr.go index 88bc626bd2..8d088e76df 100644 --- a/p2p/net/swarm/swarm_addr.go +++ b/p2p/net/swarm/swarm_addr.go @@ -3,7 +3,8 @@ package swarm import ( "time" - addrutil "github.com/libp2p/go-addr-util" + manet "github.com/multiformats/go-multiaddr/net" + ma "github.com/multiformats/go-multiaddr" ) @@ -52,8 +53,7 @@ func (s *Swarm) InterfaceListenAddresses() ([]ma.Multiaddr, error) { if len(listenAddres) > 0 { // We're actually listening on addresses. var err error - ifaceListenAddres, err = addrutil.ResolveUnspecifiedAddresses(listenAddres, nil) - + ifaceListenAddres, err = manet.ResolveUnspecifiedAddresses(listenAddres, nil) if err != nil { s.listeners.Unlock() // Lock early exit return nil, err diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index cccf80c13d..a5b9c6ba87 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -11,7 +11,6 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" - addrutil "github.com/libp2p/go-addr-util" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) @@ -562,7 +561,7 @@ func (s *Swarm) addrsForDial(ctx context.Context, p peer.ID) ([]ma.Multiaddr, er goodAddrs := s.filterKnownUndialables(p, peerAddrs) if forceDirect, _ := network.GetForceDirectDial(ctx); forceDirect { - goodAddrs = addrutil.FilterAddrs(goodAddrs, s.nonProxyAddr) + goodAddrs = ma.FilterAddrs(goodAddrs, s.nonProxyAddr) } if len(goodAddrs) == 0 { @@ -649,7 +648,7 @@ func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) []ma.Mul } } - return addrutil.FilterAddrs(addrs, + return ma.FilterAddrs(addrs, func(addr ma.Multiaddr) bool { for _, a := range ourAddrs { if a.Equal(addr) { @@ -660,7 +659,7 @@ func (s *Swarm) filterKnownUndialables(p peer.ID, addrs []ma.Multiaddr) []ma.Mul }, s.canDial, // TODO: Consider allowing link-local addresses - addrutil.AddrOverNonLocalIP, + func(addr ma.Multiaddr) bool { return !manet.IsIP6LinkLocal(addr) }, func(addr ma.Multiaddr) bool { return s.gater == nil || s.gater.InterceptAddrDial(p, addr) }, From 74f920351464e08ec0697c78b54b8d13e6cf16f3 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 2 Jan 2022 16:59:25 +0400 Subject: [PATCH 251/259] use the transport.Upgrader interface --- p2p/net/swarm/testing/testing.go | 20 +++++++++----------- p2p/net/swarm/testing/testing_test.go | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 0ed3273591..4af3925f23 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -4,9 +4,6 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - - csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/control" "github.com/libp2p/go-libp2p-core/crypto" @@ -15,6 +12,9 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/sec/insecure" + "github.com/libp2p/go-libp2p-core/transport" + + csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-peerstore/pstoremem" quic "github.com/libp2p/go-libp2p-quic-transport" swarm "github.com/libp2p/go-libp2p-swarm" @@ -25,6 +25,7 @@ import ( "github.com/libp2p/go-tcp-transport" ma "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/require" ) type config struct { @@ -81,7 +82,7 @@ func DialTimeout(t time.Duration) Option { } // GenUpgrader creates a new connection upgrader for use with this swarm. -func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { +func GenUpgrader(t *testing.T, n *swarm.Swarm, opts ...tptu.Option) transport.Upgrader { id := n.LocalPeer() pk := n.Peerstore().PrivKey(id) secMuxer := new(csms.SSMuxer) @@ -89,11 +90,9 @@ func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader { stMuxer := msmux.NewBlankTransport() stMuxer.AddTransport("/yamux/1.0.0", yamux.DefaultTransport) - - return &tptu.Upgrader{ - Secure: secMuxer, - Muxer: stMuxer, - } + u, err := tptu.New(secMuxer, stMuxer, opts...) + require.NoError(t, err) + return u } // GenSwarm generates a new test swarm. @@ -134,8 +133,7 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { s, err := swarm.NewSwarm(p.ID, ps, swarmOpts...) require.NoError(t, err) - upgrader := GenUpgrader(s) - upgrader.ConnGater = cfg.connectionGater + upgrader := GenUpgrader(t, s, tptu.WithConnectionGater(cfg.connectionGater)) if !cfg.disableTCP { var tcpOpts []tcp.Option diff --git a/p2p/net/swarm/testing/testing_test.go b/p2p/net/swarm/testing/testing_test.go index 60cd278719..ef62570224 100644 --- a/p2p/net/swarm/testing/testing_test.go +++ b/p2p/net/swarm/testing/testing_test.go @@ -9,5 +9,5 @@ import ( func TestGenSwarm(t *testing.T) { swarm := GenSwarm(t) require.NoError(t, swarm.Close()) - GenUpgrader(swarm) + GenUpgrader(t, swarm) } From 9df97000623edb2a33383028ef2541a3f434135e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 30 Dec 2021 17:00:23 +0400 Subject: [PATCH 252/259] add support for the resource manager --- p2p/net/swarm/swarm.go | 16 ++++++ p2p/net/swarm/swarm_conn.go | 26 +++++++-- p2p/net/swarm/swarm_stream.go | 15 ++++-- p2p/net/swarm/swarm_test.go | 92 ++++++++++++++++++++++++++++++-- p2p/net/swarm/testing/testing.go | 17 ++++-- 5 files changed, 151 insertions(+), 15 deletions(-) diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index 4761790769..9df1840370 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -76,6 +76,13 @@ func WithDialTimeoutLocal(t time.Duration) Option { } } +func WithResourceManager(m network.ResourceManager) Option { + return func(s *Swarm) error { + s.rcmgr = m + return nil + } +} + // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the @@ -88,6 +95,8 @@ type Swarm struct { // down before continuing. refs sync.WaitGroup + rcmgr network.ResourceManager + local peer.ID peers peerstore.Peerstore @@ -156,6 +165,9 @@ func NewSwarm(local peer.ID, peers peerstore.Peerstore, opts ...Option) (*Swarm, return nil, err } } + if s.rcmgr == nil { + s.rcmgr = network.NullResourceManager + } s.dsync = newDialSync(s.dialWorkerLoop) s.limiter = newDialLimiter(s.dialAddr) @@ -586,6 +598,10 @@ func (s *Swarm) String() string { return fmt.Sprintf("", s.LocalPeer()) } +func (s *Swarm) ResourceManager() network.ResourceManager { + return s.rcmgr +} + // Swarm is a Network. var _ network.Network = (*Swarm)(nil) var _ transport.TransportNetwork = (*Swarm)(nil) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 388e9e76e4..5099eddecb 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -9,7 +9,6 @@ import ( "time" ic "github.com/libp2p/go-libp2p-core/crypto" - "github.com/libp2p/go-libp2p-core/mux" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" @@ -42,6 +41,8 @@ type Conn struct { stat network.ConnStats } +var _ network.Conn = &Conn{} + func (c *Conn) ID() string { // format: - return fmt.Sprintf("%s-%d", c.RemotePeer().Pretty()[0:10], c.id) @@ -93,6 +94,7 @@ func (c *Conn) removeStream(s *Stream) { c.stat.NumStreams-- delete(c.streams.m, s) c.streams.Unlock() + s.scope.Done() } // listens for new streams. @@ -109,9 +111,14 @@ func (c *Conn) start() { if err != nil { return } + scope, err := c.swarm.ResourceManager().OpenStream(c.RemotePeer(), network.DirInbound) + if err != nil { + ts.Reset() + continue + } c.swarm.refs.Add(1) go func() { - s, err := c.addStream(ts, network.DirInbound) + s, err := c.addStream(ts, network.DirInbound, scope) // Don't defer this. We don't want to block // swarm shutdown on the connection handler. @@ -186,19 +193,23 @@ func (c *Conn) NewStream(ctx context.Context) (network.Stream, error) { } } + scope, err := c.swarm.ResourceManager().OpenStream(c.RemotePeer(), network.DirOutbound) + if err != nil { + return nil, err + } ts, err := c.conn.OpenStream(ctx) - if err != nil { return nil, err } - return c.addStream(ts, network.DirOutbound) + return c.addStream(ts, network.DirOutbound, scope) } -func (c *Conn) addStream(ts mux.MuxedStream, dir network.Direction) (*Stream, error) { +func (c *Conn) addStream(ts network.MuxedStream, dir network.Direction, scope network.StreamManagementScope) (*Stream, error) { c.streams.Lock() // Are we still online? if c.streams.m == nil { c.streams.Unlock() + scope.Done() ts.Reset() return nil, ErrConnClosed } @@ -207,6 +218,7 @@ func (c *Conn) addStream(ts mux.MuxedStream, dir network.Direction) (*Stream, er s := &Stream{ stream: ts, conn: c, + scope: scope, stat: network.Stats{ Direction: dir, Opened: time.Now(), @@ -244,3 +256,7 @@ func (c *Conn) GetStreams() []network.Stream { } return streams } + +func (c *Conn) Scope() network.ConnScope { + return c.conn.Scope() +} diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index b0063b1b21..5e5c965335 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -6,7 +6,6 @@ import ( "sync/atomic" "time" - "github.com/libp2p/go-libp2p-core/mux" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/protocol" ) @@ -19,8 +18,9 @@ var _ network.Stream = &Stream{} type Stream struct { id uint64 - stream mux.MuxedStream + stream network.MuxedStream conn *Conn + scope network.StreamManagementScope closeOnce sync.Once @@ -131,8 +131,13 @@ func (s *Stream) Protocol() protocol.ID { // This doesn't actually *do* anything other than record the fact that we're // speaking the given protocol over this stream. It's still up to the user to // negotiate the protocol. This is usually done by the Host. -func (s *Stream) SetProtocol(p protocol.ID) { +func (s *Stream) SetProtocol(p protocol.ID) error { + if err := s.scope.SetProtocol(p); err != nil { + return err + } + s.protocol.Store(p) + return nil } // SetDeadline sets the read and write deadlines for this stream. @@ -154,3 +159,7 @@ func (s *Stream) SetWriteDeadline(t time.Time) error { func (s *Stream) Stat() network.Stats { return s.stat } + +func (s *Stream) Scope() network.StreamScope { + return s.scope +} diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 0c5c341db2..714941f5bd 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -11,17 +11,22 @@ import ( "testing" "time" + "github.com/libp2p/go-libp2p-core/protocol" + + swarm "github.com/libp2p/go-libp2p-swarm" + . "github.com/libp2p/go-libp2p-swarm/testing" + "github.com/libp2p/go-libp2p-core/control" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" - swarm "github.com/libp2p/go-libp2p-swarm" - . "github.com/libp2p/go-libp2p-swarm/testing" - logging "github.com/ipfs/go-log/v2" + mocknetwork "github.com/libp2p/go-libp2p-testing/mocks/network" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" + + "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) @@ -453,3 +458,84 @@ func TestStreamCount(t *testing.T) { str.Close() require.Equal(t, countStreams(), 8) } + +func TestResourceManager(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + rcmgr1 := mocknetwork.NewMockResourceManager(ctrl) + s1 := GenSwarm(t, OptResourceManager(rcmgr1)) + defer s1.Close() + + rcmgr2 := mocknetwork.NewMockResourceManager(ctrl) + s2 := GenSwarm(t, OptResourceManager(rcmgr2)) + defer s2.Close() + connectSwarms(t, context.Background(), []*swarm.Swarm{s1, s2}) + + strChan := make(chan network.Stream) + s2.SetStreamHandler(func(str network.Stream) { strChan <- str }) + + streamScope1 := mocknetwork.NewMockStreamManagementScope(ctrl) + rcmgr1.EXPECT().OpenStream(s2.LocalPeer(), network.DirOutbound).Return(streamScope1, nil) + streamScope2 := mocknetwork.NewMockStreamManagementScope(ctrl) + rcmgr2.EXPECT().OpenStream(s1.LocalPeer(), network.DirInbound).Return(streamScope2, nil) + str, err := s1.NewStream(context.Background(), s2.LocalPeer()) + require.NoError(t, err) + str.Write([]byte("foobar")) + + p := protocol.ID("proto") + streamScope1.EXPECT().SetProtocol(p) + require.NoError(t, str.SetProtocol(p)) + + sstr := <-strChan + streamScope2.EXPECT().Done() + require.NoError(t, sstr.Close()) + streamScope1.EXPECT().Done() + require.NoError(t, str.Close()) +} + +func TestResourceManagerNewStream(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + rcmgr1 := mocknetwork.NewMockResourceManager(ctrl) + s1 := GenSwarm(t, OptResourceManager(rcmgr1)) + defer s1.Close() + + s2 := GenSwarm(t) + defer s2.Close() + + connectSwarms(t, context.Background(), []*swarm.Swarm{s1, s2}) + + rerr := errors.New("denied") + rcmgr1.EXPECT().OpenStream(s2.LocalPeer(), network.DirOutbound).Return(nil, rerr) + _, err := s1.NewStream(context.Background(), s2.LocalPeer()) + require.ErrorIs(t, err, rerr) +} + +func TestResourceManagerAcceptStream(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + rcmgr1 := mocknetwork.NewMockResourceManager(ctrl) + s1 := GenSwarm(t, OptResourceManager(rcmgr1)) + defer s1.Close() + + rcmgr2 := mocknetwork.NewMockResourceManager(ctrl) + s2 := GenSwarm(t, OptResourceManager(rcmgr2)) + defer s2.Close() + s2.SetStreamHandler(func(str network.Stream) { t.Fatal("didn't expect to accept a stream") }) + + connectSwarms(t, context.Background(), []*swarm.Swarm{s1, s2}) + + streamScope := mocknetwork.NewMockStreamManagementScope(ctrl) + rcmgr1.EXPECT().OpenStream(s2.LocalPeer(), network.DirOutbound).Return(streamScope, nil) + streamScope.EXPECT().Done() + rcmgr2.EXPECT().OpenStream(s1.LocalPeer(), network.DirInbound).Return(nil, errors.New("nope")) + str, err := s1.NewStream(context.Background(), s2.LocalPeer()) + require.NoError(t, err) + _, err = str.Write([]byte("foobar")) + require.NoError(t, err) + _, err = str.Read([]byte{0}) + require.EqualError(t, err, "stream reset") +} diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 4af3925f23..e1ab991324 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -13,6 +13,7 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/sec/insecure" "github.com/libp2p/go-libp2p-core/transport" + "github.com/libp2p/go-tcp-transport" csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-peerstore/pstoremem" @@ -22,8 +23,6 @@ import ( tptu "github.com/libp2p/go-libp2p-transport-upgrader" yamux "github.com/libp2p/go-libp2p-yamux" msmux "github.com/libp2p/go-stream-muxer-multistream" - "github.com/libp2p/go-tcp-transport" - ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" ) @@ -35,6 +34,7 @@ type config struct { disableQUIC bool dialTimeout time.Duration connectionGater connmgr.ConnectionGater + rcmgr network.ResourceManager sk crypto.PrivKey } @@ -68,6 +68,12 @@ func OptConnGater(cg connmgr.ConnectionGater) Option { } } +func OptResourceManager(rcmgr network.ResourceManager) Option { + return func(_ *testing.T, c *config) { + c.rcmgr = rcmgr + } +} + // OptPeerPrivateKey configures the peer private key which is then used to derive the public key and peer ID. func OptPeerPrivateKey(sk crypto.PrivKey) Option { return func(_ *testing.T, c *config) { @@ -127,6 +133,9 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { if cfg.connectionGater != nil { swarmOpts = append(swarmOpts, swarm.WithConnectionGater(cfg.connectionGater)) } + if cfg.rcmgr != nil { + swarmOpts = append(swarmOpts, swarm.WithResourceManager(cfg.rcmgr)) + } if cfg.dialTimeout != 0 { swarmOpts = append(swarmOpts, swarm.WithDialTimeout(cfg.dialTimeout)) } @@ -140,7 +149,7 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { if cfg.disableReuseport { tcpOpts = append(tcpOpts, tcp.DisableReuseport()) } - tcpTransport, err := tcp.NewTCPTransport(upgrader, tcpOpts...) + tcpTransport, err := tcp.NewTCPTransport(upgrader, nil, tcpOpts...) require.NoError(t, err) if err := s.AddTransport(tcpTransport); err != nil { t.Fatal(err) @@ -152,7 +161,7 @@ func GenSwarm(t *testing.T, opts ...Option) *swarm.Swarm { } } if !cfg.disableQUIC { - quicTransport, err := quic.NewTransport(p.PrivKey, nil, cfg.connectionGater) + quicTransport, err := quic.NewTransport(p.PrivKey, nil, cfg.connectionGater, nil) if err != nil { t.Fatal(err) } From e1de1839354db4006014de2c52ea0cfe901fbc31 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 20 Jan 2022 23:44:42 +0200 Subject: [PATCH 253/259] release the stream scope if the conn fails to open a new stream --- p2p/net/swarm/swarm_conn.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go index 5099eddecb..77b2defb52 100644 --- a/p2p/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -199,6 +199,7 @@ func (c *Conn) NewStream(ctx context.Context) (network.Stream, error) { } ts, err := c.conn.OpenStream(ctx) if err != nil { + scope.Done() return nil, err } return c.addStream(ts, network.DirOutbound, scope) From 53881b2b0bda80cd2b40ed7472293c10300117b6 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Feb 2022 12:56:04 +0200 Subject: [PATCH 254/259] refactor dialWorkerLoop into an object --- p2p/net/swarm/dial_worker.go | 351 +++++++++++++++++++++++++++++++++++ p2p/net/swarm/swarm_dial.go | 312 +------------------------------ 2 files changed, 353 insertions(+), 310 deletions(-) create mode 100644 p2p/net/swarm/dial_worker.go diff --git a/p2p/net/swarm/dial_worker.go b/p2p/net/swarm/dial_worker.go new file mode 100644 index 0000000000..7bf0d1cc41 --- /dev/null +++ b/p2p/net/swarm/dial_worker.go @@ -0,0 +1,351 @@ +package swarm + +import ( + "context" + "sync" + + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +// ///////////////////////////////////////////////////////////////////////////////// +// lo and behold, The Dialer +// TODO explain how all this works +// //////////////////////////////////////////////////////////////////////////////// + +type dialRequest struct { + ctx context.Context + resch chan dialResponse +} + +type dialResponse struct { + conn *Conn + err error +} + +type pendRequest struct { + req dialRequest // the original request + err *DialError // dial error accumulator + addrs map[ma.Multiaddr]struct{} // pending addr dials +} + +type addrDial struct { + addr ma.Multiaddr + ctx context.Context + conn *Conn + err error + requests []int + dialed bool +} + +type dialWorker struct { + s *Swarm + peer peer.ID + reqch <-chan dialRequest + reqno int + requests map[int]*pendRequest + pending map[ma.Multiaddr]*addrDial + resch chan dialResult + + active int + done bool // true when the request channel has been closed + connected bool // true when a connection has been successfully established + + nextDial []ma.Multiaddr + triggerDial <-chan struct{} + + // for testing + wg sync.WaitGroup + eval <-chan func() +} + +func newDialWorker(s *Swarm, p peer.ID, reqch <-chan dialRequest) *dialWorker { + return &dialWorker{ + s: s, + peer: p, + reqch: reqch, + requests: make(map[int]*pendRequest), + pending: make(map[ma.Multiaddr]*addrDial), + resch: make(chan dialResult), + } +} + +func (w *dialWorker) loop() { + w.wg.Add(1) + defer w.wg.Done() + defer w.s.limiter.clearAllPeerDials(w.peer) + + triggerNow := make(chan struct{}) + close(triggerNow) + +loop: + for { + select { + case req, ok := <-w.reqch: + if !ok { + // request channel has been closed, wait for pending dials to complete + if w.active > 0 { + w.done = true + w.reqch = nil + w.triggerDial = nil + continue loop + } + + // no active dials, we are done + return + } + + c := w.s.bestAcceptableConnToPeer(req.ctx, w.peer) + if c != nil { + req.resch <- dialResponse{conn: c} + continue loop + } + + addrs, err := w.s.addrsForDial(req.ctx, w.peer) + if err != nil { + req.resch <- dialResponse{err: err} + continue loop + } + + // at this point, len(addrs) > 0 or else it would be error from addrsForDial + // ranke them to process in order + addrs = w.rankAddrs(addrs) + + // create the pending request object + pr := &pendRequest{ + req: req, + err: &DialError{Peer: w.peer}, + addrs: make(map[ma.Multiaddr]struct{}), + } + for _, a := range addrs { + pr.addrs[a] = struct{}{} + } + + // check if any of the addrs has been successfully dialed and accumulate + // errors from complete dials while collecting new addrs to dial/join + var todial []ma.Multiaddr + var tojoin []*addrDial + + for _, a := range addrs { + ad, ok := w.pending[a] + if !ok { + todial = append(todial, a) + continue + } + + if ad.conn != nil { + // dial to this addr was successful, complete the request + req.resch <- dialResponse{conn: ad.conn} + continue loop + } + + if ad.err != nil { + // dial to this addr errored, accumulate the error + pr.err.recordErr(a, ad.err) + delete(pr.addrs, a) + continue + } + + // dial is still pending, add to the join list + tojoin = append(tojoin, ad) + } + + if len(todial) == 0 && len(tojoin) == 0 { + // all request applicable addrs have been dialed, we must have errored + req.resch <- dialResponse{err: pr.err} + continue loop + } + + // the request has some pending or new dials, track it and schedule new dials + w.reqno++ + w.requests[w.reqno] = pr + + for _, ad := range tojoin { + if !ad.dialed { + if simConnect, isClient, reason := network.GetSimultaneousConnect(req.ctx); simConnect { + if simConnect, _, _ := network.GetSimultaneousConnect(ad.ctx); !simConnect { + ad.ctx = network.WithSimultaneousConnect(ad.ctx, isClient, reason) + } + } + } + ad.requests = append(ad.requests, w.reqno) + } + + if len(todial) > 0 { + for _, a := range todial { + w.pending[a] = &addrDial{addr: a, ctx: req.ctx, requests: []int{w.reqno}} + } + + w.nextDial = append(w.nextDial, todial...) + w.nextDial = w.rankAddrs(w.nextDial) + + // trigger a new dial now to account for the new addrs we added + w.triggerDial = triggerNow + } + + case <-w.triggerDial: + for _, addr := range w.nextDial { + // spawn the dial + ad := w.pending[addr] + err := w.s.dialNextAddr(ad.ctx, w.peer, addr, w.resch) + if err != nil { + w.dispatchError(ad, err) + } else { + w.active++ + } + } + + w.nextDial = nil + w.triggerDial = nil + + case res := <-w.resch: + w.active-- + + if res.Conn != nil { + w.connected = true + } + + if w.done && w.active == 0 { + if res.Conn != nil { + // we got an actual connection, but the dial has been cancelled + // Should we close it? I think not, we should just add it to the swarm + _, err := w.s.addConn(res.Conn, network.DirOutbound) + if err != nil { + // well duh, now we have to close it + res.Conn.Close() + } + } + return + } + + ad := w.pending[res.Addr] + + if res.Conn != nil { + // we got a connection, add it to the swarm + conn, err := w.s.addConn(res.Conn, network.DirOutbound) + if err != nil { + // oops no, we failed to add it to the swarm + res.Conn.Close() + w.dispatchError(ad, err) + if w.active == 0 && len(w.nextDial) > 0 { + w.triggerDial = triggerNow + } + continue loop + } + + // dispatch to still pending requests + for _, reqno := range ad.requests { + pr, ok := w.requests[reqno] + if !ok { + // it has already dispatched a connection + continue + } + + pr.req.resch <- dialResponse{conn: conn} + delete(w.requests, reqno) + } + + ad.conn = conn + ad.requests = nil + + continue loop + } + + // it must be an error -- add backoff if applicable and dispatch + if res.Err != context.Canceled && !w.connected { + // we only add backoff if there has not been a successful connection + // for consistency with the old dialer behavior. + w.s.backf.AddBackoff(w.peer, res.Addr) + } + + w.dispatchError(ad, res.Err) + if w.active == 0 && len(w.nextDial) > 0 { + w.triggerDial = triggerNow + } + + case f := <-w.eval: + f() + } + } +} + +// dispatches an error to a specific addr dial +func (w *dialWorker) dispatchError(ad *addrDial, err error) { + ad.err = err + for _, reqno := range ad.requests { + pr, ok := w.requests[reqno] + if !ok { + // has already been dispatched + continue + } + + // accumulate the error + pr.err.recordErr(ad.addr, err) + + delete(pr.addrs, ad.addr) + if len(pr.addrs) == 0 { + // all addrs have erred, dispatch dial error + // but first do a last one check in case an acceptable connection has landed from + // a simultaneous dial that started later and added new acceptable addrs + c := w.s.bestAcceptableConnToPeer(pr.req.ctx, w.peer) + if c != nil { + pr.req.resch <- dialResponse{conn: c} + } else { + pr.req.resch <- dialResponse{err: pr.err} + } + delete(w.requests, reqno) + } + } + + ad.requests = nil + + // if it was a backoff, clear the address dial so that it doesn't inhibit new dial requests. + // this is necessary to support active listen scenarios, where a new dial comes in while + // another dial is in progress, and needs to do a direct connection without inhibitions from + // dial backoff. + // it is also necessary to preserve consisent behaviour with the old dialer -- TestDialBackoff + // regresses without this. + if err == ErrDialBackoff { + delete(w.pending, ad.addr) + } +} + +// ranks addresses in descending order of preference for dialing, with the following rules: +// NonRelay > Relay +// NonWS > WS +// Private > Public +// UDP > TCP +func (w *dialWorker) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { + addrTier := func(a ma.Multiaddr) (tier int) { + if isRelayAddr(a) { + tier |= 0b1000 + } + if isExpensiveAddr(a) { + tier |= 0b0100 + } + if !manet.IsPrivateAddr(a) { + tier |= 0b0010 + } + if isFdConsumingAddr(a) { + tier |= 0b0001 + } + + return tier + } + + tiers := make([][]ma.Multiaddr, 16) + for _, a := range addrs { + tier := addrTier(a) + tiers[tier] = append(tiers[tier], a) + } + + result := make([]ma.Multiaddr, 0, len(addrs)) + for _, tier := range tiers { + result = append(result, tier...) + } + + return result +} diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index c92970cdee..ca11d7326f 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -278,281 +278,10 @@ func (s *Swarm) dialPeer(ctx context.Context, p peer.ID) (*Conn, error) { return nil, err } -// ///////////////////////////////////////////////////////////////////////////////// -// lo and behold, The Dialer -// TODO explain how all this works -// //////////////////////////////////////////////////////////////////////////////// - -type dialRequest struct { - ctx context.Context - resch chan dialResponse -} - -type dialResponse struct { - conn *Conn - err error -} - // dialWorkerLoop synchronizes and executes concurrent dials to a single peer func (s *Swarm) dialWorkerLoop(p peer.ID, reqch <-chan dialRequest) { - defer s.limiter.clearAllPeerDials(p) - - type pendRequest struct { - req dialRequest // the original request - err *DialError // dial error accumulator - addrs map[ma.Multiaddr]struct{} // pending addr dials - } - - type addrDial struct { - addr ma.Multiaddr - ctx context.Context - conn *Conn - err error - requests []int - dialed bool - } - - reqno := 0 - requests := make(map[int]*pendRequest) - pending := make(map[ma.Multiaddr]*addrDial) - - dispatchError := func(ad *addrDial, err error) { - ad.err = err - for _, reqno := range ad.requests { - pr, ok := requests[reqno] - if !ok { - // has already been dispatched - continue - } - - // accumulate the error - pr.err.recordErr(ad.addr, err) - - delete(pr.addrs, ad.addr) - if len(pr.addrs) == 0 { - // all addrs have erred, dispatch dial error - // but first do a last one check in case an acceptable connection has landed from - // a simultaneous dial that started later and added new acceptable addrs - c := s.bestAcceptableConnToPeer(pr.req.ctx, p) - if c != nil { - pr.req.resch <- dialResponse{conn: c} - } else { - pr.req.resch <- dialResponse{err: pr.err} - } - delete(requests, reqno) - } - } - - ad.requests = nil - - // if it was a backoff, clear the address dial so that it doesn't inhibit new dial requests. - // this is necessary to support active listen scenarios, where a new dial comes in while - // another dial is in progress, and needs to do a direct connection without inhibitions from - // dial backoff. - // it is also necessary to preserve consisent behaviour with the old dialer -- TestDialBackoff - // regresses without this. - if err == ErrDialBackoff { - delete(pending, ad.addr) - } - } - - var triggerDial <-chan struct{} - triggerNow := make(chan struct{}) - close(triggerNow) - - var nextDial []ma.Multiaddr - active := 0 - done := false // true when the request channel has been closed - connected := false // true when a connection has been successfully established - - resch := make(chan dialResult) - -loop: - for { - select { - case req, ok := <-reqch: - if !ok { - // request channel has been closed, wait for pending dials to complete - if active > 0 { - done = true - reqch = nil - triggerDial = nil - continue loop - } - - // no active dials, we are done - return - } - - c := s.bestAcceptableConnToPeer(req.ctx, p) - if c != nil { - req.resch <- dialResponse{conn: c} - continue loop - } - - addrs, err := s.addrsForDial(req.ctx, p) - if err != nil { - req.resch <- dialResponse{err: err} - continue loop - } - - // at this point, len(addrs) > 0 or else it would be error from addrsForDial - // ranke them to process in order - addrs = s.rankAddrs(addrs) - - // create the pending request object - pr := &pendRequest{ - req: req, - err: &DialError{Peer: p}, - addrs: make(map[ma.Multiaddr]struct{}), - } - for _, a := range addrs { - pr.addrs[a] = struct{}{} - } - - // check if any of the addrs has been successfully dialed and accumulate - // errors from complete dials while collecting new addrs to dial/join - var todial []ma.Multiaddr - var tojoin []*addrDial - - for _, a := range addrs { - ad, ok := pending[a] - if !ok { - todial = append(todial, a) - continue - } - - if ad.conn != nil { - // dial to this addr was successful, complete the request - req.resch <- dialResponse{conn: ad.conn} - continue loop - } - - if ad.err != nil { - // dial to this addr errored, accumulate the error - pr.err.recordErr(a, ad.err) - delete(pr.addrs, a) - continue - } - - // dial is still pending, add to the join list - tojoin = append(tojoin, ad) - } - - if len(todial) == 0 && len(tojoin) == 0 { - // all request applicable addrs have been dialed, we must have errored - req.resch <- dialResponse{err: pr.err} - continue loop - } - - // the request has some pending or new dials, track it and schedule new dials - reqno++ - requests[reqno] = pr - - for _, ad := range tojoin { - if !ad.dialed { - if simConnect, isClient, reason := network.GetSimultaneousConnect(req.ctx); simConnect { - if simConnect, _, _ := network.GetSimultaneousConnect(ad.ctx); !simConnect { - ad.ctx = network.WithSimultaneousConnect(ad.ctx, isClient, reason) - } - } - } - ad.requests = append(ad.requests, reqno) - } - - if len(todial) > 0 { - for _, a := range todial { - pending[a] = &addrDial{addr: a, ctx: req.ctx, requests: []int{reqno}} - } - - nextDial = append(nextDial, todial...) - nextDial = s.rankAddrs(nextDial) - - // trigger a new dial now to account for the new addrs we added - triggerDial = triggerNow - } - - case <-triggerDial: - for _, addr := range nextDial { - // spawn the dial - ad := pending[addr] - err := s.dialNextAddr(ad.ctx, p, addr, resch) - if err != nil { - dispatchError(ad, err) - } else { - active++ - } - } - - nextDial = nil - triggerDial = nil - - case res := <-resch: - active-- - - if res.Conn != nil { - connected = true - } - - if done && active == 0 { - if res.Conn != nil { - // we got an actual connection, but the dial has been cancelled - // Should we close it? I think not, we should just add it to the swarm - _, err := s.addConn(res.Conn, network.DirOutbound) - if err != nil { - // well duh, now we have to close it - res.Conn.Close() - } - } - return - } - - ad := pending[res.Addr] - - if res.Conn != nil { - // we got a connection, add it to the swarm - conn, err := s.addConn(res.Conn, network.DirOutbound) - if err != nil { - // oops no, we failed to add it to the swarm - res.Conn.Close() - dispatchError(ad, err) - if active == 0 && len(nextDial) > 0 { - triggerDial = triggerNow - } - continue loop - } - - // dispatch to still pending requests - for _, reqno := range ad.requests { - pr, ok := requests[reqno] - if !ok { - // it has already dispatched a connection - continue - } - - pr.req.resch <- dialResponse{conn: conn} - delete(requests, reqno) - } - - ad.conn = conn - ad.requests = nil - - continue loop - } - - // it must be an error -- add backoff if applicable and dispatch - if res.Err != context.Canceled && !connected { - // we only add backoff if there has not been a successful connection - // for consistency with the old dialer behavior. - s.backf.AddBackoff(p, res.Addr) - } - - dispatchError(ad, res.Err) - if active == 0 && len(nextDial) > 0 { - triggerDial = triggerNow - } - } - } + w := newDialWorker(s, p, reqch) + w.loop() } func (s *Swarm) addrsForDial(ctx context.Context, p peer.ID) ([]ma.Multiaddr, error) { @@ -597,43 +326,6 @@ func (s *Swarm) nonProxyAddr(addr ma.Multiaddr) bool { return !t.Proxy() } -// ranks addresses in descending order of preference for dialing, with the following rules: -// NonRelay > Relay -// NonWS > WS -// Private > Public -// UDP > TCP -func (s *Swarm) rankAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { - addrTier := func(a ma.Multiaddr) (tier int) { - if isRelayAddr(a) { - tier |= 0b1000 - } - if isExpensiveAddr(a) { - tier |= 0b0100 - } - if !manet.IsPrivateAddr(a) { - tier |= 0b0010 - } - if isFdConsumingAddr(a) { - tier |= 0b0001 - } - - return tier - } - - tiers := make([][]ma.Multiaddr, 16) - for _, a := range addrs { - tier := addrTier(a) - tiers[tier] = append(tiers[tier], a) - } - - result := make([]ma.Multiaddr, 0, len(addrs)) - for _, tier := range tiers { - result = append(result, tier...) - } - - return result -} - // filterKnownUndialables takes a list of multiaddrs, and removes those // that we definitely don't want to dial: addresses configured to be blocked, // IPv6 link-local addresses, addresses without a dial-capable transport, From f50fa5792ceebc52fc3b95a7165b44cb248cd610 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Feb 2022 13:56:55 +0200 Subject: [PATCH 255/259] remove stale comment --- p2p/net/swarm/dial_sync.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/p2p/net/swarm/dial_sync.go b/p2p/net/swarm/dial_sync.go index 73d8af6b1f..f24ecd3c41 100644 --- a/p2p/net/swarm/dial_sync.go +++ b/p2p/net/swarm/dial_sync.go @@ -74,8 +74,6 @@ func (ds *dialSync) getActiveDial(p peer.ID) (*activeDial, error) { if !ok { // This code intentionally uses the background context. Otherwise, if the first call // to Dial is canceled, subsequent dial calls will also be canceled. - // XXX: this also breaks direct connection logic. We will need to pipe the - // information through some other way. ctx, cancel := context.WithCancel(context.Background()) actd = &activeDial{ ctx: ctx, From ebb9e45a598043585efe3c909a617e53a14cbda6 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Feb 2022 14:55:10 +0200 Subject: [PATCH 256/259] RIP active dial tracking It turns out it is not actually necessary if the limiter short-circuits. Point being is that if the request channel has been closed, there is no active dial any more, at which point we might as well consider it done. This fixes the goroutine leak. --- p2p/net/swarm/dial_worker.go | 51 ++++++------------------------------ p2p/net/swarm/limiter.go | 2 +- 2 files changed, 9 insertions(+), 44 deletions(-) diff --git a/p2p/net/swarm/dial_worker.go b/p2p/net/swarm/dial_worker.go index 7bf0d1cc41..8d5d434197 100644 --- a/p2p/net/swarm/dial_worker.go +++ b/p2p/net/swarm/dial_worker.go @@ -50,16 +50,15 @@ type dialWorker struct { pending map[ma.Multiaddr]*addrDial resch chan dialResult - active int - done bool // true when the request channel has been closed connected bool // true when a connection has been successfully established - nextDial []ma.Multiaddr + nextDial []ma.Multiaddr + + // ready when we have more addresses to dial (nextDial is not empty) triggerDial <-chan struct{} // for testing - wg sync.WaitGroup - eval <-chan func() + wg sync.WaitGroup } func newDialWorker(s *Swarm, p peer.ID, reqch <-chan dialRequest) *dialWorker { @@ -78,23 +77,15 @@ func (w *dialWorker) loop() { defer w.wg.Done() defer w.s.limiter.clearAllPeerDials(w.peer) - triggerNow := make(chan struct{}) - close(triggerNow) + // used to signal readiness to dial and completion of the dial + ready := make(chan struct{}) + close(ready) loop: for { select { case req, ok := <-w.reqch: if !ok { - // request channel has been closed, wait for pending dials to complete - if w.active > 0 { - w.done = true - w.reqch = nil - w.triggerDial = nil - continue loop - } - - // no active dials, we are done return } @@ -183,7 +174,7 @@ loop: w.nextDial = w.rankAddrs(w.nextDial) // trigger a new dial now to account for the new addrs we added - w.triggerDial = triggerNow + w.triggerDial = ready } case <-w.triggerDial: @@ -193,8 +184,6 @@ loop: err := w.s.dialNextAddr(ad.ctx, w.peer, addr, w.resch) if err != nil { w.dispatchError(ad, err) - } else { - w.active++ } } @@ -202,25 +191,10 @@ loop: w.triggerDial = nil case res := <-w.resch: - w.active-- - if res.Conn != nil { w.connected = true } - if w.done && w.active == 0 { - if res.Conn != nil { - // we got an actual connection, but the dial has been cancelled - // Should we close it? I think not, we should just add it to the swarm - _, err := w.s.addConn(res.Conn, network.DirOutbound) - if err != nil { - // well duh, now we have to close it - res.Conn.Close() - } - } - return - } - ad := w.pending[res.Addr] if res.Conn != nil { @@ -230,9 +204,6 @@ loop: // oops no, we failed to add it to the swarm res.Conn.Close() w.dispatchError(ad, err) - if w.active == 0 && len(w.nextDial) > 0 { - w.triggerDial = triggerNow - } continue loop } @@ -262,12 +233,6 @@ loop: } w.dispatchError(ad, res.Err) - if w.active == 0 && len(w.nextDial) > 0 { - w.triggerDial = triggerNow - } - - case f := <-w.eval: - f() } } } diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go index ac72279bfe..6b49d8ec0b 100644 --- a/p2p/net/swarm/limiter.go +++ b/p2p/net/swarm/limiter.go @@ -220,7 +220,7 @@ func (dl *dialLimiter) executeDial(j *dialJob) { select { case j.resp <- dialResult{Conn: con, Addr: j.addr, Err: err}: case <-j.ctx.Done(): - if err == nil { + if con != nil { con.Close() } } From 3828fa3570797f913b1df04c18fc0bfc1994f718 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Feb 2022 15:00:58 +0200 Subject: [PATCH 257/259] dial worker tests --- p2p/net/swarm/dial_worker_test.go | 327 ++++++++++++++++++++++++++++++ p2p/net/swarm/limiter_test.go | 2 +- 2 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 p2p/net/swarm/dial_worker_test.go diff --git a/p2p/net/swarm/dial_worker_test.go b/p2p/net/swarm/dial_worker_test.go new file mode 100644 index 0000000000..d9aa115e2f --- /dev/null +++ b/p2p/net/swarm/dial_worker_test.go @@ -0,0 +1,327 @@ +package swarm + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + csms "github.com/libp2p/go-conn-security-multistream" + "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/sec/insecure" + "github.com/libp2p/go-libp2p-core/transport" + "github.com/libp2p/go-libp2p-peerstore/pstoremem" + quic "github.com/libp2p/go-libp2p-quic-transport" + tnet "github.com/libp2p/go-libp2p-testing/net" + tptu "github.com/libp2p/go-libp2p-transport-upgrader" + yamux "github.com/libp2p/go-libp2p-yamux" + msmux "github.com/libp2p/go-stream-muxer-multistream" + tcp "github.com/libp2p/go-tcp-transport" + ma "github.com/multiformats/go-multiaddr" +) + +func makeSwarm(t *testing.T) *Swarm { + p := tnet.RandPeerNetParamsOrFatal(t) + + ps, err := pstoremem.NewPeerstore() + require.NoError(t, err) + ps.AddPubKey(p.ID, p.PubKey) + ps.AddPrivKey(p.ID, p.PrivKey) + t.Cleanup(func() { ps.Close() }) + + s, err := NewSwarm(p.ID, ps, WithDialTimeout(time.Second)) + require.NoError(t, err) + + upgrader := makeUpgrader(t, s) + + var tcpOpts []tcp.Option + tcpOpts = append(tcpOpts, tcp.DisableReuseport()) + tcpTransport, err := tcp.NewTCPTransport(upgrader, nil, tcpOpts...) + require.NoError(t, err) + if err := s.AddTransport(tcpTransport); err != nil { + t.Fatal(err) + } + if err := s.Listen(p.Addr); err != nil { + t.Fatal(err) + } + + quicTransport, err := quic.NewTransport(p.PrivKey, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := s.AddTransport(quicTransport); err != nil { + t.Fatal(err) + } + if err := s.Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/quic")); err != nil { + t.Fatal(err) + } + + return s +} + +func makeUpgrader(t *testing.T, n *Swarm) transport.Upgrader { + id := n.LocalPeer() + pk := n.Peerstore().PrivKey(id) + secMuxer := new(csms.SSMuxer) + secMuxer.AddTransport(insecure.ID, insecure.NewWithIdentity(id, pk)) + + stMuxer := msmux.NewBlankTransport() + stMuxer.AddTransport("/yamux/1.0.0", yamux.DefaultTransport) + u, err := tptu.New(secMuxer, stMuxer) + require.NoError(t, err) + return u +} + +func TestDialWorkerLoopBasic(t *testing.T) { + s1 := makeSwarm(t) + s2 := makeSwarm(t) + defer s1.Close() + defer s2.Close() + + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) + + reqch := make(chan dialRequest) + resch := make(chan dialResponse) + worker := newDialWorker(s1, s2.LocalPeer(), reqch) + go worker.loop() + + var conn *Conn + reqch <- dialRequest{ctx: context.Background(), resch: resch} + select { + case res := <-resch: + require.NoError(t, res.err) + conn = res.conn + case <-time.After(time.Minute): + t.Fatal("dial didn't complete") + } + + s, err := conn.NewStream(context.Background()) + require.NoError(t, err) + s.Close() + + var conn2 *Conn + reqch <- dialRequest{ctx: context.Background(), resch: resch} + select { + case res := <-resch: + require.NoError(t, res.err) + conn2 = res.conn + case <-time.After(time.Minute): + t.Fatal("dial didn't complete") + } + + require.Equal(t, conn, conn2) + + close(reqch) + worker.wg.Wait() +} + +func TestDialWorkerLoopConcurrent(t *testing.T) { + s1 := makeSwarm(t) + s2 := makeSwarm(t) + defer s1.Close() + defer s2.Close() + + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) + + reqch := make(chan dialRequest) + worker := newDialWorker(s1, s2.LocalPeer(), reqch) + go worker.loop() + + const dials = 100 + var wg sync.WaitGroup + resch := make(chan dialResponse, dials) + for i := 0; i < dials; i++ { + wg.Add(1) + go func() { + defer wg.Done() + reschgo := make(chan dialResponse, 1) + reqch <- dialRequest{ctx: context.Background(), resch: reschgo} + select { + case res := <-reschgo: + resch <- res + case <-time.After(time.Minute): + resch <- dialResponse{err: errors.New("timed out!")} + } + }() + } + wg.Wait() + + for i := 0; i < dials; i++ { + res := <-resch + require.NoError(t, res.err) + } + + t.Log("all concurrent dials done") + + close(reqch) + worker.wg.Wait() +} + +func TestDialWorkerLoopFailure(t *testing.T) { + s1 := makeSwarm(t) + defer s1.Close() + + p2 := tnet.RandPeerNetParamsOrFatal(t) + + s1.Peerstore().AddAddrs(p2.ID, []ma.Multiaddr{ma.StringCast("/ip4/11.0.0.1/tcp/1234"), ma.StringCast("/ip4/11.0.0.1/udp/1234/quic")}, peerstore.PermanentAddrTTL) + + reqch := make(chan dialRequest) + resch := make(chan dialResponse) + worker := newDialWorker(s1, p2.ID, reqch) + go worker.loop() + + reqch <- dialRequest{ctx: context.Background(), resch: resch} + select { + case res := <-resch: + require.Error(t, res.err) + case <-time.After(time.Minute): + t.Fatal("dial didn't complete") + } + + close(reqch) + worker.wg.Wait() +} + +func TestDialWorkerLoopConcurrentFailure(t *testing.T) { + s1 := makeSwarm(t) + defer s1.Close() + + p2 := tnet.RandPeerNetParamsOrFatal(t) + + s1.Peerstore().AddAddrs(p2.ID, []ma.Multiaddr{ma.StringCast("/ip4/11.0.0.1/tcp/1234"), ma.StringCast("/ip4/11.0.0.1/udp/1234/quic")}, peerstore.PermanentAddrTTL) + + reqch := make(chan dialRequest) + worker := newDialWorker(s1, p2.ID, reqch) + go worker.loop() + + const dials = 100 + var errTimeout = errors.New("timed out!") + var wg sync.WaitGroup + resch := make(chan dialResponse, dials) + for i := 0; i < dials; i++ { + wg.Add(1) + go func() { + defer wg.Done() + reschgo := make(chan dialResponse, 1) + reqch <- dialRequest{ctx: context.Background(), resch: reschgo} + + select { + case res := <-reschgo: + resch <- res + case <-time.After(time.Minute): + resch <- dialResponse{err: errTimeout} + } + }() + } + wg.Wait() + + for i := 0; i < dials; i++ { + res := <-resch + require.Error(t, res.err) + if res.err == errTimeout { + t.Fatal("dial response timed out") + } + } + + t.Log("all concurrent dials done") + + close(reqch) + worker.wg.Wait() +} + +func TestDialWorkerLoopConcurrentMix(t *testing.T) { + s1 := makeSwarm(t) + s2 := makeSwarm(t) + defer s1.Close() + defer s2.Close() + + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.PermanentAddrTTL) + s1.Peerstore().AddAddrs(s2.LocalPeer(), []ma.Multiaddr{ma.StringCast("/ip4/11.0.0.1/tcp/1234"), ma.StringCast("/ip4/11.0.0.1/udp/1234/quic")}, peerstore.PermanentAddrTTL) + + reqch := make(chan dialRequest) + worker := newDialWorker(s1, s2.LocalPeer(), reqch) + go worker.loop() + + const dials = 100 + var wg sync.WaitGroup + resch := make(chan dialResponse, dials) + for i := 0; i < dials; i++ { + wg.Add(1) + go func() { + defer wg.Done() + reschgo := make(chan dialResponse, 1) + reqch <- dialRequest{ctx: context.Background(), resch: reschgo} + select { + case res := <-reschgo: + resch <- res + case <-time.After(time.Minute): + resch <- dialResponse{err: errors.New("timed out!")} + } + }() + } + wg.Wait() + + for i := 0; i < dials; i++ { + res := <-resch + require.NoError(t, res.err) + } + + t.Log("all concurrent dials done") + + close(reqch) + worker.wg.Wait() +} + +func TestDialWorkerLoopConcurrentFailureStress(t *testing.T) { + s1 := makeSwarm(t) + defer s1.Close() + + p2 := tnet.RandPeerNetParamsOrFatal(t) + + var addrs []ma.Multiaddr + for i := 0; i < 200; i++ { + addrs = append(addrs, ma.StringCast(fmt.Sprintf("/ip4/11.0.0.%d/tcp/%d", i%256, 1234+i))) + } + s1.Peerstore().AddAddrs(p2.ID, addrs, peerstore.PermanentAddrTTL) + + reqch := make(chan dialRequest) + worker := newDialWorker(s1, p2.ID, reqch) + go worker.loop() + + const dials = 100 + var errTimeout = errors.New("timed out!") + var wg sync.WaitGroup + resch := make(chan dialResponse, dials) + for i := 0; i < dials; i++ { + wg.Add(1) + go func() { + defer wg.Done() + reschgo := make(chan dialResponse, 1) + reqch <- dialRequest{ctx: context.Background(), resch: reschgo} + select { + case res := <-reschgo: + resch <- res + case <-time.After(5 * time.Minute): + resch <- dialResponse{err: errTimeout} + } + }() + } + wg.Wait() + + for i := 0; i < dials; i++ { + res := <-resch + require.Error(t, res.err) + if res.err == errTimeout { + t.Fatal("dial response timed out") + } + } + + t.Log("all concurrent dials done") + + close(reqch) + worker.wg.Wait() +} diff --git a/p2p/net/swarm/limiter_test.go b/p2p/net/swarm/limiter_test.go index 87f9e930d2..47918ce565 100644 --- a/p2p/net/swarm/limiter_test.go +++ b/p2p/net/swarm/limiter_test.go @@ -328,7 +328,7 @@ func TestStressLimiter(t *testing.T) { for i := 0; i < 20; i++ { select { case <-success: - case <-time.After(time.Second * 5): + case <-time.After(time.Minute): t.Fatal("expected a success within five seconds") } } From b53bbc51a499b154c06124de8fa29112de860005 Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 6 Apr 2022 14:51:34 +0200 Subject: [PATCH 258/259] fix: call to (*T).Fatal from a non-test goroutine --- p2p/net/swarm/dial_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index e5557d369b..48fd30bbef 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -90,12 +90,13 @@ func TestSimultDials(t *testing.T) { // connect everyone { var wg sync.WaitGroup + errs := make(chan error, 20) // 2 connect calls in each of the 10 for-loop iterations connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // copy for other peer log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.LocalPeer(), dst, addr) s.Peerstore().AddAddr(dst, addr, peerstore.TempAddrTTL) if _, err := s.DialPeer(ctx, dst); err != nil { - t.Fatal("error swarm dialing to peer", err) + errs <- err } wg.Done() } @@ -116,6 +117,13 @@ func TestSimultDials(t *testing.T) { go connect(swarms[1], swarms[0].LocalPeer(), ifaceAddrs0[0]) } wg.Wait() + close(errs) + + for err := range errs { + if err != nil { + t.Fatal("error swarm dialing to peer", err) + } + } } // should still just have 1, at most 2 connections :) From 3effab121a17361e118785fd0cd0f86576bd9d20 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 21 Apr 2022 13:35:04 +0100 Subject: [PATCH 259/259] switch from github.com/libp2p/go-libp2p-swarm to p2p/net/swarm --- config/config.go | 2 +- config/muxer_test.go | 6 +++--- go.mod | 6 +++--- p2p/discovery/backoff/backoffcache_test.go | 3 +-- p2p/discovery/backoff/backoffconnector_test.go | 3 +-- p2p/discovery/mdns_legacy/mdns_test.go | 9 +++++---- p2p/discovery/routing/routing_test.go | 2 +- p2p/host/autonat/autonat_test.go | 2 +- p2p/host/autonat/dialpolicy_test.go | 6 ++++-- p2p/host/autonat/svc_test.go | 3 +-- p2p/host/basic/basic_host_test.go | 9 +++++---- p2p/host/basic/peer_connectedness_test.go | 3 ++- p2p/host/blank/peer_connectedness_test.go | 3 ++- p2p/net/swarm/dial_test.go | 6 +++--- p2p/net/swarm/peers_test.go | 2 +- p2p/net/swarm/simul_test.go | 8 ++++---- p2p/net/swarm/swarm_addr_test.go | 6 +++--- p2p/net/swarm/swarm_net_test.go | 4 ++-- p2p/net/swarm/swarm_notif_test.go | 4 ++-- p2p/net/swarm/swarm_test.go | 4 ++-- p2p/net/swarm/testing/testing.go | 3 ++- p2p/net/swarm/transport_test.go | 4 ++-- p2p/protocol/circuitv2/relay/relay_test.go | 6 +++--- p2p/protocol/identify/id_glass_test.go | 3 +-- p2p/protocol/identify/id_test.go | 2 +- p2p/protocol/identify/peer_loop_test.go | 3 +-- p2p/protocol/ping/ping_test.go | 5 +++-- p2p/test/backpressure/backpressure_test.go | 9 +++++---- p2p/test/reconnects/reconnect_test.go | 3 +-- 29 files changed, 66 insertions(+), 63 deletions(-) diff --git a/config/config.go b/config/config.go index 7b11c2e357..0a68736f64 100644 --- a/config/config.go +++ b/config/config.go @@ -23,11 +23,11 @@ import ( bhost "github.com/libp2p/go-libp2p/p2p/host/basic" blankhost "github.com/libp2p/go-libp2p/p2p/host/blank" routed "github.com/libp2p/go-libp2p/p2p/host/routed" + "github.com/libp2p/go-libp2p/p2p/net/swarm" circuitv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/client" relayv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" "github.com/libp2p/go-libp2p/p2p/protocol/holepunch" - swarm "github.com/libp2p/go-libp2p-swarm" tptu "github.com/libp2p/go-libp2p-transport-upgrader" logging "github.com/ipfs/go-log/v2" diff --git a/config/muxer_test.go b/config/muxer_test.go index 50e32ef07d..76aada1061 100644 --- a/config/muxer_test.go +++ b/config/muxer_test.go @@ -3,12 +3,12 @@ package config import ( "testing" - "github.com/libp2p/go-libp2p-core/network" + bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - bhost "github.com/libp2p/go-libp2p/p2p/host/basic" yamux "github.com/libp2p/go-libp2p-yamux" ) diff --git a/go.mod b/go.mod index 655228de67..4dcf718c66 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/libp2p/go-libp2p-peerstore v0.6.0 github.com/libp2p/go-libp2p-quic-transport v0.17.0 github.com/libp2p/go-libp2p-resource-manager v0.2.1 - github.com/libp2p/go-libp2p-swarm v0.10.2 github.com/libp2p/go-libp2p-testing v0.9.2 github.com/libp2p/go-libp2p-tls v0.4.1 github.com/libp2p/go-libp2p-transport-upgrader v0.7.1 @@ -35,12 +34,14 @@ require ( github.com/libp2p/zeroconf/v2 v2.1.1 github.com/multiformats/go-multiaddr v0.5.0 github.com/multiformats/go-multiaddr-dns v0.3.1 + github.com/multiformats/go-multiaddr-fmt v0.1.0 github.com/multiformats/go-multihash v0.1.0 github.com/multiformats/go-multistream v0.3.0 github.com/multiformats/go-varint v0.0.6 github.com/raulk/go-watchdog v1.2.0 github.com/stretchr/testify v1.7.0 github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 + github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) @@ -76,6 +77,7 @@ require ( github.com/libp2p/go-flow-metrics v0.0.3 // indirect github.com/libp2p/go-libp2p-blankhost v0.3.0 // indirect github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect + github.com/libp2p/go-libp2p-swarm v0.10.2 // indirect github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-openssl v0.0.7 // indirect github.com/libp2p/go-reuseport v0.1.0 // indirect @@ -96,7 +98,6 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.0.4 // indirect github.com/multiformats/go-base36 v0.1.0 // indirect - github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.0.3 // indirect github.com/multiformats/go-multicodec v0.4.1 // indirect github.com/nxadm/tail v1.4.8 // indirect @@ -112,7 +113,6 @@ require ( github.com/raulk/clock v1.1.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect diff --git a/p2p/discovery/backoff/backoffcache_test.go b/p2p/discovery/backoff/backoffcache_test.go index f297ddfbb0..920b45924c 100644 --- a/p2p/discovery/backoff/backoffcache_test.go +++ b/p2p/discovery/backoff/backoffcache_test.go @@ -9,11 +9,10 @@ import ( "github.com/libp2p/go-libp2p/p2p/discovery/mocks" bhost "github.com/libp2p/go-libp2p/p2p/host/blank" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/discovery" "github.com/libp2p/go-libp2p-core/peer" - - swarmt "github.com/libp2p/go-libp2p-swarm/testing" ) type delayedDiscovery struct { diff --git a/p2p/discovery/backoff/backoffconnector_test.go b/p2p/discovery/backoff/backoffconnector_test.go index b421658dac..6bf958f9c9 100644 --- a/p2p/discovery/backoff/backoffconnector_test.go +++ b/p2p/discovery/backoff/backoffconnector_test.go @@ -8,12 +8,11 @@ import ( "time" bhost "github.com/libp2p/go-libp2p/p2p/host/blank" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - "github.com/stretchr/testify/require" ) diff --git a/p2p/discovery/mdns_legacy/mdns_test.go b/p2p/discovery/mdns_legacy/mdns_test.go index 0e667e149f..17a5fe77a0 100644 --- a/p2p/discovery/mdns_legacy/mdns_test.go +++ b/p2p/discovery/mdns_legacy/mdns_test.go @@ -5,12 +5,13 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" + bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + + "github.com/stretchr/testify/require" ) type DiscoveryNotifee struct { @@ -22,7 +23,7 @@ func (n *DiscoveryNotifee) HandlePeerFound(pi peer.AddrInfo) { } func TestMdnsDiscovery(t *testing.T) { - //TODO: re-enable when the new lib will get integrated + // TODO: re-enable when the new lib will get integrated t.Skip("TestMdnsDiscovery fails randomly with current lib") ctx, cancel := context.WithCancel(context.Background()) diff --git a/p2p/discovery/routing/routing_test.go b/p2p/discovery/routing/routing_test.go index 13f4da3adf..3f88297237 100644 --- a/p2p/discovery/routing/routing_test.go +++ b/p2p/discovery/routing/routing_test.go @@ -9,12 +9,12 @@ import ( "github.com/libp2p/go-libp2p/p2p/discovery/mocks" "github.com/libp2p/go-libp2p/p2p/discovery/util" bhost "github.com/libp2p/go-libp2p/p2p/host/blank" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/discovery" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" ) type mockRoutingTable struct { diff --git a/p2p/host/autonat/autonat_test.go b/p2p/host/autonat/autonat_test.go index 7cf6a8efd7..7f111bd30b 100644 --- a/p2p/host/autonat/autonat_test.go +++ b/p2p/host/autonat/autonat_test.go @@ -7,13 +7,13 @@ import ( pb "github.com/libp2p/go-libp2p/p2p/host/autonat/pb" bhost "github.com/libp2p/go-libp2p/p2p/host/blank" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/event" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-msgio/protoio" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" diff --git a/p2p/host/autonat/dialpolicy_test.go b/p2p/host/autonat/dialpolicy_test.go index 8ee70dbb6a..afa047eb73 100644 --- a/p2p/host/autonat/dialpolicy_test.go +++ b/p2p/host/autonat/dialpolicy_test.go @@ -6,10 +6,12 @@ import ( "net" "testing" + blankhost "github.com/libp2p/go-libp2p/p2p/host/blank" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" + "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - blankhost "github.com/libp2p/go-libp2p/p2p/host/blank" + "github.com/multiformats/go-multiaddr" ) diff --git a/p2p/host/autonat/svc_test.go b/p2p/host/autonat/svc_test.go index 56d8dd7d92..f8768689d9 100644 --- a/p2p/host/autonat/svc_test.go +++ b/p2p/host/autonat/svc_test.go @@ -7,13 +7,12 @@ import ( "time" bhost "github.com/libp2p/go-libp2p/p2p/host/blank" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/event" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" ) diff --git a/p2p/host/basic/basic_host_test.go b/p2p/host/basic/basic_host_test.go index 9b229b4c03..bfda432f55 100644 --- a/p2p/host/basic/basic_host_test.go +++ b/p2p/host/basic/basic_host_test.go @@ -11,7 +11,10 @@ import ( "testing" "time" - "github.com/libp2p/go-eventbus" + "github.com/libp2p/go-libp2p/p2p/host/autonat" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" + "github.com/libp2p/go-libp2p/p2p/protocol/identify" + "github.com/libp2p/go-libp2p-core/event" "github.com/libp2p/go-libp2p-core/helpers" "github.com/libp2p/go-libp2p-core/host" @@ -21,10 +24,8 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" "github.com/libp2p/go-libp2p-core/record" "github.com/libp2p/go-libp2p-core/test" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - "github.com/libp2p/go-libp2p/p2p/host/autonat" - "github.com/libp2p/go-libp2p/p2p/protocol/identify" + "github.com/libp2p/go-eventbus" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" diff --git a/p2p/host/basic/peer_connectedness_test.go b/p2p/host/basic/peer_connectedness_test.go index 818499de19..fc8ecb60a8 100644 --- a/p2p/host/basic/peer_connectedness_test.go +++ b/p2p/host/basic/peer_connectedness_test.go @@ -5,10 +5,11 @@ import ( "testing" "time" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" + "github.com/libp2p/go-libp2p-core/event" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/stretchr/testify/require" ) diff --git a/p2p/host/blank/peer_connectedness_test.go b/p2p/host/blank/peer_connectedness_test.go index 88c8f2861a..cabf3fe14a 100644 --- a/p2p/host/blank/peer_connectedness_test.go +++ b/p2p/host/blank/peer_connectedness_test.go @@ -5,10 +5,11 @@ import ( "testing" "time" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" + "github.com/libp2p/go-libp2p-core/event" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/stretchr/testify/require" ) diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go index 48fd30bbef..0a98d69e43 100644 --- a/p2p/net/swarm/dial_test.go +++ b/p2p/net/swarm/dial_test.go @@ -7,15 +7,15 @@ import ( "testing" "time" - . "github.com/libp2p/go-libp2p-swarm" + . "github.com/libp2p/go-libp2p/p2p/net/swarm" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" testutil "github.com/libp2p/go-libp2p-core/test" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - "github.com/libp2p/go-libp2p-testing/ci" + "github.com/libp2p/go-libp2p-testing/ci" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" diff --git a/p2p/net/swarm/peers_test.go b/p2p/net/swarm/peers_test.go index d7c19b4f15..a2d8f820e2 100644 --- a/p2p/net/swarm/peers_test.go +++ b/p2p/net/swarm/peers_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - . "github.com/libp2p/go-libp2p-swarm" + . "github.com/libp2p/go-libp2p/p2p/net/swarm" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" diff --git a/p2p/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go index 0e7d6da705..5f3d6d34ae 100644 --- a/p2p/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -7,14 +7,14 @@ import ( "testing" "time" + . "github.com/libp2p/go-libp2p/p2p/net/swarm" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" + "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" - ma "github.com/multiformats/go-multiaddr" - - . "github.com/libp2p/go-libp2p-swarm" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-libp2p-testing/ci" + ma "github.com/multiformats/go-multiaddr" ) func TestSimultOpen(t *testing.T) { diff --git a/p2p/net/swarm/swarm_addr_test.go b/p2p/net/swarm/swarm_addr_test.go index 21ecafa2ad..2bdc1ba3ae 100644 --- a/p2p/net/swarm/swarm_addr_test.go +++ b/p2p/net/swarm/swarm_addr_test.go @@ -4,13 +4,13 @@ import ( "context" "testing" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" + "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/test" - "github.com/stretchr/testify/require" ma "github.com/multiformats/go-multiaddr" - - swarmt "github.com/libp2p/go-libp2p-swarm/testing" + "github.com/stretchr/testify/require" ) func TestDialBadAddrs(t *testing.T) { diff --git a/p2p/net/swarm/swarm_net_test.go b/p2p/net/swarm/swarm_net_test.go index 3096574d7a..1fa08bb7ff 100644 --- a/p2p/net/swarm/swarm_net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -7,11 +7,11 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" + . "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/network" - . "github.com/libp2p/go-libp2p-swarm/testing" + "github.com/stretchr/testify/require" ) // TestConnectednessCorrect starts a few networks, connects a few diff --git a/p2p/net/swarm/swarm_notif_test.go b/p2p/net/swarm/swarm_notif_test.go index c0c6f82db8..8b1011bee7 100644 --- a/p2p/net/swarm/swarm_notif_test.go +++ b/p2p/net/swarm/swarm_notif_test.go @@ -5,14 +5,14 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" + . "github.com/libp2p/go-libp2p/p2p/net/swarm" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" ma "github.com/multiformats/go-multiaddr" - . "github.com/libp2p/go-libp2p-swarm" + "github.com/stretchr/testify/require" ) func TestNotifications(t *testing.T) { diff --git a/p2p/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go index 714941f5bd..c7db03e994 100644 --- a/p2p/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -13,8 +13,8 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" - swarm "github.com/libp2p/go-libp2p-swarm" - . "github.com/libp2p/go-libp2p-swarm/testing" + "github.com/libp2p/go-libp2p/p2p/net/swarm" + . "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/control" "github.com/libp2p/go-libp2p-core/network" diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index e1ab991324..30d8f22055 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -4,6 +4,8 @@ import ( "testing" "time" + "github.com/libp2p/go-libp2p/p2p/net/swarm" + "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/control" "github.com/libp2p/go-libp2p-core/crypto" @@ -18,7 +20,6 @@ import ( csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-peerstore/pstoremem" quic "github.com/libp2p/go-libp2p-quic-transport" - swarm "github.com/libp2p/go-libp2p-swarm" tnet "github.com/libp2p/go-libp2p-testing/net" tptu "github.com/libp2p/go-libp2p-transport-upgrader" yamux "github.com/libp2p/go-libp2p-yamux" diff --git a/p2p/net/swarm/transport_test.go b/p2p/net/swarm/transport_test.go index 6d5913cf58..3c863b23e7 100644 --- a/p2p/net/swarm/transport_test.go +++ b/p2p/net/swarm/transport_test.go @@ -4,8 +4,8 @@ import ( "context" "testing" - swarm "github.com/libp2p/go-libp2p-swarm" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" + "github.com/libp2p/go-libp2p/p2p/net/swarm" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" diff --git a/p2p/protocol/circuitv2/relay/relay_test.go b/p2p/protocol/circuitv2/relay/relay_test.go index f7b0f35e1d..5bd840406c 100644 --- a/p2p/protocol/circuitv2/relay/relay_test.go +++ b/p2p/protocol/circuitv2/relay/relay_test.go @@ -10,19 +10,19 @@ import ( "time" bhost "github.com/libp2p/go-libp2p/p2p/host/blank" + "github.com/libp2p/go-libp2p/p2p/net/swarm" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/client" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" - "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-peerstore/pstoremem" - swarm "github.com/libp2p/go-libp2p-swarm" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-tcp-transport" ma "github.com/multiformats/go-multiaddr" ) diff --git a/p2p/protocol/identify/id_glass_test.go b/p2p/protocol/identify/id_glass_test.go index 7111f70ba6..de83d8be6b 100644 --- a/p2p/protocol/identify/id_glass_test.go +++ b/p2p/protocol/identify/id_glass_test.go @@ -6,12 +6,11 @@ import ( "time" blhost "github.com/libp2p/go-libp2p/p2p/host/blank" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/p2p/protocol/identify/id_test.go b/p2p/protocol/identify/id_test.go index 0d946c2f8e..10f274374f 100644 --- a/p2p/protocol/identify/id_test.go +++ b/p2p/protocol/identify/id_test.go @@ -12,6 +12,7 @@ import ( "github.com/libp2p/go-libp2p" blhost "github.com/libp2p/go-libp2p/p2p/host/blank" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p/p2p/protocol/identify" pb "github.com/libp2p/go-libp2p/p2p/protocol/identify/pb" @@ -27,7 +28,6 @@ import ( "github.com/libp2p/go-eventbus" "github.com/libp2p/go-libp2p-peerstore/pstoremem" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" "github.com/libp2p/go-libp2p-testing/race" "github.com/libp2p/go-msgio/protoio" diff --git a/p2p/protocol/identify/peer_loop_test.go b/p2p/protocol/identify/peer_loop_test.go index 560495f21d..c6bbbd3fc4 100644 --- a/p2p/protocol/identify/peer_loop_test.go +++ b/p2p/protocol/identify/peer_loop_test.go @@ -6,12 +6,11 @@ import ( "time" blhost "github.com/libp2p/go-libp2p/p2p/host/blank" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - "github.com/stretchr/testify/require" ) diff --git a/p2p/protocol/ping/ping_test.go b/p2p/protocol/ping/ping_test.go index e7b5964780..75021b1406 100644 --- a/p2p/protocol/ping/ping_test.go +++ b/p2p/protocol/ping/ping_test.go @@ -7,10 +7,11 @@ import ( "github.com/stretchr/testify/require" - "github.com/libp2p/go-libp2p-core/peer" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p/p2p/protocol/ping" + + "github.com/libp2p/go-libp2p-core/peer" ) func TestPing(t *testing.T) { diff --git a/p2p/test/backpressure/backpressure_test.go b/p2p/test/backpressure/backpressure_test.go index b5d6a548c4..20336f02e8 100644 --- a/p2p/test/backpressure/backpressure_test.go +++ b/p2p/test/backpressure/backpressure_test.go @@ -6,13 +6,14 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" + bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" - logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/protocol" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" ) var log = logging.Logger("backpressure") diff --git a/p2p/test/reconnects/reconnect_test.go b/p2p/test/reconnects/reconnect_test.go index 0d71ec5c02..72523ffd26 100644 --- a/p2p/test/reconnects/reconnect_test.go +++ b/p2p/test/reconnects/reconnect_test.go @@ -10,14 +10,13 @@ import ( "time" bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" - swarmt "github.com/libp2p/go-libp2p-swarm/testing" - "github.com/stretchr/testify/require" )