WIP: ON HOLD: Allchannel 2.0

Tribler/Core/APIImplementation/LaunchManyCore.py

@@ -242,6 +242,20 @@ def load_ipv8_overlays(self):
         if not self.session.config.get_dispersy_enabled():
             self.ipv8.strategies.append((RandomWalk(discovery_community), 20))
 
+        # AllChannel2 Community
+        if self.session.config.get_channel_search_enabled():
+            triblerchain_peer = Peer(self.session.trustchain_keypair)
+            from Tribler.community.allchannel2.community import AllChannel2Community
+
+            channel_directory = os.path.join(self.session.config.get_state_dir(), u"channels")
+            self.allchannel2_community = AllChannel2Community(triblerchain_peer, self.ipv8.endpoint,
+                                                              self.ipv8.network,
+                                                              tribler_session=self.session,
+                                                              working_directory=channel_directory)
+            self.allchannel2_community.load_channels()
+            self.ipv8.overlays.append(self.allchannel2_community)
+            self.ipv8.strategies.append((RandomWalk(self.allchannel2_community), 20))
+
         # TriblerChain Community
         if self.session.config.get_trustchain_enabled():
             triblerchain_peer = Peer(self.session.trustchain_keypair)

Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py

@@ -151,6 +151,7 @@ def __init__(self, session, tdef):
         self.deferreds_handle = []
         self.deferred_added = Deferred()
         self.deferred_removed = Deferred()
+        self.deferred_finished = Deferred()
 
         self.handle_check_lc = self.register_task("handle_check", LoopingCall(self.check_handle))
 
@@ -663,6 +664,8 @@ def reset_priorities():
             if self.endbuffsize:
                 self.set_byte_priority([(self.get_vod_fileindex(), 0, -1)], 1)
                 self.endbuffsize = 0
+        if not self.deferred_finished.called:
+            self.deferred_finished.callback(self)
 
     def update_lt_status(self, lt_status):
         """ Update libtorrent stats and check if the download should be stopped."""

Tribler/Test/Community/Allchannel2/test_community.py

@@ -0,0 +1,102 @@
+import os
+import shutil
+
+from twisted.internet.defer import succeed
+
+from Tribler.community.allchannel2.community import AllChannel2Community
+from Tribler.pyipv8.ipv8.keyvault.crypto import ECCrypto
+from Tribler.pyipv8.ipv8.peer import Peer
+import Tribler.Test as super_module
+from Tribler.pyipv8.ipv8.test.base import TestBase
+from Tribler.pyipv8.ipv8.test.util import twisted_wrapper
+
+
+class FakeDownloadConfig(object):
+
+    def __init__(self, download_dir):
+        self.download_dir = download_dir
+
+    def get_dest_dir(self):
+        return self.download_dir
+
+
+class FakeDownload(object):
+
+    def __init__(self, download_dir):
+        self.deferred_finished = succeed(self)
+        self.dlconfig = FakeDownloadConfig(download_dir)
+
+
+class FakeSession(object):
+
+    def __init__(self, download_dir):
+        self.magnet_links = {}
+        self.download_dir = download_dir
+
+    def start_download_from_uri(self, magnetlink, _):
+        self.magnet_links[magnetlink] = ''
+        return succeed(FakeDownload(self.download_dir))
+
+
+class TestAllChannel2(TestBase):
+
+    def setUp(self):
+        super(TestAllChannel2, self).setUp()
+
+        mocked_community = AllChannel2Community
+        key = ECCrypto().generate_key(u"very-low")
+        mocked_community.master_peer = Peer(key)
+        self.initialize(mocked_community, 2)
+
+        data_dir = os.path.abspath(os.path.join(os.path.dirname(super_module.__file__), 'data', 'channels'))
+        for node in self.nodes:
+            node.overlay.working_directory = data_dir
+
+    def tearDown(self):
+        super(TestAllChannel2, self).tearDown()
+
+        for node in self.nodes:
+            channel_dir = os.path.abspath(os.path.join(node.overlay.working_directory, node.overlay.my_channel_name))
+            if os.path.isdir(channel_dir):
+                shutil.rmtree(channel_dir)
+
+    def test_write_channel(self):
+        """
+        Check if we can add a magnet link to our channel and write it to file.
+        """
+        magnet_link = 'a'*20
+        self.nodes[0].overlay.add_magnetlink(magnet_link)
+        channel_name = self.nodes[0].overlay.my_channel_name
+
+        self.assertListEqual([channel_name], self.nodes[0].overlay.get_channels())
+        self.assertListEqual([magnet_link], self.nodes[0].overlay.get_magnetlinks(channel_name))
+        self.assertEqual('\xc9\x18[5\x1c\x99=\xe9\x17\xd7|\x0ee\xf6E=ia\xb5W',
+                         self.nodes[0].overlay.my_channel_info_hash)
+
+    def test_read_channel(self):
+        """
+        Check if we can read a channel from disk.
+        """
+        channel_name = "testcase0"
+        magnet_link = 'a' * 20
+        self.nodes[0].overlay.load_channel(channel_name)
+
+        self.assertListEqual([channel_name], self.nodes[0].overlay.get_channels())
+        self.assertListEqual([magnet_link], self.nodes[0].overlay.get_magnetlinks(channel_name))
+
+    @twisted_wrapper
+    def test_share_channel(self):
+        """
+        Check if peers start downloading each others channel after introducing.
+        """
+        magnet_link = 'a' * 20
+        for node in self.nodes:
+            node.overlay.tribler_session = FakeSession(node.overlay.working_directory)
+            node.overlay.add_magnetlink(magnet_link)
+
+        yield self.introduce_nodes()
+        yield self.deliver_messages()
+
+        for node in self.nodes:
+            self.assertListEqual(['magnet:?xt=urn:btih:\xc9\x18[5\x1c\x99=\xe9\x17\xd7|\x0ee\xf6E=ia\xb5W'],
+                                 node.overlay.tribler_session.magnet_links.keys())

Tribler/Test/Community/Allchannel2/test_structures.py

@@ -0,0 +1,144 @@
+import unittest
+
+from Tribler.community.allchannel2.structures import Chunk, ChunkedTable
+
+
+class TestChunk(unittest.TestCase):
+
+    def setUp(self):
+        self.chunk = Chunk()
+
+    def test_empty(self):
+        """
+        Test if a Chunk is initially empty.
+        """
+        self.assertDictEqual(self.chunk.data, {})
+
+    def test_add(self):
+        """
+        Test if we can add a key value pair to the Chunk.
+        """
+        self.assertTrue(self.chunk.add("key", "value"))
+        self.assertDictEqual(self.chunk.data, {"key": "value"})
+
+    def test_remove(self):
+        """
+        Test if we can remove a key value pair from the Chunk.
+        """
+        self.assertTrue(self.chunk.add("key", "value"))
+        self.chunk.remove("key")
+        self.assertDictEqual(self.chunk.data, {})
+
+    def test_add_full(self):
+        """
+        Test if we cannot add blocks to an already full Chunk.
+        """
+        self.chunk.max_length = 0
+        self.assertFalse(self.chunk.add("key", "value"))
+
+    def test_serialize_empty(self):
+        """
+        Test if we can serialize and deserialize an empty Chunk.
+        """
+        data = self.chunk.serialize()
+        chunk = Chunk.unserialize(data)
+        self.assertDictEqual(self.chunk.data, chunk.data)
+
+    def test_serialize(self):
+        """
+        Test if we can serialize and deserialize a Chunk.
+        """
+        self.chunk.data = {"key": "value"}
+        data = self.chunk.serialize()
+        chunk = Chunk.unserialize(data)
+        self.assertDictEqual(self.chunk.data, chunk.data)
+
+
+class TestChunkedTable(unittest.TestCase):
+
+    def setUp(self):
+        self.chunked_table = ChunkedTable()
+
+    def test_empty(self):
+        """
+        Test if a ChunkedTable is initially empty.
+        """
+        self.assertDictEqual(self.chunked_table.chunklist, {})
+
+    def test_add(self):
+        """
+        Test if we can add a key value pair to the ChunkedTable.
+        """
+        self.chunked_table.add("key", "value")
+        self.assertEqual(len(self.chunked_table.chunklist.keys()), 1)
+        self.assertDictEqual(self.chunked_table.get_all(), {"key": "value"})
+
+    def test_add_second(self):
+        """
+        Test if we can add a key value pair to the ChunkedTable in an existing Chunk.
+        """
+        self.chunked_table.add("key", "value")
+        self.chunked_table.add("key2", "value2")
+        self.assertEqual(len(self.chunked_table.chunklist.keys()), 1)
+        self.assertDictEqual(self.chunked_table.get_all(), {"key": "value", "key2": "value2"})
+
+    def test_add_second_spill(self):
+        """
+        Test if we can add a key value pair to the ChunkedTable if the existing Chunk is full.
+        """
+        self.chunked_table.add("key", "value")
+        self.chunked_table.chunklist.values()[0].max_length = 0
+        self.chunked_table.add("key2", "value2")
+        self.assertEqual(len(self.chunked_table.chunklist.keys()), 2)
+        self.assertDictEqual(self.chunked_table.get_all(), {"key": "value", "key2": "value2"})
+
+    def test_remove(self):
+        """
+        Test if we can remove a key value pair from the ChunkedTable.
+        """
+        self.chunked_table.add("key", "value")
+        self.chunked_table.remove("key")
+        self.assertEqual(len(self.chunked_table.chunklist.keys()), 0)
+        self.assertDictEqual(self.chunked_table.get_all(), {})
+
+    def test_remove_hole(self):
+        """
+        Test if we do not remove middle Chunks in the ChunkTable.
+        """
+        fake_chunklist = {0: Chunk(), 1:Chunk(), 2:Chunk()}
+        fake_chunklist[0].add("key0", "value0")
+        fake_chunklist[1].add("key1", "value1")
+        fake_chunklist[2].add("key2", "value2")
+        self.chunked_table.chunklist = fake_chunklist
+        self.chunked_table.remove("key1")
+        self.assertEqual(len(self.chunked_table.chunklist.keys()), 3)
+        self.assertDictEqual(self.chunked_table.get_all(), {"key0": "value0", "key2": "value2"})
+
+    def test_remove_hole_recover(self):
+        """
+        Test if we remove all trailing empty Chunks in the ChunkTable.
+        """
+        fake_chunklist = {0: Chunk(), 1:Chunk(), 2:Chunk()}
+        fake_chunklist[0].add("key0", "value0")
+        fake_chunklist[2].add("key2", "value2")
+        self.chunked_table.chunklist = fake_chunklist
+        self.chunked_table.remove("key2")
+        self.assertEqual(len(self.chunked_table.chunklist.keys()), 1)
+        self.assertDictEqual(self.chunked_table.get_all(), {"key0": "value0"})
+
+    def test_serialize_empty(self):
+        """
+        Test if we can serialize and deserialize an empty ChunkedTable.
+        """
+        data = self.chunked_table.serialize()
+        chunked_table = ChunkedTable.unserialize(data)
+        self.assertDictEqual(self.chunked_table.get_all(), chunked_table.get_all())
+
+    def test_serialize(self):
+        """
+        Test if we can serialize and deserialize a ChunkedTable.
+        """
+        self.chunked_table.add("key", "value")
+        data = self.chunked_table.serialize()
+        chunked_table = ChunkedTable.unserialize(data)
+        self.assertDictEqual(self.chunked_table.get_all(), chunked_table.get_all())

Tribler/Test/data/channels/testcase0/0

@@ -0,0 +1 @@
+d20:aaaaaaaaaaaaaaaaaaaa0:e