diff --git a/README.md b/README.md index bb599c9734..95d5f90d32 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ We make use of submodules, so remember using the --recursive argument when cloni ## Dependencies ### Debian/Ubuntu/Mint -sudo apt-get install scons build-essential libevent-dev python-libtorrent python-apsw python-wxgtk2.8 python-netifaces python-m2crypto vlc +sudo apt-get install scons build-essential libevent-dev python-libtorrent python-apsw python-wxgtk2.8 python-netifaces python-m2crypto vlc python-igraph ### Windows TODO diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index 5cd8532ab3..06ae6569f5 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -10,6 +10,9 @@ from threading import Event, Thread, enumerate as enumerate_threads, currentThread from Tribler.Core.ServerPortHandler import MultiHandler from Tribler.Core.Utilities.configparser import CallbackConfigParser +from Tribler.community.anontunnel.endpoint import DispersyBypassEndpoint +from Tribler.community.privatesemantic.crypto.elgamalcrypto import ElgamalCrypto, \ + NoElgamalCrypto import logging from traceback import print_exc @@ -121,12 +124,12 @@ def register(self, session, sesslock): if self.session.get_dispersy_tunnel_over_swift() and self.swift_process: endpoint = TunnelEndpoint(self.swift_process) else: - endpoint = RawserverEndpoint(self.rawserver, self.session.get_dispersy_port()) + endpoint = DispersyBypassEndpoint(self.rawserver, self.session.get_dispersy_port()) callback = Callback("Dispersy") # WARNING NAME SIGNIFICANT working_directory = unicode(self.session.get_state_dir()) - self.dispersy = Dispersy(callback, endpoint, working_directory) + self.dispersy = Dispersy(callback, endpoint, working_directory, crypto=ElgamalCrypto()) # TODO: see if we can postpone dispersy.start to improve GUI responsiveness. # However, for now we must start self.dispersy.callback before running @@ -925,14 +928,17 @@ def sessconfig_changed_callback(self, section, name, new_value, old_value): self.ltmgr.set_utp(new_value) elif section == 'libtorrent' and name == 'lt_proxyauth': if self.ltmgr: - self.ltmgr.set_proxy_settings(*self.session.get_libtorrent_proxy_settings()) + self.ltmgr.set_proxy_settings(self.ltmgr.ltsession, *self.session.get_libtorrent_proxy_settings()) elif section == 'torrent_checking' and name == 'torrent_checking_period': if self.rtorrent_handler and value_changed: self.rtorrent_handler.set_max_num_torrents(new_value) # Return True/False, depending on whether or not the config value can be changed at runtime. elif (section == 'general' and name in ['nickname', 'mugshot', 'videoanalyserpath']) or \ - (section == 'libtorrent' and name in ['lt_proxytype', 'lt_proxyserver']) or \ + (section == 'libtorrent' and name in ['lt_proxytype', 'lt_proxyserver', + 'anon_proxyserver', 'anon_proxytype', 'anon_proxyauth', + 'anon_listen_port']) or \ (section == 'torrent_collecting' and name in ['stop_collecting_threshold']) or \ + (section == 'proxy_community' and name in ['socks5_listen_port']) or \ (section == 'swift' and name in ['swiftmetadir']): return True else: diff --git a/Tribler/Core/CacheDB/Notifier.py b/Tribler/Core/CacheDB/Notifier.py index ffba97828f..1647554225 100644 --- a/Tribler/Core/CacheDB/Notifier.py +++ b/Tribler/Core/CacheDB/Notifier.py @@ -2,21 +2,21 @@ # see LICENSE.txt for license information import threading -import logging from Tribler.Core.simpledefs import NTFY_MISC, NTFY_PEERS, NTFY_TORRENTS, \ NTFY_PLAYLISTS, NTFY_COMMENTS, NTFY_MODIFICATIONS, NTFY_MODERATIONS, \ NTFY_MARKINGS, NTFY_MYPREFERENCES, NTFY_ACTIVITIES, NTFY_REACHABLE, \ NTFY_CHANNELCAST, NTFY_VOTECAST, NTFY_DISPERSY, NTFY_TRACKERINFO, \ - NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE + NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE, NTFY_ANONTUNNEL +import logging class Notifier: SUBJECTS = [NTFY_MISC, NTFY_PEERS, NTFY_TORRENTS, NTFY_PLAYLISTS, NTFY_COMMENTS, NTFY_MODIFICATIONS, NTFY_MODERATIONS, NTFY_MARKINGS, NTFY_MYPREFERENCES, NTFY_ACTIVITIES, NTFY_REACHABLE, NTFY_CHANNELCAST, - NTFY_VOTECAST, NTFY_DISPERSY, NTFY_TRACKERINFO] + NTFY_VOTECAST, NTFY_DISPERSY, NTFY_TRACKERINFO, NTFY_ANONTUNNEL] # . . . # todo: add all datahandler types+other observables diff --git a/Tribler/Core/DownloadConfig.py b/Tribler/Core/DownloadConfig.py index c366e85aaa..cf37ee7f8e 100644 --- a/Tribler/Core/DownloadConfig.py +++ b/Tribler/Core/DownloadConfig.py @@ -92,6 +92,12 @@ def get_mode(self): @return DLMODE_NORMAL/DLMODE_VOD """ return self.dlconfig.get('downloadconfig', 'mode') + def set_anon_mode(self, anon_mode): + self.dlconfig.set('downloadconfig', 'anon_mode', anon_mode) + + def get_anon_mode(self): + return self.dlconfig.get('downloadconfig', 'anon_mode') + def set_selected_files(self, files): """ Select which files in the torrent to download. The filenames must be the names as they appear in the content def, including encoding. diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index 374762b70d..cd6e936657 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -255,7 +255,8 @@ def create_engine_wrapper(self, lm_network_engine_wrapper_created_callback, psta with self.dllock: if not self.cew_scheduled: self.ltmgr = self.session.lm.ltmgr - if not self.ltmgr or (isinstance(self.tdef, TorrentDefNoMetainfo) and not self.ltmgr.is_dht_ready()): + if not self.ltmgr or (isinstance(self.tdef, TorrentDefNoMetainfo) and not self.ltmgr.is_dht_ready()) or \ + (self.get_anon_mode() and not self.ltmgr.is_anon_ready()): self._logger.info("LibtorrentDownloadImpl: LTMGR or DHT not ready, rescheduling create_engine_wrapper") create_engine_wrapper_lambda = lambda: self.create_engine_wrapper(lm_network_engine_wrapper_created_callback, pstate, initialdlstatus=initialdlstatus) self.session.lm.rawserver.add_task(create_engine_wrapper_lambda, 5) @@ -275,6 +276,7 @@ def network_create_engine_wrapper(self, lm_network_engine_wrapper_created_callba atp["paused"] = True atp["auto_managed"] = False atp["duplicate_is_error"] = True + atp["anon_mode"] = self.get_anon_mode() resume_data = pstate.get('state', 'engineresumedata') if pstate else None if not isinstance(self.tdef, TorrentDefNoMetainfo): @@ -500,6 +502,9 @@ def on_file_renamed_alert(self, alert): os.rmdir(self.unwanteddir_abs) def on_performance_alert(self, alert): + if self.get_anon_mode(): + return + # When the send buffer watermark is too low, double the buffer size to a maximum of 50MiB. This is the same mechanism as Deluge uses. if alert.message().endswith("send buffer watermark too low (upload rate will suffer)"): settings = self.ltmgr.ltsession.settings() diff --git a/Tribler/Core/Libtorrent/LibtorrentMgr.py b/Tribler/Core/Libtorrent/LibtorrentMgr.py index dc8fce66eb..83b0ed3295 100644 --- a/Tribler/Core/Libtorrent/LibtorrentMgr.py +++ b/Tribler/Core/Libtorrent/LibtorrentMgr.py @@ -1,5 +1,6 @@ # Written by Egbert Bouman +from libtorrent import proxy_type import os import time import binascii @@ -75,7 +76,7 @@ def __init__(self, trsession, ignore_singleton=False): self.ltsession.add_dht_router('router.bitcomet.com', 6881) # Load proxy settings - self.set_proxy_settings(*self.trsession.get_libtorrent_proxy_settings()) + self.set_proxy_settings(self.ltsession, *self.trsession.get_libtorrent_proxy_settings()) self.set_utp(self.trsession.get_libtorrent_utp()) @@ -99,11 +100,14 @@ def __init__(self, trsession, ignore_singleton=False): if not os.path.exists(self.metadata_tmpdir): os.mkdir(self.metadata_tmpdir) + self.ltsession_anon = None + def getInstance(*args, **kw): if LibtorrentMgr.__single is None: LibtorrentMgr(*args, **kw) return LibtorrentMgr.__single getInstance = staticmethod(getInstance) + ''' :type : () -> LibtorrentMgr ''' def delInstance(): del LibtorrentMgr.__single @@ -114,6 +118,32 @@ def hasInstance(): return LibtorrentMgr.__single != None hasInstance = staticmethod(hasInstance) + def is_anon_ready(self): + return self.ltsession_anon is not None + + def create_anonymous_session(self): + settings = lt.session_settings() + settings.enable_outgoing_utp = True + settings.enable_incoming_utp = True + settings.enable_outgoing_tcp = False + settings.enable_incoming_tcp = False + settings.anonymous_mode = True + ltsession = lt.session(flags=1) + ltsession.set_settings(settings) + ltsession.set_alert_mask(lt.alert.category_t.stats_notification | + lt.alert.category_t.error_notification | + lt.alert.category_t.status_notification | + lt.alert.category_t.storage_notification | + lt.alert.category_t.performance_warning | + lt.alert.category_t.debug_notification) + + + self.set_proxy_settings(ltsession, *self.trsession.get_anon_proxy_settings()) + self.ltsession_anon = ltsession + + ltsession.listen_on(self.trsession.get_anon_listen_port(), self.trsession.get_anon_listen_port()+10) + self._logger.info("Started ANON LibTorrent session on port %d", ltsession.listen_port()) + def shutdown(self): # Save DHT state dhtstate_file = open(os.path.join(self.trsession.get_state_dir(), DHTSTATE_FILENAME), 'w') @@ -127,7 +157,7 @@ def shutdown(self): if os.path.exists(self.metadata_tmpdir): rmtree(self.metadata_tmpdir) - def set_proxy_settings(self, ptype, server=None, auth=None): + def set_proxy_settings(self, ltsession, ptype, server=None, auth=None): proxy_settings = lt.proxy_settings() proxy_settings.type = lt.proxy_type(ptype) if server: @@ -138,10 +168,7 @@ def set_proxy_settings(self, ptype, server=None, auth=None): proxy_settings.password = auth[1] proxy_settings.proxy_hostnames = True proxy_settings.proxy_peer_connections = True - self.ltsession.set_peer_proxy(proxy_settings) - self.ltsession.set_web_seed_proxy(proxy_settings) - self.ltsession.set_tracker_proxy(proxy_settings) - self.ltsession.set_dht_proxy(proxy_settings) + ltsession.set_proxy(proxy_settings) def set_utp(self, enable): settings = self.ltsession.settings() @@ -173,41 +200,11 @@ def get_dht_nodes(self): def is_dht_ready(self): return self.dht_ready - def queue_position_up(self, infohash): - with self.torlock: - download = self.torrents.get(hexlify(infohash), None) - if download: - download.handle.queue_position_up() - self._refresh_queue_positions() - - def queue_position_down(self, infohash): - with self.torlock: - download = self.torrents.get(hexlify(infohash), None) - if download: - download.handle.queue_position_down() - self._refresh_queue_positions() - - def queue_position_top(self, infohash): - with self.torlock: - download = self.torrents.get(hexlify(infohash), None) - if download: - download.handle.queue_position_top() - self._refresh_queue_positions() - - def queue_position_bottom(self, infohash): - with self.torlock: - download = self.torrents.get(hexlify(infohash), None) - if download: - download.handle.queue_position_bottom() - self._refresh_queue_positions() - - def _refresh_queue_positions(self): - for d in self.torrents.values(): - d.queue_position = d.handle.queue_position() - def add_torrent(self, torrentdl, atp): # If we are collecting the torrent for this infohash, abort this first. with self.metainfo_lock: + anon_mode = atp.pop('anon_mode', False) + ltsession = self.ltsession_anon if anon_mode else self.ltsession if atp.has_key('ti'): infohash = str(atp['ti'].info_hash()) @@ -220,14 +217,14 @@ def add_torrent(self, torrentdl, atp): self._logger.info("LibtorrentMgr: killing get_metainfo request for %s", infohash) handle, _, _ = self.metainfo_requests.pop(infohash) if handle: - self.ltsession.remove_torrent(handle, 0) + ltsession.remove_torrent(handle, 0) - handle = self.ltsession.add_torrent(encode_atp(atp)) + handle = ltsession.add_torrent(encode_atp(atp)) infohash = str(handle.info_hash()) with self.torlock: if infohash in self.torrents: raise DuplicateDownloadException() - self.torrents[infohash] = torrentdl + self.torrents[infohash] = (torrentdl, ltsession) self._logger.debug("LibtorrentMgr: added torrent %s", infohash) @@ -239,7 +236,7 @@ def remove_torrent(self, torrentdl, removecontent=False): infohash = str(handle.info_hash()) with self.torlock: if infohash in self.torrents: - self.ltsession.remove_torrent(handle, int(removecontent)) + self.torrents[infohash][1].remove_torrent(handle, int(removecontent)) del self.torrents[infohash] self._logger.debug("LibtorrentMgr: remove torrent %s", infohash) else: @@ -263,31 +260,36 @@ def delete_mappings(self): self.upnp_mapper.delete_mapping(mapping) def process_alerts(self): - if self.ltsession: - alert = self.ltsession.pop_alert() - while alert: - alert_type = str(type(alert)).split("'")[1].split(".")[-1] - if alert_type == 'external_ip_alert': - external_ip = str(alert).split()[-1] - if self.external_ip != external_ip: - self.external_ip = external_ip - self._logger.info('LibtorrentMgr: external IP is now %s', self.external_ip) - handle = getattr(alert, 'handle', None) - if handle: - if handle.is_valid(): - infohash = str(handle.info_hash()) - with self.torlock: - if infohash in self.torrents: - self.torrents[infohash].process_alert(alert, alert_type) - elif infohash in self.metainfo_requests: - if type(alert) == lt.metadata_received_alert: - self.got_metainfo(infohash) - else: - self._logger.debug("LibtorrentMgr: could not find torrent %s", infohash) + for ltsession in [self.ltsession, self.ltsession_anon]: + if ltsession: + alert = ltsession.pop_alert() + while alert: + self.process_alert(alert) + alert = ltsession.pop_alert() + + self.trsession.lm.rawserver.add_task(self.process_alerts, 1) + + def process_alert(self, alert): + alert_type = str(type(alert)).split("'")[1].split(".")[-1] + if alert_type == 'external_ip_alert': + external_ip = str(alert).split()[-1] + if self.external_ip != external_ip: + self.external_ip = external_ip + self._logger.info('LibtorrentMgr: external IP is now %s', self.external_ip) + handle = getattr(alert, 'handle', None) + if handle: + if handle.is_valid(): + infohash = str(handle.info_hash()) + with self.torlock: + if infohash in self.torrents: + self.torrents[infohash][0].process_alert(alert, alert_type) + elif infohash in self.metainfo_requests: + if type(alert) == lt.metadata_received_alert: + self.got_metainfo(infohash) else: - self._logger.debug("LibtorrentMgr: alert for invalid torrent") - alert = self.ltsession.pop_alert() - self.trsession.lm.rawserver.add_task(self.process_alerts, 1) + self._logger.debug("LibtorrentMgr: could not find torrent %s", infohash) + else: + self._logger.debug("LibtorrentMgr: alert for invalid torrent") def reachability_check(self): if self.ltsession and self.ltsession.status().has_incoming_connections: diff --git a/Tribler/Core/RawServer/RawServer.py b/Tribler/Core/RawServer/RawServer.py index 5aad294cee..9a4c7f7ef3 100644 --- a/Tribler/Core/RawServer/RawServer.py +++ b/Tribler/Core/RawServer/RawServer.py @@ -106,14 +106,14 @@ def scan_for_timeouts(self): self.sockethandler.scan_for_timeouts() def bind(self, port, bind='', reuse= False, - ipv6_socket_style=1): - self.sockethandler.bind(port, bind, reuse, ipv6_socket_style) + ipv6_socket_style=1, handler=None): + self.sockethandler.bind(port, bind, reuse, ipv6_socket_style, handler) def find_and_bind(self, first_try, minport, maxport, bind='', reuse = False, - ipv6_socket_style=1, randomizer= False): + ipv6_socket_style=1, randomizer= False, handler=None): # 2fastbt_ result = self.sockethandler.find_and_bind(first_try, minport, maxport, bind, reuse, - ipv6_socket_style, randomizer) + ipv6_socket_style, randomizer, handler) # _2fastbt return result diff --git a/Tribler/Core/RawServer/SocketHandler.py b/Tribler/Core/RawServer/SocketHandler.py index d77d529654..b3ffcd4f10 100644 --- a/Tribler/Core/RawServer/SocketHandler.py +++ b/Tribler/Core/RawServer/SocketHandler.py @@ -247,6 +247,7 @@ def __init__(self, timeout, ipv6_enable, readsize=100000): self.dead_from_write = [] self.max_connects = 1000 self.servers = {} + self.interfaces = [] self.btengine_said_reachable = False self.interrupt_socket = None self.udp_sockets = {} @@ -263,11 +264,9 @@ def scan_for_timeouts(self): self._logger.debug("SocketHandler: scan_timeout closing connection %s", k.get_ip()) self._close_socket(k) - def bind(self, port, bind=[], reuse= False, ipv6_socket_style = 1): + def bind(self, port, bind=[], reuse= False, ipv6_socket_style = 1, handler=None): port = int(port) addrinfos = [] - self.servers = {} - self.interfaces = [] # if bind != [] bind to all specified addresses (can be IPs or hostnames) # else bind to default ipv6 and ipv4 address if bind: @@ -294,14 +293,14 @@ def bind(self, port, bind=[], reuse= False, ipv6_socket_style = 1): server.setblocking(0) self._logger.debug("SocketHandler: Try to bind socket on %s ...", addrinfo[4]) server.bind(addrinfo[4]) - self.servers[server.fileno()] = server + self.servers[server.fileno()] = (server, handler) if bind: self.interfaces.append(server.getsockname()[0]) self._logger.debug("SocketHandler: OK") server.listen(64) self.poll.register(server, POLLIN) except socket.error as e: - for server in self.servers.values(): + for server, _ in self.servers.values(): try: server.close() except: @@ -314,7 +313,7 @@ def bind(self, port, bind=[], reuse= False, ipv6_socket_style = 1): self.port = port def find_and_bind(self, first_try, minport, maxport, bind='', reuse= False, - ipv6_socket_style=1, randomizer= False): + ipv6_socket_style=1, randomizer= False, handler=None): e = 'maxport less than minport - no ports to check' if maxport - minport < 50 or not randomizer: portrange = range(minport, maxport + 1) @@ -330,7 +329,7 @@ def find_and_bind(self, first_try, minport, maxport, bind='', reuse= False, if first_try != 0: # try 22 first, because TU only opens port 22 for SSH... try: self.bind(first_try, bind, reuse=reuse, - ipv6_socket_style=ipv6_socket_style) + ipv6_socket_style=ipv6_socket_style, handler=handler) return first_try except socket.error as e: pass @@ -338,7 +337,7 @@ def find_and_bind(self, first_try, minport, maxport, bind='', reuse= False, try: # print >> sys.stderr, listen_port, bind, reuse self.bind(listen_port, bind, reuse=reuse, - ipv6_socket_style=ipv6_socket_style) + ipv6_socket_style=ipv6_socket_style, handler=handler) return listen_port except socket.error as e: raise @@ -456,7 +455,7 @@ def _sleep(self): def handle_events(self, events): for sock, event in events: # print >>sys.stderr,"SocketHandler: event on sock#",sock - s = self.servers.get(sock) # socket.socket + s, h = self.servers.get(sock, (None, None)) # socket.socket if s: if event & (POLLHUP | POLLERR) != 0: self._logger.debug("SocketHandler: Got event, close server socket") @@ -474,10 +473,10 @@ def handle_events(self, events): # the connection. if len(self.single_sockets) < self.max_connects: newsock.setblocking(0) - nss = SingleSocket(self, newsock, self.handler) # create socket for incoming peers and tracker + nss = SingleSocket(self, newsock, (h or self.handler)) # create socket for incoming peers and tracker self.single_sockets[newsock.fileno()] = nss self.poll.register(newsock, POLLIN) - self.handler.external_connection_made(nss) + (h or self.handler).external_connection_made(nss) else: self._logger.info("SocketHandler: too many connects") newsock.close() @@ -588,7 +587,7 @@ def shutdown(self): ss.close() except: pass - for server in self.servers.values(): + for server, _ in self.servers.values(): try: server.close() except: diff --git a/Tribler/Core/SessionConfig.py b/Tribler/Core/SessionConfig.py index b4f85b99bf..2fc6f8f8fb 100644 --- a/Tribler/Core/SessionConfig.py +++ b/Tribler/Core/SessionConfig.py @@ -177,6 +177,12 @@ def set_listen_port_runtime(self, port): """ self.selected_ports['~'.join(('general', 'minport'))] = port + def set_proxy_community_socks5_listen_port(self, port): + self.sessconfig.set(u'proxy_community', u'socks5_listen_port', port) + + def get_proxy_community_socks5_listen_port(self): + return self._obtain_port(u'proxy_community', u'socks5_listen_port') + def get_listen_port(self): """ Returns the current UDP/TCP listen port. @return Port number. """ @@ -244,6 +250,31 @@ def get_libtorrent_proxy_settings(self): self.sessconfig.get(u'libtorrent', u'lt_proxyserver'), \ self.sessconfig.get(u'libtorrent', u'lt_proxyauth')) + def set_anon_proxy_settings(self, ptype, server=None, auth=None): + """ + @param ptype Integer (0 = no proxy server, 1 = SOCKS4, 2 = SOCKS5, 3 = SOCKS5 + auth, 4 = HTTP, 5 = HTTP + auth) + @param server (host, port) tuple or None + @param auth (username, password) tuple or None + """ + self.sessconfig.set(u'libtorrent', u'anon_proxytype', ptype) + self.sessconfig.set(u'libtorrent', u'anon_proxyserver', server if ptype else None) + self.sessconfig.set(u'libtorrent', u'anon_proxyauth', auth if ptype in [3, 5] else None) + + def get_anon_proxy_settings(self): + """ + @return: libtorrent anonymous settings + """ + return (self.sessconfig.get(u'libtorrent', u'anon_proxytype'), + self.sessconfig.get(u'libtorrent', u'anon_proxyserver'), + self.sessconfig.get(u'libtorrent', u'anon_proxyauth')) + + + def set_anon_listen_port(self, listen_port=None): + self.sessconfig.set(u'libtorrent', u'anon_listen_port', listen_port) + + def get_anon_listen_port(self): + return self._obtain_port(u'libtorrent', u'anon_listen_port') + def set_libtorrent_utp(self, value): """ Enable or disable LibTorrent uTP (default = True). @param value Boolean. diff --git a/Tribler/Core/Video/utils.py b/Tribler/Core/Video/utils.py index 7c655175cb..dca9d58cda 100644 --- a/Tribler/Core/Video/utils.py +++ b/Tribler/Core/Video/utils.py @@ -126,6 +126,8 @@ def return_feasible_playback_modes(): raise Exception("Incorrect vlc version. We require at least version 0.9, this is %s" % version) l.append(PLAYBACKMODE_INTERNAL) + except NameError: + logger.error("libvlc_get_version couldn't be called, no playback possible") except Exception: print_exc() @@ -186,4 +188,4 @@ def get_ranges(headervalue, content_length): # Negative subscript (last N bytes) result.append((content_length - int(stop), content_length)) - return result \ No newline at end of file + return result diff --git a/Tribler/Core/defaults.py b/Tribler/Core/defaults.py index 29249ae617..cf2b622c31 100644 --- a/Tribler/Core/defaults.py +++ b/Tribler/Core/defaults.py @@ -57,6 +57,10 @@ sessdefaults['general']['peer_icon_path'] = None sessdefaults['general']['live_aux_seeders'] = [] +# Proxy community section +sessdefaults['proxy_community'] = OrderedDict() +sessdefaults['proxy_community']['socks5_listen_port'] = -1 + # Mainline DHT settings sessdefaults['mainline_dht'] = OrderedDict() sessdefaults['mainline_dht']['enabled'] = True @@ -83,6 +87,12 @@ sessdefaults['libtorrent']['lt_proxyauth'] = None sessdefaults['libtorrent']['utp'] = True +# Anonymous libtorrent +sessdefaults['libtorrent']['anon_listen_port'] = -1 +sessdefaults['libtorrent']['anon_proxytype'] = 0 +sessdefaults['libtorrent']['anon_proxyserver'] = None +sessdefaults['libtorrent']['anon_proxyauth'] = None + # SWIFTPROC config sessdefaults['swift'] = OrderedDict() sessdefaults['swift']['enabled'] = True @@ -110,6 +120,8 @@ sessdefaults['video']['port'] = -1 sessdefaults['video']['preferredmode'] = PLAYBACKMODE_INTERNAL + + # # BT per download opts # diff --git a/Tribler/Core/simpledefs.py b/Tribler/Core/simpledefs.py index 1f0d7df04b..b5dd8e6527 100644 --- a/Tribler/Core/simpledefs.py +++ b/Tribler/Core/simpledefs.py @@ -62,6 +62,7 @@ NTFY_SEEDINGSTATSSETTINGS = 'seedingstatssettings' NTFY_VOTECAST = 'votecast' NTFY_CHANNELCAST = 'channelcast' +NTFY_ANONTUNNEL = 'anontunnel' NTFY_TRACKERINFO = 'trackerinfo' # non data handler subjects @@ -86,6 +87,12 @@ NTFY_VIDEO_STOPPED = 'video_stopped' NTFY_VIDEO_ENDED = 'video_ended' NTFY_VIDEO_BUFFERING = 'video_bufering' +NTFY_CREATED = 'created' +NTFY_EXTENDED = 'extended' +NTFY_EXTENDED_FOR = 'extended_for' +NTFY_BROKEN = 'broken' +NTFY_SELECT = 'select' +NTFY_JOINED = 'joined' # object IDs for NTFY_ACTIVITIES subject NTFY_ACT_NONE = 0 diff --git a/Tribler/Main/Dialogs/AddTorrent.py b/Tribler/Main/Dialogs/AddTorrent.py index 13943bb750..ddded28843 100644 --- a/Tribler/Main/Dialogs/AddTorrent.py +++ b/Tribler/Main/Dialogs/AddTorrent.py @@ -104,21 +104,21 @@ def OnAdd(self, event): if input.startswith("http://"): destdir = self.defaultDLConfig.get_dest_dir() if self.choose and self.choose.IsChecked(): - destdir = self._GetDestPath(torrenturl=input) + destdir, anon_mode = self._GetDestPath(torrenturl=input) if not destdir: return - if self.frame.startDownloadFromUrl(str(input), destdir): + if self.frame.startDownloadFromUrl(str(input), destdir, anon_mode=anon_mode): self.EndModal(wx.ID_OK) elif input.startswith("magnet:"): destdir = self.defaultDLConfig.get_dest_dir() if self.choose and self.choose.IsChecked(): - destdir = self._GetDestPath() + destdir, anon_mode = self._GetDestPath() if not destdir: return - if self.frame.startDownloadFromMagnet(str(input), destdir): + if self.frame.startDownloadFromMagnet(str(input), destdir, anon_mode=anon_mode): self.EndModal(wx.ID_OK) def OnLibrary(self, event): @@ -149,15 +149,15 @@ def __processPaths(self, paths): torrentfilename = None if len(filenames) == 1: torrentfilename = filenames[0] - destdir = self._GetDestPath(torrentfilename) + destdir, anon_mode = self._GetDestPath(torrentfilename) if not destdir: return if getattr(self.frame, 'startDownloads', False): - self.frame.startDownloads(filenames, fixtorrent=True, destdir=destdir) + self.frame.startDownloads(filenames, fixtorrent=True, destdir=destdir, anon_mode=anon_mode) else: for filename in filenames: - self.frame.startDownload(filename, fixtorrent=True, destdir=destdir) + self.frame.startDownload(filename, fixtorrent=True, destdir=destdir, anon_mode=anon_mode) def OnBrowse(self, event): dlg = wx.FileDialog(None, "Please select the .torrent file(s).", wildcard="torrent (*.torrent)|*.torrent", style=wx.FD_OPEN | wx.FD_MULTIPLE) @@ -207,7 +207,7 @@ def OnCreate(self, event): def _GetDestPath(self, torrentfilename=None, torrenturl=None): destdir = None - + anon_mode = False tdef = None if torrentfilename: tdef = TorrentDef.load(torrentfilename) @@ -220,5 +220,6 @@ def _GetDestPath(self, torrentfilename=None, torrenturl=None): if id == wx.ID_OK: destdir = dlg.GetPath() + anon_mode = dlg.GetAnonMode() dlg.Destroy() - return destdir + return (destdir, anon_mode) diff --git a/Tribler/Main/Dialogs/SaveAs.py b/Tribler/Main/Dialogs/SaveAs.py index 19d8163fa6..b9575d5e94 100644 --- a/Tribler/Main/Dialogs/SaveAs.py +++ b/Tribler/Main/Dialogs/SaveAs.py @@ -73,6 +73,9 @@ def __init__(self, parent, tdef, defaultdir, defaultname, selectedFiles=None): vSizer.Add(hSizer, 0, wx.EXPAND | wx.BOTTOM, 3) + # self.anon_check = wx.CheckBox(self, -1, 'Use anonymous downloading mode') + # vSizer.Add(self.anon_check, 0, wx.TOP | wx.BOTTOM, 5) + if tdef and tdef.get_files(): self.AddFileList(tdef, selectedFiles, vSizer, len(vSizer.GetChildren())) @@ -87,7 +90,7 @@ def __init__(self, parent, tdef, defaultdir, defaultname, selectedFiles=None): sizer.Add(ag, 0, wx.ALIGN_CENTER_VERTICAL) sizer.AddStretchSpacer() vSizer.Add(sizer, 1, wx.EXPAND | wx.BOTTOM, 3) - self.SetSize((600, 150)) + self.SetSize((600, 185)) # convert tdef into guidbtuple, and collect it using torrentsearch_manager.getTorrent torrent = Torrent.fromTorrentDef(tdef) @@ -167,7 +170,7 @@ def OnKeyUp(event): def SetCollected(self, tdef): self.collected = tdef - self.SetSize((600, 450)) + self.SetSize((600, 475)) vSizer = self.GetSizer().GetItem(0).GetSizer() hsizer = vSizer.GetItem(len(vSizer.GetChildren()) - 2).GetSizer() self.Freeze() @@ -196,6 +199,9 @@ def GetSelectedFiles(self): return files return None + def GetAnonMode(self): + return False # self.anon_check.GetValue() + def OnOk(self, event=None): if self.listCtrl: nrSelected = len(self.listCtrl.GetSelectedItems()) diff --git a/Tribler/Main/tribler_main.py b/Tribler/Main/tribler_main.py index 3b38d358f0..d43f97e535 100644 --- a/Tribler/Main/tribler_main.py +++ b/Tribler/Main/tribler_main.py @@ -449,6 +449,9 @@ def define_communities(): from Tribler.community.channel.community import ChannelCommunity from Tribler.community.channel.preview import PreviewChannelCommunity from Tribler.community.metadata.community import MetadataCommunity + from Tribler.community.anontunnel.community import ProxyCommunity + from Tribler.community.anontunnel import exitstrategies + from Tribler.community.anontunnel.Socks5.server import Socks5Server self._logger.info("tribler: Preparing communities...") now = time() @@ -472,9 +475,24 @@ def define_communities(): dispersy.define_auto_load(ChannelCommunity, s.dispersy_member, load=True) dispersy.define_auto_load(PreviewChannelCommunity, s.dispersy_member) + keypair = dispersy.crypto.generate_key(u"NID_secp160k1") + dispersy_member = dispersy.get_member( + private_key=dispersy.crypto.key_to_bin(keypair), + ) + + + proxy_community = dispersy.define_auto_load(ProxyCommunity, dispersy_member, (None, s), load=True)[0] + + socks_server = Socks5Server(proxy_community, s.lm.rawserver, s.get_proxy_community_socks5_listen_port()) + socks_server.start() + exit_strategy = exitstrategies.DefaultExitStrategy(s.lm.rawserver, proxy_community) + proxy_community.observers.append(exit_strategy) + diff = time() - now self._logger.info("tribler: communities are ready in %.2f seconds", diff) + s.set_anon_proxy_settings(2, ("127.0.0.1", s.get_proxy_community_socks5_listen_port())) + swift_process = s.get_swift_proc() and s.get_swift_process() dispersy = s.get_dispersy_instance() dispersy.callback.call(define_communities) diff --git a/Tribler/Main/vwxGUI/GuiUtility.py b/Tribler/Main/vwxGUI/GuiUtility.py index bef0c466a8..d0e987fe00 100644 --- a/Tribler/Main/vwxGUI/GuiUtility.py +++ b/Tribler/Main/vwxGUI/GuiUtility.py @@ -238,6 +238,11 @@ def ShowPage(self, page, *args): elif self.guiPage == 'stats': self.frame.stats.Show(False) + if page == 'anonymity': + self.frame.anonymity.Show() + elif self.guiPage == 'anonymity': + self.frame.anonymity.Show(False) + if self.frame.videoparentpanel: if page == 'videoplayer': self.frame.videoparentpanel.Show(True) diff --git a/Tribler/Main/vwxGUI/MainFrame.py b/Tribler/Main/vwxGUI/MainFrame.py index 89a622b3dd..d9da79c55b 100644 --- a/Tribler/Main/vwxGUI/MainFrame.py +++ b/Tribler/Main/vwxGUI/MainFrame.py @@ -59,7 +59,7 @@ TorrentDetails, LibraryDetails, ChannelDetails, PlaylistDetails from Tribler.Main.vwxGUI.TopSearchPanel import TopSearchPanel, \ TopSearchPanelStub -from Tribler.Main.vwxGUI.home import Home, Stats +from Tribler.Main.vwxGUI.home import Home, Stats, Anonymity from Tribler.Main.vwxGUI.channel import SelectedChannelList, Playlist, \ ManageChannel from Tribler.Main.vwxGUI.SRstatusbar import SRstatusbar @@ -261,6 +261,8 @@ def OnShowSplitter(event): self.stats = Stats(self) self.stats.Show(False) + self.anonymity = Anonymity(self) + self.anonymity.Show(False) self.managechannel = ManageChannel(self) self.managechannel.Show(False) @@ -281,6 +283,7 @@ def OnShowSplitter(event): hSizer.Add(separator, 0, wx.EXPAND) hSizer.Add(self.home, 1, wx.EXPAND) hSizer.Add(self.stats, 1, wx.EXPAND) + hSizer.Add(self.anonymity, 1, wx.EXPAND) hSizer.Add(self.splitter, 1, wx.EXPAND) else: vSizer = wx.BoxSizer(wx.VERTICAL) @@ -439,7 +442,7 @@ def startDownloadFromUrl(self, url, destdir=None, cmdline=False, selectedFiles=[ self.guiUtility.Notify("Download from url failed", icon=wx.ART_WARNING) return False - def startDownload(self, torrentfilename=None, destdir=None, sdef=None, tdef=None, cmdline=False, clicklog=None, name=None, vodmode=False, doemode=None, fixtorrent=False, selectedFiles=None, correctedFilename=None, hidden=False): + def startDownload(self, torrentfilename=None, destdir=None, sdef=None, tdef=None, cmdline=False, clicklog=None, name=None, vodmode=False, anon_mode=False, fixtorrent=False, selectedFiles=None, correctedFilename=None, hidden=False): self._logger.debug("mainframe: startDownload: %s %s %s %s %s %s", torrentfilename, destdir, sdef, tdef, vodmode, selectedFiles) if fixtorrent and torrentfilename: @@ -510,12 +513,23 @@ def do_gui(): selectedFiles = dlg.GetSelectedFiles() else: destdir = dlg.GetPath() + anon_mode = dlg.GetAnonMode() else: cancelDownload = True dlg.Destroy() else: raise Exception("cannot create dialog, not on wx thread") + if anon_mode: + if not tdef: + raise Exception('Currently only torrents can be downloaded in anonymous mode') + elif sdef: + sdef = None + cdef = tdef + monitorSwiftProgress = False + + dscfg.set_anon_mode(anon_mode) + if not cancelDownload: if destdir is not None: dscfg.set_dest_dir(destdir) diff --git a/Tribler/Main/vwxGUI/arflayout_fb.py b/Tribler/Main/vwxGUI/arflayout_fb.py new file mode 100644 index 0000000000..ec2be21c61 --- /dev/null +++ b/Tribler/Main/vwxGUI/arflayout_fb.py @@ -0,0 +1,128 @@ +import random +import math + +layout = {} + + +def arf_remove(toRemove): + global layout + + for vertexid in toRemove: + if vertexid in layout: + layout.pop(vertexid) + + new_layout = {} + for index, vertexid in enumerate(sorted(layout)): + new_layout[index] = layout[vertexid] + layout = new_layout + +def arf_layout(toInsert, graph): + # We just run one iteration, as we want the algorithm to use as little resources as possible. + + numNodes = graph.vcount() + arf_advance(toInsert, numNodes, graph) + + # Calculate positions in advance + x, y = zip(*layout.values()) + minX, maxX, minY, maxY = min(x), max(x), min(y), max(y) + xRange, yRange = maxX - minX, maxY - minY + + positions = {} + + if xRange != 0 and yRange != 0: + for vertexid in range(graph.vcount()): + if not layout.has_key(vertexid): + continue + x, y = layout[vertexid][0], layout[vertexid][1] + x_scaled, y_scaled = (x - minX) / xRange, (y - minY) / yRange + positions[vertexid] = (x_scaled, y_scaled) + + return positions + +def arf_advance(toInsert, numNodes, graph): + global layout + + for vertexid in toInsert: + layout[vertexid] = arf_get_position(vertexid, numNodes, graph) + toInsert.clear() + + for index in range(numNodes): + pos = layout.get(index, None) + if pos: + fX, fY = arf_get_force(index, numNodes, graph) + layout[index] = [ pos[0] + fX * 2, pos[1] + fY * 2 ] + +def arf_get_force(vertexid, numNodes, graph): + global layout + + x, y = layout.get(vertexid, (0.0, 0.0)) + + forceX, forceY = (0, 0) + + if x == 0 and y == 0: + return (forceX, forceY) + + for otherVertexid in range(0, numNodes): + if vertexid != otherVertexid: + otherX, otherY = layout.get(otherVertexid, (0, 0)) + if otherX == 0 and otherY == 0: + continue + + tempX = otherX - x + tempY = otherY - y + + mult = 3 if graph.are_connected(vertexid, otherVertexid) else 1 + mult *= 0.2 / math.sqrt(numNodes) + addX = tempX * mult + addY = tempY * mult + forceX += addX + forceY += addY + + mult = 8 / math.sqrt(tempX ** 2 + tempY ** 2) + addX = tempX * mult + addY = tempY * mult + forceX -= addX + forceY -= addY + + return (forceX, forceY) + +def arf_get_position(vertexid, numNodes, graph): + global layout + + nvertices = [] + if vertexid < numNodes: + for otherVertexid in graph.neighbors(vertexid): + if otherVertexid in layout.keys(): + nvertices.append(otherVertexid) + + pos = layout.get(vertexid, None) + if not pos: + pos = [random.random(), random.random()] + + if nvertices: + for otherVertexid in nvertices: + x2, y2 = layout[otherVertexid] + pos[0] += x2 + pos[1] += y2 + mult = 1.0 / len(nvertices) + pos[0] = pos[0] * mult + pos[1] = pos[1] * mult + return pos + +def CubicHermite(t, p0, p1, m0, m1): + t2 = t * t + t3 = t2 * t + return (2 * t3 - 3 * t2 + 1) * p0 + (t3 - 2 * t2 + t) * m0 + (-2 * t3 + 3 * t2) * p1 + (t3 - t2) * m1 + +def CubicHermiteInterpolate(t1, t2, t3, x1, x2, t): + v = (x2 - x1) / (t1 / 2.0 + t2 + t3 / 2.0) + d1 = v * t1 / 2.0 + d2 = v * t2 + + if t <= t1: + interpolate = CubicHermite(t / t1, x1, x1 + d1, 0, d2 / t2 * t1) + elif t <= t1 + t2: + interpolate = x1 + d1 + d2 * (t - t1) / t2 + else: + interpolate = CubicHermite((t - t1 - t2) / t3, x1 + d1 + d2, x2, d2 / t2 * t3, 0) + return interpolate diff --git a/Tribler/Main/vwxGUI/home.py b/Tribler/Main/vwxGUI/home.py index 9e9fa1aba0..d89d4fd84b 100644 --- a/Tribler/Main/vwxGUI/home.py +++ b/Tribler/Main/vwxGUI/home.py @@ -1,20 +1,40 @@ # Written by Niels Zeilemaker +import threading import wx import sys import os +import copy + +import wx +import igraph +from Tribler.Main.Dialogs.GUITaskQueue import GUITaskQueue +from Tribler.Utilities.TimedTaskQueue import TimedTaskQueue +from Tribler.community.anontunnel.community import ProxyCommunity +import datetime +from Tribler.community.anontunnel.routing import Hop + +try: + import igraph.vendor.texttable +except: + pass +import random import logging import binascii -from time import strftime +from time import strftime, time +from collections import defaultdict from traceback import print_exc from Tribler.Category.Category import Category -from Tribler.Core.simpledefs import NTFY_TORRENTS, NTFY_INSERT +from Tribler.Core.Tag.Extraction import TermExtraction +from Tribler.Core.simpledefs import NTFY_TORRENTS, NTFY_INSERT, NTFY_ANONTUNNEL, \ + NTFY_CREATED, NTFY_EXTENDED, NTFY_BROKEN, NTFY_SELECT, NTFY_JOINED, \ + NTFY_EXTENDED_FOR from Tribler.Core.Session import Session from Tribler.Core.CacheDB.SqliteCacheDBHandler import MiscDBHandler, \ TorrentDBHandler, ChannelCastDBHandler from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler -from Tribler.Main.vwxGUI import SEPARATOR_GREY, DEFAULT_BACKGROUND +from Tribler.Main.vwxGUI import SEPARATOR_GREY, DEFAULT_BACKGROUND, LIST_BLUE from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY from Tribler.Main.vwxGUI.list_header import DetailHeader @@ -24,6 +44,13 @@ from Tribler.Main.vwxGUI.widgets import SelectableListCtrl, \ TextCtrlAutoComplete, BetterText as StaticText, LinkStaticText +try: + # C(ython) module + import arflayout +except ImportError, e: + # Python fallback module + import arflayout_fb as arflayout + class Home(wx.Panel): @@ -550,6 +577,439 @@ def onActivity(self, msg): self.list.DeleteItem(size - 1) +class Anonymity(wx.Panel): + def __init__(self, parent): + wx.Panel.__init__(self, parent, -1) + + self.SetBackgroundColour(wx.WHITE) + self.guiutility = GUIUtility.getInstance() + self.utility = self.guiutility.utility + self.session = self.utility.session + + dispersy = self.utility.session.lm.dispersy + ''':type : Dispersy''' + + self.proxy_community = (c for c in dispersy.get_communities() if isinstance(c, ProxyCommunity)).next() + ''':type : ProxyCommunity ''' + + self.AddComponents() + + self.my_address = Hop(self.proxy_community.my_member._ec.pub()) + self.my_address.address = ('127.0.0.1', "SELF") + + self.vertices = {} + self.edges = [] + + self.selected_edges = [] + + self.vertex_active = -1 + self.vertex_hover = -1 + self.vertex_hover_evt = None + self.vertex_active_evt = None + + self.vertex_to_colour = {} + self.colours = [wx.RED, wx.Colour(156, 18, 18), wx.Colour(183, 83, 83), wx.Colour(254, 134, 134), wx.Colour(254, 190, 190)] + + self.step = 0 + self.fps = 20 + + self.last_keyframe = 0 + self.time_step = 5.0 + self.radius = 32 + + self.layout_busy = False + self.new_data = False + + self.peers = [] + self.toInsert = set() + + self.refresh_timer = wx.Timer(self) + self.Bind(wx.EVT_TIMER, lambda evt: self.graph_panel.Refresh(), self.refresh_timer) + self.refresh_timer.Start(1000.0 / self.fps) + + self.circuit_timer = wx.Timer(self) + self.Bind(wx.EVT_TIMER, self.OnUpdateCircuits, self.circuit_timer) + self.circuit_timer.Start(5000) + + self.taskqueue = GUITaskQueue.getInstance() + + self.lock = threading.RLock() + + self.session.add_observer(self.OnExtended, NTFY_ANONTUNNEL, [NTFY_CREATED, NTFY_EXTENDED, NTFY_BROKEN]) + self.session.add_observer(self.OnSelect, NTFY_ANONTUNNEL, [NTFY_SELECT]) + self.session.add_observer(self.OnJoined, NTFY_ANONTUNNEL, [NTFY_JOINED]) + self.session.add_observer(self.OnExtendedFor, NTFY_ANONTUNNEL, [NTFY_EXTENDED_FOR]) + + def AddComponents(self): + self.graph_panel = wx.Panel(self, -1) + self.graph_panel.Bind(wx.EVT_MOTION, self.OnMouse) + self.graph_panel.Bind(wx.EVT_LEFT_UP, self.OnMouse) + self.graph_panel.Bind(wx.EVT_PAINT, self.OnPaint) + self.graph_panel.Bind(wx.EVT_ERASE_BACKGROUND, self.OnEraseBackground) + self.graph_panel.Bind(wx.EVT_SIZE, self.OnSize) + + self.circuit_list = SelectableListCtrl(self, style=wx.LC_REPORT | wx.BORDER_SIMPLE) + self.circuit_list.InsertColumn(0, 'Circuit', wx.LIST_FORMAT_LEFT, 30) + self.circuit_list.InsertColumn(1, 'Online', wx.LIST_FORMAT_RIGHT, 50) + self.circuit_list.InsertColumn(2, 'Hops', wx.LIST_FORMAT_RIGHT, 45) + self.circuit_list.InsertColumn(3, 'Bytes up', wx.LIST_FORMAT_RIGHT, 65) + self.circuit_list.InsertColumn(4, 'Bytes down', wx.LIST_FORMAT_RIGHT, 65) + self.circuit_list.setResizeColumn(0) + self.circuit_list.Bind(wx.EVT_LIST_ITEM_SELECTED, self.OnItemSelected) + self.circuit_list.Bind(wx.EVT_LIST_ITEM_DESELECTED, self.OnItemSelected) + self.circuit_to_listindex = {} + + self.log_text = wx.TextCtrl(self, style=wx.TE_MULTILINE | wx.BORDER_SIMPLE | wx.HSCROLL & wx.VSCROLL) + self.log_text.SetEditable(False) + + vSizer = wx.BoxSizer(wx.VERTICAL) + vSizer.Add(self.circuit_list, 1, wx.EXPAND | wx.BOTTOM, 20) + vSizer.Add(self.log_text, 1, wx.EXPAND) + self.main_sizer = wx.BoxSizer(wx.HORIZONTAL) + self.main_sizer.Add(self.graph_panel, 3, wx.EXPAND | wx.ALL, 20) + self.main_sizer.Add(vSizer, 2, wx.EXPAND | wx.ALL, 20) + self.SetSizer(self.main_sizer) + + def OnItemSelected(self, event): + selected = [] + item = self.circuit_list.GetFirstSelected() + while item != -1: + selected.append(item) + item = self.circuit_list.GetNextSelected(item) + + selected_edges = [] + for item in selected: + for circuit_id, listindex in self.circuit_to_listindex.iteritems(): + if listindex == item: + circuit = self.circuits.get(circuit_id, None) + + if circuit: + hops = [self.my_address] + list(copy.copy(circuit.hops)) + for index in range(len(hops) - 1): + vertexid1 = self.peers.index(hops[index]) if hops[index] in self.peers else None + vertexid2 = self.peers.index(hops[index + 1]) if hops[index + 1] in self.peers else None + edge = set([vertexid1, vertexid2]) + selected_edges.append(edge) + + self.selected_edges = selected_edges + + def OnUpdateCircuits(self, event): + self.circuits = dict(self.proxy_community.circuits) + stats = self.proxy_community.global_stats.circuit_stats + + # Add new circuits & update existing circuits + for circuit_id, circuit in self.circuits.iteritems(): + if circuit_id not in self.circuit_to_listindex: + pos = self.circuit_list.InsertStringItem(sys.maxsize, str(circuit_id)) + self.circuit_to_listindex[circuit_id] = pos + else: + pos = self.circuit_to_listindex[circuit_id] + self.circuit_list.SetStringItem(pos, 1, str(circuit.state)) + self.circuit_list.SetStringItem(pos, 2, str(len(circuit.hops)) + "/" + str(circuit.goal_hops)) + + bytes_uploaded = stats[circuit_id].bytes_uploaded + bytes_downloaded = stats[circuit_id].bytes_downloaded + + self.circuit_list.SetStringItem(pos, 3, self.utility.size_format(bytes_uploaded)) + self.circuit_list.SetStringItem(pos, 4, self.utility.size_format(bytes_downloaded)) + + # Remove old circuits + old_circuits = [circuit_id for circuit_id in self.circuit_to_listindex if circuit_id not in self.circuits] + for circuit_id in old_circuits: + listindex = self.circuit_to_listindex[circuit_id] + self.circuit_list.DeleteItem(listindex) + self.circuit_to_listindex.pop(circuit_id) + for k, v in self.circuit_to_listindex.items(): + if v > listindex: + self.circuit_to_listindex[k] = v - 1 + + # Update graph + old_edges = getattr(self, 'old_edges', []) + new_edges = [] + + for circuit in self.circuits.values(): + hops = [self.my_address] + list(circuit.hops) + for index in range(len(hops) - 1): + edge = set([hops[index], hops[index + 1]]) + if edge not in new_edges: + new_edges.append(edge) + + for edge in new_edges: + if edge not in old_edges: + self.AddEdge(*edge) + + for edge in old_edges: + if edge not in new_edges: + self.RemoveEdge(*edge) + + self.old_edges = new_edges + + def AppendToLog(self, msg): + self.log_text.AppendText('[%s]: %s' % (datetime.datetime.now().strftime("%H:%M:%S"), msg)) + + @forceWxThread + def OnExtended(self, subject, changeType, circuit): + if changeType == NTFY_CREATED: + self.AppendToLog("Created circuit %s\n" % (circuit.circuit_id)) + if changeType == NTFY_EXTENDED: + self.AppendToLog("Extended circuit %s\n" % (circuit.circuit_id)) + if changeType == NTFY_BROKEN: + self.AppendToLog("Circuit %d has been broken\n" % circuit) + + @forceWxThread + def OnSelect(self, subject, changeType, circuit, address): + self.AppendToLog("Circuit %d has been selected for destination %s\n" % (circuit, address)) + + @forceWxThread + def OnJoined(self, subject, changeType, address, circuit_id): + self.AppendToLog("Joined an external circuit %d with %s:%d\n" % (circuit_id, address[0], address[1])) + + @forceWxThread + def OnExtendedFor(self, subject, changeType, extended_for, extended_with): + self.AppendToLog("Extended an external circuit (%s:%d, %d) with (%s:%d, %d)\n" % ( + extended_for[0].sock_addr[0], extended_for[0].sock_addr[1], extended_for[1], extended_with[0].sock_addr[0], + extended_with[0].sock_addr[1], extended_with[1])) + + def AddEdge(self, from_addr, to_addr): + with self.lock: + # Convert from_addr/to_addr to from_id/to_id + if from_addr not in self.peers: + self.peers.append(from_addr) + from_id = self.peers.index(from_addr) + if to_addr not in self.peers: + self.peers.append(to_addr) + to_id = self.peers.index(to_addr) + + # Add id's to graph + for peer_id in (from_id, to_id): + if peer_id not in self.vertices: + self.toInsert.add(peer_id) + self.vertices[peer_id] = {} + self.edges.append([to_id, from_id]) + self.new_data = True + + def RemoveEdge(self, from_addr, to_addr): + with self.lock: + if from_addr in self.peers and to_addr in self.peers: + from_id = self.peers.index(from_addr) + to_id = self.peers.index(to_addr) + if [to_id, from_id] in self.edges: + self.edges.remove([to_id, from_id]) + if [from_id, to_id] in self.edges: + self.edges.remove([from_id, to_id]) + self.RemoveUnconnectedVertices() + self.new_data = True + + def RemoveUnconnectedVertices(self): + # Build a list of vertices and their number of neighbors, and delete the unconnected ones. + for vertex_id, num_neighbors in self.CountNeighbors().iteritems(): + if num_neighbors == 0: + self.RemoveVertex(vertex_id) + + def CountNeighbors(self): + with self.lock: + num_neighbors = dict([(k, 0) for k in self.vertices]) + for edge in self.edges: + for vertexid in edge: + num_neighbors[vertexid] = num_neighbors.get(vertexid, 0) + 1 + return num_neighbors + + def RemoveVertex(self, toremove_id): + with self.lock: + if toremove_id in self.vertices: + self.vertices.pop(toremove_id) + if toremove_id in self.vertex_to_colour: + self.vertex_to_colour.pop(toremove_id) + if toremove_id < len(self.peers): + self.peers.pop(toremove_id) + self.edges = [edge for edge in self.edges if toremove_id not in edge] + self.toInsert = set([id - 1 if id > toremove_id else id for id in self.toInsert if id != toremove_id]) + self.vertex_active = self.vertex_active - 1 if self.vertex_active > toremove_id else self.vertex_active + self.vertex_hover = self.vertex_hover - 1 if self.vertex_hover > toremove_id else self.vertex_hover + + # We want the vertex id's to be 0, 1, 2 etc., so we need to correct for the vertex that we just removed. + vertices = {} + for index, vertexid in enumerate(sorted(self.vertices)): + vertices[index] = self.vertices[vertexid] + self.vertices = vertices + vertex_to_colour = {} + for index, vertexid in enumerate(sorted(self.vertex_to_colour)): + vertex_to_colour[index] = self.vertex_to_colour[vertexid] + self.vertex_to_colour = vertex_to_colour + for edge in self.edges: + if edge[0] >= toremove_id: + edge[0] -= 1 + if edge[1] >= toremove_id: + edge[1] -= 1 + + # The arflayout module keeps the vertex positions from the latest iteration in memory. So we need to notify arflayout. + arflayout.arf_remove([toremove_id]) + + def CalculateLayout(self): + with self.lock: + edges = copy.copy(self.edges) + toInsert = self.toInsert + self.toInsert = set() + + graph = igraph.Graph(edges, directed=False) + positions = arflayout.arf_layout(toInsert, graph) + + with self.lock: + self.step += 1 + for vertexid, pos in positions.iteritems(): + self.SetVertexPosition(vertexid, self.step, *pos) + self.time_step = 5 + max(0, len(self.vertices) - 75) / 9 + self.last_keyframe = time() + self.layout_busy = False + + def OnMouse(self, event): + if event.Moving(): + self.vertex_hover_evt = event.GetPosition() + elif event.LeftUp(): + self.vertex_active_evt = event.GetPosition() + + def OnSize(self, evt): + size = min(*evt.GetEventObject().GetSize()) + self.graph_panel.SetSize((size, size)) + + def OnEraseBackground(self, event): + pass + + def OnPaint(self, event): + eo = event.GetEventObject() + dc = wx.BufferedPaintDC(eo) + dc.Clear() + gc = wx.GraphicsContext.Create(dc) + + w, h = eo.GetSize().x - 2 * self.radius - 1, eo.GetSize().y - 2 * self.radius - 1 + + schedule_layout = not self.layout_busy and self.new_data and time() - self.last_keyframe >= self.time_step + if schedule_layout: + task = lambda : self.CalculateLayout() + self.taskqueue.add_task(task) + self.new_data = False + self.layout_busy = True + + if len(self.vertices) > 0: + + int_points = {} + + with self.lock: + + # Get current vertex positions using interpolation + for vertexid in self.vertices.iterkeys(): + if self.GetVertexPosition(vertexid, self.step): + if self.GetVertexPosition(vertexid, self.step - 1): + scaled_x, scaled_y = self.InterpolateVertexPosition(vertexid, self.step - 1, self.step) + else: + scaled_x, scaled_y = self.GetVertexPosition(vertexid, self.step) + int_points[vertexid] = (scaled_x * w + self.radius, scaled_y * h + self.radius) + + # Draw edges + for vertexid1, vertexid2 in self.edges: + if int_points.has_key(vertexid1) and int_points.has_key(vertexid2): + if set([vertexid1, vertexid2]) in self.selected_edges: + gc.SetPen(wx.Pen(wx.BLUE, 4)) + else: + gc.SetPen(wx.Pen(wx.Colour(229, 229, 229), 4)) + x1, y1 = int_points[vertexid1] + x2, y2 = int_points[vertexid2] + gc.DrawLines([(x1, y1), (x2, y2)]) + + # Draw vertices + gc.SetPen(wx.TRANSPARENT_PEN) + for vertexid in self.vertices.iterkeys(): + colour = self.vertex_to_colour.get(vertexid, None) + if not colour: + colour = self.colours[0] if self.peers[vertexid] == self.my_address else random.choice(self.colours[1:]) + self.vertex_to_colour[vertexid] = colour + gc.SetBrush(wx.Brush(colour)) + + if int_points.has_key(vertexid): + x, y = int_points[vertexid] + gc.DrawEllipse(x - self.radius / 2, y - self.radius / 2, self.radius, self.radius) + + if len(self.vertices.get(vertexid, {})) <= 2: + gc.SetBrush(wx.WHITE_BRUSH) + gc.DrawEllipse(x - self.radius / 4, y - self.radius / 4, self.radius / 2, self.radius / 2) + + # Draw circle around active vertex + gc.SetBrush(wx.TRANSPARENT_BRUSH) + + if self.vertex_hover_evt: + self.vertex_hover = self.PositionToVertex(self.vertex_hover_evt, int_points) + self.vertex_hover_evt = None + + if self.vertex_hover >= 0: + x, y = int_points[self.vertex_hover] + pen = wx.Pen(wx.Colour(229, 229, 229), 1, wx.USER_DASH) + pen.SetDashes([8, 4]) + gc.SetPen(pen) + gc.DrawEllipse(x - self.radius, y - self.radius, self.radius * 2, self.radius * 2) + + if self.vertex_active_evt: + self.vertex_active = self.PositionToVertex(self.vertex_active_evt, int_points) + self.vertex_active_evt = None + + if self.vertex_active in int_points: + x, y = int_points[self.vertex_active] + pen = wx.Pen(self.vertex_to_colour.get(self.vertex_active, wx.BLACK), 1, wx.USER_DASH) + pen.SetDashes([8, 4]) + gc.SetPen(pen) + gc.DrawEllipse(x - self.radius, y - self.radius, self.radius * 2, self.radius * 2) + + text_height = dc.GetTextExtent('gG')[1] + box_height = text_height + 3 + + # Draw status box + x = x - 150 - 1.1 * self.radius if x > self.graph_panel.GetSize()[0] / 2 else x + 1.1 * self.radius + y = y - box_height - 1.1 * self.radius if y > self.graph_panel.GetSize()[1] / 2 else y + 1.1 * self.radius + gc.SetBrush(wx.Brush(wx.Colour(216, 237, 255, 50))) + gc.SetPen(wx.Pen(LIST_BLUE)) + gc.DrawRectangle(x, y, 150, box_height) + + # Draw status text + dc.SetFont(self.GetFont()) + for index, text in enumerate(['IP %s:%s' % (self.peers[self.vertex_active].host, self.peers[self.vertex_active].port)]): + dc.DrawText(text, x + 5, y + index * text_height + 5) + + # Draw vertex count + gc.SetFont(self.GetFont()) + gc.DrawText("|V| = %d" % len(int_points), w - 50, h - 20) + + def PositionToVertex(self, position, key_to_position): + for vertexid, vposition in key_to_position.iteritems(): + if (position[0] - vposition[0]) ** 2 + (position[1] - vposition[1]) ** 2 < self.radius ** 2: + return vertexid + return -1 + + def InterpolateVertexPosition(self, vertexid, s1, s2): + x0, y0 = self.GetVertexPosition(vertexid, s1) + x1, y1 = self.GetVertexPosition(vertexid, s2) + + t = min(time() - self.last_keyframe, self.time_step) + t1 = 1.0 / 5 * self.time_step + t2 = 3.0 / 5 * self.time_step + t3 = 1.0 / 5 * self.time_step + x = arflayout.CubicHermiteInterpolate(t1, t2, t3, x0, x1, t) + y = arflayout.CubicHermiteInterpolate(t1, t2, t3, y0, y1, t) + return (x, y) + + def GetVertexPosition(self, vertexid, t): + if self.vertices.has_key(vertexid): + return self.vertices[vertexid].get(t, None) + return None + + def SetVertexPosition(self, vertexid, t, x, y): + if self.vertices.has_key(vertexid): + self.vertices[vertexid][t] = (x, y) + else: + self.vertices[vertexid] = {t: (x, y)} + + def ResetSearchBox(self): + pass + + class ArtworkPanel(wx.Panel): def __init__(self, parent): diff --git a/Tribler/Main/vwxGUI/list.py b/Tribler/Main/vwxGUI/list.py index a3beeba445..a538b76a75 100644 --- a/Tribler/Main/vwxGUI/list.py +++ b/Tribler/Main/vwxGUI/list.py @@ -9,8 +9,12 @@ import wx from wx.lib.wordwrap import wordwrap +from time import time +from datetime import date, datetime +from colorsys import hsv_to_rgb, rgb_to_hsv from Tribler.Category.Category import Category +from Tribler.Core.Libtorrent.LibtorrentMgr import LibtorrentMgr from Tribler.Core.simpledefs import NTFY_MISC, DLSTATUS_STOPPED, \ DLSTATUS_STOPPED_ON_ERROR, DLSTATUS_WAITING4HASHCHECK, \ @@ -1641,7 +1645,8 @@ def __init__(self, parent): {'name': 'Ratio', 'width': '15em', 'fmt': self._format_ratio, 'autoRefresh': False}, {'name': 'Time seeding', 'width': '25em', 'fmt': self._format_seedingtime, 'autoRefresh': False}, {'name': 'Swift ratio', 'width': '15em', 'fmt': self._format_ratio, 'autoRefresh': False}, - {'name': 'Swift time seeding', 'width': '30em', 'fmt': self._format_seedingtime, 'autoRefresh': False}] + {'name': 'Swift time seeding', 'width': '30em', 'fmt': self._format_seedingtime, 'autoRefresh': False}, + {'name': 'Anonymous', 'width': '15em', 'autoRefresh': False}] columns = self.guiutility.SetColumnInfo(LibraryListItem, columns, hide_defaults=[2, 7, 8, 9, 10]) ColumnsManager.getInstance().setColumns(LibraryListItem, columns) @@ -2197,11 +2202,11 @@ def _PostInit(self): def __SetData(self): self.list.SetData([(1, ['Home'], None, ActivityListItem), (2, ['Results'], None, ActivityListItem), (3, ['Channels'], None, ActivityListItem), - (4, ['Downloads'], None, ActivityListItem), (5, ['Videoplayer'], None, ActivityListItem)]) + (4, ['Downloads'], None, ActivityListItem), (5, ['Anonymity'], None, ActivityListItem), (6, ['Videoplayer'], None, ActivityListItem)]) self.ResizeListItems() self.DisableItem(2) if not self.guiutility.frame.videoparentpanel: - self.DisableItem(5) + self.DisableItem(6) self.DisableCollapse() self.selectTab('home') @@ -2287,6 +2292,8 @@ def OnExpand(self, item): return self.expandedPanel_channels elif item.data[0] == 'Downloads': self.guiutility.ShowPage('my_files') + elif item.data[0] == 'Anonymity': + self.guiutility.ShowPage('anonymity') elif item.data[0] == 'Videoplayer': if self.guiutility.guiPage not in ['videoplayer']: self.guiutility.ShowPage('videoplayer') @@ -2352,8 +2359,10 @@ def selectTab(self, tab): itemKey = 3 elif tab == 'my_files': itemKey = 4 - elif tab == 'videoplayer': + elif tab == 'anonymity': itemKey = 5 + elif tab == 'videoplayer': + itemKey = 6 if itemKey: wx.CallAfter(self.Select, itemKey, True) return diff --git a/Tribler/Test/test_anontunnel_community.py b/Tribler/Test/test_anontunnel_community.py new file mode 100644 index 0000000000..b0105ba7e8 --- /dev/null +++ b/Tribler/Test/test_anontunnel_community.py @@ -0,0 +1,128 @@ +# Written by Niels Zeilemaker +# see LICENSE.txt for license information + +import os +import sys +import time + +from Tribler.Test.test_as_server import TestGuiAsServer, BASE_DIR +from Tribler.Core.simpledefs import dlstatus_strings + + +class TestAnonTunnelCommunity(TestGuiAsServer): + + def test_anon_download(self): + def take_second_screenshot(): + self.screenshot() + self.quit() + + def take_screenshot(download_time): + self.screenshot("After an anonymous libtorrent download (took %.2f s)" % download_time) + self.guiUtility.ShowPage('anonymity') + self.Call(1, take_second_screenshot) + + def do_create_local_torrent(): + torrentfilename = self.setupSeeder() + start_time = time.time() + download = self.guiUtility.frame.startDownload(torrentfilename=torrentfilename, destdir=self.getDestDir(), anon_mode=True) + + self.guiUtility.ShowPage('my_files') + self.Call(5, lambda: download.add_peer(("127.0.0.1", self.session2.get_listen_port()))) + self.CallConditional( + 100, + lambda: download.get_progress() == 1.0, + lambda: take_screenshot(time.time() - start_time), + 'Download should be finished after 100 seconds' + ) + + self.startTest(do_create_local_torrent) + + def startTest(self, callback, min_timeout=5): + def setup_proxies(): + for i in range(3, 7): + create_proxy(i) + + callback() + + def create_proxy(index): + from Tribler.Core.Session import Session + from Tribler.community.anontunnel.community import ProxyCommunity, ProxySettings + from Tribler.community.anontunnel import exitstrategies, crypto + + self.setUpPreSession() + config = self.config.copy() + config.set_libtorrent(True) + config.set_dispersy(True) + config.set_state_dir(self.getStateDir(index)) + + session = Session(config, ignore_singleton=True) + session.start() + self.sessions.append(session) + + while not session.lm.initComplete: + time.sleep(1) + + dispersy = session.lm.dispersy + + def load_community(session): + keypair = dispersy.crypto.generate_key(u"NID_secp160k1") + dispersy_member = dispersy.get_member(private_key=dispersy.crypto.key_to_bin(keypair)) + + settings = ProxySettings() + + proxy_community = dispersy.define_auto_load(ProxyCommunity, dispersy_member, (settings, None), load=True)[0] + exit_strategy = exitstrategies.DefaultExitStrategy(session.lm.rawserver, proxy_community) + proxy_community.observers.append(exit_strategy) + + return proxy_community + + self.community = dispersy.callback.call(load_community, (session,)) + + TestGuiAsServer.startTest(self, setup_proxies) + + def setupSeeder(self): + from Tribler.Core.Session import Session + from Tribler.Core.TorrentDef import TorrentDef + from Tribler.Core.DownloadConfig import DownloadStartupConfig + + self.setUpPreSession() + self.config.set_libtorrent(True) + + self.config2 = self.config.copy() + self.config2.set_state_dir(self.getStateDir(2)) + self.session2 = Session(self.config2, ignore_singleton=True) + self.session2.start() + + tdef = TorrentDef() + tdef.add_content(os.path.join(BASE_DIR, "data", "video.avi")) + tdef.set_tracker("http://fake.net/announce") + tdef.finalize() + torrentfn = os.path.join(self.session2.get_state_dir(), "gen.torrent") + tdef.save(torrentfn) + + dscfg = DownloadStartupConfig() + dscfg.set_dest_dir(os.path.join(BASE_DIR, "data")) # basedir of the file we are seeding + d = self.session2.start_download(tdef, dscfg) + d.set_state_callback(self.seeder_state_callback) + + return torrentfn + + def seeder_state_callback(self, ds): + d = ds.get_download() + print >> sys.stderr, "test: seeder:", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress() + return (5.0, False) + + def setUp(self): + TestGuiAsServer.setUp(self) + self.sessions = [] + self.session2 = None + + def tearDown(self): + if self.session2: + self._shutdown_session(self.session2) + + for session in self.sessions: + self._shutdown_session(session) + + time.sleep(10) + TestGuiAsServer.tearDown(self) diff --git a/Tribler/community/anontunnel/Main.py b/Tribler/community/anontunnel/Main.py new file mode 100644 index 0000000000..f94be9e346 --- /dev/null +++ b/Tribler/community/anontunnel/Main.py @@ -0,0 +1,338 @@ +""" +AnonTunnel CLI interface +""" + +import logging.config +import threading +import os +from Tribler.community.anontunnel.Socks5.server import Socks5Server +from Tribler.community.anontunnel.stats import StatsCrawler + +import sys +import argparse +from threading import Thread, Event +from traceback import print_exc +import time +from Tribler.Core.RawServer.RawServer import RawServer +from Tribler.community.anontunnel import exitstrategies +from Tribler.community.anontunnel.community import ProxyCommunity, \ + ProxySettings +from Tribler.community.anontunnel.endpoint import DispersyBypassEndpoint +from Tribler.community.privatesemantic.crypto.elgamalcrypto import \ + ElgamalCrypto, NoElgamalCrypto +from Tribler.dispersy.callback import Callback +from Tribler.dispersy.dispersy import Dispersy +from Tribler.community.anontunnel.extendstrategies import TrustThyNeighbour, \ + NeighbourSubset +from Tribler.community.anontunnel.lengthstrategies import \ + RandomCircuitLengthStrategy, ConstantCircuitLength +from Tribler.community.anontunnel.selectionstrategies import \ + RandomSelectionStrategy, LengthSelectionStrategy + + +logging.config.fileConfig( + os.path.dirname(os.path.realpath(__file__)) + "/logger.conf") + +logger = logging.getLogger(__name__) + +try: + import yappi +except ImportError: + logger.warning("Yappi not installed, profiling options won't be available") + + +class AnonTunnel(Thread): + """ + The standalone AnonTunnel application. Does not depend on Tribler Session + or LaunchManyCore but creates all dependencies by itself. + + @param int socks5_port: the SOCKS5 port to listen on, or None to disable + the SOCKS5 server + @param ProxySettings settings: the settings to pass to the ProxyCommunity + @param bool crawl: whether to store incoming Stats messages using the + StatsCrawler + """ + + def __init__(self, socks5_port, settings=None, crawl=False): + Thread.__init__(self) + self.crawl = crawl + self.settings = settings + self.server_done_flag = Event() + self.raw_server = RawServer(self.server_done_flag, + 1, + 600.0, + ipv6_enable=False, + failfunc=lambda (e): print_exc(), + errorfunc=lambda (e): print_exc()) + + callback = Callback() + self.socks5_port = socks5_port + self.socks5_server = None + + endpoint = DispersyBypassEndpoint(self.raw_server, port=10000) + self.dispersy = Dispersy(callback, endpoint, u".", + u":memory:", crypto=ElgamalCrypto()) + + self.community = None + ''' @type: ProxyCommunity ''' + + def __calc_diff(self, then, bytes_exit0, bytes_enter0, bytes_relay0): + now = time.time() + + if not self.community or not then: + return now, 0, 0, 0, 0, 0, 0 + + diff = now - then + + stats = self.community.global_stats.stats + relay_stats = self.community.global_stats.relay_stats + + speed_exit = (stats['bytes_exit'] - bytes_exit0) / diff if then else 0 + bytes_exit = stats['bytes_exit'] + + speed_enter = (stats['bytes_enter'] - bytes_enter0) / diff if then else 0 + bytes_enter = stats['bytes_enter'] + + relay_2 = sum([r.bytes[1] for r in relay_stats.values()]) + + speed_relay = (relay_2 - bytes_relay0) / diff if then else 0 + bytes_relay = relay_2 + + return now, speed_exit, speed_enter, speed_relay, \ + bytes_exit, bytes_enter, bytes_relay + + def __speed_stats(self): + time = None + bytes_exit = 0 + bytes_enter = 0 + bytes_relay = 0 + + while True: + stats = self.__calc_diff(time, bytes_exit, bytes_enter, bytes_relay) + time, speed_exit, speed_enter, speed_relay, bytes_exit, bytes_enter, bytes_relay = stats + + active_circuits = len(self.community.circuits) + num_routes = len(self.community.relay_from_to) / 2 + + print "CIRCUITS %d RELAYS %d EXIT %.2f KB/s ENTER %.2f KB/s RELAY %.2f KB/s\n" % ( + active_circuits, num_routes, speed_exit / 1024.0, + speed_enter / 1024.0, speed_relay / 1024.0), + + yield 3.0 + + def run(self): + """ + Start the standalone AnonTunnel + """ + + self.dispersy.start() + logger.error( + "Dispersy is listening on port %d" % self.dispersy.lan_address[1]) + + def _join_overlay(raw_server, dispersy): + member = self.dispersy.get_new_member(u"NID_secp160k1") + proxy_community = dispersy.define_auto_load(ProxyCommunity, member, (self.settings, False), load=True)[0] + ''' @type: ProxyCommunity ''' + + if self.socks5_server: + self.socks5_server = Socks5Server( + proxy_community, self.raw_server, self.socks5_port) + self.socks5_server.start() + + self.community = proxy_community + exit_strategy = exitstrategies.DefaultExitStrategy(raw_server, self.community) + proxy_community.observers.append(exit_strategy) + + if self.crawl: + self.community.observers.append(StatsCrawler(self.dispersy, self.raw_server)) + + return proxy_community + + proxy_args = self.raw_server, self.dispersy + self.community = self.dispersy.callback.call(_join_overlay, proxy_args) + ''' :type : Tribler.community.anontunnel.community.ProxyCommunity ''' + + self.dispersy.callback.register(self.__speed_stats) + self.raw_server.listen_forever(None) + + def stop(self): + """ + Stop the standalone AnonTunnel + """ + if self.dispersy: + self.dispersy.stop() + + self.server_done_flag.set() + + if self.raw_server: + self.raw_server.shutdown() + + +def main(argv): + """ + Start CLI interface of the AnonTunnel + @param argv: the CLI arguments, except the first + """ + parser = argparse.ArgumentParser( + description='Anonymous Tunnel CLI interface') + + try: + parser.add_argument('-p', '--socks5', help='Socks5 port') + parser.add_argument('-y', '--yappi', + help="Profiling mode, either 'wall' or 'cpu'") + parser.add_argument('-l', '--length-strategy', default=[], nargs='*', + help='Circuit length strategy') + parser.add_argument('-s', '--select-strategy', default=[], nargs='*', + help='Circuit selection strategy') + parser.add_argument('-e', '--extend-strategy', default='subset', + help='Circuit extend strategy') + parser.add_argument('--max-circuits', nargs=1, default=10, + help='Maximum number of circuits to create') + parser.add_argument('--crawl', default=False, + help='Record stats from others in results.db') + parser.add_help = True + args = parser.parse_args(sys.argv[1:]) + + except argparse.ArgumentError: + parser.print_help() + sys.exit(2) + + socks5_port = None + + if args.yappi == 'wall': + profile = "wall" + elif args.yappi == 'cpu': + profile = "cpu" + else: + profile = None + + if args.socks5: + socks5_port = int(args.socks5) + + if profile: + yappi.set_clock_type(profile) + yappi.start(builtins=True) + print "Profiling using %s time" % yappi.get_clock_type()['type'] + + crawl = True if args.crawl else False + proxy_settings = ProxySettings() + + # Set extend strategy + if args.extend_strategy == 'delegate': + logger.error("EXTEND STRATEGY DELEGATE: We delegate the selection of " + "hops to the rest of the circuit") + proxy_settings.extend_strategy = TrustThyNeighbour + elif args.extend_strategy == 'subset': + logger.error("SUBSET STRATEGY DELEGATE: We delegate the selection of " + "hops to the rest of the circuit") + proxy_settings.extend_strategy = NeighbourSubset + else: + raise ValueError("extend_strategy must be either random or delegate") + + # Circuit length strategy + if args.length_strategy[:1] == ['random']: + strategy = RandomCircuitLengthStrategy(*args.length_strategy[1:]) + proxy_settings.length_strategy = strategy + logger.error("Using RandomCircuitLengthStrategy with arguments %s", + ', '.join(args.length_strategy[1:])) + + elif args.length_strategy[:1] == ['constant']: + strategy = ConstantCircuitLength(*args.length_strategy[1:]) + proxy_settings.length_strategy = strategy + logger.error( + "Using ConstantCircuitLength with arguments %s", + ', '.join(args.length_strategy[1:])) + + # Circuit selection strategies + if args.select_strategy[:1] == ['random']: + strategy = RandomSelectionStrategy(*args.select_strategy[1:]) + proxy_settings.selection_strategy = strategy + logger.error("Using RandomCircuitLengthStrategy with arguments %s" + ', '.join(args.select_strategy[1:])) + + elif args.select_strategy[:1] == ['length']: + strategy = LengthSelectionStrategy(*args.select_strategy[1:]) + proxy_settings.selection_strategy = strategy + logger.error("Using LengthSelectionStrategy with arguments %s", + ', '.join(args.select_strategy[1:])) + + anon_tunnel = AnonTunnel(socks5_port, proxy_settings, crawl) + ''' @type: AnonTunnel ''' + + anon_tunnel.start() + + + + while 1: + try: + line = sys.stdin.readline() + except KeyboardInterrupt: + anon_tunnel.stop() + os._exit(0) + break + + if not line: + break + + if line == 'threads\n': + for thread in threading.enumerate(): + print "%s \t %d" % (thread.name, thread.ident) + elif line == 'p\n': + if profile: + + for func_stats in yappi.get_func_stats().sort("subtime")[:50]: + print "YAPPI: %10dx %10.3fs" % ( + func_stats.ncall, func_stats.tsub), func_stats.name + else: + print >> sys.stderr, "Profiling disabled!" + + elif line == 'P\n': + if profile: + filename = 'callgrindc_%d.yappi' % \ + anon_tunnel.dispersy.lan_address[1] + yappi.get_func_stats().save(filename, type='callgrind') + else: + print >> sys.stderr, "Profiling disabled!" + + elif line == 't\n': + if profile: + yappi.get_thread_stats().sort("totaltime").print_all() + + else: + print >> sys.stderr, "Profiling disabled!" + + elif line == 'c\n': + stats = anon_tunnel.community.global_stats.circuit_stats + + print "========\nCircuits\n========\n" \ + "id\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)" + for circuit_id, circuit in anon_tunnel.community.circuits.items(): + print "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f" % ( + circuit.circuit_id, circuit.candidate.sock_addr[0], + circuit.candidate.sock_addr[1], + circuit.goal_hops, len(circuit.hops), + stats[circuit_id].bytes_downloaded / 1024.0 / 1024.0, + stats[circuit_id].bytes_uploaded / 1024.0 / 1024.0 + ) + elif line == 'q\n': + anon_tunnel.stop() + os._exit(0) + break + elif line == 'r\n': + print "circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)" + + from_to = anon_tunnel.community.relay_from_to + + for key in from_to.keys(): + relay = from_to[key] + + print "%s-->\t%s\t\t%.2f" % ( + (key[0], key[1]), + (relay.sock_addr, relay.circuit_id), + relay.bytes[1] / 1024.0 / 1024.0, + ) + + +if __name__ == "__main__": + main(sys.argv[1:]) + diff --git a/Tribler/community/anontunnel/Socks5/__init__.py b/Tribler/community/anontunnel/Socks5/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Tribler/community/anontunnel/Socks5/connection.py b/Tribler/community/anontunnel/Socks5/connection.py new file mode 100644 index 0000000000..68044bb0de --- /dev/null +++ b/Tribler/community/anontunnel/Socks5/connection.py @@ -0,0 +1,314 @@ +""" +Created on 3 jun. 2013 + +@author: Chris +""" +import logging +from socket import socket +from Tribler.community.anontunnel.Socks5 import conversion + + +class ConnectionState: + """ + Enumeration of possible SOCKS5 connection states + """ + def __init__(self): + pass + + BEFORE_METHOD_REQUEST = 'BEFORE_METHOD_REQUEST' + METHOD_REQUESTED = 'METHOD_REQUESTED' + CONNECTED = 'CONNECTED' + PROXY_REQUEST_RECEIVED = 'PROXY_REQUEST_RECEIVED' + PROXY_REQUEST_ACCEPTED = 'PROXY_REQUEST_ACCEPTED' + TCP_RELAY = 'TCP_RELAY' + + +class Socks5ConnectionObserver: + """ + Socks5 Connection observer + """ + + def __init__(self): + pass + + def on_udp_associate_request(self, connection, request): + """ + + @param Socks5Connection connection: the Socks5 connection we got the + request on + @param Request request: the request received + """ + pass + + +class Socks5Connection(object): + """ + SOCKS5 TCP Connection handler + + Supports a subset of the SOCKS5 protocol, no authentication and no support + for TCP BIND requests + """ + + def __init__(self, single_socket, socks5_server): + self.state = ConnectionState.BEFORE_METHOD_REQUEST + self._logger = logging.getLogger(__name__) + + self.observers = [] + ''' :type : list[Socks5ConnectionObserver] ''' + + self.single_socket = single_socket + ''' :type : SingleSocket ''' + + self.socks5_server = socks5_server + """ :type : TcpConnectionHandler """ + + self.buffer = '' + self.tcp_relay = None + self.udp_associate = None + ''' :type : (Socks5Connection) -> socket ''' + + def data_came_in(self, data): + """ + Called by the TcpConnectionHandler when new data has been received. + + Processes the incoming buffer by attempting to read messages defined + in the SOCKS5 protocol + + :param data: the data received + :return: None + """ + + if len(self.buffer) == 0: + self.buffer = data + else: + self.buffer = self.buffer + data + + if self.tcp_relay: + self._try_tcp_relay() + else: + self._process_buffer() + + def _try_handshake(self): + + # Try to read a HANDSHAKE request + offset, request = conversion.decode_methods_request(0, self.buffer) + + # No (complete) HANDSHAKE received, so dont do anything + if request is None: + return None + + # Consume the buffer + self.buffer = self.buffer[offset:] + + assert isinstance(request, conversion.MethodRequest) + + # Only accept NO AUTH + if request.version != 0x05 or 0x00 not in request.methods: + self._logger.error("Client has sent INVALID METHOD REQUEST") + self.buffer = '' + self.close() + return + + self._logger.info("Client {0} has sent METHOD REQUEST".format( + (self.single_socket.get_ip(), self.single_socket.get_port()) + )) + + # Respond that we would like to use NO AUTHENTICATION (0x00) + if self.state is not ConnectionState.CONNECTED: + response = conversion.encode_method_selection_message( + conversion.SOCKS_VERSION, 0x00) + self.write(response) + + # We are connected now, the next incoming message will be a REQUEST + self.state = ConnectionState.CONNECTED + + def _try_tcp_relay(self): + """ + Forward the complete buffer to the paired TCP socket + + :return: None + """ + self._logger.info("Relaying TCP data") + self.tcp_relay.sendall(self.buffer) + self.buffer = '' + + def deny_request(self, request, drop_connection=True): + """ + Deny SOCKS5 request + @param Request request: the request to deny + @param bool drop_connection: whether to drop the connection or not + """ + if self.state != ConnectionState.PROXY_REQUEST_RECEIVED: + raise ValueError("There must be a request before we can deny it") + + self.state = ConnectionState.CONNECTED + + response = conversion.encode_reply( + 0x05, conversion.REP_COMMAND_NOT_SUPPORTED, 0x00, + conversion.ADDRESS_TYPE_IPV4, "0.0.0.0", 0) + + self.write(response) + + if self.single_socket: + self._logger.error( + "DENYING SOCKS5 Request from {0}".format( + (self.single_socket.get_ip(), + self.single_socket.get_port()) + ) + ) + + if drop_connection: + self.close() + + def accept_udp_associate(self, request, udp_socket): + """ + Accept the UDP request given and redirect the client to the udp_socket + @param request: + @param socket udp_socket: the udp relay socket + """ + + if not isinstance(udp_socket, socket): + raise ValueError("Parameter udp_socket must be of socket type") + + if self.state != ConnectionState.PROXY_REQUEST_RECEIVED: + raise ValueError("SOCKS5 connection has not been established yet!") + + # We use same IP as the single socket, but the port number + # comes from the newly created UDP listening socket + ip = self.single_socket.get_myip() + port = udp_socket.getsockname()[1] + + self._logger.warning( + "Accepting UDP ASSOCIATE request from %s:%d, " + "direct client to %s:%d", + self.single_socket.get_ip(), self.single_socket.get_port(), + ip, port) + + response = conversion.encode_reply( + 0x05, conversion.REP_SUCCEEDED, 0x00, + conversion.ADDRESS_TYPE_IPV4, ip, port) + self.write(response) + + def _try_request(self): + """ + Try to consume a REQUEST message and respond whether we will accept the + request. + + Will setup a TCP relay or an UDP socket to accommodate TCP RELAY and + UDP ASSOCIATE requests. After a TCP relay is set up the handler will + deactivate itself and change the Connection to a TcpRelayConnection. + Further data will be passed on to that handler. + + :return: None + """ + self._logger.debug("Client {0} has sent PROXY REQUEST".format( + (self.single_socket.get_ip(), self.single_socket.get_port()) + )) + offset, request = conversion.decode_request(0, self.buffer) + + if request is None: + return None + + self.buffer = self.buffer[offset:] + + assert isinstance(request, conversion.Request) + self.state = ConnectionState.PROXY_REQUEST_RECEIVED + + accept = True + + try: + if request.cmd == conversion.REQ_CMD_UDP_ASSOCIATE: + for observer in self.observers: + observer.on_udp_associate_request(self, request) + + accept = False + elif request.cmd == conversion.REQ_CMD_BIND: + response = conversion.encode_reply( + 0x05, conversion.REP_SUCCEEDED, 0x00, + conversion.ADDRESS_TYPE_IPV4, "127.0.0.1", 1081) + self.write(response) + self.state = ConnectionState.PROXY_REQUEST_ACCEPTED + elif request.cmd == conversion.REQ_CMD_CONNECT: + self._logger.warning( + "TCP req to %s:%d support it. Returning HOST UNREACHABLE", + *request.destination) + response = conversion.encode_reply( + 0x05, conversion.REP_HOST_UNREACHABLE, 0x00, + conversion.ADDRESS_TYPE_IPV4, "0.0.0.0", 0) + self.write(response) + accept = False + else: + self.deny_request(request, False) + accept = False + except: + response = conversion.encode_reply( + 0x05, conversion.REP_COMMAND_NOT_SUPPORTED, 0x00, + conversion.ADDRESS_TYPE_IPV4, "0.0.0.0", 0) + self.write(response) + self._logger.exception("Exception thrown. Returning unsupported " + "command response") + accept = False + + if accept: + self.state = ConnectionState.PROXY_REQUEST_ACCEPTED + + return accept + + def _process_buffer(self): + """ + Processes the buffer by attempting to messages which are to be expected + in the current state + """ + while len(self.buffer) > 0: + self.state = self._guess_state() + + # We are at the initial state, so we expect a handshake request. + if self.state == ConnectionState.BEFORE_METHOD_REQUEST: + if not self._try_handshake(): + break # Not enough bytes so wait till we got more + + # We are connected so the + elif self.state == ConnectionState.CONNECTED: + if not self._try_request(): + break # Not enough bytes so wait till we got more + else: + self.buffer = '' + + def _guess_state(self): + if len(self.buffer) < 3: + return self.state + + data = self.buffer + is_version = ord(data[0]) == 0x05 + if is_version and data[1] == chr(0x01) and chr(0x00) == data[2]: + guessed_state = ConnectionState.BEFORE_METHOD_REQUEST + + if self.state != guessed_state: + self._logger.error("GUESSING SOCKS5 state %s should be %s!", guessed_state, self.state) + + return guessed_state + + has_valid_command = ord(data[1]) in [0x01, 0x02, 0x03] + has_valid_address = ord(data[2]) in [0x01, 0x03, 0x04] + + if is_version and has_valid_command and has_valid_address: + return ConnectionState.CONNECTED + + return self.state + + def write(self, data): + if self.single_socket is not None: + self.single_socket.write(data) + + def close(self): + if self.single_socket is not None: + self._logger.error( + "On close() of %s:%d", self.single_socket.get_ip(), + self.single_socket.get_port()) + + self.single_socket.close() + self.single_socket = None + ''' :type : SingleSocket ''' + + if self.tcp_relay: + self.tcp_relay.close() diff --git a/Tribler/community/anontunnel/Socks5/conversion.py b/Tribler/community/anontunnel/Socks5/conversion.py new file mode 100644 index 0000000000..cde62e1fb6 --- /dev/null +++ b/Tribler/community/anontunnel/Socks5/conversion.py @@ -0,0 +1,248 @@ +import struct +import socket + +# Some constants used in the RFC 1928 specification +SOCKS_VERSION = 0x05 + +ADDRESS_TYPE_IPV4 = 0x01 +ADDRESS_TYPE_DOMAIN_NAME = 0x03 +ADDRESS_TYP_IPV6 = 0x04 + +REQ_CMD_CONNECT = 0x01 +REQ_CMD_BIND = 0x02 +REQ_CMD_UDP_ASSOCIATE = 0x03 + +REP_SUCCEEDED = 0x00 +REP_GENERAL_SOCKS_SERVER_FAIL = 0x01 +REP_CONNECTION_NOT_ALLOWED_BY_RULE_SET = 0x02 +REP_NETWORK_UNREACHABLE = 0x03 +REP_HOST_UNREACHABLE = 0x04 +REP_CONNECTION_REFUSED = 0x05 +REP_TTL_EXPIRED = 0x06 +REP_COMMAND_NOT_SUPPORTED = 0x07 +REP_ADDRESS_TYPE_NOT_SUPPORTED = 0x08 + + +class MethodRequest(object): + def __init__(self, version, methods): + self.version = version + self.methods = methods + + +class Request(object): + def __init__(self, version, cmd, rsv, address_type, destination_address, + destination_port): + self.version = version + self.cmd = cmd + self.rsv = rsv + self.address_type = address_type + self.destination_host = destination_address + self.destination_port = destination_port + + @property + def destination(self): + """ + The destination address as a tuple + @rtype: (str, int) + """ + return self.destination_host, self.destination_port + + +class UdpRequest(object): + """ + + @param rsv: the reserved bits in the SOCKS protocol + @param frag: + @param address_type: whether we deal with an IPv4 or IPv6 address + @param str destination_address: the destination host + @param int destination_port: the destination port + @param str payload: the payload + """ + + def __init__(self, rsv, frag, address_type, destination_address, + destination_port, payload): + self.rsv = rsv + self.frag = frag + self.address_type = address_type + self.destination_host = destination_address + self.destination_port = destination_port + self.payload = payload + + @property + def destination(self): + """ + The destination address as a tuple + @rtype: (str, int) + """ + return self.destination_host, self.destination_port + + +def decode_methods_request(offset, data): + """ + Try to decodes a METHOD request + @param int offset: the offset to start in the data + @param str data: the serialised data to decode from + @return: Tuple (offset, None) on failure, else (new_offset, MethodRequest) + @rtype: (int, None|MethodRequest) + """ + # Check if we have enough bytes + if len(data) - offset < 2: + return offset, None + + (version, number_of_methods) = struct.unpack_from("BB", data, offset) + + # We only know how to handle Socks5 protocol + if not version == SOCKS_VERSION: + return offset, None + + offset += 2 + + methods = set([]) + for i in range(number_of_methods): + method, = struct.unpack_from("B", data, offset) + methods.add(method) + offset += 1 + + return offset, MethodRequest(version, methods) + + +def encode_method_selection_message(version, method): + """ + Serialise a Method Selection message + @param version: the SOCKS5 version + @param method: the authentication method to select + @return: the serialised format + @rtype: str + """ + return struct.pack("BB", version, method) + + +def __encode_address(address_type, address): + if address_type == ADDRESS_TYPE_IPV4: + data = socket.inet_aton(address) + elif address_type == ADDRESS_TYP_IPV6: + raise ValueError("IPv6 not implemented") + elif address_type == ADDRESS_TYPE_DOMAIN_NAME: + data = struct.pack("B", len(address)) + address + else: + raise ValueError( + "address_type must be either IPv4, IPv6 or a domain name") + + return data + + +def __decode_address(address_type, offset, data): + if address_type == ADDRESS_TYPE_IPV4: + destination_address = socket.inet_ntoa(data[offset:offset + 4]) + offset += 4 + elif address_type == ADDRESS_TYPE_DOMAIN_NAME: + domain_length, = struct.unpack_from("B", data, offset) + offset += 1 + destination_address = data[offset:offset + domain_length] + offset += domain_length + elif address_type == ADDRESS_TYP_IPV6: + return offset, None + else: + raise ValueError("Unsupported address type") + + return offset, destination_address + + +def decode_request(orig_offset, data): + """ + Try to decode a SOCKS5 request + @param int orig_offset: the offset to start decoding in the data + @param str data: the raw data + @return: tuple (new_offset, Request) or (original_offset, None) on failure + @rtype: (int, Request|None) + """ + offset = orig_offset + + # Check if we have enough bytes + if len(data) - offset < 4: + return orig_offset, None + + version, cmd, rsv, address_type = struct.unpack_from("BBBB", data, offset) + offset += 4 + + assert version == SOCKS_VERSION + assert rsv == 0 + + offset, destination_address = __decode_address(address_type, offset, data) + + # Check if we could decode address, if not bail out + if not destination_address: + return orig_offset, None + + # Check if we have enough bytes + if len(data) - offset < 2: + return orig_offset, None + + destination_port, = struct.unpack_from("!H", data, offset) + offset += 2 + + return offset, Request(version, cmd, rsv, address_type, + destination_address, destination_port) + + +def encode_reply(version, rep, rsv, address_type, bind_address, bind_port): + """ + Encode a REPLY + @param int version: SOCKS5 version + @param int rep: the response + @param int rsv: reserved bytes + @param address_type: the address type of the bind address + @param bind_address: the bind address host + @param bind_port: the bind address port + @return: + """ + data = struct.pack("BBBB", version, rep, rsv, address_type) + + data += __encode_address(address_type, bind_address) + + data += struct.pack("!H", bind_port) + return data + + +def decode_udp_packet(data): + """ + Decodes a SOCKS5 UDP packet + @param str data: the raw packet data + @return: An UdpRequest object containing the parsed data + @rtype: UdpRequest + """ + offset = 0 + (rsv, frag, address_type) = struct.unpack_from("!HBB", data, offset) + offset += 4 + + offset, destination_address = __decode_address(address_type, offset, data) + + destination_port, = struct.unpack_from("!H", data, offset) + offset += 2 + + payload = data[offset:] + + return UdpRequest(rsv, frag, address_type, destination_address, + destination_port, payload) + + +def encode_udp_packet(rsv, frag, address_type, address, port, payload): + """ + Encodes a SOCKS5 UDP packet + @param rsv: reserved bytes + @param frag: fragment + @param address_type: the address's type + @param address: address host + @param port: address port + @param payload: the original UDP payload + @return: serialised byte string + @rtype: str + """ + strings = [ + struct.pack("!HBB", rsv, frag, address_type), + __encode_address(address_type, address), + struct.pack("!H", port), + payload + ] + + return ''.join(strings) \ No newline at end of file diff --git a/Tribler/community/anontunnel/Socks5/server.py b/Tribler/community/anontunnel/Socks5/server.py new file mode 100644 index 0000000000..db9731b2f8 --- /dev/null +++ b/Tribler/community/anontunnel/Socks5/server.py @@ -0,0 +1,145 @@ +from Tribler.community.anontunnel.events import TunnelObserver + +import logging +import socket +from Tribler.Core.RawServer.SocketHandler import SingleSocket +from Tribler.community.anontunnel.globals import CIRCUIT_STATE_READY +from Tribler.community.anontunnel.routing import CircuitPool +from .session import Socks5Session +from .connection import Socks5Connection + +__author__ = 'chris' + + +class Socks5Server(TunnelObserver): + """ + The SOCKS5 server which allows clients to proxy UDP over Circuits in the + ProxyCommunity + + @param ProxyCommunity tunnel: the ProxyCommunity to request circuits from + @param RawServer raw_server: the RawServer instance to bind on + @param int socks5_port: the port to listen on + """ + def __init__(self, tunnel, raw_server, socks5_port=1080, num_circuits=4, min_circuits=4, min_session_circuits=4): + super(Socks5Server, self).__init__() + self._logger = logging.getLogger(__name__) + + self.tunnel = tunnel + self.socks5_port = socks5_port + self.raw_server = raw_server + ''' @type : RawServer ''' + self.reserved_circuits = [] + ''' @type : list[Circuit] ''' + self.awaiting_circuits = 0 + ''' @type : int ''' + self.tcp2session = {} + ''' @type : dict[Socks5Connection, Socks5Session] ''' + + self.circuit_pool = CircuitPool(num_circuits, "SOCKS5(master)") + self.tunnel.observers.append(self.circuit_pool) + self.tunnel.circuit_pools.append(self.circuit_pool) + + self.min_circuits = min_circuits + self.min_session_circuits = min_session_circuits + + raw_server.add_task(self.__start_anon_session, 5.0) + + def __start_anon_session(self): + if len(self.circuit_pool.available_circuits) >= self.min_circuits: + self._logger.warning("Creating ANON session") + + try: + from Tribler.Core.Libtorrent.LibtorrentMgr import LibtorrentMgr + LibtorrentMgr.getInstance().create_anonymous_session() + except ImportError: + self._logger.exception("Cannot create anonymous session!") + + return True + else: + self.raw_server.add_task(self.__start_anon_session, delay=1.0) + return False + + def start(self): + try: + self.raw_server.bind(self.socks5_port, reuse=True, handler=self) + self._logger.info("SOCKS5 listening on port %d", self.socks5_port) + self.tunnel.observers.append(self) + except socket.error: + self._logger.error( + "Cannot listen on SOCK5 port %s:%d, perhaps another " + "instance is running?", + "0.0.0.0", self.socks5_port) + except: + self._logger.exception("Exception trying to reserve circuits") + + def external_connection_made(self, single_socket): + """ + Called by the RawServer when a new connection has been made + + @param SingleSocket single_socket: the new connection + """ + self._logger.info("accepted SOCKS5 new connection") + s5con = Socks5Connection(single_socket, self) + + try: + session_pool = CircuitPool(4, "SOCKS5(%s:%d)" % (single_socket.get_ip(), single_socket.get_port())) + session = Socks5Session(self.raw_server, s5con, self, session_pool, min_circuits=self.min_session_circuits) + self.tunnel.observers.append(session) + self.tunnel.observers.append(session_pool) + self.tunnel.circuit_pools.insert(0, session_pool) + + self.tcp2session[single_socket] = session + except: + self._logger.exception("Error while accepting SOCKS5 connection") + s5con.close() + + def connection_flushed(self, single_socket): + pass + + def connection_lost(self, single_socket): + self._logger.info("SOCKS5 TCP connection lost") + + if single_socket not in self.tcp2session: + return + + session = self.tcp2session[single_socket] + self.tunnel.observers.remove(session) + self.tunnel.circuit_pools.remove(session.circuit_pool) + + # Reclaim good circuits + good_circuits = [circuit for circuit in session.circuit_pool.available_circuits if circuit.state == CIRCUIT_STATE_READY] + + self._logger.warning( + "Reclaiming %d good circuits due to %s:%d", + len(good_circuits), + single_socket.get_ip(), single_socket.get_port()) + + for circuit in good_circuits: + self.circuit_pool.fill(circuit) + + s5con = session.connection + del self.tcp2session[single_socket] + + try: + s5con.close() + except: + pass + + def data_came_in(self, single_socket, data): + """ + Data is in the READ buffer, depending on MODE the Socks5 or + Relay mechanism will be used + + :param single_socket: + :param data: + :return: + """ + tcp_connection = self.tcp2session[single_socket].connection + try: + tcp_connection.data_came_in(data) + except: + self._logger.exception("Error while handling incoming TCP data") + + def on_break_circuit(self, circuit): + if circuit in self.reserved_circuits: + self.reserved_circuits.remove(circuit) diff --git a/Tribler/community/anontunnel/Socks5/session.py b/Tribler/community/anontunnel/Socks5/session.py new file mode 100644 index 0000000000..a729a6a987 --- /dev/null +++ b/Tribler/community/anontunnel/Socks5/session.py @@ -0,0 +1,174 @@ +import logging +from Tribler.community.anontunnel.Socks5 import conversion +from Tribler.community.anontunnel.Socks5.connection import \ + Socks5ConnectionObserver +from Tribler.community.anontunnel.events import TunnelObserver, \ + CircuitPoolObserver +from Tribler.community.anontunnel.globals import CIRCUIT_STATE_READY +from Tribler.community.anontunnel.routing import NotEnoughCircuitsException +from Tribler.community.anontunnel.selectionstrategies import RoundRobin + + +class Socks5Session(TunnelObserver, Socks5ConnectionObserver): + """ + A SOCKS5 session, composed by a TCP connection, an UDP proxy port and a + list of circuits where data can be tunneled over + + @param Socks5Connection connection: the Socks5Connection + @param RawServer raw_server: The raw server, used to create and listen on + UDP-sockets + @param CircuitPool circuit_pool: the circuit pool + """ + def __init__(self, raw_server, connection, server, circuit_pool, min_circuits=4): + TunnelObserver.__init__(self) + self.raw_server = raw_server + self._logger = logging.getLogger(__name__) + self.connection = connection + self.connection.observers.append(self) + self.circuit_pool = circuit_pool + + self.min_circuits = min_circuits + + self.server = server + + self.destinations = {} + ''' :type: dict[(str, int), Circuit] ''' + + self.selection_strategy = RoundRobin() + + self.remote_udp_address = None + self._udp_socket = None + + def on_udp_associate_request(self, connection, request): + """ + @param Socks5Connection connection: the connection + @param request: + @return: + """ + if not self.circuit_pool.available_circuits: + try: + for _ in range(self.min_circuits): + circuit = self.server.circuit_pool.allocate() + # Move from main pool to session pool + self.server.circuit_pool.remove_circuit(circuit) + self.circuit_pool.fill(circuit) + except NotEnoughCircuitsException: + self.close_session("not enough circuits") + connection.deny_request(request) + return + + self._udp_socket = self.raw_server.create_udpsocket(0, "0.0.0.0") + self.raw_server.start_listening_udp(self._udp_socket, self) + connection.accept_udp_associate(request, self._udp_socket) + + def close_session(self, reason='unspecified'): + """ + Closes the session and the linked TCP connection + @param str reason: the reason why the session should be closed + """ + self._logger.error("Closing session, reason = {0}".format(reason)) + self.connection.close() + + def on_break_circuit(self, broken_circuit): + """ + When a circuit breaks and it affects our operation we should re-add the + peers when a new circuit is available to reinitiate the 3-way handshake + + @param Circuit broken_circuit: the circuit that has been broken + @return: + """ + affected_destinations = set(destination + for destination, tunnel_circuit + in self.destinations.iteritems() + if tunnel_circuit == broken_circuit) + + # We are not affected by the circuit that has been broken, continue + # without any changes + if not affected_destinations: + return + + from Tribler.Core.Libtorrent.LibtorrentMgr import LibtorrentMgr + mgr = LibtorrentMgr.getInstance() + anon_session = mgr.ltsession_anon + ''' :type : libtorrent.session ''' + + affected_torrents = dict((download, affected_destinations.intersection(peer.ip for peer in download.handle.get_peer_info())) + for infohash, (download, session) + in mgr.torrents.items() + if session == anon_session + ) + ''' :type : dict[LibtorrentDownloadImpl, set[(str, int)] ''' + + def _peer_add(): + for destination in affected_destinations: + if destination in self.destinations: + del self.destinations[destination] + self._logger.error("Deleting peer %s from destination list", destination) + + for torrent, peers in affected_torrents.items(): + for peer in peers: + self._logger.error("Readding peer %s to torrent %s", peer, torrent.tdef.get_infohash().encode("HEX")) + torrent.add_peer(peer) + + # Observer that waits for a new circuit before re-adding the peers + # is used only when there are no other circuits left + class _peer_adder(CircuitPoolObserver): + def on_circuit_added(self, pool, circuit): + _peer_add() + pool.observers.remove(self) + + # If there are any other circuits we will just map them to any + # new circuit + if [circuit for circuit in self.circuit_pool.available_circuits if circuit != broken_circuit]: + _peer_add() + else: + self._logger.warning("Waiting for new circuits before re-adding peers") + self.circuit_pool.observers.append(_peer_adder()) + + def _select(self, destination): + if not destination in self.destinations: + selected_circuit = self.selection_strategy.select(self.circuit_pool.available_circuits) + self.destinations[destination] = selected_circuit + + self._logger.warning("SELECT circuit {0} for {1}".format( + self.destinations[destination].circuit_id, + destination + )) + + return self.destinations[destination] + + def data_came_in(self, packets): + for source_address, packet in packets: + if self.remote_udp_address and \ + self.remote_udp_address != source_address: + self.close_session('invalid source_address!') + return + + self.remote_udp_address = source_address + + request = conversion.decode_udp_packet(packet) + + circuit = self._select(request.destination) + + if circuit.state != CIRCUIT_STATE_READY: + self._logger.error("Circuit is not ready, dropping %d bytes to %s", len(request.payload), request.destination) + else: + circuit.tunnel_data(request.destination, request.payload) + + def on_incoming_from_tunnel(self, community, circuit, origin, data): + if circuit not in self.circuit_pool.circuits: + return + + if not self.remote_udp_address: + self._logger.warning("No return address yet, dropping packet!") + return + + self.destinations[origin] = circuit + + socks5_udp = conversion.encode_udp_packet( + 0, 0, conversion.ADDRESS_TYPE_IPV4, origin[0], origin[1], data) + + bytes_written = self._udp_socket.sendto(socks5_udp, + self.remote_udp_address) + if bytes_written < len(socks5_udp): + self._logger.error("Packet drop on return!") diff --git a/Tribler/community/anontunnel/__init__.py b/Tribler/community/anontunnel/__init__.py new file mode 100644 index 0000000000..a0ee4ead29 --- /dev/null +++ b/Tribler/community/anontunnel/__init__.py @@ -0,0 +1,8 @@ +""" +The AnonTunnel community package + +Defines a ProxyCommunity which discovers other proxies and offers an API to +create and reserve circuits. A basic SOCKS5 server is included which reserves +circuits for its client connections and tunnels UDP packets over them back and +forth +""" diff --git a/Tribler/community/anontunnel/cache.py b/Tribler/community/anontunnel/cache.py new file mode 100644 index 0000000000..187a22a0ba --- /dev/null +++ b/Tribler/community/anontunnel/cache.py @@ -0,0 +1,123 @@ +""" +Cache module for the ProxyCommunity. + +Keeps track of outstanding PING and EXTEND requests and of candidates used in +CREATE and CREATED requests. + +""" + +import logging +from Tribler.dispersy.requestcache import NumberCache + +__author__ = 'chris' + + +class CircuitRequestCache(NumberCache): + PREFIX = u"anon-circuit" + + """ + Circuit request cache is used to keep track of circuit building. It + succeeds when the circuit reaches full length. + + On timeout the circuit is removed + + @param ProxyCommunity community: the instance of the ProxyCommunity + @param int force_number: + """ + + def __init__(self, community, circuit): + number = self.create_identifier(circuit) + + NumberCache.__init__(self, community.request_cache, self.PREFIX, number) + self._logger = logging.getLogger(__name__) + self.community = community + self.circuit = circuit + ''' :type : Tribler.community.anontunnel.community.Circuit ''' + + @property + def timeout_delay(self): + return 10.0 + + def on_success(self): + """ + Mark the Request as successful, cancelling the timeout + """ + + from Tribler.community.anontunnel.globals \ + import CIRCUIT_STATE_READY + + if self.circuit.state == CIRCUIT_STATE_READY: + self._logger.info("Circuit %d is ready", self.number) + self.community.dispersy.callback.register( + self.community.request_cache.pop, args=(self.prefix, self.number,)) + + def on_timeout(self): + from Tribler.community.anontunnel.globals \ + import CIRCUIT_STATE_READY + + if not self.circuit.state == CIRCUIT_STATE_READY: + reason = 'timeout on CircuitRequestCache, state = %s' % \ + self.circuit.state + self.community.remove_circuit(self.number, reason) + + @classmethod + def create_identifier(cls, circuit): + return circuit.circuit_id + + +class PingRequestCache(NumberCache): + PREFIX = u"anon-ping" + + """ + Request cache that is used to time-out PING messages + + @param ProxyCommunity community: instance of the ProxyCommunity + @param force_number: + """ + def __init__(self, community, circuit): + NumberCache.__init__(self, community.request_cache, self.PREFIX, circuit.circuit_id) + + self.circuit = circuit + self.community = community + + @property + def timeout_delay(self): + return 10.0 + + def on_pong(self, message): + self.community.circuits[self.number].beat_heart() + self.community.dispersy.callback.register( + self.community.request_cache.pop, args=(self.PREFIX, self.number,)) + + def on_timeout(self): + self.community.remove_circuit(self.number, 'RequestCache') + + +class CreatedRequestCache(NumberCache): + PREFIX = u"anon-created" + + def __init__(self, community, circuit_id, candidate, candidates): + """ + + @param int circuit_id: the circuit's id + @param WalkCandidate candidate: the candidate from which we got the CREATE + @param dict[str, WalkCandidate] candidates: we sent to the candidate to pick from + """ + + number = self.create_identifier(circuit_id, candidate) + super(CreatedRequestCache, self).__init__(community.request_cache, self.PREFIX, number) + + self.circuit_id = circuit_id + self.candidate = candidate + self.candidates = dict(candidates) + + @property + def timeout_delay(self): + return 10.0 + + def on_timeout(self): + pass + + @classmethod + def create_identifier(cls, circuit_id, candidate): + return circuit_id \ No newline at end of file diff --git a/Tribler/community/anontunnel/community.py b/Tribler/community/anontunnel/community.py new file mode 100644 index 0000000000..94878b436f --- /dev/null +++ b/Tribler/community/anontunnel/community.py @@ -0,0 +1,995 @@ +""" +AnonTunnel community module +""" + +# Python imports +import threading +import random +import time +from collections import defaultdict + +# Tribler and Dispersy imports +from Tribler.community.anontunnel.cache import CircuitRequestCache, \ + PingRequestCache, CreatedRequestCache +from Tribler.community.anontunnel.routing import Circuit, Hop, RelayRoute +from Tribler.community.anontunnel.tests.test_libtorrent import LibtorrentTest +from Tribler.dispersy.authentication import MemberAuthentication, \ + NoAuthentication +from Tribler.dispersy.conversion import DefaultConversion +from Tribler.dispersy.destination import CommunityDestination +from Tribler.dispersy.distribution import LastSyncDistribution +from Tribler.dispersy.message import Message, DelayMessageByProof +from Tribler.dispersy.resolution import PublicResolution +from Tribler.dispersy.candidate import Candidate, WalkCandidate +from Tribler.dispersy.community import Community + +# AnonTunnel imports +from Tribler.community.anontunnel import crypto +from Tribler.community.anontunnel import extendstrategies +from Tribler.community.anontunnel import selectionstrategies +from Tribler.community.anontunnel import lengthstrategies +from Tribler.community.anontunnel.payload import StatsPayload, CreateMessage, \ + CreatedMessage, ExtendedMessage, \ + PongMessage, PingMessage, DataMessage +from Tribler.community.anontunnel.conversion import CustomProxyConversion, \ + ProxyConversion +from Tribler.community.anontunnel.globals import MESSAGE_EXTEND, \ + MESSAGE_CREATE, MESSAGE_CREATED, MESSAGE_DATA, MESSAGE_EXTENDED, \ + MESSAGE_PING, MESSAGE_PONG, MESSAGE_TYPE_STRING, \ + CIRCUIT_STATE_READY, CIRCUIT_STATE_EXTENDING, \ + ORIGINATOR, PING_INTERVAL, ENDPOINT + +__author__ = 'chris' + +import logging + + +class ProxySettings: + """ + Data structure containing settings, including some defaults, + for the ProxyCommunity + """ + + def __init__(self): + length = random.randint(3, 3) + + self.max_circuits = 1 + self.delay = 300 + + self.extend_strategy = extendstrategies.NeighbourSubset + self.select_strategy = selectionstrategies.RoundRobin() + self.length_strategy = lengthstrategies.ConstantCircuitLength(length) + self.crypto = crypto.DefaultCrypto() + + +class ProxyCommunity(Community): + """ + The dispersy community which discovers other proxies on the internet and + creates TOR-like circuits together with them + + @type dispersy: Tribler.dispersy.dispersy.Dispersy + @type master_member: Tribler.dispersy.member.Member + @type settings: ProxySettings or unknown + @type tribler_session: Tribler.Core.Session.Session + """ + + def __init__(self, dispersy, master_member, my_member, settings=None, + tribler_session=None): + super(ProxyCommunity, self).__init__(dispersy, master_member, my_member) + + self.lock = threading.RLock() + self._logger = logging.getLogger(__name__) + + self.settings = settings if settings else ProxySettings() + # Custom conversion + self.__packet_prefix = "fffffffe".decode("HEX") + + self.observers = [] + ''' :type : list of TunnelObserver''' + + self.proxy_conversion = CustomProxyConversion() + self._message_handlers = defaultdict(lambda: lambda *args: None) + ''' :type : dict[ + str, + ( + int, Candidate, + StatsPayload|Tribler.community.anontunnel.payload.BaseMessage + ) -> bool] + ''' + + self.circuits = {} + """ :type : dict[int, Circuit] """ + self.directions = {} + + self.relay_from_to = {} + """ :type : dict[((str, int),int),RelayRoute] """ + + self.waiting_for = {} + """ :type : dict[((str, int),int), bool] """ + + # Map destination address to the circuit to be used + self.destination_circuit = {} + ''' @type: dict[(str, int), int] ''' + + self.circuit_pools = [] + ''' :type : list[CircuitPool] ''' + + # Attach message handlers + self._initiate_message_handlers() + + # add self to crypto + self.settings.crypto.set_proxy(self) + + # Enable global counters + from Tribler.community.anontunnel.stats import StatsCollector + + self.global_stats = StatsCollector(self, "global") + self.global_stats.start() + + # Listen to prefix endpoint + try: + dispersy.endpoint.listen_to(self.__packet_prefix, self.__handle_packet) + except AttributeError: + self._logger.error("Cannot listen to our prefix, are you sure that you are using the DispersyBypassEndpoint?") + + dispersy.callback.register(self.__ping_circuits) + + if tribler_session: + from Tribler.Core.CacheDB.Notifier import Notifier + self.tribler_session = tribler_session + self.notifier = Notifier.getInstance() + delay = self.settings.delay if self.settings.delay is not None else 300 + tribler_session.lm.rawserver.add_task(lambda: LibtorrentTest(self, self.tribler_session, delay)) + else: + self.notifier = None + + def __loop_discover(): + while True: + try: + self.__discover() + finally: + yield 5.0 + + self.dispersy.callback.register(__loop_discover) + + @classmethod + def get_master_members(cls, dispersy): + # generated: Wed Sep 18 22:47:22 2013 + # curve: high <<< NID_sect571r1 >>> + # len: 571 bits ~ 144 bytes signature + # pub: 170 3081a7301006072a8648ce3d020106052b81040027038192000404608 + # 29f9bb72f0cb094904aa6f885ff70e1e98651e81119b1e7b42402f3c5cfa183d8d + # 96738c40ffd909a70020488e3b59b67de57bb1ac5dec351d172fe692555898ac94 + # 4b68c730590f850ab931c5732d5a9d573a7fe1f9dc8a9201bc3cb63ab182c9e485 + # d08ff4ac294f09e16d3925930946f87e91ef9c40bbb4189f9c5af6696f57eec3b8 + # f2f77e7ab56fd8d6d63 + # pub-sha1 089515d307ed31a25eec2c54667ddcd2d402c041 + #-----BEGIN PUBLIC KEY----- + # MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQEYIKfm7cvDLCUkEqm+IX/cOHphlHo + # ERmx57QkAvPFz6GD2NlnOMQP/ZCacAIEiOO1m2feV7saxd7DUdFy/mklVYmKyUS2 + # jHMFkPhQq5McVzLVqdVzp/4fncipIBvDy2OrGCyeSF0I/0rClPCeFtOSWTCUb4fp + # HvnEC7tBifnFr2aW9X7sO48vd+erVv2NbWM= + #-----END PUBLIC KEY----- + master_key = "3081a7301006072a8648ce3d020106052b810400270381920004" \ + "0460829f9bb72f0cb094904aa6f885ff70e1e98651e81119b1e7" \ + "b42402f3c5cfa183d8d96738c40ffd909a70020488e3b59b67de" \ + "57bb1ac5dec351d172fe692555898ac944b68c730590f850ab93" \ + "1c5732d5a9d573a7fe1f9dc8a9201bc3cb63ab182c9e485d08ff" \ + "4ac294f09e16d3925930946f87e91ef9c40bbb4189f9c5af6696" \ + "f57eec3b8f2f77e7ab56fd8d6d63".decode("HEX") + + master = dispersy.get_member(public_key=master_key) + return [master] + + @property + def crypto(self): + """ + @rtype: ElgamalCrypto + """ + return self.dispersy.crypto + + @property + def packet_crypto(self): + return self.settings.crypto + + def __discover(self): + circuits_needed = lambda: max( + sum(pool.lacking for pool in self.circuit_pools), + self.settings.max_circuits - len(self.circuits) + ) + + with self.lock: + for i in range(0, circuits_needed()): + self._logger.debug("Need %d new circuits!", circuits_needed()) + goal_hops = self.settings.length_strategy.circuit_length() + + if goal_hops == 0: + circuit_id = self._generate_circuit_id() + self.circuits[circuit_id] = Circuit(circuit_id, self) + + first_pool = next((pool for pool in self.circuit_pools if pool.lacking), None) + if first_pool: + first_pool.fill(self.circuits[circuit_id]) + + else: + circuit_candidates = set([c.candidate for c in self.circuits.values()]) + candidate = next((c for c in self.dispersy_yield_verified_candidates() + if c not in circuit_candidates), None) + + if candidate is None: + return + else: + try: + self.create_circuit(candidate, goal_hops) + except: + self._logger.exception("Error creating circuit while running __discover") + + def unload_community(self): + """ + Called by dispersy when the ProxyCommunity is being unloaded + @return: + """ + for observer in self.observers: + observer.on_unload() + Community.unload_community(self) + + def _initiate_message_handlers(self): + self._message_handlers[MESSAGE_CREATE] = self.on_create + self._message_handlers[MESSAGE_CREATED] = self.on_created + self._message_handlers[MESSAGE_DATA] = self.on_data + self._message_handlers[MESSAGE_EXTEND] = self.on_extend + self._message_handlers[MESSAGE_EXTENDED] = self.on_extended + self._message_handlers[MESSAGE_PING] = self.on_ping + self._message_handlers[MESSAGE_PONG] = self.on_pong + + def initiate_conversions(self): + """ + Called by dispersy when we need to return our message Conversions + @rtype: list[Tribler.dispersy.conversion.Conversion] + """ + return [DefaultConversion(self), ProxyConversion(self)] + + def initiate_meta_messages(self): + """ + Called by dispersy when we need to define the messages we would like + to use in the community + @rtype: list[Message] + """ + return super(ProxyCommunity, self).initiate_meta_messages() + [ + Message( + self, + u"stats", + MemberAuthentication(), + PublicResolution(), + LastSyncDistribution(synchronization_direction=u"DESC", + priority=128, history_size=1), + CommunityDestination(node_count=10), + StatsPayload(), + self._generic_timeline_check, + self._on_stats + )] + + def _generic_timeline_check(self, messages): + meta = messages[0].meta + if isinstance(meta.authentication, NoAuthentication): + # we can not timeline.check this message because it uses the NoAuthentication policy + for message in messages: + yield message + + else: + for message in messages: + allowed, proofs = self.timeline.check(message) + if allowed: + yield message + else: + yield DelayMessageByProof(message) + + def _on_stats(self, messages): + for observer in self.observers: + for message in messages: + observer.on_tunnel_stats(self, message.authentication.member, message.candidate, message.payload.stats) + + def send_stats(self, stats): + """ + Send a stats message to the community + @param dict stats: the statistics dictionary to share + """ + + def __send_stats(): + meta = self.get_meta_message(u"stats") + record = meta.impl(authentication=(self._my_member,), + distribution=(self.claim_global_time(),), + payload=(stats,)) + + self._logger.warning("Sending stats") + self.dispersy.store_update_forward([record], True, False, True) + + self.dispersy.callback.register(__send_stats) + + def __handle_incoming(self, circuit_id, am_originator, candidate, data): + # Let packet_crypto handle decrypting the incoming packet + data = self.packet_crypto.handle_incoming_packet(candidate, circuit_id, data) + + if not data: + self._logger.error("Circuit ID {0} doesn't talk crypto language, dropping packet".format(circuit_id)) + return False + + # Try to parse the packet + _, payload = self.proxy_conversion.decode(data) + + packet_type = self.proxy_conversion.get_type(data) + str_type = MESSAGE_TYPE_STRING.get(packet_type) + + # Let packet_crypto handle decrypting packet contents + payload = self.packet_crypto.handle_incoming_packet_content(candidate, circuit_id, payload, packet_type) + + # If un-decrypt-able, drop packet + if not payload: + self._logger.warning("IGNORED %s from %s:%d over circuit %d", + str_type, candidate.sock_addr[0], + candidate.sock_addr[1], circuit_id) + return False + + if am_originator: + self.circuits[circuit_id].beat_heart() + + handler = self._message_handlers[packet_type] + result = handler(circuit_id, candidate, payload) + + if result: + self.__dict_inc(self.dispersy.statistics.success, str_type) + else: + self.__dict_inc(self.dispersy.statistics.success, + str_type + '-ignored') + self._logger.debug("Prev message was IGNORED") + + return True + + def __relay(self, circuit_id, data, relay_key, sock_addr): + # First, relay packet if we know whom to forward message to for + # this circuit. This happens only when the circuit is already + # established with both parent and child and if the node is not + # waiting for a CREATED message from the child + + direction = self.directions[relay_key] + next_relay = self.relay_from_to[relay_key] + + # let packet_crypto handle en-/decrypting relay packet + data = self.packet_crypto.handle_relay_packet(direction, sock_addr, circuit_id, data) + + if not data: + return False + + this_relay_key = (next_relay.sock_addr, next_relay.circuit_id) + + if this_relay_key in self.relay_from_to: + this_relay = self.relay_from_to[this_relay_key] + this_relay.last_incoming = time.time() + + # TODO: check whether direction is set correctly here! + for observer in self.observers: + observer.on_relay(this_relay_key, relay_key, direction, data) + + packet_type = self.proxy_conversion.get_type(data) + + str_type = MESSAGE_TYPE_STRING.get( + packet_type, 'unknown-type-%d' % ord(packet_type) + ) + + self.send_packet( + destination=Candidate(next_relay.sock_addr, False), + circuit_id=next_relay.circuit_id, + message_type=packet_type, + packet=data, + relayed=True + ) + + self.__dict_inc(self.dispersy.statistics.success, + str_type + '-relayed') + + return True + + def __handle_packet(self, sock_addr, orig_packet): + """ + @param (str, int) sock_addr: socket address in tuple format + @param orig_packet: + @return: + """ + packet = orig_packet[len(self.__packet_prefix):] + circuit_id, data = self.proxy_conversion.get_circuit_and_data(packet) + relay_key = (sock_addr, circuit_id) + + is_relay = circuit_id > 0 and relay_key in self.relay_from_to \ + and not relay_key in self.waiting_for + is_originator = not is_relay and circuit_id in self.circuits + is_initial = not is_relay and not is_originator + + result = False + + try: + if is_relay: + result = self.__relay(circuit_id, data, relay_key, sock_addr) + else: + candidate = self._candidates.get(sock_addr) + if isinstance(candidate, WalkCandidate) and candidate.get_members(): + result = self.__handle_incoming(circuit_id, is_originator, candidate, data) + else: + candidate = None + self._logger.error("Unknown candidate at %s, drop!", sock_addr) + result = False + except: + result = False + self._logger.exception( + "Incoming from %s on %d message error." + "INITIAL=%s, ORIGINATOR=%s, RELAY=%s", + sock_addr, circuit_id, is_initial, is_originator, is_relay) + + if not result: + if is_relay: + self.remove_relay(relay_key, "error on incoming packet!") + elif is_originator: + self.remove_circuit(circuit_id, "error on incoming packet!") + + def create_circuit(self, first_hop, goal_hops, extend_strategy=None, + deferred=None): + """ Create a new circuit, with one initial hop + + @param WalkCandidate first_hop: The first hop of our circuit, needs to + be a candidate. + @param int goal_hops: The number of hops the circuit should reach + @param T <= extendstrategies.ExtendStrategy extend_strategy: + The extend strategy used + + @rtype: Tribler.community.anontunnel.routing.Circuit + """ + + if not (goal_hops > 0): + raise ValueError("We can only create circuits with more than 0 hops using create_circuit()!") + + with self.lock: + circuit_id = self._generate_circuit_id(first_hop.sock_addr) + circuit = Circuit( + circuit_id=circuit_id, + goal_hops=goal_hops, + candidate=first_hop, + proxy=self) + + self.dispersy.callback.call(lambda: self._request_cache.add(CircuitRequestCache(self, circuit))) + + if extend_strategy: + circuit.extend_strategy = extend_strategy + else: + circuit.extend_strategy = self.settings.extend_strategy( + self, circuit) + + hop_public_key = iter(first_hop.get_members()).next()._ec + circuit.unverified_hop = Hop(hop_public_key) + circuit.unverified_hop.address = first_hop.sock_addr + + self._logger.warning("Creating circuit %d of %d hops. Fist hop: %s:%d", + circuit_id, circuit.goal_hops, + first_hop.sock_addr[0], + first_hop.sock_addr[1] + ) + + self.circuits[circuit_id] = circuit + self.waiting_for[(first_hop.sock_addr, circuit_id)] = True + + self.send_message(first_hop, circuit_id, MESSAGE_CREATE, + CreateMessage("", self.my_member.public_key)) + + return circuit + + def remove_circuit(self, circuit_id, additional_info=''): + """ + Removes a circuit from our pool, destroying it + @param int circuit_id: the id of the circuit to destroy + @param str additional_info: optional reason, useful for logging + @return: whether the removal was successful + """ + assert isinstance(circuit_id, (long, int)), type(circuit_id) + + if circuit_id in self.circuits: + self._logger.error("Breaking circuit %d " + additional_info, + circuit_id) + circuit = self.circuits[circuit_id] + + circuit.destroy() + del self.circuits[circuit_id] + for observer in self.observers: + observer.on_break_circuit(circuit) + + return True + return False + + def remove_relay(self, relay_key, additional_info=''): + """ + Removes a relay from our routing table, will drop any incoming packets + and eventually cause a timeout at the originator + + @param ((str, int) int) relay_key: the key of the relay to remove + @param str additional_info: optional reason, useful for logging + @return: whether the removal was successful + """ + if relay_key in self.relay_from_to: + self._logger.error( + ("Breaking relay %s:%d %d " + additional_info) % ( + relay_key[0][0], relay_key[0][1], relay_key[1])) + + # Only remove one side of the relay, this isn't as pretty but + # both sides have separate incoming timer, hence + # after removing one side the other will follow. + del self.relay_from_to[relay_key] + + for observer in self.observers: + observer.on_break_relay(relay_key) + + return True + return False + + def on_create(self, circuit_id, candidate, message): + """ + Handle incoming CREATE message, acknowledge the CREATE request with a + CREATED reply + + @param int circuit_id: The circuit's identifier + @param Candidate candidate: The candidate we got a CREATE message from + @param CreateMessage message: The message's payload + """ + relay_key = (candidate.sock_addr, circuit_id) + self.directions[relay_key] = ENDPOINT + self._logger.info('We joined circuit %d with neighbour %s' + , circuit_id, candidate) + + candidates = {} + ''' :type : dict[str, WalkCandidate] ''' + + for _ in range(1, 5): + candidate_temp = next(self.dispersy_yield_verified_candidates(), None) + " :type: WalkCandidate" + + if not candidate_temp: + break + + candidates[iter(candidate_temp.get_members()).next().public_key] = candidate_temp + + candidate_list = [next(iter(c.get_members())).public_key for c in candidates.itervalues()] + + self.create_created_cache(circuit_id, candidate, candidates) + + if self.notifier: + from Tribler.Core.simpledefs import NTFY_ANONTUNNEL, NTFY_JOINED + + self.notifier.notify(NTFY_ANONTUNNEL, NTFY_JOINED, + candidate.sock_addr, circuit_id) + + return self.send_message( + destination=candidate, + circuit_id=circuit_id, + message_type=MESSAGE_CREATED, + message=CreatedMessage(candidate_list, reply_to=message) + ) + + def on_created(self, circuit_id, candidate, message): + """ Handle incoming CREATED messages relay them backwards towards + the originator if necessary + + @param int circuit_id: The circuit's id we got a CREATED message on + @param Candidate candidate: The candidate we got the message from + @param CreatedMessage message: The message we received + + @return: whether the message could be handled correctly + + """ + relay_key = (candidate.sock_addr, circuit_id) + + del self.waiting_for[relay_key] + self.directions[relay_key] = ORIGINATOR + if relay_key in self.relay_from_to: + self._logger.debug("Got CREATED message, " + "forward as EXTENDED to origin.") + extended_message = ExtendedMessage(message.key, + message.candidate_list) + forwarding_relay = self.relay_from_to[relay_key] + + candidate = Candidate(forwarding_relay.sock_addr, False) + return self.send_message(candidate, forwarding_relay.circuit_id, + MESSAGE_EXTENDED, extended_message) + + # This is ours! + if circuit_id in self.circuits: + circuit = self.circuits[circuit_id] + return self._ours_on_created_extended(circuit, message) + + return False + + def _ours_on_created_extended(self, circuit, message): + """ + @param ExtendedMessage | CreatedMessage message: the CREATED or + EXTENDED message we received + """ + + request = self.dispersy.callback.call( + self.request_cache.get, + args=( + CircuitRequestCache.PREFIX, + CircuitRequestCache.create_identifier(circuit),)) + + candidate_list = message.candidate_list + + circuit.add_hop(circuit.unverified_hop) + circuit.unverified_hop = None + + if self.my_member.public_key in candidate_list: + candidate_list.remove(self.my_member.public_key) + + for hop in circuit.hops: + if hop.public_key in candidate_list: + candidate_list.remove(hop.public_key) + + if circuit.state == CIRCUIT_STATE_EXTENDING: + try: + if not circuit.extend_strategy.extend(candidate_list): + raise ValueError("Extend strategy returned False") + except BaseException as e: + self.remove_circuit(circuit.circuit_id, e.message) + return False + + elif circuit.state == CIRCUIT_STATE_READY: + request.on_success() + + first_pool = next((pool for pool in self.circuit_pools if pool.lacking), None) + if first_pool: + first_pool.fill(circuit) + else: + return False + + if self.notifier: + from Tribler.Core.simpledefs import NTFY_ANONTUNNEL, \ + NTFY_CREATED, NTFY_EXTENDED + + if len(circuit.hops) == 1: + self.notifier.notify( + NTFY_ANONTUNNEL, NTFY_CREATED, circuit) + else: + self.notifier.notify( + NTFY_ANONTUNNEL, NTFY_EXTENDED, circuit) + + return True + + def on_data(self, circuit_id, candidate, message): + """ + Handles incoming DATA message + + Determines whether the data comes from the outside world (origin set) + or whether the data came from the origin (destination set) + + If the data comes from the outside world the on_incoming_from_tunnel + method is called on the observers and the circuit is marked as active + + When the data comes from the origin we need to EXIT to the outside + world. This is left to the observers as well, by calling the + on_exiting_from_tunnel method. + + @param int circuit_id: the circuit's id we received the DATA message on + @param Candidate|None candidate: the messenger of the packet + @param DataMessage message: the message's content + + @return: whether the message could be handled correctly + """ + + # If its our circuit, the messenger is the candidate assigned to that + # circuit and the DATA's destination is set to the zero-address then + # the packet is from the outside world and addressed to us from + if circuit_id in self.circuits and message.origin \ + and candidate == self.circuits[circuit_id].candidate: + + self.circuits[circuit_id].beat_heart() + for observer in self.observers: + observer.on_incoming_from_tunnel(self, self.circuits[circuit_id], message.origin, message.data) + + return True + # It is not our circuit so we got it from a relay, we need to EXIT it! + if message.destination != ('0.0.0.0', 0): + + for observer in self.observers: + observer.on_exiting_from_tunnel(circuit_id, candidate, message.destination, message.data) + + return True + return False + + def on_extend(self, circuit_id, candidate, message): + """ + Upon reception of a EXTEND message the message is forwarded over the + Circuit if possible. At the end of the circuit a CREATE request is + send to the Proxy to extend the circuit with. It's CREATED reply will + eventually be received and propagated back along the Circuit. + + @param int circuit_id: the circuit's id we got the EXTEND message on + @param Candidate candidate: the relay which sent us the EXTEND + @param ExtendMessage message: the message's content + + @return: whether the message could be handled correctly + """ + + if message.extend_with: + cache = self.get_created_cache(circuit_id, candidate) + extend_candidate = cache.candidates[message.extend_with] + + self._logger.warning( + "ON_EXTEND send CREATE for circuit (%s, %d) to %s:%d!", + candidate, + circuit_id, + extend_candidate.sock_addr[0], + extend_candidate.sock_addr[1]) + else: + self._logger.error("Cancelling EXTEND, no candidate!") + return + + relay_key = (candidate.sock_addr, circuit_id) + if relay_key in self.relay_from_to: + current_relay = self.relay_from_to[relay_key] + assert not current_relay.online, \ + "shouldn't be called whenever relay is online, " \ + "the extend message should have been forwarded" + + # We will just forget the attempt and try again, possible with + # another candidate + old_to_key = current_relay.sock_addr, current_relay.circuit_id + del self.relay_from_to[old_to_key] + del self.relay_from_to[relay_key] + + new_circuit_id = self._generate_circuit_id(extend_candidate.sock_addr) + to_key = (extend_candidate.sock_addr, new_circuit_id) + + self.waiting_for[to_key] = True + self.relay_from_to[to_key] = RelayRoute(circuit_id, + candidate.sock_addr) + self.relay_from_to[relay_key] = RelayRoute(new_circuit_id, + extend_candidate.sock_addr) + + key = message.key + + self.directions[to_key] = ORIGINATOR + self.directions[relay_key] = ENDPOINT + + self._logger.info("Extending circuit, got candidate with IP %s:%d from cache", *extend_candidate.sock_addr) + return self.send_message(extend_candidate, new_circuit_id, + MESSAGE_CREATE, CreateMessage(key, self.my_member.public_key)) + + def on_extended(self, circuit_id, candidate, message): + """ + A circuit has been extended, forward the acknowledgment back to the + origin of the EXTEND. If we are the origin update our records. + + @param int circuit_id: the circuit's id we got the EXTENDED message on + @param Candidate candidate: the relay which sent us the EXTENDED + @param ExtendedMessage message: the message's content + + @return: whether the message could be handled correctly + """ + + circuit = self.circuits[circuit_id] + return self._ours_on_created_extended(circuit, message) + + def create_ping(self, candidate, circuit): + """ + Creates, sends and keeps track of a PING message to given candidate on + the specified circuit. + + @param Candidate candidate: the candidate to which we want to sent a + ping + @param Circuit circuit: the circuit id to sent the ping over + """ + + circuit_id = circuit.circuit_id + + def __do_add(): + if not self._request_cache.has(PingRequestCache.PREFIX, circuit_id): + cache = PingRequestCache(self, circuit) + self._request_cache.add(cache) + + self._dispersy.callback.register(__do_add) + + self._logger.debug("SEND PING TO CIRCUIT {0}".format(circuit_id)) + self.send_message(candidate, circuit_id, MESSAGE_PING, PingMessage()) + + def on_ping(self, circuit_id, candidate, message): + """ + Upon reception of a PING message we respond with a PONG message + + @param int circuit_id: the circuit's id we got the PING from + @param Candidate candidate: the relay we got the PING from + @param PingMessage message: the message's content + + @return: whether the message could be handled correctly + """ + self._logger.debug("GOT PING FROM CIRCUIT {0}".format(circuit_id)) + return self.send_message( + destination=candidate, + circuit_id=circuit_id, + message_type=MESSAGE_PONG, + message=PongMessage()) + + def on_pong(self, circuit_id, candidate, message): + """ + When we receive a PONG message on our circuit we can be sure that the + circuit is alive and well. + + @param int circuit_id: the circuit's id we got the PONG message on + @param Candidate candidate: the relay which sent us the PONG + @param PongMessage message: the message's content + + @return: whether the message could be handled correctly + """ + request = self.dispersy.callback.call( + self._request_cache.get, + args=(PingRequestCache.PREFIX, circuit_id,)) + + if request: + request.on_pong(message) + return True + return False + + def _generate_circuit_id(self, neighbour=None): + circuit_id = random.randint(1, 255000) + + # prevent collisions + while circuit_id in self.circuits or \ + (neighbour and (neighbour, circuit_id) in self.relay_from_to): + circuit_id = random.randint(1, 255000) + + return circuit_id + + def send_message(self, destination, circuit_id, message_type, message): + """ + Send a message to a specified destination and circuit + @param Candidate destination: the relay's candidate + @param int circuit_id: the circuit to sent over + @param str message_type: the messages type, used to determine how to + serialize it + @param BaseMessage message: the messages content in object form + @return: + """ + message = self.packet_crypto.handle_outgoing_packet_content(destination, circuit_id, message, message_type) + + if message is None: + return False + + content = self.proxy_conversion.encode(message_type, message) + content = self.packet_crypto.handle_outgoing_packet(destination, circuit_id, message_type, message, content) + + if content is None: + return False + + return self.send_packet(destination, circuit_id, message_type, content) + + def send_packet(self, destination, circuit_id, message_type, packet, + relayed=False): + """ + Sends a packet to a relay over the specified circuit + @param Candidate destination: the relay's candidate structure + @param int circuit_id: the circuit to sent over + @param str message_type: the messages type, for logging purposes + @param str packet: the messages content in serialised form + @param bool relayed: whether this is a relay packet or not + @return: whether the send was successful + """ + assert isinstance(destination, Candidate), type(destination) + assert isinstance(packet, str), type(packet) + + packet = self.proxy_conversion.add_circuit(packet, circuit_id) + + str_type = MESSAGE_TYPE_STRING.get( + message_type, "unknown-type-" + str(ord(message_type))) + + self.__dict_inc(self.dispersy.statistics.outgoing, + str_type + ('-relayed' if relayed else ''), 1) + + # we need to make sure that this endpoint is thread safe + return self.dispersy.endpoint.send_packet(destination, self.__packet_prefix + packet) + + def __dict_inc(self, statistics_dict, key, inc=1): + key = u"anontunnel-" + key + self._dispersy.statistics.dict_inc(statistics_dict, key, inc) + + @property + def active_circuits(self): + """ + Dict of active circuits, a circuit is active when its state is + CIRCUIT_STATE_READY + @rtype: dict[int, Tribler.community.anontunnel.routing.Circuit] + """ + return dict((circuit_id, circuit) + for circuit_id, circuit in self.circuits.items() + if circuit.state == CIRCUIT_STATE_READY) + + def __ping_circuits(self): + while True: + try: + to_be_removed = [ + self.remove_relay(relay_key, 'no activity') + for relay_key, relay in self.relay_from_to.items() + if relay.ping_time_remaining == 0] + + self._logger.info("removed %d relays", len(to_be_removed)) + assert all(to_be_removed) + + to_be_pinged = [ + circuit for circuit in self.circuits.values() + if circuit.ping_time_remaining < PING_INTERVAL + and circuit.candidate] + + #self._logger.info("pinging %d circuits", len(to_be_pinged)) + for circuit in to_be_pinged: + self.create_ping(circuit.candidate, circuit) + except Exception: + self._logger.error("Ping error") + + yield PING_INTERVAL + + def tunnel_data_to_end(self, ultimate_destination, payload, circuit): + """ + Tunnel data to the end and request an EXIT to the outside world + + @param (str, int) ultimate_destination: The destination outside the + tunnel community + @param str payload: The raw payload to send to the ultimate destination + @param Tribler.community.anontunnel.routing.Circuit circuit: The + circuit id to tunnel data over + + @return: Whether the request has been handled successfully + """ + + if circuit.goal_hops == 0: + for observer in self.observers: + observer.on_exiting_from_tunnel(circuit.circuit_id, None, ultimate_destination, payload) + else: + self.send_message( + circuit.candidate, circuit.circuit_id, MESSAGE_DATA, + DataMessage(ultimate_destination, payload, None)) + + for observer in self.observers: + observer.on_send_data(circuit.circuit_id, circuit.candidate, ultimate_destination, payload) + + def tunnel_data_to_origin(self, circuit_id, candidate, source_address, + payload): + """ + Tunnel data to originator + + @param int circuit_id: The circuit's id to return data over + @param Candidate candidate: The relay to return data over + @param (str, int) source_address: The source outside the tunnel + community + @param str payload: The raw payload to return to the originator + + @return: Whether the request has been handled successfully + """ + with self.lock: + result = self.send_message( + candidate, circuit_id, MESSAGE_DATA, + DataMessage(None, payload, source_address)) + + if result: + for observer in self.observers: + observer.on_enter_tunnel(circuit_id, candidate, source_address, payload) + + return result + + def create_created_cache(self, circuit_id, candidate, candidates): + """ + + @param int circuit_id: the circuit id we received the CREATE from + @param WalkCandidate candidate: the candidate we got the CREATE from + @param dict[str, WalkCandidate] candidates: list of extend candidates we sent back + """ + self.dispersy.callback.call(lambda: self._request_cache.add(CreatedRequestCache(self, circuit_id, candidate, candidates))) + + def get_created_cache(self, circuit_id, candidate): + return self.dispersy.callback.call( + self.request_cache.get, + ( + CreatedRequestCache.PREFIX, + CreatedRequestCache.create_identifier(circuit_id, candidate), + ) + ) diff --git a/Tribler/community/anontunnel/conversion.py b/Tribler/community/anontunnel/conversion.py new file mode 100644 index 0000000000..64d44543d5 --- /dev/null +++ b/Tribler/community/anontunnel/conversion.py @@ -0,0 +1,260 @@ +import logging +import sys +from Tribler.Core.Utilities.encoding import encode, decode +from Tribler.community.anontunnel.globals import MESSAGE_CREATE, \ + MESSAGE_CREATED, MESSAGE_EXTEND, MESSAGE_EXTENDED, MESSAGE_DATA, \ + MESSAGE_PING, MESSAGE_PONG +from Tribler.community.anontunnel.payload import ExtendMessage, DataMessage, \ + PingMessage, PongMessage, CreatedMessage, ExtendedMessage, CreateMessage +from Tribler.dispersy.conversion import BinaryConversion +import struct + + +class ProxyConversion(BinaryConversion): + """ + The dispersy conversion used for the STATS message in the ProxyCommunity + @param ProxyCommunity community: the instance of the ProxyCommunity + """ + + def __init__(self, community): + super(ProxyConversion, self).__init__(community, "\x01") + + self._logger = logging.getLogger(__name__) + + self.define_meta_message( + chr(1), + community.get_meta_message(u"stats"), + self._encode_stats, + self._decode_stats + ) + + @staticmethod + def _encode_stats(message): + return encode(message.payload.stats), + + @staticmethod + def _decode_stats(placeholder, offset, data): + offset, stats = decode(data, offset) + + return offset, placeholder.meta.payload.implement(stats) + + +class CustomProxyConversion(): + """ + Custom conversion for Proxy messages. This conversion encodes objects + to bytes and vice versa + """ + def __init__(self): + self._logger = logging.getLogger(__name__) + + self.encode_functions = { + MESSAGE_CREATE: self.__encode_create, + MESSAGE_CREATED: self.__encode_created, + MESSAGE_EXTEND: self.__encode_extend, + MESSAGE_EXTENDED: self.__encode_extended, + MESSAGE_DATA: self.__encode_data, + MESSAGE_PING: lambda _: '', + MESSAGE_PONG: lambda _: '' + } + + self.decode_functions = { + MESSAGE_CREATE: self.__decode_create, + MESSAGE_CREATED: self.__decode_created, + MESSAGE_EXTEND: self.__decode_extend, + MESSAGE_EXTENDED: self.__decode_extended, + MESSAGE_DATA: self.__decode_data, + MESSAGE_PING: lambda offset, data: PingMessage(), + MESSAGE_PONG: lambda offset, data: PongMessage(), + } + + def encode(self, message_type, message): + """ + Encodes an object into a byte string + @param str message_type: the messages type (see the constants) + @param BaseMessage message: the message to serialize + @return: the message in byte format + @rtype: str + """ + return message_type + self.encode_functions[message_type](message) + + def decode(self, data, offset=0): + """ + Decode a byte string to a message + @param str data: raw byte string to decode + @param int offset: the offset to start at + @return: the message in object format + @rtype: BaseMessage + """ + message_type = data[offset] + assert message_type > 0 + return message_type, self.decode_functions[message_type]( + data, offset + 1) + + def get_circuit_and_data(self, message_buffer, offset=0): + """ + Get the circuit id and the payload byte string from a byte string + @param str message_buffer: the byte string to parse + @param int offset: the offset to start decoding from + @rtype (int, str) + """ + circuit_id, = struct.unpack_from("!L", message_buffer[offset:offset+4]) + offset += 4 + + return circuit_id, message_buffer[offset:] + + def get_type(self, data): + """ + Gets the type from a raw byte string + + @param str data: the raw byte string to get the type of + @rtype: str + """ + return data[0] + + def add_circuit(self, data, new_id): + """ + Prepends the circuit id to the raw byte string + @param str data: the raw byte string to prepend the circuit id to + @param int new_id: the circuit id to prepend + @rtype: str + """ + return struct.pack("!L", new_id) + data + + def __encode_extend(self, extend_message): + extend_with = extend_message.extend_with + key = extend_message.key + + data = "".join([ + struct.pack("!HH", len(extend_with), len(key)), + extend_with, + key + ]) + + return data + + def __decode_extend(self, message_buffer, offset=0): + if len(message_buffer) < offset + 4: + raise ValueError( + "Cannot unpack extend_with, insufficient packet size") + + extendwith_length, key_length = struct.unpack_from("!HH", message_buffer[offset:offset+4]) + offset += 4 + + extend_with = message_buffer[offset:offset + extendwith_length] + offset += extendwith_length + + key = message_buffer[offset:offset+key_length] + + message = ExtendMessage(extend_with) + message.key = key + return message + + def __encode_data(self, data_message): + if data_message.destination is None: + (host, port) = ("0.0.0.0", 0) + else: + (host, port) = data_message.destination + + if data_message.origin is None: + origin = ("0.0.0.0", 0) + else: + origin = data_message.origin + + return ''.join([ + struct.pack( + "!HHHHL", len(host), port, len(origin[0]), + origin[1], len(data_message.data) + ), + host, + origin[0], + data_message.data + ]) + + def __decode_data(self, message_buffer, offset=0): + host_length, port, origin_host_length, origin_port, payload_length = \ + struct.unpack_from("!HHHHL", message_buffer, offset) + offset += 12 + + if len(message_buffer) < offset + host_length: + raise ValueError("Cannot unpack Host, insufficient packet size") + host = message_buffer[offset:offset + host_length] + offset += host_length + + destination = (host, port) + + if len(message_buffer) < offset + origin_host_length: + raise ValueError( + "Cannot unpack Origin Host, insufficient packet size") + origin_host = message_buffer[offset:offset + origin_host_length] + offset += origin_host_length + + origin = (origin_host, origin_port) + + if origin == ("0.0.0.0", 0): + origin = None + + if destination == ("0.0.0.0", 0): + destination = None + + if payload_length == 0: + payload = None + else: + if len(message_buffer) < offset + payload_length: + raise ValueError( + "Cannot unpack Data, insufficient packet size") + payload = message_buffer[offset:offset + payload_length] + + return DataMessage(destination, payload, origin) + + def __encode_created(self, message): + #key = long_to_bytes(messages.key, DIFFIE_HELLMAN_MODULUS_SIZE / 8) + return struct.pack("!H", len(message.key)) + message.key + \ + message.candidate_list + + def __decode_created(self, message_buffer, offset=0): + key_length, = struct.unpack_from("!H", + message_buffer[offset:offset + 2]) + offset += 2 + key = message_buffer[offset:offset + key_length] + offset += key_length + + encrypted_candidate_list = message_buffer[offset:] + message = CreatedMessage(encrypted_candidate_list) + message.key = key + return message + + def __encode_extended(self, message): + return struct.pack("!H", len(message.key)) + message.key + \ + message.candidate_list + + def __decode_extended(self, message_buffer, offset=0): + key_length, = struct.unpack_from("!H", message_buffer[offset:]) + offset += 2 + key = message_buffer[offset:offset+key_length] + offset += key_length + + encrypted_candidate_list = message_buffer[offset:] + + return ExtendedMessage(key, encrypted_candidate_list) + + def __encode_create(self, create_message): + """ + :type create_message : CreateMessage + """ + return "".join([ + struct.pack("!HH", len(create_message.key), len(create_message.public_key)), + create_message.key, + create_message.public_key + ]) + + def __decode_create(self, message_buffer, offset=0): + len_key, len_pub_key = struct.unpack_from("!HH", message_buffer[offset:offset+4]) + offset += 4 + + key = message_buffer[offset:offset+len_key] + offset += len_key + + public_key = message_buffer[offset:offset + len_pub_key] + offset += len_pub_key + + return CreateMessage(key, public_key) \ No newline at end of file diff --git a/Tribler/community/anontunnel/crypto.py b/Tribler/community/anontunnel/crypto.py new file mode 100644 index 0000000000..5c8331efc9 --- /dev/null +++ b/Tribler/community/anontunnel/crypto.py @@ -0,0 +1,589 @@ +from Crypto.Util.number import bytes_to_long, long_to_bytes +from Tribler.community.privatesemantic.crypto.optional_crypto import mpz, rand, aes_encrypt_str, \ + aes_decrypt_str +from collections import defaultdict +import hashlib +import logging +from Tribler.Core.Utilities import encoding +from Tribler.Core.Utilities.encoding import encode, decode +from Tribler.community.anontunnel.events import TunnelObserver + + +from Tribler.community.anontunnel.globals import MESSAGE_CREATED, ORIGINATOR, \ + ENDPOINT, MESSAGE_CREATE, MESSAGE_EXTEND, MESSAGE_EXTENDED, \ + DIFFIE_HELLMAN_MODULUS, DIFFIE_HELLMAN_MODULUS_SIZE, \ + DIFFIE_HELLMAN_GENERATOR + + +class CryptoError(Exception): + pass + + +class Crypto(TunnelObserver): + + def __init__(self): + TunnelObserver.__init__(self) + self.encrypt_outgoing_packet_content = defaultdict() + self.decrypt_incoming_packet_content = defaultdict() + self._logger = logging.getLogger(__name__) + + def outgoing_packet_crypto(self, candidate, circuit, message_type, message, payload): + return payload + + def incoming_packet_crypto(self, candidate, circuit, payload): + return payload + + def relay_packet_crypto(self, destination, circuit, message_type, content): + return content + + def handle_incoming_packet(self, candidate, circuit_id, data): + """ + As soon as a packet comes in it has to be decrypted depending on candidate / circuit id + @param candidate: The originator of the packet + @param circuit_id: The circuit ID in the packet + @param data: The packet data + @return: The unencrypted data + """ + + data = self.incoming_packet_crypto(candidate, circuit_id, data) + if not data: + return None + return data + + def handle_incoming_packet_content(self, candidate, circuit_id, payload, packet_type): + """ + As soon as an incoming packet is decrypted, the content has to be decrypted + depending on the packet type + @param candidate: The originator of the packet + @param circuit_id: The circuit ID in the packet + @param payload: The packet data + @param packet_type: The type of the packet + @return: The payload with unencrypted content + """ + + if packet_type in self.decrypt_incoming_packet_content: + payload = self.decrypt_incoming_packet_content[packet_type](candidate, circuit_id, payload) + if not payload: + return None + return payload + + def handle_outgoing_packet(self, destination, circuit_id, message_type, message, content): + """ + Outgoing packets have to be encrypted according to the destination, packet type and + circuit identifier + @param destination: The originator of the packet + @param circuit_id: The circuit ID in the packet + @param message_type: The type of the packet + @param content: The packet data + @return: The encrypted content + """ + try: + content = self.outgoing_packet_crypto(destination, circuit_id, message_type, message, content) + return content + except: + self._logger.exception("Cannot encrypt outgoing packet content") + return None + + def handle_outgoing_packet_content(self, destination, circuit_id, message, message_type): + """ + Content of outgoing packets have to be encrypted according to the destination, 4 + message type and circuit identifier + @param destination: The originator of the packet + @param circuit_id: The circuit ID in the packet + @param message_type: The type of the packet + @param message: The message + @return: The message with encrypted content + """ + + try: + if message_type in self.encrypt_outgoing_packet_content: + message = self.encrypt_outgoing_packet_content[message_type](destination, circuit_id, message) + return message + except: + self._logger.exception("Cannot encrypt outgoing packet content") + return None + + def handle_relay_packet(self, direction, sock_addr, circuit_id, data): + """ + Relayed messages have to be encrypted / decrypted depending on direction, sock address and + circuit identifier + @param direction: direction of the packet + @param circuit_id: The circuit ID in the packet + @param sock_addr: socket address of the originator of the message + @param data: The message data + @return: The message data, en- / decrypted according to the circuitdirection + """ + try: + data = self.relay_packet_crypto(direction, sock_addr, circuit_id, data) + return data + except: + self._logger.error("Cannot crypt relay packet") + return None + + +class NoCrypto(Crypto): + def __init__(self): + Crypto.__init__(self) + self.proxy = None + self.key_to_forward = None + self.encrypt_outgoing_packet_content[MESSAGE_CREATED] = self._encrypt_created_content + self.decrypt_incoming_packet_content[MESSAGE_CREATED] = self._decrypt_created_content + self.decrypt_incoming_packet_content[MESSAGE_EXTENDED] = self._decrypt_extended_content + + + def set_proxy(self, proxy): + """ + Method which enables the "nocrypto" cryptography settings for the given + community. NoCrypto encodes and decodes candidate lists, everything is + passed as a string in the messages + + @param ProxyCommunity proxy: Proxy community to which this crypto + object is coupled + """ + self.proxy = proxy + + def disable(self): + """ + Disables the crypto settings + """ + self.outgoing_packet_crypto = lambda candidate, circuit, message, payload: payload + self.incoming_packet_crypto = lambda candidate, circuit, payload: payload + self.relay_packet_crypto = lambda destination, circuit, message_type, content: content + self.encrypt_outgoing_packet_content = defaultdict() + self.decrypt_incoming_packet_content = defaultdict() + + def _encrypt_created_content(self, candidate, circuit_id, message): + """ + Candidate list must be converted to a string in nocrypto + + @param Candidate candidate: Destination of the message + @param int circuit_id: Circuit identifier + @param CreatedMessage message: Message as passed from the community + @return CreatedMessage: Version of the message with candidate string + """ + message.candidate_list = encode(message.candidate_list) + return message + + def _decrypt_created_content(self, candidate, circuit_id, message): + """ + If created is for us, decode candidate list from string to dict + + @param Candidate candidate: Sender of the message + @param int circuit_id: Circuit identifier + @param CreatedMessage message: Message as passed from the community + @return CreatedMessage: Message with candidates as dict + """ + if circuit_id in self.proxy.circuits: + _, message.candidate_list = decode(message.candidate_list) + return message + + def _decrypt_extended_content(self, candidate, circuit_id, message): + """ + Convert candidate list from string to dict + + @param Candidate candidate: Sender of the message + @param int circuit_id: Circuit identifier + @param ExtendedMessage message: Message as passed from the community + @return ExtendedMessage: Extended message with candidate list as dict + """ + _, message.candidate_list = decode(message.candidate_list) + return message + + +class DefaultCrypto(Crypto): + + @staticmethod + def _generate_diffie_secret(): + """ + Generates a new Diffie Hellman g^x. Note the mpz lib used for Windows + @return: tuple of x and g^x + """ + dh_secret = 0 + while dh_secret >= DIFFIE_HELLMAN_MODULUS or dh_secret < 2: + dh_secret = rand("next", DIFFIE_HELLMAN_MODULUS) + dh_secret = mpz(dh_secret) + + dh_first_part = mpz(pow(DIFFIE_HELLMAN_GENERATOR, dh_secret, DIFFIE_HELLMAN_MODULUS)) + return dh_secret, dh_first_part + + def __init__(self): + Crypto.__init__(self) + self.proxy = None + """ :type proxy: ProxyCommunity """ + self._logger = logging.getLogger(__name__) + self._received_secrets = {} + self.session_keys = {} + self.encrypt_outgoing_packet_content[MESSAGE_CREATE] = self._encrypt_create_content + self.encrypt_outgoing_packet_content[MESSAGE_CREATED] = self._encrypt_created_content + self.encrypt_outgoing_packet_content[MESSAGE_EXTEND] = self._encrypt_extend_content + self.encrypt_outgoing_packet_content[MESSAGE_EXTENDED] = self._encrypt_extended_content + self.decrypt_incoming_packet_content[MESSAGE_CREATE] = self._decrypt_create_content + self.decrypt_incoming_packet_content[MESSAGE_CREATED] = self._decrypt_created_content + self.decrypt_incoming_packet_content[MESSAGE_EXTEND] = self._decrypt_extend_content + self.decrypt_incoming_packet_content[MESSAGE_EXTENDED] = self._decrypt_extended_content + + def on_break_relay(self, relay_key): + """ + Method called whenever a relay is broken after for example a timeout + or an invalid packet. Callback from the community to remove the session + key + @param relay_key: + """ + if relay_key in self.session_keys: + del self.session_keys[relay_key] + + def set_proxy(self, proxy): + """ + Method which enables the "defaultcrypto" cryptography settings for + the given community. Default crypto uses cryptography for all message + types, based on exchanged secrets established with DIFFIE HELLMAN and + Elliptic Curve Elgamal. See documentation for extra info. + + @param ProxyCommunity proxy: Proxy community to which this crypto + object is coupled + """ + self.proxy = proxy + proxy.observers.append(self) + + def _encrypt_create_content(self, candidate, circuit_id, message): + """ + Method which encrypts the contents of a CREATE message before it + is being sent. The only thing in a CREATE message that needs to be + encrypted is the first part of the DIFFIE HELLMAN handshake, which is + created in this method. + + @param Candidate candidate: Destination of the message + @param int circuit_id: Circuit identifier + @param CreateMessage message: Message as passed from the community + @return CreateMessage: Version of the message with encrypted contents + """ + + pub_key = iter(candidate.get_members()).next()._ec + message.public_key = self.proxy.crypto.key_to_bin(self.proxy.my_member._ec.pub()) + + if circuit_id in self.proxy.circuits: + dh_secret, dh_first_part = self._generate_diffie_secret() + + encrypted_dh_first_part = self.proxy.crypto.encrypt( + pub_key, long_to_bytes(dh_first_part)) + message.key = encrypted_dh_first_part + + hop = self.proxy.circuits[circuit_id].unverified_hop + hop.dh_secret = dh_secret + hop.dh_first_part = dh_first_part + hop.public_key = pub_key + else: + message.key = self.key_to_forward + self.key_to_forward = None + + return message + + def _decrypt_create_content(self, candidate, circuit_id, message): + """ + The first part of the DIFFIE HELLMAN handshake is encrypted with + Elgamal and is decrypted here + + @param Candidate candidate: Sender of the message + @param int circuit_id: Circuit identifier + @param CreateMessage message: Message as passed from the community + @return CreateMessage: Message with decrypted contents + """ + relay_key = (candidate.sock_addr, circuit_id) + my_key = self.proxy.my_member._ec + dh_second_part = mpz(bytes_to_long(self.proxy.crypto.decrypt(my_key, message.key))) + message.key = dh_second_part + + if dh_second_part < 2 or dh_second_part > DIFFIE_HELLMAN_MODULUS - 1: + self._logger.warning("Invalid DH data received over circuit {}.".format(circuit_id)) + return None + + self._received_secrets[relay_key] = dh_second_part + return message + + def _encrypt_extend_content(self, candidate, circuit_id, message): + """ + Method which encrypts the contents of an EXTEND message before it + is being sent. The only thing in an EXTEND message that needs to be + encrypted is the first part of the DIFFIE HELLMAN handshake, which is + created in this method. + + @param Candidate candidate: Destination of the message + @param int circuit_id: Circuit identifier + @param ExtendMessage message: Message as passed from the community + @return ExtendMessage: Version of the message with encrypted contents + """ + dh_secret, dh_first_part = self._generate_diffie_secret() + + pub_key = self.proxy.circuits[circuit_id].unverified_hop.public_key + + encrypted_dh_first_part = self.proxy.crypto.encrypt( + pub_key, long_to_bytes(dh_first_part)) + message.key = encrypted_dh_first_part + + hop = self.proxy.circuits[circuit_id].unverified_hop + hop.dh_first_part = dh_first_part + hop.dh_secret = dh_secret + + return message + + def _decrypt_extend_content(self, candidate, circuit_id, message): + """ + Nothing is encrypted in an Extend message + + @param Candidate candidate: Sender of the message + @param int circuit_id: Circuit identifier + @param ExtendMessage message: Message as passed from the community + @return ExtendMessage: Message with decrypted contents + """ + self.key_to_forward = message.key + return message + + def _encrypt_created_content(self, candidate, circuit_id, message): + """ + Method which encrypts the contents of a CREATED message before it + is being sent. There are two things that need to be encrypted in a + CREATED message. The second part of the DIFFIE HELLMAN handshake, which + is being generated and encrypted in this method, and the candidate + list, which is passed from the community. + + @param Candidate candidate: Destination of the message + @param int circuit_id: Circuit identifier + @param CreatedMessage message: Message as passed from the community + @return CreatedMessage: Version of the message with encrypted contents + """ + relay_key = (candidate.sock_addr, circuit_id) + dh_secret, _ = self._generate_diffie_secret() + + key = pow(self._received_secrets[relay_key], + dh_secret, DIFFIE_HELLMAN_MODULUS) + + m = hashlib.sha256() + m.update(str(key)) + key = m.digest()[0:16] + + self.session_keys[relay_key] = key + return_key = pow(DIFFIE_HELLMAN_GENERATOR, dh_secret, + DIFFIE_HELLMAN_MODULUS) + message.key = long_to_bytes(return_key) + + encoded_dict = encoding.encode(message.candidate_list) + message.candidate_list = aes_encrypt_str(self.session_keys[relay_key], encoded_dict) + + return message + + def _decrypt_created_content(self, candidate, circuit_id, message): + """ + Nothing to decrypt if you're not the originator of the circuit. Else, + The candidate list should be decrypted as if it was an Extended + message. + + @param Candidate candidate: Sender of the message + @param int circuit_id: Circuit identifier + @param CreatedMessage message: Message as passed from the community + @return CreatedMessage: Message with decrypted contents + """ + if circuit_id in self.proxy.circuits: + return self._decrypt_extended_content( + candidate, circuit_id, message) + return message + + def _encrypt_extended_content(self, candidate, circuit_id, message): + """ + Everything is already encrypted in an Extended message + + @param Candidate candidate: Destination of the message + @param int circuit_id: Circuit identifier + @param ExtendedMessage | CreatedMessage message: Message as passed + from the community + @return ExtendedMessage: Same + """ + return message + + def _decrypt_extended_content(self, candidate, circuit_id, message): + """ + This method decrypts the contents of an encrypted Extended message. + If the candidate list is undecryptable, the message is malformed and + the circuit should be broken. + + @param Candidate candidate: Sender of the message + @param int circuit_id: Circuit identifier + @param ExtendedMessage|CreatedMessage message: Message as passed from + the community + @return ExtendedMessage|CreatedMessage: Extended message with + unencrypted contents + """ + unverified_hop = self.proxy.circuits[circuit_id].unverified_hop + + dh_second_part = mpz(bytes_to_long(message.key)) + + if dh_second_part < 2 or dh_second_part > DIFFIE_HELLMAN_MODULUS - 1: + self._logger.warning("Invalid DH data received over circuit {}.".format(circuit_id)) + return None + + session_key = pow(dh_second_part, + unverified_hop.dh_secret, + DIFFIE_HELLMAN_MODULUS) + m = hashlib.sha256() + m.update(str(session_key)) + key = m.digest()[0:16] + unverified_hop.session_key = key + try: + encoded_dict = aes_decrypt_str(unverified_hop.session_key, message.candidate_list) + _, cand_dict = encoding.decode(encoded_dict) + message.candidate_list = cand_dict + except: + reason = "Can't decrypt candidate list!" + self._logger.error(reason) + self.proxy.remove_circuit(circuit_id, reason) + return None + + return message + + def outgoing_packet_crypto(self, candidate, circuit_id, + message_type, message, content): + """ + Apply crypto to outgoing messages. The current protocol handles 3 + distinct cases: CREATE/CREATED, ORIGINATOR, ENDPOINT / RELAY. + + Messages of type CREATE or CREATED are encrypted using Elgamal, when + these messages are received they need to be encrypted using the + recipients PUBLIC KEY. + + If you have a SESSION KEY for the outgoing message use it to encrypt + the packet, in this case you are a hop of the circuit. This adds a + layer to the onion. + + If you do not have a SESSION KEY for the outgoing message but created + the circuit encrypt it with the SESSION KEY linked to the circuit + itself + + @param Candidate candidate: the recipient of the message + @param int circuit_id: the circuit to sent the message over + @param str message_type: the message's type + @param str content: the raw (serialized) content of the message + @rtype: str + @return: the encrypted payload + """ + + relay_key = (candidate.sock_addr, circuit_id) + + # CREATE and CREATED have to be Elgamal encrypted + if message_type == MESSAGE_CREATED or message_type == MESSAGE_CREATE: +# candidate_pub_key = message.public_key if message_type == MESSAGE_CREATE else message.reply_to.public_key +# candidate_pub_key = self.proxy.crypto.key_from_public_bin(candidate_pub_key) + + candidate_pub_key = next(iter(candidate.get_members()))._ec + + content = self.proxy.crypto.encrypt(candidate_pub_key, content) + # Else add AES layer + elif relay_key in self.session_keys: + content = aes_encrypt_str(self.session_keys[relay_key], content) + # If own circuit, AES layers have to be added + elif circuit_id in self.proxy.circuits: + # I am the originator so I have to create the full onion + circuit = self.proxy.circuits[circuit_id] + hops = circuit.hops + for hop in reversed(hops): + content = aes_encrypt_str(hop.session_key, content) + else: + raise CryptoError("Don't know how to encrypt outgoing message") + + return content + + def relay_packet_crypto(self, direction, sock_addr, circuit_id, data): + """ + Crypto RELAY messages. Two distinct cases are considered: relaying to + the ENDPOINT and relaying back to the ORIGINATOR. + + When relaying to the ENDPOINT we need to strip one layer of the onion, + before forwarding the packet to the next hop. This is done by + decrypting with our SESSION_KEY. + + In the case that we relay towards the ORIGINATOR an additional onion + layer must be added. This is done by encrypting with our SESSION KEY + + @param str direction: the direction of the relay + @param sock_addr: the destination of the relay message + @param circuit_id: the destination circuit + @param data: the data to relay + @return: the onion encrypted payload + @rtype: str + """ + relay_key = (sock_addr, circuit_id) + next_relay = self.proxy.relay_from_to[relay_key] + next_relay_key = (next_relay.sock_addr, next_relay.circuit_id) + + # Message is going downstream so I have to add my onion layer + if direction == ORIGINATOR: + data = aes_encrypt_str( + self.session_keys[next_relay_key], data) + + # Message is going upstream so I have to remove my onion layer + elif direction == ENDPOINT: + data = aes_decrypt_str( + self.session_keys[relay_key], data) + else: + raise ValueError("The parameter 'direction' must be either" + "ORIGINATOR or ENDPOINT") + + return data + + def incoming_packet_crypto(self, candidate, circuit_id, data): + """ + Decrypt incoming packets. Three cases are considered. The case that + we are the ENDPOINT of the circuit, the case that we are the ORIGINATOR + and finally the case that we are neither. This means that this is our + first packet on this circuit and that it MUST be a CREATE or CREATED + message + + @todo: Check whether it really is a CREATE or CREATED message ? + + @param Candidate candidate: the candidate we got the message from + @param int circuit_id: the circuit we got the message on + @param str data: the raw payload we received + @return: the decrypted payload + @rtype: str + """ + + relay_key = (candidate.sock_addr, circuit_id) + # I'm the last node in the circuit, probably an EXTEND message, + # decrypt with AES + if relay_key in self.session_keys: + + try: + # last node in circuit, circuit already exists + return aes_decrypt_str(self.session_keys[relay_key], data) + except: + self._logger.warning("Cannot decrypt a message destined for us, the end of a circuit.") + return None + + # If I am the circuits originator I want to peel layers + elif circuit_id in self.proxy.circuits and len( + self.proxy.circuits[circuit_id].hops) > 0: + + try: + # I am the originator so I'll peel the onion skins + for hop in self.proxy.circuits[circuit_id].hops: + data = aes_decrypt_str(hop.session_key, data) + + return data + except: + self._logger.warning("Cannot decrypt packet. It should be a packet coming of our own circuit, but we cannot peel the onion.") + return None + + # I don't know the sender! Let's decrypt with my private Elgamal key + else: + try: + # last node in circuit, circuit does not exist yet, + # decrypt with Elgamal key + self._logger.debug( + "Circuit does not yet exist, decrypting with my Elgamal key") + my_key = self.proxy.my_member._ec + data = self.proxy.crypto.decrypt(my_key, data) + + return data + except: + self._logger.warning("Cannot decrypt packet, should be an initial packet encrypted with our public Elgamal key"); + return None + + diff --git a/Tribler/community/anontunnel/endpoint.py b/Tribler/community/anontunnel/endpoint.py new file mode 100644 index 0000000000..b886e73e16 --- /dev/null +++ b/Tribler/community/anontunnel/endpoint.py @@ -0,0 +1,61 @@ +""" +Contains the DispersyBypassEndpoint to be used as Dispersy endpoint when the +ProxyCommunity is being used +""" + + +from Queue import Queue, Full +from threading import Thread +from Tribler.dispersy.endpoint import RawserverEndpoint +import logging +__author__ = 'chris' + + +class DispersyBypassEndpoint(RawserverEndpoint): + """ + Creates an Dispersy Endpoint which bypasses the Dispersy message handling + system for incoming packets matching set prefixes + + @type raw_server: Tribler.Core.RawServer.RawServer.RawServer + @type port: int + @type ip: str + """ + def __init__(self, raw_server, port, ip="0.0.0.0"): + super(DispersyBypassEndpoint, self).__init__(raw_server, port, ip) + self.packet_handlers = {} + self.queue = Queue() + + self._logger = logging.getLogger(__name__) + + def listen_to(self, prefix, handler): + """ + Register a prefix to a handler + + @param str prefix: the prefix of a packet to register to the handler + @param ((str, int), str) -> None handler: the handler that will be + called for packets starting with the set prefix + """ + self.packet_handlers[prefix] = handler + + def data_came_in(self, packets, cache=True): + """ + Called by the RawServer when UDP packets arrive + @type packets: list[((str, int), str)] + @return: + """ + normal_packets = [] + try: + for packet in packets: + + prefix = next((p for p in self.packet_handlers if + packet[1].startswith(p)), None) + if prefix: + self.packet_handlers[prefix](*packet) + else: + normal_packets.append(packet) + except Full: + self._logger.warning( + "DispersyBypassEndpoint cant keep up with incoming packets!") + + if normal_packets: + super(DispersyBypassEndpoint, self).data_came_in(normal_packets, cache) \ No newline at end of file diff --git a/Tribler/community/anontunnel/events.py b/Tribler/community/anontunnel/events.py new file mode 100644 index 0000000000..2249f38fa0 --- /dev/null +++ b/Tribler/community/anontunnel/events.py @@ -0,0 +1,129 @@ +""" +Event handling related module + +Can be safely imported as it does not import any dependencies in the global + namespace +""" + +__author__ = 'chris' + + +class TunnelObserver(object): + """ + The TunnelObserver class is being notified by the ProxyCommunity in case + a circuit / relay breaks, the global state changes or when data is being + sent and received + """ + + def __init__(self): + pass + + def on_break_circuit(self, circuit): + """ + Called when a circuit has been broken and removed from the + ProxyCommunity + @param Circuit circuit: the circuit that has been broken + """ + pass + + # noinspection PyMethodMayBeStatic + def on_break_relay(self, relay_key): + """ + Called when a relay has been broken due to inactivity + @param ((str, int), int) relay_key: the identifier in + (sock_addr, circuit) format + """ + pass + + def on_incoming_from_tunnel(self, community, circuit, origin, data): + """ + Called when we are receiving data from our circuit + + @type community: Tribler.community.anontunnel.community.ProxyCommunity + @param Circuit circuit: the circuit the data was received on + @param (str, int) origin: the origin of the packet in sock_addr format + @param str data: the data received + """ + pass + + def on_exiting_from_tunnel(self, circuit_id, candidate, destination, data): + """ + Called when a DATA message has been received destined for the outside + world + + @param int circuit_id: the circuit id where the the DATA message was + received on + @param Candidate candidate: the relay candidate who relayed the message + @param (str, int) destination: the packet's ultimate destination + @param data: the payload + """ + pass + + def on_tunnel_stats(self, community, member, candidate, stats): + """ + Called when a STATS message has been received + @param Member member: member according to authentication + @type community: Tribler.community.anontunnel.community.ProxyCommunity + @type candidate: Candidate + @type stats: dict + """ + pass + + def on_enter_tunnel(self, circuit_id, candidate, origin, payload): + """ + Called when we received a packet from the outside world + + @param int circuit_id: the circuit for which we received data from the + outside world + @param Candidate candidate: the known relay for this circuit + @param (str, int) origin: the outside origin of the packet + @param str payload: the packet's payload + """ + pass + + def on_send_data(self, circuit_id, candidate, destination, payload): + """ + Called when uploading data over a circuit + + @param int circuit_id: the circuit where data being uploaded over + @param Candidate candidate: the relay used to send data over + @param (str, int) destination: the destination of the packet + @param str payload: the packet's payload + """ + pass + + def on_relay(self, from_key, to_key, direction, data): + """ + Called when we are relaying data from_key to to_key + + @param ((str, int), int) from_key: the relay we are getting data from + @param ((str, int), int) to_key: the relay we are sending data to + @param direction: ENDPOINT if we are relaying towards the end of the + tunnel, ORIGINATOR otherwise + @type data: str + @return: + """ + pass + + def on_unload(self): + """ + Called when the ProxyCommunity is being unloaded + """ + pass + + +class CircuitPoolObserver(object): + """ + An observer interface for circuit pools. Contains the event triggered when + a new circuit has been added to the pool + """ + def __init__(self): + pass + + def on_circuit_added(self, pool, circuit): + """ + A circuit has been added to the pool + @param CircuitPool pool: the pool to which the circuit has been added + @param Circuit circuit: the circuit that has been added + """ + pass diff --git a/Tribler/community/anontunnel/exitsocket.py b/Tribler/community/anontunnel/exitsocket.py new file mode 100644 index 0000000000..9378f11359 --- /dev/null +++ b/Tribler/community/anontunnel/exitsocket.py @@ -0,0 +1,105 @@ +from Tribler.community.anontunnel.payload import DataMessage + +__author__ = 'Chris' + +import logging + + +class TunnelExitSocket(object): + """ + Sends incoming UDP packets back over the DispersyTunnelProxy. + """ + + def __init__(self, raw_server, proxy, circuit_id, destination_address): + """ + Instantiate a new return handler + + :param proxy: instance to use to push packets back into upon reception + of an external UDP packet + :param circuit_id: the circuit to use to pass messages over in the + tunnel proxy + :param destination_address: the first hop of the circuit + + :type proxy: Tribler.community.anontunnel.community.ProxyCommunity + + """ + + socket = raw_server.create_udpsocket(0, "0.0.0.0") + raw_server.start_listening_udp(socket, self) + + self.proxy = proxy + self.destination_address = destination_address + self.circuit_id = circuit_id + self.socket = socket + self._logger = logging.getLogger(__name__) + + def sendto(self, data, destination): + """ + Sends data to the destination over an UDP socket + @param str data: the data to send + @param (str, int) destination: the destination to send to + """ + self.socket.sendto(data, destination) + + def data_came_in(self, packets): + """ + Method called by the server when a new UDP packet has been received + :param packets: list of tuples (source address, packet) of the UDP + packets received + """ + + for source_address, packet in packets: + self.proxy.tunnel_data_to_origin( + circuit_id=self.circuit_id, + candidate=self.destination_address, + source_address=source_address, + payload=packet) + + +class ShortCircuitExitSocket(object): + """ + Only used when there are no circuits, it will be a 0-hop tunnel. So there + is no anonymity at all. + """ + + def __init__(self, raw_server, proxy, circuit_id, destination_address): + """ + Instantiate a new return handler + + :param proxy: instance to use to push packets back into upon reception + of an external UDP packet + :param destination_address: the first hop of the circuit + + :type proxy: ProxyCommunity + + """ + + socket = raw_server.create_udpsocket(0, "0.0.0.0") + raw_server.start_listening_udp(socket, self) + + self.proxy = proxy + self.destination_address = destination_address + self.socket = socket + self.circuit_id = circuit_id + self._logger = logging.getLogger(__name__) + + def data_came_in(self, packets): + """ + Method called by the server when a new UDP packet has been received + + :param packets: list of tuples (source address, packet) of the UDP + packets received + """ + + for source_address, packet in packets: + message = DataMessage(("0.0.0.0", 0), packet, source_address) + self.proxy.on_data(self.circuit_id, None, message) + + def sendto(self, data, destination): + """ + Sends data to the destination over an UDP socket + @param str data: the data to send + @param (str, int) destination: the destination to send to + """ + + self.socket.sendto(data, destination) \ No newline at end of file diff --git a/Tribler/community/anontunnel/exitstrategies.py b/Tribler/community/anontunnel/exitstrategies.py new file mode 100644 index 0000000000..b5c81bd610 --- /dev/null +++ b/Tribler/community/anontunnel/exitstrategies.py @@ -0,0 +1,56 @@ +import socket +import logging +from Tribler.community.anontunnel import exitsocket +from Tribler.community.anontunnel.events import TunnelObserver + +__author__ = 'chris' + + +class DefaultExitStrategy(TunnelObserver): + def __init__(self, raw_server, proxy): + """ + @type proxy: ProxyCommunity + """ + + TunnelObserver.__init__(self) + self.raw_server = raw_server + self._logger = logging.getLogger(__name__) + + self.proxy = proxy + self._exit_sockets = {} + + def on_exiting_from_tunnel(self, circuit_id, return_candidate, destination, + data): + try: + exit_socket = self.get_exit_socket(circuit_id, return_candidate) + exit_socket.sendto(data, destination) + except socket.error: + self._logger.error("Dropping packets while EXITing data") + + @staticmethod + def create(proxy, raw_server, circuit_id, address): + # There is a special case where the circuit_id is None, then we act as + # EXIT node ourselves. In this case we create a ShortCircuitHandler + # that bypasses dispersy by patching ENTER packets directly into the + # Proxy's on_data event. + + if circuit_id in proxy.circuits and \ + proxy.circuits[circuit_id].goal_hops == 0: + return_handler = exitsocket.ShortCircuitExitSocket( + raw_server, proxy, circuit_id, address) + else: + # Otherwise incoming ENTER packets should propagate back over the + # Dispersy tunnel, we use the CircuitReturnHandler. It will use the + # DispersyTunnelProxy.send_data method to forward the data packet + return_handler = exitsocket.TunnelExitSocket(raw_server, proxy, + circuit_id, address) + + return return_handler + + def get_exit_socket(self, circuit_id, address): + # If we don't have an exit socket yet for this socket, create one + if not (circuit_id in self._exit_sockets): + return_handler = self.create(self.proxy, self.raw_server, + circuit_id, address) + self._exit_sockets[circuit_id] = return_handler + return self._exit_sockets[circuit_id] \ No newline at end of file diff --git a/Tribler/community/anontunnel/extendstrategies.py b/Tribler/community/anontunnel/extendstrategies.py new file mode 100644 index 0000000000..84efe02159 --- /dev/null +++ b/Tribler/community/anontunnel/extendstrategies.py @@ -0,0 +1,94 @@ +import logging +from Tribler.community.anontunnel.globals import CIRCUIT_STATE_EXTENDING, \ + MESSAGE_EXTEND +from Tribler.community.anontunnel.payload import ExtendMessage +from Tribler.community.anontunnel.routing import Hop + +__author__ = 'chris' + + +class NoCandidatesException(ValueError): + pass + + +class ExtendStrategy: + def __init__(self): + self._logger = logging.getLogger(__name__) + + def extend(self, candidate_list=None): + if not candidate_list: + candidate_list = [] + + raise NotImplementedError() + + +class TrustThyNeighbour(ExtendStrategy): + def __init__(self, proxy, circuit): + """ + :type proxy: Tribler.community.anontunnel.community.ProxyCommunity + :param circuit: + """ + ExtendStrategy.__init__(self) + self.proxy = proxy + self.circuit = circuit + + def extend(self, candidate_list=None): + if not candidate_list: + candidate_list = [] + + assert self.circuit.state == CIRCUIT_STATE_EXTENDING, \ + "Only circuits with state CIRCUIT_STATE_EXTENDING can be extended" + assert self.circuit.goal_hops > len(self.circuit.hops), \ + "Circuits with correct length cannot be extended" + + self._logger.info("Trusting our tunnel to extend circuit %d", + self.circuit.circuit_id) + self.proxy.send_message(self.circuit.candidate, + self.circuit.circuit_id, MESSAGE_EXTEND, + ExtendMessage(None)) + + +class NeighbourSubset(ExtendStrategy): + def __init__(self, proxy, circuit): + """ + :type proxy: Tribler.community.anontunnel.community.ProxyCommunity + :param circuit: + """ + ExtendStrategy.__init__(self) + self.proxy = proxy + self.circuit = circuit + + def extend(self, candidate_list=None): + if not candidate_list: + candidate_list = [] + + assert self.circuit.state == CIRCUIT_STATE_EXTENDING, \ + "Only circuits with state CIRCUIT_STATE_EXTENDING can be extended" + assert self.circuit.goal_hops > len(self.circuit.hops), \ + "Circuits with correct length cannot be extended" + + extend_hop_public_bin = next(iter(candidate_list), None) + + if not extend_hop_public_bin: + raise NoCandidatesException("No candidates (with key) to extend, bailing out.") + + extend_hop_public_key = self.proxy.dispersy.crypto.key_from_public_bin(extend_hop_public_bin) + hashed_public_key = self.proxy.dispersy.crypto.key_to_hash(extend_hop_public_key) + + self.circuit.candidate.pub_key = extend_hop_public_key + self.circuit.unverified_hop = Hop(extend_hop_public_key) + + try: + self._logger.info( + "We chose %s from the list to extend circuit %d", + hashed_public_key, self.circuit.circuit_id) + + self.proxy.send_message( + self.circuit.candidate, self.circuit.circuit_id, + MESSAGE_EXTEND, + ExtendMessage(extend_hop_public_bin)) + except BaseException: + self._logger.exception("Encryption error") + return False + + return True \ No newline at end of file diff --git a/Tribler/community/anontunnel/globals.py b/Tribler/community/anontunnel/globals.py new file mode 100644 index 0000000000..b175184980 --- /dev/null +++ b/Tribler/community/anontunnel/globals.py @@ -0,0 +1,41 @@ +from Tribler.community.privatesemantic.crypto.optional_crypto import mpz +from Tribler.dispersy.candidate import CANDIDATE_WALK_LIFETIME + +ORIGINATOR = "originator" +ENDPOINT = "endpoint" + +ANON_DOWNLOAD_DELAY = 300 + +CIRCUIT_STATE_READY = 'READY' +CIRCUIT_STATE_EXTENDING = 'EXTENDING' +CIRCUIT_STATE_TO_BE_EXTENDED = 'TO_BE_EXTENDED' +CIRCUIT_STATE_BROKEN = 'BROKEN' + +MESSAGE_CREATE = chr(1) +MESSAGE_CREATED = chr(2) +MESSAGE_EXTEND = chr(3) +MESSAGE_EXTENDED = chr(4) +MESSAGE_DATA = chr(5) +MESSAGE_PING = chr(6) +MESSAGE_PONG = chr(7) +MESSAGE_STATS = chr(10) + +AES_KEY_SIZE = 16 + +MESSAGE_TYPE_STRING = { + MESSAGE_CREATE: u'create', + MESSAGE_CREATED: u'created', + MESSAGE_EXTEND: u'extend', + MESSAGE_EXTENDED: u'extended', + MESSAGE_DATA: u'data', + MESSAGE_PING: u'ping', + MESSAGE_PONG: u'pong', + MESSAGE_STATS: u'stats' +} + +PING_INTERVAL = 10.0 +# we use group 14 of the IETF rfc3526 with a 2048 bit modulus +# http://tools.ietf.org/html/rfc3526 +DIFFIE_HELLMAN_GENERATOR = 2 +DIFFIE_HELLMAN_MODULUS = mpz(0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF) +DIFFIE_HELLMAN_MODULUS_SIZE = 2048 \ No newline at end of file diff --git a/Tribler/community/anontunnel/lengthstrategies.py b/Tribler/community/anontunnel/lengthstrategies.py new file mode 100644 index 0000000000..fb52cd21c6 --- /dev/null +++ b/Tribler/community/anontunnel/lengthstrategies.py @@ -0,0 +1,30 @@ +from random import randint + +__author__ = 'chris' + + +class CircuitLengthStrategy(object): + def __init__(self): + pass + + def circuit_length(self): + raise NotImplementedError() + + +class RandomCircuitLengthStrategy(CircuitLengthStrategy): + def __init__(self, minimum_length, maximum_length): + super(RandomCircuitLengthStrategy, self).__init__() + self.min = int(minimum_length) + self.max = int(maximum_length) + + def circuit_length(self): + return randint(self.min, self.max) + + +class ConstantCircuitLength(CircuitLengthStrategy): + def __init__(self, desired_length): + super(ConstantCircuitLength, self).__init__() + self.desired_length = int(desired_length) + + def circuit_length(self): + return self.desired_length \ No newline at end of file diff --git a/Tribler/community/anontunnel/logger.conf b/Tribler/community/anontunnel/logger.conf new file mode 100644 index 0000000000..840ee38171 --- /dev/null +++ b/Tribler/community/anontunnel/logger.conf @@ -0,0 +1,38 @@ +[loggers] +keys=root,candidates + +[handlers] +keys=debugging,default + +[formatters] +keys=debugging,default + +[logger_root] +level=ERROR +handlers=default + +[logger_candidates] +level=ERROR +qualname=dispersy-stats-detailed-candidates +handlers=default +propagate=0 + +[handler_default] +class=StreamHandler +level=NOTSET +formatter=debugging +args=(sys.stderr,) + +[formatter_default] +format=%(asctime)s %(levelname)s %(message)s +class=logging.Formatter + +[handler_debugging] +class=StreamHandler +level=NOTSET +formatter=debugging +args=(sys.stderr,) + +[formatter_debugging] +format=%(levelname)-7s %(created).2f %(module)15s:%(lineno)-4d %(message)s +class=logging.Formatter diff --git a/Tribler/community/anontunnel/payload.py b/Tribler/community/anontunnel/payload.py new file mode 100644 index 0000000000..09709ced2e --- /dev/null +++ b/Tribler/community/anontunnel/payload.py @@ -0,0 +1,73 @@ +from Tribler.dispersy.payload import Payload + +__author__ = 'Chris' + + +#noinspection PyClassHasNoInit +class BaseMessage: + pass + + +class PingMessage(BaseMessage): + def __init__(self): + pass + + +class PongMessage(BaseMessage): + def __init__(self): + pass + + +class CreateMessage(BaseMessage): + def __init__(self, key="\0"*336, public_key=""): + assert isinstance(key, basestring) + assert isinstance(public_key, basestring) + + self.key = key + self.public_key = public_key + + +class CreatedMessage(BaseMessage): + def __init__(self, candidate_list, reply_to=None): + # Assert candidate_list is a list and that all items are strings + assert all(isinstance(key, basestring) for key in candidate_list) + assert reply_to is None or isinstance(reply_to, CreateMessage) + + self.key = "" + self.candidate_list = candidate_list + self.reply_to = reply_to + + +class ExtendMessage(BaseMessage): + def __init__(self, extend_with): + assert extend_with is None or isinstance(extend_with, basestring) + + self.extend_with = extend_with + self.key = "" + + +class ExtendedMessage(BaseMessage): + def __init__(self, key, candidate_list): + assert isinstance(key, basestring) + assert all(isinstance(key, basestring) for key in candidate_list) + + self.key = key + self.candidate_list = candidate_list + + +class DataMessage(BaseMessage): + def __init__(self, destination, data, origin=None): + assert destination is None or isinstance(destination[0], basestring) and isinstance(destination[1], int) + assert isinstance(data, basestring) + assert origin is None or isinstance(origin[0], basestring) and isinstance(origin[1], int) + + self.destination = destination + self.data = data + self.origin = origin + + +class StatsPayload(Payload): + class Implementation(Payload.Implementation): + def __init__(self, meta, stats): + super(StatsPayload.Implementation, self).__init__(meta) + self.stats = stats \ No newline at end of file diff --git a/Tribler/community/anontunnel/plot_results.plt b/Tribler/community/anontunnel/plot_results.plt new file mode 100644 index 0000000000..5bd6950650 --- /dev/null +++ b/Tribler/community/anontunnel/plot_results.plt @@ -0,0 +1,13 @@ +#!/usr/bin/gnuplot +# set terminal postscript eps enhanced colour size 10cm,8cm font 'Arial-Bold,14' +# set output 'plot.eps' + +set term pngcairo +set output 'plot.png' + +set title 'Download speed over circuits with different hop lengths' +set xlabel 'Number of hops' +set ylabel 'Speed [KB/s]' +set xtics 1 + +plot "hop_speed.txt" using 1:5 w lines title "Average speed [KB/s]" diff --git a/Tribler/community/anontunnel/routing.py b/Tribler/community/anontunnel/routing.py new file mode 100644 index 0000000000..df450c820e --- /dev/null +++ b/Tribler/community/anontunnel/routing.py @@ -0,0 +1,246 @@ +import hashlib +import logging +import threading +import time +from M2Crypto.EC import EC_pub +from Tribler.community.anontunnel.events import TunnelObserver +from Tribler.community.anontunnel.globals import CIRCUIT_STATE_READY, \ + CIRCUIT_STATE_BROKEN, CIRCUIT_STATE_EXTENDING, PING_INTERVAL +from Tribler.dispersy.candidate import CANDIDATE_WALK_LIFETIME, Candidate + +__author__ = 'chris' + + +class Circuit: + """ Circuit data structure storing the id, state and hops """ + + def __init__(self, circuit_id, goal_hops=0, candidate=None, proxy=None): + """ + Instantiate a new Circuit data structure + :type proxy: ProxyCommunity + :param int circuit_id: the id of the candidate circuit + :param WalkCandidate candidate: the first hop of the circuit + :return: Circuit + """ + + self._broken = False + self._hops = [] + self._logger = logging.getLogger(__name__) + + self.circuit_id = circuit_id + self.candidate = candidate + self.goal_hops = goal_hops + self.extend_strategy = None + self.last_incoming = time.time() + + self.proxy = proxy + + self.unverified_hop = None + ''' :type : Hop ''' + + @property + def hops(self): + """ + Return a read only tuple version of the hop-list of this circuit + @rtype tuple[Hop] + """ + return tuple(self._hops) + + def add_hop(self, hop): + """ + Adds a hop to the circuits hop collection + @param Hop hop: the hop to add + """ + self._hops.append(hop) + + @property + def state(self): + """ + The circuit state, can be either: + CIRCUIT_STATE_BROKEN, CIRCUIT_STATE_EXTENDING or CIRCUIT_STATE_READY + @rtype: str + """ + if self._broken: + return CIRCUIT_STATE_BROKEN + + if len(self.hops) < self.goal_hops: + return CIRCUIT_STATE_EXTENDING + else: + return CIRCUIT_STATE_READY + + @property + def ping_time_remaining(self): + """ + The time left before we consider the circuit inactive, when it returns + 0 a PING must be sent to keep the circuit, including relays at its hop, + alive. + """ + too_old = time.time() - 2 * PING_INTERVAL + diff = self.last_incoming - too_old + return diff if diff > 0 else 0 + + def __contains__(self, other): + if isinstance(other, Candidate): + # TODO: should compare to a list here + return other == self.candidate + + def beat_heart(self): + """ + Mark the circuit as active + """ + self.last_incoming = time.time() + + def tunnel_data(self, destination, payload): + """ + Convenience method to tunnel data over this circuit + @param (str, int) destination: the destination of the packet + @param str payload: the packet's payload + @return bool: whether the tunnel request has succeeded, this is in no + way an acknowledgement of delivery! + """ + + return self.proxy.tunnel_data_to_end(destination, payload, self) + + def destroy(self, reason='unknown'): + """ + Destroys the circuit and calls the error callback of the circuit's + deferred if it has not been called before + + @param str reason: the reason why the circuit is being destroyed + """ + self._broken = True + + +class Hop: + """ + Circuit Hop containing the address, its public key and the first part of + the Diffie-Hellman handshake + """ + + def __init__(self, public_key=None): + """ + @param None|EC_pub public_key: public key object of the hop + """ + + assert public_key is None or isinstance(public_key, EC_pub) + + self.session_key = None + self.dh_first_part = None + self.dh_secret = None + self.address = None + self.public_key = public_key + + @property + def host(self): + """ + The hop's hostname + """ + if self.address: + return self.address[0] + return " UNKNOWN HOST " + + @property + def port(self): + """ + The hop's port + """ + if self.address: + return self.address[1] + return " UNKNOWN PORT " + + +class RelayRoute(object): + """ + Relay object containing the destination circuit, socket address and whether + it is online or not + """ + + def __init__(self, circuit_id, sock_addr): + """ + @type sock_addr: (str, int) + @type circuit_id: int + @return: + """ + + self.sock_addr = sock_addr + self.circuit_id = circuit_id + self.online = False + self.last_incoming = time.time() + + @property + def ping_time_remaining(self): + """ + The time left before we consider the relay inactive + """ + too_old = time.time() - CANDIDATE_WALK_LIFETIME - 5.0 + diff = self.last_incoming - too_old + return diff if diff > 0 else 0 + + +class CircuitPool(TunnelObserver): + def __init__(self, size, name): + super(CircuitPool, self).__init__() + + self._logger = logging.getLogger(__name__) + self._logger.info("Creating a circuit pool of size %d with name '%s'", size, name) + + self.lock = threading.RLock() + self.size = size + self.circuits = set() + self.allocated_circuits = set() + self.name = name + + self.observers = [] + + def on_break_circuit(self, circuit): + if circuit in self.circuits: + self.remove_circuit(circuit) + + @property + def lacking(self): + return max(0, self.size - len(self.circuits)) + + @property + def available_circuits(self): + return [circuit + for circuit in self.circuits + if circuit not in self.allocated_circuits] + + def remove_circuit(self, circuit): + self._logger.info("Removing circuit %d from pool '%s'", circuit.circuit_id, self.name) + with self.lock: + self.circuits.remove(circuit) + + def fill(self, circuit): + self._logger.info("Adding circuit %d to pool '%s'", circuit.circuit_id, self.name) + + with self.lock: + self.circuits.add(circuit) + for observer in self.observers: + observer.on_circuit_added(self, circuit) + + def deallocate(self, circuit): + self._logger.info("Deallocate circuit %d from pool '%s'", circuit.circuit_id, self.name) + + with self.lock: + self.allocated_circuits.remove(circuit) + + def allocate(self): + with self.lock: + try: + circuit = next((c for c in self.circuits if c not in self.allocated_circuits)) + self.allocated_circuits.add(circuit) + self._logger.info("Allocate circuit %d from pool %s", circuit.circuit_id, self.name) + + return circuit + + except StopIteration: + if not self.lacking: + self._logger.warning("Growing size of pool %s from %d to %d", self.name, self.size, self.size*2) + self.size *= 2 + + raise NotEnoughCircuitsException() + + +class NotEnoughCircuitsException(Exception): + pass \ No newline at end of file diff --git a/Tribler/community/anontunnel/selectionstrategies.py b/Tribler/community/anontunnel/selectionstrategies.py new file mode 100644 index 0000000000..55fe4dcb26 --- /dev/null +++ b/Tribler/community/anontunnel/selectionstrategies.py @@ -0,0 +1,69 @@ +import random +import logging + +__author__ = 'chris' + + +class SelectionStrategy: + """ + Base class for selection strategies + """ + def __init__(self): + self._logger = logging.getLogger(__name__) + + def select(self, circuits_to_select_from): + """ + Selects a circuit from a list of candidates + @param list[Circuit] circuits_to_select_from: the circuits to pick from + @rtype: Circuit + """ + pass + + +class RoundRobin(SelectionStrategy): + """ + Selects circuits in round robin fashion + """ + def __init__(self): + SelectionStrategy.__init__(self) + self.index = -1 + + def select(self, circuits_to_select_from): + self.index = (self.index + 1) % len(circuits_to_select_from) + return circuits_to_select_from[self.index] + + +class RandomSelectionStrategy(SelectionStrategy): + """ + Strategy that selects a circuit at random + """ + + def select(self, circuits_to_select_from): + if not circuits_to_select_from: + raise ValueError("Variable circuits_to_select must be a list of circuits") + + circuit = random.choice(circuits_to_select_from) + return circuit + + +class LengthSelectionStrategy(SelectionStrategy): + """ + Selects a circuit which length is between the min and max given (inclusive) + """ + def __init__(self, minimum_length, maximum_length, random_selection=True): + SelectionStrategy.__init__(self) + self.min = int(minimum_length) + self.max = int(maximum_length) + self.random = random_selection + + def select(self, circuits_to_select_from): + candidates = [c for c in circuits_to_select_from if + self.min <= len(c.hops) <= self.max] + + if not candidates: + raise ValueError("No circuit matching the criteria") + + if self.random: + return random.choice(candidates) + else: + return candidates[0] \ No newline at end of file diff --git a/Tribler/community/anontunnel/stats.py b/Tribler/community/anontunnel/stats.py new file mode 100644 index 0000000000..e2d824d348 --- /dev/null +++ b/Tribler/community/anontunnel/stats.py @@ -0,0 +1,385 @@ +from collections import defaultdict +import logging +import os +import sqlite3 +import uuid +import time +from Tribler.community.anontunnel.crypto import NoCrypto + +from Tribler.community.anontunnel.events import TunnelObserver +from Tribler.dispersy.database import Database + +__author__ = 'chris' + +sqlite3.register_converter('GUID', lambda b: uuid.UUID(bytes_le=b)) +sqlite3.register_adapter(uuid.UUID, lambda u: buffer(u.bytes_le)) + +class CircuitStats: + def __init__(self): + self.timestamp = None + self.times = [] + self.bytes_up_list = [] + self.bytes_down_list = [] + + self.bytes_down = [0, 0] + self.bytes_up = [0, 0] + + self.speed_up = 0.0 + self.speed_down = 0.0 + + @property + def bytes_downloaded(self): + return self.bytes_down[1] + + @property + def bytes_uploaded(self): + return self.bytes_up[1] + + +class RelayStats: + def __init__(self): + self.timestamp = None + + self.times = [] + self.bytes_list = [] + self.bytes = [0, 0] + self.speed = 0 + + +class StatsCollector(TunnelObserver): + def __init__(self, proxy, name): + """ + @type proxy: Tribler.community.anontunnel.community.ProxyCommunity + """ + + TunnelObserver.__init__(self) + + self._logger = logging.getLogger(__name__) + self.name = name + + self.stats = { + 'bytes_returned': 0, + 'bytes_exit': 0, + 'bytes_enter': 0, + 'broken_circuits': 0 + } + self.download_stats = {} + self.session_id = uuid.uuid4() + self.proxy = proxy + self.running = False + self.circuit_stats = defaultdict(lambda: CircuitStats()) + ''':type : dict[int, CircuitStats] ''' + self.relay_stats = defaultdict(lambda: RelayStats()) + ''':type : dict[((str,int),int), RelayStats] ''' + self._circuit_cache = {} + ''':type : dict[int, Circuit] ''' + + def pause(self): + """ + Pause stats collecting + """ + self._logger.info("Removed StatsCollector %s as observer", self.name) + self.running = False + self.proxy.observers.remove(self) + + def clear(self): + """ + Clear collected stats + """ + + self.circuit_stats.clear() + self.relay_stats.clear() + + def stop(self): + self.pause() + self.clear() + + def start(self): + if self.running: + raise ValueError("Cannot start collector {0} since it is already running".format(self.name)) + + self._logger.info("Resuming stats collector {0}!".format(self.name)) + self.running = True + self.proxy.observers.append(self) + self.proxy.dispersy.callback.register(self.__calc_speeds) + + def on_break_circuit(self, circuit): + if len(circuit.hops) == circuit.goal_hops: + self.stats['broken_circuits'] += 1 + + def __calc_speeds(self): + while self.running: + t2 = time.time() + self._circuit_cache.update(self.proxy.circuits) + + for circuit_id in self.proxy.circuits.keys(): + c = self.circuit_stats[circuit_id] + + if c.timestamp is None: + c.timestamp = time.time() + elif c.timestamp < t2: + + if len(c.bytes_up_list) == 0 or c.bytes_up[-1] != \ + c.bytes_up_list[-1] and c.bytes_down[-1] != \ + c.bytes_down_list[-1]: + c.bytes_up_list.append(c.bytes_up[-1]) + c.bytes_down_list.append(c.bytes_down[-1]) + c.times.append(t2) + + c.speed_up = 1.0 * (c.bytes_up[1] - c.bytes_up[0]) / ( + t2 - c.timestamp) + c.speed_down = 1.0 * ( + c.bytes_down[1] - c.bytes_down[0]) / (t2 - c.timestamp) + + c.timestamp = t2 + c.bytes_up = [c.bytes_up[1], c.bytes_up[1]] + c.bytes_down = [c.bytes_down[1], c.bytes_down[1]] + + for relay_key in self.proxy.relay_from_to.keys(): + r = self.relay_stats[relay_key] + + if r.timestamp is None: + r.timestamp = time.time() + elif r.timestamp < t2: + changed = len(r.bytes_list) == 0 \ + or r.bytes[-1] != r.bytes_list[-1] + + if changed: + r.bytes_list.append(r.bytes[-1]) + r.times.append(t2) + + r.speed = 1.0 * (r.bytes[1] - r.bytes[0]) / ( + t2 - r.timestamp) + r.timestamp = t2 + r.bytes = [r.bytes[1], r.bytes[1]] + + yield 1.0 + + def on_enter_tunnel(self, circuit_id, candidate, origin, payload): + self.stats['bytes_enter'] += len(payload) + + def on_incoming_from_tunnel(self, community, circuit, origin, data): + self.stats['bytes_returned'] += len(data) + self.circuit_stats[circuit.circuit_id].bytes_down[1] += len(data) + + def on_exiting_from_tunnel(self, circuit_id, candidate, destination, data): + self.stats['bytes_exit'] += len(data) + + valid = False if circuit_id not in self.proxy.circuits \ + else self.proxy.circuits[circuit_id].goal_hops == 0 + + if valid: + self.circuit_stats[circuit_id].bytes_up[-1] += len(data) + + def on_send_data(self, circuit_id, candidate, destination, + payload): + self.circuit_stats[circuit_id].bytes_up[-1] += len(payload) + + def on_relay(self, from_key, to_key, direction, data): + self.relay_stats[from_key].bytes[-1] += len(data) + self.relay_stats[to_key].bytes[-1] += len(data) + + def _create_stats(self): + stats = { + 'uuid': self.session_id.get_bytes_le(), + 'encryption': 0 if isinstance(self.proxy.settings.crypto, NoCrypto) else 1, + 'swift': self.download_stats, + 'bytes_enter': self.stats['bytes_enter'], + 'bytes_exit': self.stats['bytes_exit'], + 'bytes_return': self.stats['bytes_returned'], + 'broken_circuits': self.stats['broken_circuits'], + 'circuits': [ + { + 'hops': self._circuit_cache[circuit_id].goal_hops, + 'bytes_down': c.bytes_down_list[-1] - c.bytes_down_list[0], + 'bytes_up': c.bytes_up_list[-1] - c.bytes_up_list[0], + 'time': c.times[-1] - c.times[0] + } + for circuit_id, c in self.circuit_stats.items() + if len(c.times) >= 2 + ], + 'relays': [ + { + 'bytes': r.bytes_list[-1], + 'time': r.times[-1] - r.times[0] + } + for r in self.relay_stats.values() + if r.times and len(r.times) >= 2 + ] + } + + return stats + + def on_unload(self): + if self.download_stats: + self._logger.error("Sharing statistics now!") + self.share_stats() + + def share_stats(self): + self.proxy.send_stats(self._create_stats()) + + +class StatsDatabase(Database): + LATEST_VERSION = 1 + schema = u""" + CREATE TABLE result ( + "result_id" INTEGER PRIMARY KEY AUTOINCREMENT, + "mid" BLOB, + "session_id" GUID, + "time" DATETIME, + "host" NULL, + "port" NULL, + "swift_size" NULL, + "swift_time" NULL, + "bytes_enter" NULL, + "bytes_exit" NULL, + "bytes_returned" NULL, + "encryption" INTEGER NOT NULL DEFAULT ('0') + , "broken_circuits" INTEGER); + + CREATE TABLE IF NOT EXISTS result_circuit ( + result_circuit_id INTEGER PRIMARY KEY AUTOINCREMENT, + result_id, + hops, + bytes_up, + bytes_down, + time + ); + + CREATE TABLE IF NOT EXISTS result_relay( + result_relay_id INTEGER PRIMARY KEY AUTOINCREMENT, + result_id, + bytes, + time + ); + + CREATE TABLE option(key TEXT PRIMARY KEY, value BLOB); + INSERT INTO option(key, value) VALUES('database_version', '""" + str(LATEST_VERSION) + """'); + """ + + if __debug__: + __doc__ = schema + + def __init__(self, dispersy): + self._dispersy = dispersy + + super(StatsDatabase, self).__init__(os.path.join(dispersy.working_directory, u"anontunnel.db")) + + def open(self, initial_statements=True, prepare_visioning=True): + self._dispersy.database.attach_commit_callback(self.commit) + return super(StatsDatabase, self).open(initial_statements, prepare_visioning) + + def close(self, commit=True): + self._dispersy.database.detach_commit_callback(self.commit) + return super(StatsDatabase, self).close(commit) + + def check_database(self, database_version): + assert isinstance(database_version, unicode) + assert database_version.isdigit() + assert int(database_version) >= 0 + database_version = int(database_version) + + # setup new database with current database_version + if database_version < 1: + self.executescript(self.schema) + self.commit() + + else: + # upgrade to version 2 + if database_version < 2: + # there is no version 2 yet... + # if __debug__: dprint("upgrade database ", database_version, " -> ", 2) + # self.executescript(u"""UPDATE option SET value = '2' WHERE key = 'database_version';""") + # self.commit() + # if __debug__: dprint("upgrade database ", database_version, " -> ", 2, " (done)") + pass + + return self.LATEST_VERSION + + def add_stat(self, member, candidate, stats): + """ + @param Member member: + @param Candidate candidate: + @param stats: + """ + + sock_addr = candidate.sock_addr + + self.execute( + u'''INSERT OR FAIL INTO result + ( + mid, encryption, session_id, time, + host, port, swift_size, swift_time, + bytes_enter, bytes_exit, bytes_returned, broken_circuits + ) + VALUES (?, ?, ?,DATETIME('now'),?,?,?,?,?,?,?,?)''', + ( + buffer(member.mid), + stats['encryption'] or 0, + uuid.UUID(bytes_le=stats['uuid']), + unicode(sock_addr[0]), sock_addr[1], + stats['swift']['size'], stats['swift']['download_time'], + stats['bytes_enter'], stats['bytes_exit'], + (stats['bytes_return'] or 0), + (stats['broken_circuits'] or 0) + ) + ) + + result_id = self.last_insert_rowid + + for circuit in stats['circuits']: + self.execute(u''' + INSERT INTO result_circuit ( + result_id, hops, bytes_up, bytes_down, time + ) VALUES (?, ?, ?, ?, ?)''', + ( + result_id, circuit['hops'], + circuit['bytes_up'], + circuit['bytes_down'], + circuit['time'] + )) + + for relay in stats['relays']: + self.execute(u''' + INSERT INTO result_relay (result_id, bytes, time) + VALUES (?, ?, ?) + ''', (result_id, relay['bytes'], relay['time'])) + + self.commit() + + def get_num_stats(self): + ''' + @rtype: int + @return: number of stats + ''' + + return self.execute(u''' + SELECT COUNT(*) + FROM result + ''').fetchone()[0] + + +class StatsCrawler(TunnelObserver): + """ + Stores incoming stats in a SQLite database + @param RawServer raw_server: the RawServer instance to queue database tasks + on + """ + + def __init__(self, dispersy, raw_server): + TunnelObserver.__init__(self) + self._logger = logging.getLogger(__name__) + self._logger.warning("Running StatsCrawler") + self.raw_server = raw_server + self.database = StatsDatabase(dispersy) + self.raw_server.add_task(lambda: self.database.open()) + + def on_tunnel_stats(self, community, member, candidate, stats): + self.raw_server.add_task(lambda: self.database.add_stat(member, candidate, stats)) + + def get_num_stats(self): + return self.database.get_num_stats() + + def stop(self): + self._logger.error("Stopping crawler") + self.raw_server.add_task(lambda: self.database.close()) \ No newline at end of file diff --git a/Tribler/community/anontunnel/tests/__init__.py b/Tribler/community/anontunnel/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Tribler/community/anontunnel/tests/test_circuit.py b/Tribler/community/anontunnel/tests/test_circuit.py new file mode 100644 index 0000000000..92ee40544a --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_circuit.py @@ -0,0 +1,48 @@ +from unittest import TestCase +import time +from Tribler.community.anontunnel.globals import CIRCUIT_STATE_EXTENDING, \ + CIRCUIT_STATE_READY, CIRCUIT_STATE_BROKEN +from Tribler.community.anontunnel.routing import Circuit, Hop +from Tribler.dispersy.candidate import Candidate + +__author__ = 'chris' + + +class TestCircuit(TestCase): + def test_destroy(self): + candidate = Candidate(("127.0.0.1", 1000), False) + circuit = Circuit(1, 1, candidate) + + self.assertNotEqual(CIRCUIT_STATE_BROKEN, circuit.state, "Newly created circuit should not be considered broken") + circuit.destroy("Because we want to") + self.assertEqual(CIRCUIT_STATE_BROKEN, circuit.state, "Destroyed circuit should be considered broken") + + def test_beat_heart(self): + candidate = Candidate(("127.0.0.1", 1000), False) + circuit = Circuit(1, 1, candidate) + circuit.add_hop(Hop(None)) + + circuit.beat_heart() + self.assertAlmostEqual(time.time(), circuit.last_incoming, delta=0.1, msg="Beat heart should update the last_incoming time") + + def test_state(self): + candidate = Candidate(("127.0.0.1", 1000), False) + + circuit = Circuit(1, 2, candidate) + self.assertNotEqual( + CIRCUIT_STATE_READY, circuit.state, + "Circuit should not be online when goal hops not reached") + + circuit = Circuit(1, 1, candidate) + self.assertNotEqual( + CIRCUIT_STATE_READY, circuit.state, + "Single hop circuit without confirmed first hop should always be offline") + + circuit.add_hop(Hop(None)) + self.assertEqual( + CIRCUIT_STATE_READY, circuit.state, + "Single hop circuit with candidate should always be online") + + circuit = Circuit(0) + self.assertEqual(CIRCUIT_STATE_READY, circuit.state, + "Zero hop circuit should always be online") diff --git a/Tribler/community/anontunnel/tests/test_circuitPool.py b/Tribler/community/anontunnel/tests/test_circuitPool.py new file mode 100644 index 0000000000..d25c1fda79 --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_circuitPool.py @@ -0,0 +1,74 @@ +from unittest import TestCase +from Tribler.community.anontunnel.routing import CircuitPool, Circuit + +__author__ = 'Chris' + + +class TestCircuitPool(TestCase): + def setUp(self): + self.pool = CircuitPool(5, 'test pool') + + def test_on_break_circuit(self): + for i in range(1, 5): + circuit = Circuit(0, 0) + self.pool.fill(circuit) + self.pool.on_break_circuit(circuit) + self.assertNotIn(circuit, self.pool.circuits) + self.assertNotIn(circuit, self.pool.available_circuits) + + def test_lacking(self): + for i in range(1, 5): + circuit = Circuit(0, 0) + self.pool.fill(circuit) + + self.assertEqual(self.pool.lacking, 5-i) + + def test_available_circuits(self): + circuits = [Circuit(0, 0) for _ in range(5)] + + for circuit in circuits: + self.pool.fill(circuit) + self.assertIn(circuit, self.pool.available_circuits) + self.pool.remove_circuit(circuit) + self.assertNotIn(circuit, self.pool.available_circuits) + + def test_remove_circuit(self): + circuits = [Circuit(0, 0) for _ in range(5)] + + for circuit in circuits: + self.pool.fill(circuit) + self.assertIn(circuit, self.pool.circuits) + self.pool.remove_circuit(circuit) + self.assertNotIn(circuit, self.pool.circuits) + + def test_fill(self): + for i in range(1, 5): + circuit = Circuit(0, 0) + self.pool.fill(circuit) + + self.assertIn(circuit, self.pool.circuits) + self.assertIn(circuit, self.pool.available_circuits) + + def test_deallocate(self): + circuits = [Circuit(0, 0) for _ in range(5)] + + for circuit in circuits: + self.pool.fill(circuit) + + for i in range(1, 5): + circuit = self.pool.allocate() + self.assertIn(circuit, self.pool.circuits) + self.assertNotIn(circuit, self.pool.available_circuits) + self.pool.deallocate(circuit) + self.assertIn(circuit, self.pool.available_circuits) + + def test_allocate(self): + circuits = [Circuit(0, 0) for _ in range(5)] + + for circuit in circuits: + self.pool.fill(circuit) + + for i in range(1, 5): + circuit = self.pool.allocate() + self.assertIn(circuit, self.pool.circuits) + self.assertNotIn(circuit, self.pool.available_circuits) \ No newline at end of file diff --git a/Tribler/community/anontunnel/tests/test_defaultCrypto.py b/Tribler/community/anontunnel/tests/test_defaultCrypto.py new file mode 100644 index 0000000000..54b9fa6964 --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_defaultCrypto.py @@ -0,0 +1,223 @@ +import logging.config +import os +import random +from mock import Mock +import time +from Tribler.Test.test_as_server import TestAsServer +from Tribler.community.anontunnel import exitstrategies +from Tribler.community.anontunnel.community import ProxyCommunity, ProxySettings +from Tribler.community.anontunnel.crypto import NoCrypto, DefaultCrypto +from Tribler.community.anontunnel.payload import CreateMessage, ExtendMessage, CreatedMessage +from Tribler.community.anontunnel.routing import Circuit, Hop +from Tribler.community.privatesemantic.conversion import long_to_bytes +from Tribler.dispersy.candidate import WalkCandidate, CANDIDATE_ELIGIBLE_DELAY +from Tribler.dispersy.endpoint import NullEndpoint + +logging.config.fileConfig( + os.path.dirname(os.path.realpath(__file__)) + "/../logger.conf") + +__author__ = 'rutger' + +class DummyEndpoint(NullEndpoint): + def send_simple(self, *args): + pass + +class DummyCandidate(): + def __init__(self, key=None): + #super(DummyCandidate, self).__init__(self) + self.members = [] + self.sock_addr = Mock() + member = Mock() + if not key: + key = self.dispersy.crypto.generate_key(u"NID_secp160k1") + member._ec = key + self.members.append(member) + + def get_members(self): + return self.members + + +class TestDefaultCrypto(TestAsServer): + @property + def crypto(self): + return self.community.settings.crypto + + def setUp(self): + super(TestDefaultCrypto, self).setUp() + self.__candidate_counter = 0 + self.dispersy = self.session.lm.dispersy + + dispersy = self.dispersy + + def load_community(): + keypair = dispersy.crypto.generate_key(u"NID_secp160k1") + dispersy_member = dispersy.get_member(private_key=dispersy.crypto.key_to_bin(keypair)) + + settings = ProxySettings() + settings.crypto = DefaultCrypto() + + proxy_community = dispersy.define_auto_load(ProxyCommunity, dispersy_member, (settings, None), load=True)[0] + exitstrategies.DefaultExitStrategy(self.session.lm.rawserver, proxy_community) + + return proxy_community + + self.community = dispersy.callback.call(load_community) + ''' :type : ProxyCommunity ''' + + def setUpPreSession(self): + super(TestDefaultCrypto, self).setUpPreSession() + self.config.set_dispersy(True) + + def __create_walk_candidate(self): + def __create(): + self.__candidate_counter += 1 + wan_address = ("8.8.8.{0}".format(self.__candidate_counter), self.__candidate_counter) + lan_address = ("0.0.0.0", 0) + candidate = WalkCandidate(wan_address, False, lan_address, wan_address, u'unknown') + + + key = self.dispersy.crypto.generate_key(u"NID_secp160k1") + member = self.dispersy.get_member(public_key=self.dispersy.crypto.key_to_bin(key.pub())) + candidate.associate(member) + + now = time.time() + candidate.walk(now - CANDIDATE_ELIGIBLE_DELAY) + candidate.walk_response(now) + return candidate + + return self.dispersy.callback.call(__create) + + def __prepare_for_create(self): + self.crypto.key_to_forward = '0' * 16 + + def __prepare_for_created(self, candidate, circuit_id): + dh_key = self.crypto._generate_diffie_secret() + hop = Hop(self.community.my_member._ec.pub()) + hop.dh_secret = dh_key[0] + hop.dh_first_part = dh_key[1] + self.community.circuits[circuit_id] = Circuit(circuit_id, 1, candidate, self.community) + self.community.circuits[circuit_id].unverified_hop = hop + self.crypto._received_secrets[(candidate.sock_addr, circuit_id)] = dh_key[1] + + def __generate_candidate_list(self): + list = {} + list['a'] = 'A' + list['b'] = 'B' + return list + + def __add_circuit(self, circuit_id): + self.crypto.proxy.circuits[circuit_id] = Circuit(circuit_id) + + def __add_relay(self, relay_key=None): + if not relay_key: + circuit_id = random.randint(1000) + relay_key = (Mock(), circuit_id) + self.crypto.session_keys[relay_key] = "1" + + def test_on_break_relay_existing_key(self): + relay_key = ("a", "b") + self.crypto.session_keys[relay_key] = "test" + self.crypto.on_break_relay(relay_key) + self.assertNotIn(relay_key, self.crypto.session_keys) + + def test_on_break_relay_non_existing_key(self): + relay_key = ("a", "b") + self.assertNotIn(relay_key, self.crypto.session_keys) + + def test_on_break_relay_different_keys(self): + relay_key = ("a", "b") + second_relay_key = ("b", "a") + self.crypto.session_keys[relay_key] = "test" + self.crypto.on_break_relay(second_relay_key) + self.assertIn(relay_key, self.crypto.session_keys) + self.assertNotIn(second_relay_key, self.crypto.session_keys) + + def test__encrypt_decrypt_create_content(self): + #test own circuit create + candidate = DummyCandidate(self.community.my_member._ec) + + create_message = CreateMessage() + circuit_id = 123 + self.community.circuits[123] = Circuit(123, 1, candidate, self.community) + hop = Hop(self.community.my_member._ec.pub()) + self.community.circuits[123].unverified_hop = hop + + encrypted_create_message = \ + self.crypto._encrypt_create_content(candidate, circuit_id, create_message) + + unverified_hop = self.community.circuits[123].unverified_hop + unencrypted_key = unverified_hop.dh_first_part + unencrypted_pub_key = self.community.crypto.key_to_bin(self.community.my_member._ec.pub()) + self.assertNotEquals(unencrypted_key, encrypted_create_message.key) + + decrypted_create_message = self.crypto._decrypt_create_content(candidate, circuit_id, encrypted_create_message) + + self.assertEquals(unencrypted_key, decrypted_create_message.key) + self.assertEquals(unencrypted_pub_key, decrypted_create_message.public_key) + + #test other circuit create + self.__prepare_for_create() + del self.community.circuits[123] + candidate = DummyCandidate(self.community.my_member._ec) + + create_message = CreateMessage() + circuit_id = 123 + self.community.circuits[123] = Circuit(123, 1, candidate, self.community) + hop = Hop(self.community.my_member._ec.pub()) + self.community.circuits[123].unverified_hop = hop + + encrypted_create_message = \ + self.crypto._encrypt_create_content(candidate, circuit_id, create_message) + + unverified_hop = self.community.circuits[123].unverified_hop + unencrypted_key = unverified_hop.dh_first_part + unencrypted_pub_key = self.community.crypto.key_to_bin(self.community.my_member._ec.pub()) + self.assertNotEquals(unencrypted_key, encrypted_create_message.key) + + decrypted_create_message = self.crypto._decrypt_create_content(candidate, circuit_id, encrypted_create_message) + + self.assertEquals(unencrypted_key, decrypted_create_message.key) + self.assertEquals(unencrypted_pub_key, decrypted_create_message.public_key) + + + def test__encrypt_decrypt_extend_content(self): + candidate = DummyCandidate(self.community.my_member._ec) + + extend_message = ExtendMessage(self.community.my_member.mid) + circuit_id = 123 + self.community.circuits[123] = Circuit(123, 1, candidate, self.community) + hop = Hop(self.community.my_member._ec.pub()) + self.community.circuits[123].unverified_hop = hop + + encrypted_extend_message = \ + self.crypto._encrypt_extend_content(candidate, circuit_id, extend_message) + + unverified_hop = self.community.circuits[123].unverified_hop + unencrypted_key = unverified_hop.dh_first_part + self.assertNotEquals(unencrypted_key, encrypted_extend_message.key) + + decrypted_extend_message = self.crypto._decrypt_create_content(candidate, circuit_id, encrypted_extend_message) + + self.assertEquals(unencrypted_key, decrypted_extend_message.key) + self.assertEquals(self.community.my_member.mid, decrypted_extend_message.extend_with) + + + def test__encrypt_decrypt_created_content(self): + candidate = DummyCandidate(self.community.my_member._ec) + candidate_list = self.__generate_candidate_list() + + circuit_id = 123 + self.__prepare_for_created(candidate, circuit_id) + + created_message = CreatedMessage(candidate_list) + + encrypted_created_message = \ + self.crypto._encrypt_created_content(candidate, circuit_id, created_message) + + unverified_hop = self.community.circuits[circuit_id].unverified_hop + unencrypted_key = unverified_hop.dh_first_part + self.assertNotEquals(unencrypted_key, encrypted_created_message.key) + + decrypted_created_message = self.crypto._decrypt_created_content(candidate, circuit_id, encrypted_created_message) + + self.assertEquals(candidate_list, decrypted_created_message.candidate_list) diff --git a/Tribler/community/anontunnel/tests/test_defaultExitStrategy.py b/Tribler/community/anontunnel/tests/test_defaultExitStrategy.py new file mode 100644 index 0000000000..51a905fbf1 --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_defaultExitStrategy.py @@ -0,0 +1,80 @@ +from threading import Event +from unittest import TestCase +from mock import Mock +from Tribler.Core.RawServer.RawServer import RawServer +from Tribler.community.anontunnel.exitsocket import ShortCircuitExitSocket, TunnelExitSocket +from Tribler.community.anontunnel.exitstrategies import DefaultExitStrategy +from Tribler.community.anontunnel.payload import DataMessage +from Tribler.community.anontunnel.routing import Circuit, Hop + +__author__ = 'Chris' + + +class TestDefaultExitStrategy(TestCase): + def setUp(self): + self.socket = Mock() + self.socket.sendto = Mock(return_value=True) + + raw_server = Mock() + raw_server.create_udpsocket = Mock(return_value=self.socket) + raw_server.start_listening_udp = Mock(return_value=None) + + self.raw_server = raw_server + self.__create_counter = 0 + + def __create_circuit(self, hops): + circuit = Circuit(self.__create_counter, hops) + for _ in range(hops): + circuit.add_hop(Hop(None)) + + self.__create_counter += 1 + return circuit + + def test_on_exiting_from_tunnel(self): + proxy = Mock() + proxy.circuits = [ + self.__create_circuit(0), + self.__create_circuit(1), + ] + + proxy.on_data = Mock() + + return_candidate = Mock() + destination = ("google.com", 80) + data = "Hello world" + + strategy = DefaultExitStrategy(self.raw_server, proxy) + strategy.on_exiting_from_tunnel(proxy.circuits[0].circuit_id, return_candidate, destination, data) + self.socket.sendto.assert_called_with(data, destination) + + def on_create(self): + proxy = Mock() + proxy.circuits = [ + self.__create_circuit(0), + self.__create_circuit(1), + ] + + destination = ("google.com", 80) + + strategy = DefaultExitStrategy(self.raw_server, proxy) + + exit_socket = strategy.create(proxy, self.raw_server, proxy.circuits[0].circuit_id, destination) + self.assertIsInstance(exit_socket, ShortCircuitExitSocket) + + exit_socket = strategy.create(proxy, self.raw_server, proxy.circuits[1].circuit_id, destination) + self.assertIsInstance(exit_socket, TunnelExitSocket) + + def on_get_socket(self): + proxy = Mock() + proxy.circuits = [ + self.__create_circuit(0), + self.__create_circuit(1), + ] + + destination = ("google.com", 80) + + strategy = DefaultExitStrategy(self.raw_server, proxy) + + exit_socket = strategy.get_exit_socket(proxy.circuits[0].circuit_id, destination) + exit_socket2 = strategy.get_exit_socket(proxy.circuits[0].circuit_id, destination) + self.assertEqual(exit_socket, exit_socket2, "Subsequent exit packets to the same destination should use the exact same exit socket") \ No newline at end of file diff --git a/Tribler/community/anontunnel/tests/test_dispersyBypassEndpoint.py b/Tribler/community/anontunnel/tests/test_dispersyBypassEndpoint.py new file mode 100644 index 0000000000..cdb2fa3a1f --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_dispersyBypassEndpoint.py @@ -0,0 +1,36 @@ +import threading +from unittest import TestCase +from Tribler.Core.RawServer.RawServer import RawServer +from Tribler.community.anontunnel.endpoint import DispersyBypassEndpoint + +__author__ = 'chris' + + +class TestDispersyBypassEndpoint(TestCase): + def on_bypass_message(self, sock_addr, payload): + """ + + @param (str, int) sock_addr: ip address + @param str payload: the payload + """ + self.bypass_message = (sock_addr, payload) + + def setUp(self): + self.succeed = False + + done_flag = threading.Event() + raw_server = RawServer(done_flag, 10, 5) + + self.endpoint = DispersyBypassEndpoint(raw_server, 0) + + def test_data_came_in(self): + prefix = str(('f' * 23 + 'e').decode("HEX")) + self.endpoint.listen_to(prefix, self.on_bypass_message) + + packet = ( + ("127.0.0.1", 100), + prefix + "Hello world!" + ) + + self.endpoint.data_came_in([packet]) + self.assertEqual(self.bypass_message, packet) \ No newline at end of file diff --git a/Tribler/community/anontunnel/tests/test_extendStrategies.py b/Tribler/community/anontunnel/tests/test_extendStrategies.py new file mode 100644 index 0000000000..b2893cd507 --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_extendStrategies.py @@ -0,0 +1,60 @@ +from unittest import TestCase +from Tribler.community.anontunnel.extendstrategies import TrustThyNeighbour +from Tribler.community.anontunnel.globals import MESSAGE_EXTEND, \ + CIRCUIT_STATE_BROKEN +from Tribler.community.anontunnel.payload import ExtendMessage +from Tribler.community.anontunnel.routing import Circuit, Hop +from Tribler.dispersy.candidate import Candidate + +__author__ = 'chris' + + +class ProxyMock: + def __init__(self): + self.message = None + + def send_message(self, *args): + self.message = args + + +#noinspection PyTypeChecker,PyTypeChecker +class TestTrustThyNeighbour(TestCase): + def setUp(self): + self.proxy = ProxyMock() + + def test_extend_ready_circuit(self): + circuit_candidate = Candidate(("127.0.0.1", 1000), False) + circuit = Circuit(1, 1, circuit_candidate) + circuit.add_hop(Hop(None)) + + es = TrustThyNeighbour(self.proxy, circuit) + self.assertRaises(AssertionError, es.extend) + + def test_extend_broken_circuit(self): + circuit_candidate = Candidate(("127.0.0.1", 1000), False) + circuit = Circuit(1, 1, circuit_candidate) + + # Break circuit + circuit.destroy() + self.assertEqual(circuit.state, CIRCUIT_STATE_BROKEN) + + es = TrustThyNeighbour(self.proxy, circuit) + self.assertRaises(AssertionError, es.extend) + + def test_extend_extending_circuit(self): + circuit_candidate = Candidate(("127.0.0.1", 1000), False) + circuit = Circuit(1, 2, circuit_candidate) + es = TrustThyNeighbour(self.proxy, circuit) + es.extend() + + self.assertIsInstance(self.proxy.message, tuple) + + candidate, circuit_id, message_type, message = self.proxy.message + + self.assertEqual(candidate, circuit.candidate, + "Candidate should be first hop of circuit") + self.assertEqual(circuit_id, circuit.circuit_id, + "Circuit_id should be circuit's id") + self.assertEqual(message_type, MESSAGE_EXTEND, + "Send message should be an extend type") + self.assertIsInstance(message, ExtendMessage) diff --git a/Tribler/community/anontunnel/tests/test_lengthStrategies.py b/Tribler/community/anontunnel/tests/test_lengthStrategies.py new file mode 100644 index 0000000000..42f91c669e --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_lengthStrategies.py @@ -0,0 +1,22 @@ +from unittest import TestCase +from Tribler.community.anontunnel.lengthstrategies import \ + RandomCircuitLengthStrategy, ConstantCircuitLength + +__author__ = 'chris' + + +class TestRandomCircuitLengthStrategy(TestCase): + def test_circuit_length(self): + ls = RandomCircuitLengthStrategy(3, 100) + self.assertGreaterEqual(ls.circuit_length(), 3, "Should be at least 3") + self.assertLessEqual(ls.circuit_length(), 100, + "Should be at least 100") + + ls = RandomCircuitLengthStrategy(3, 3) + self.assertEqual(ls.circuit_length(), 3, "Should be 3 exactly") + + +class TestConstantCircuitLengthStrategy(TestCase): + def test_circuit_length(self): + ls = ConstantCircuitLength(42) + self.assertEqual(ls.circuit_length(), 42, "Should be 42 exactly") diff --git a/Tribler/community/anontunnel/tests/test_libtorrent.py b/Tribler/community/anontunnel/tests/test_libtorrent.py new file mode 100644 index 0000000000..ee7da5afcb --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_libtorrent.py @@ -0,0 +1,128 @@ +import logging +import time +from Tribler.community.anontunnel.events import TunnelObserver +import shutil + + +class LibtorrentTest(TunnelObserver): + """ + @param ProxyCommunity proxy : The proxy community instance + @param Tribler.Core.Session.Session tribler_session: The Tribler Session + """ + + def __init__(self, proxy, tribler_session, delay): + super(LibtorrentTest, self).__init__() + + self._logger = logging.getLogger(__name__) + self.proxy = proxy + self.tribler_session = tribler_session + self.delay = delay + self.tribler_session.lm.rawserver.add_task(self.schedule) + self.download = None + self.stopping = False + + self.download_started_at = None + self.download_finished_at = None + + def schedule(self): + self._logger.debug("Scheduling Anonymous LibTorrent download") + self.tribler_session.lm.rawserver.add_task(self.start, self.delay) + + def _mark_test_completed(self): + filename = self.tribler_session.get_state_dir() + "/anon_test.txt" + handle = open(filename, "w") + + try: + handle.write("Delete this file to redo the anonymous download test") + finally: + handle.close() + + def on_unload(self): + self.stop() + + def stop(self, delay=0.0): + if self.download: + def remove_download(): + self.tribler_session.remove_download(self.download, True, True) + self.download = None + self._logger.error("Removed test download") + + self.tribler_session.lm.rawserver.add_task(remove_download, delay=delay) + + def _has_completed_before(self): + return False # os.path.isfile(self.tribler_session.get_state_dir() + "/anon_test.txt") + + def start(self): + from Tribler.community.anontunnel.stats import StatsCollector + import wx + from Tribler.Core.TorrentDef import TorrentDef + from Tribler.Core.simpledefs import DLSTATUS_DOWNLOADING, DLSTATUS_SEEDING + from Tribler.Main.globals import DefaultDownloadStartupConfig + from Tribler.Main.vwxGUI import forceWxThread + + hosts = [("94.23.38.156", 51413), ("95.211.198.147", 51413), ("95.211.198.142", 51413), ("95.211.198.140", 51413), ("95.211.198.141", 51413)] + + @forceWxThread + def thank_you(file_size, start_time, end_time): + avg_speed_KBps = 1.0 * file_size / (end_time - start_time) / 1024.0 + wx.MessageBox('Your average speed was %.2f KB/s' % (avg_speed_KBps) , 'Download Completed', wx.OK | wx.ICON_INFORMATION) + + def state_call(): + stats_collector = StatsCollector(self.proxy, "AnonTest") + + def _callback(ds): + if self.stopping: + return 1.0, False + + if ds.get_status() == DLSTATUS_DOWNLOADING: + if not self.download_started_at: + self.download_started_at = time.time() + stats_collector.start() + + stats_collector.download_stats = { + 'size': ds.get_progress() * ds.get_length(), + 'download_time': time.time() - self.download_started_at + } + + elif ds.get_status() == DLSTATUS_SEEDING and self.download_started_at and not self.download_finished_at: + self.download_finished_at = time.time() + stats_collector.download_stats = { + 'size': ds.get_length(), + 'download_time': self.download_finished_at - self.download_started_at + } + + stats_collector.share_stats() + stats_collector.stop() + self.stop(5.0) + + self._mark_test_completed() + + thank_you(ds.get_length(), self.download_started_at, self.download_finished_at) + return 1.0, False + + return _callback + + if self._has_completed_before(): + self._logger.warning("Skipping Anon Test since it has been run before") + return False + + destination_dir = self.tribler_session.get_state_dir() + "/anon_test/" + + try: + shutil.rmtree(destination_dir) + except: + pass + + tdef = TorrentDef.load("anon_test.torrent") + defaultDLConfig = DefaultDownloadStartupConfig.getInstance() + dscfg = defaultDLConfig.copy() + ''' :type : DefaultDownloadStartupConfig ''' + + dscfg.set_anon_mode(True) + dscfg.set_dest_dir(destination_dir) + + self.download = self.tribler_session.start_download(tdef, dscfg) + self.download.set_state_callback(state_call(), delay=1) + + for peer in hosts: + self.download.add_peer(peer) diff --git a/Tribler/community/anontunnel/tests/test_proxyCommunity.py b/Tribler/community/anontunnel/tests/test_proxyCommunity.py new file mode 100644 index 0000000000..e3bd63d088 --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_proxyCommunity.py @@ -0,0 +1,269 @@ +import logging.config +import os +import time +from mock import Mock +from Tribler.Test.test_as_server import TestAsServer +from Tribler.community.anontunnel import exitstrategies +from Tribler.community.anontunnel.community import ProxyCommunity, \ + ProxySettings +from Tribler.community.anontunnel.crypto import NoCrypto +from Tribler.community.anontunnel.events import TunnelObserver +from Tribler.community.anontunnel.globals import MESSAGE_CREATED, MESSAGE_CREATE, \ + CIRCUIT_STATE_READY, CIRCUIT_STATE_EXTENDING, CIRCUIT_STATE_BROKEN, MESSAGE_EXTEND, \ + MESSAGE_PONG +from Tribler.community.anontunnel.payload import CreateMessage, CreatedMessage, ExtendedMessage, ExtendMessage, \ + DataMessage, PingMessage, PongMessage +from Tribler.community.anontunnel.routing import Circuit +from Tribler.dispersy.candidate import WalkCandidate, CANDIDATE_ELIGIBLE_DELAY +from Tribler.dispersy.endpoint import NullEndpoint + +__author__ = 'Chris' + +logging.config.fileConfig( + os.path.dirname(os.path.realpath(__file__)) + "/../logger.conf") + + +class DummyEndpoint(NullEndpoint): + def send_simple(self, *args): + pass + + +class TestProxyCommunity(TestAsServer): + def setUp(self): + super(TestProxyCommunity, self).setUp() + self.__candidate_counter = 0 + self.dispersy = self.session.lm.dispersy + + dispersy = self.dispersy + + def load_community(): + keypair = dispersy.crypto.generate_key(u"NID_secp160k1") + dispersy_member = dispersy.get_member(private_key=dispersy.crypto.key_to_bin(keypair)) + + settings = ProxySettings() + settings.crypto = NoCrypto() + + proxy_community = dispersy.define_auto_load(ProxyCommunity, dispersy_member, (settings, None), load=True)[0] + exit_strategy = exitstrategies.DefaultExitStrategy(self.session.lm.rawserver, proxy_community) + proxy_community.observers.append(exit_strategy) + + return proxy_community + + self.community = dispersy.callback.call(load_community) + ''' :type : ProxyCommunity ''' + + def setUpPreSession(self): + super(TestProxyCommunity, self).setUpPreSession() + self.config.set_dispersy(True) + + def __create_walk_candidate(self): + def __create(): + self.__candidate_counter += 1 + wan_address = ("8.8.8.{0}".format(self.__candidate_counter), self.__candidate_counter) + lan_address = ("0.0.0.0", 0) + candidate = WalkCandidate(wan_address, False, lan_address, wan_address, u'unknown') + + key = self.dispersy.crypto.generate_key(u"NID_secp160k1") + member = self.dispersy.get_member(public_key=self.dispersy.crypto.key_to_bin(key.pub())) + candidate.associate(member) + + now = time.time() + candidate.walk(now - CANDIDATE_ELIGIBLE_DELAY) + candidate.walk_response(now) + return candidate + + return self.dispersy.callback.call(__create) + + def test_on_create(self): + create_sender = self.__create_walk_candidate() + + create_message = CreateMessage() + circuit_id = 1337 + + self.community.send_message = send_message = Mock() + self.community.on_create(circuit_id, create_sender, create_message) + + args, keyargs = send_message.call_args + + self.assertEqual(create_sender, keyargs['destination']) + self.assertEqual(circuit_id, keyargs['circuit_id']) + self.assertEqual(MESSAGE_CREATED, keyargs['message_type']) + self.assertIsInstance(keyargs['message'], CreatedMessage) + + def test_create_circuit(self): + create_sender = self.__create_walk_candidate() + + self.assertRaises(ValueError, self.community.create_circuit, create_sender, 0) + + self.community.send_message = send_message = Mock() + + hops = 1 + circuit = self.community.create_circuit(create_sender, hops) + + # Newly created circuit should be stored in circuits dict + self.assertIsInstance(circuit, Circuit) + self.assertEqual(create_sender, circuit.candidate) + self.assertEqual(hops, circuit.goal_hops) + self.assertIn(circuit.circuit_id, self.community.circuits) + self.assertEqual(circuit, self.community.circuits[circuit.circuit_id]) + self.assertEqual(CIRCUIT_STATE_EXTENDING, circuit.state) + + # We must have sent a CREATE message to the candidate in question + args, kwargs = send_message.call_args + destination, reply_circuit, message_type, created_message = args + self.assertEqual(circuit.circuit_id, reply_circuit) + self.assertEqual(create_sender, destination) + self.assertEqual(MESSAGE_CREATE, message_type) + self.assertIsInstance(created_message, CreateMessage) + + def test_on_created(self): + first_hop = self.__create_walk_candidate() + circuit = self.community.create_circuit(first_hop, 1) + + self.community.on_created(circuit.circuit_id, first_hop, CreatedMessage([])) + self.assertEqual(CIRCUIT_STATE_READY, circuit.state) + + def test_on_extended(self): + # 2 Hop - should fail due to no extend candidates + first_hop = self.__create_walk_candidate() + circuit = self.community.create_circuit(first_hop, 2) + + result = self.community.on_created(circuit.circuit_id, first_hop, CreatedMessage([])) + self.assertFalse(result) + self.assertEqual(CIRCUIT_STATE_BROKEN, circuit.state) + + # 2 Hop - should succeed + second_hop = self.__create_walk_candidate() + + public_bin = next(iter(second_hop.get_members())).public_key + key = next(iter(second_hop.get_members()))._ec + + candidate_list = [] + candidate_list.append(self.community.crypto.key_to_bin(key)) + circuit = self.community.create_circuit(first_hop, 2) + + self.community.send_message = send_message = Mock() + + result = self.community.on_created(circuit.circuit_id, first_hop, CreatedMessage(candidate_list)) + self.assertTrue(result) + + # ProxyCommunity should send an EXTEND message with the hash of second_hop's pub-key + args, kwargs = send_message.call_args + circuit_candidate, circuit_id, message_type, message = args + + self.assertEqual(first_hop, circuit_candidate) + self.assertEqual(circuit.circuit_id, circuit_id) + self.assertEqual(MESSAGE_EXTEND, message_type) + self.assertIsInstance(message, ExtendMessage) + self.assertEqual(message.extend_with, public_bin) + + # Upon reception of the ON_EXTENDED the circuit should reach it full 2-hop length and thus be ready for use + result = self.community.on_extended(circuit.circuit_id, first_hop, ExtendedMessage("", [])) + self.assertTrue(result) + self.assertEqual(CIRCUIT_STATE_READY, circuit.state) + + def test_remove_circuit(self): + first_hop = self.__create_walk_candidate() + circuit = self.community.create_circuit(first_hop, 1) + + self.assertIn(circuit.circuit_id, self.community.circuits) + self.community.remove_circuit(circuit.circuit_id) + self.assertNotIn(circuit, self.community.circuits) + + def test_on_data(self): + first_hop = self.__create_walk_candidate() + circuit = self.community.create_circuit(first_hop, 1) + self.community.on_created(circuit.circuit_id, first_hop, CreatedMessage([])) + + payload = "Hello world" + origin = ("google.com", 80) + data_message = DataMessage(None, payload, origin=origin) + + observer = TunnelObserver() + observer.on_incoming_from_tunnel = on_incoming_from_tunnel = Mock() + self.community.observers.append(observer) + + # Its on our own circuit so it should trigger the on_incoming_from_tunnel event + self.community.on_data(circuit.circuit_id, first_hop, data_message) + on_incoming_from_tunnel.assert_called_with(self.community, circuit, origin, payload) + + # Not our own circuit so we need to exit it + destination = ("google.com", 80) + exit_message = DataMessage(destination, payload, origin=None) + observer.on_exiting_from_tunnel = on_exiting_from_tunnel = Mock() + self.community.on_data(1337, first_hop, exit_message) + on_exiting_from_tunnel.assert_called_with(1337, first_hop, destination, payload) + + def test_on_extend(self): + # We mimick the intermediary hop ( ORIGINATOR - INTERMEDIARY - NODE_TO_EXTEND_ORIGINATORS_CIRCUIT_WITH ) + originator = self.__create_walk_candidate() + node_to_extend_with = self.__create_walk_candidate() + originator_circuit_id = 1337 + + extend_pub_key = next(iter(node_to_extend_with.get_members()))._ec + extend_pub_key = self.dispersy.crypto.key_to_bin(extend_pub_key) + + # make sure our node_to_extend_with comes up when yielding verified candidates + self.dispersy.callback.call(self.community.add_candidate, (node_to_extend_with,)) + self.assertIn(node_to_extend_with, self.community._candidates.itervalues()) + + self.community.send_message = send_message = Mock() + self.community.on_create(originator_circuit_id, originator, CreateMessage()) + + # Check whether we are sending node_to_extend_with in the CreatedMessage reply + args, kwargs = send_message.call_args + created_message = kwargs['message'] + candidate_dict = created_message.candidate_list + self.assertIsInstance(created_message, CreatedMessage) + self.assertIn(extend_pub_key, candidate_dict) + + self.community.on_extend(originator_circuit_id, originator, ExtendMessage(extend_pub_key)) + + # Check whether we are sending a CREATE to node_to_extend_with + args, kwargs = send_message.call_args + create_destination, circuit_id, message_type, message = args + self.assertEqual(node_to_extend_with, create_destination) + self.assertEqual(MESSAGE_CREATE, message_type) + self.assertIsInstance(message, CreateMessage) + + # Check whether the routing table has been updated + relay_from_originator = (originator.sock_addr, originator_circuit_id) + relay_from_endpoint = (node_to_extend_with.sock_addr, circuit_id) + + self.assertIn(relay_from_originator, self.community.relay_from_to) + self.assertIn(relay_from_endpoint, self.community.relay_from_to) + + def test_on_pong(self): + first_hop = self.__create_walk_candidate() + circuit = self.community.create_circuit(first_hop, 1) + self.community.on_created(circuit.circuit_id, first_hop, CreatedMessage({})) + + result = self.community.on_pong(circuit.circuit_id, first_hop, PongMessage()) + self.assertFalse(result, "Cannot handle a pong when we never sent a PING") + + self.community.create_ping(first_hop, circuit) + + # Check whether the circuit last incoming time is correct after the pong + circuit.last_incoming = 0 + result = self.community.on_pong(circuit.circuit_id, first_hop, PongMessage()) + self.assertTrue(result) + + self.assertAlmostEqual(circuit.last_incoming, time.time(), delta=0.5) + + def test_on_ping(self): + circuit_id = 1337 + first_hop = self.__create_walk_candidate() + self.community.add_candidate(first_hop) + + self.community.on_create(circuit_id, first_hop, CreateMessage()) + + self.community.send_message = send_message = Mock() + self.community.on_ping(circuit_id, first_hop, PingMessage()) + + # Check whether we responded with a pong + args, kwargs = send_message.call_args + + self.assertEqual(first_hop, kwargs['destination']) + self.assertEqual(circuit_id, kwargs['circuit_id']) + self.assertEqual(MESSAGE_PONG, kwargs['message_type']) + self.assertIsInstance(kwargs['message'], PongMessage) \ No newline at end of file diff --git a/Tribler/community/anontunnel/tests/test_selectionStrategies.py b/Tribler/community/anontunnel/tests/test_selectionStrategies.py new file mode 100644 index 0000000000..ac00129230 --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_selectionStrategies.py @@ -0,0 +1,63 @@ +from random import randint +from unittest import TestCase +from Tribler.community.anontunnel.routing import Circuit, Hop + +from Tribler.community.anontunnel.selectionstrategies import \ + LengthSelectionStrategy, RandomSelectionStrategy +from Tribler.dispersy.candidate import Candidate + + +__author__ = 'chris' + + +class TestLengthSelectionStrategy(TestCase): + def setUp(self): + self.circuits = [self.__circuit(0), self.__circuit(1), + self.__circuit(2), self.__circuit(3), + self.__circuit(4)] + + @staticmethod + def __circuit(hops): + candidate = Candidate(("127.0.0.1", 1000), False) + + circuit = Circuit(randint(0, 1000), hops, candidate) + for c in [candidate] * hops: + circuit.add_hop(Hop(None)) + + return circuit + + def test_select(self): + cs = LengthSelectionStrategy(3, 3) + self.assertEqual(cs.select(self.circuits), self.circuits[3]) + + cs = LengthSelectionStrategy(0, 3) + self.assertIn(cs.select(self.circuits), self.circuits[0:4]) + + cs = LengthSelectionStrategy(1, 3) + self.assertIn(cs.select(self.circuits), self.circuits[1:3]) + + cs = LengthSelectionStrategy(5, 10) + self.assertRaises(ValueError, cs.select, self.circuits) + + +class TestRandomSelectionStrategy(TestCase): + def setUp(self): + self.circuits = [self.__circuit(1), self.__circuit(2), + self.__circuit(3), self.__circuit(4)] + + @staticmethod + def __circuit(hops): + candidate = Candidate(("127.0.0.1", 1000), False) + + circuit = Circuit(randint(0, 1000), hops, candidate) + for c in [candidate] * hops: + circuit.add_hop = c + + return circuit + + def test_select(self): + cs = RandomSelectionStrategy() + self.assertIsInstance(cs.select(self.circuits), Circuit) + + # Cannot select from empty list + self.assertRaises(ValueError, cs.select, []) \ No newline at end of file diff --git a/Tribler/community/anontunnel/tests/test_shortCircuitExitSocket.py b/Tribler/community/anontunnel/tests/test_shortCircuitExitSocket.py new file mode 100644 index 0000000000..03b07b1926 --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_shortCircuitExitSocket.py @@ -0,0 +1,48 @@ +from unittest import TestCase +from mock import Mock +from Tribler.community.anontunnel.exitsocket import ShortCircuitExitSocket +from Tribler.community.anontunnel.payload import DataMessage + +__author__ = 'Chris' + + +class TestShortCircuitExitSocket(TestCase): + def setUp(self): + + self.socket = Mock() + self.socket.sendto = Mock(return_value=True) + + raw_server = Mock() + raw_server.create_udpsocket = Mock(return_value=self.socket) + raw_server.start_listening_udp = Mock(return_value=None) + + self.circuit_id = 123 + + self.proxy = Mock() + self.proxy.on_data = Mock() + + self.return_address = ("127.0.0.1", 1337) + self.exit_socket = ShortCircuitExitSocket(raw_server, self.proxy, self.circuit_id, self.return_address) + + def test_data_came_in(self): + packet = "Hello world" + source_address = ("google.com", 80) + self.exit_socket.data_came_in([(source_address, packet)]) + + args, kwargs = self.proxy.on_data.call_args + circuit_id, none, message = args + expected_message = DataMessage(("0.0.0.0", 0), packet, source_address) + + self.assertEqual(self.circuit_id, circuit_id) + self.assertIsNone(none) + self.assertEqual(expected_message.data, message.data) + self.assertEqual(expected_message.destination, message.destination) + self.assertEqual(expected_message.origin, message.origin) + + def test_sendto(self): + data = "Hello world" + destination = ("google.com", 80) + self.exit_socket.sendto(data, destination) + + # The underlying socket must be called by the ExitSocket + self.socket.sendto.assert_called_with(data, destination) \ No newline at end of file diff --git a/Tribler/community/anontunnel/tests/test_tunnelExitSocket.py b/Tribler/community/anontunnel/tests/test_tunnelExitSocket.py new file mode 100644 index 0000000000..35f95e4375 --- /dev/null +++ b/Tribler/community/anontunnel/tests/test_tunnelExitSocket.py @@ -0,0 +1,45 @@ +from unittest import TestCase +from mock import Mock +from Tribler.community.anontunnel.exitsocket import TunnelExitSocket + +__author__ = 'Chris' + + +class TestTunnelExitSocket(TestCase): + def setUp(self): + + self.socket = Mock() + self.socket.sendto = Mock(return_value=True) + + raw_server = Mock() + raw_server.create_udpsocket = Mock(return_value=self.socket) + raw_server.start_listening_udp = Mock(return_value=None) + + self.circuit_id = 123 + + self.proxy = Mock() + self.proxy.tunnel_data_to_origin = Mock() + + self.return_address = ("127.0.0.1", 1337) + self.exit_socket = TunnelExitSocket(raw_server, self.proxy, self.circuit_id, self.return_address) + + def test_data_came_in(self): + packet = "Hello world" + source_address = ("google.com", 80) + self.exit_socket.data_came_in([(source_address, packet)]) + + # Incoming packets must be routed back using the proxy + self.proxy.tunnel_data_to_origin.assert_called_with( + circuit_id=self.circuit_id, + candidate=self.return_address, + source_address=source_address, + payload=packet + ) + + def test_sendto(self): + data = "Hello world" + destination = ("google.com", 80) + self.exit_socket.sendto(data, destination) + + # The underlying socket must be called by the ExitSocket + self.socket.sendto.assert_called_with(data, destination) \ No newline at end of file diff --git a/Tribler/community/privatesemantic/crypto/optional_crypto.py b/Tribler/community/privatesemantic/crypto/optional_crypto.py index 6f7bfb6aa2..768e7fc894 100644 --- a/Tribler/community/privatesemantic/crypto/optional_crypto.py +++ b/Tribler/community/privatesemantic/crypto/optional_crypto.py @@ -37,25 +37,34 @@ def invert(x, m): from Crypto.Cipher import AES def aes_encrypt_str(aes_key, plain_str): - cipher = AES.new(long_to_bytes(aes_key, 16), AES.MODE_CFB, '\x00' * 16) + if isinstance(aes_key, long): + aes_key = long_to_bytes(aes_key, 16) + cipher = AES.new(aes_key, AES.MODE_CFB, '\x00' * 16) return cipher.encrypt(plain_str) def aes_decrypt_str(aes_key, encr_str): - cipher = AES.new(long_to_bytes(aes_key, 16), AES.MODE_CFB, '\x00' * 16) + if isinstance(aes_key, long): + aes_key = long_to_bytes(aes_key, 16) + cipher = AES.new(aes_key, AES.MODE_CFB, '\x00' * 16) return cipher.decrypt(encr_str) except ImportError: from random import Random as StrongRandom from Tribler.community.privatesemantic.conversion import long_to_bytes + from Tribler.dispersy.decorator import attach_runtime_statistics from M2Crypto import EVP def aes_encrypt_str(aes_key, plain_str): - cipher = EVP.Cipher(alg='aes_128_cfb', key=long_to_bytes(aes_key, 16), iv='\x00' * 16, op=1) + if isinstance(aes_key, long): + aes_key = long_to_bytes(aes_key, 16) + cipher = EVP.Cipher(alg='aes_128_cfb', key=aes_key, iv='\x00' * 16, op=1) ret = cipher.update(plain_str) return ret + cipher.final() def aes_decrypt_str(aes_key, encr_str): - cipher = EVP.Cipher(alg='aes_128_cfb', key=long_to_bytes(aes_key, 16), iv='\x00' * 16, op=0) + if isinstance(aes_key, long): + aes_key = long_to_bytes(aes_key, 16) + cipher = EVP.Cipher(alg='aes_128_cfb', key=aes_key, iv='\x00' * 16, op=0) ret = cipher.update(encr_str) return ret + cipher.final() diff --git a/anon_test.torrent b/anon_test.torrent new file mode 100644 index 0000000000..e07bfd723b Binary files /dev/null and b/anon_test.torrent differ