From 1fccd7a7b51934a23942b4daac77196aeebb63f1 Mon Sep 17 00:00:00 2001 From: "Bjarni R. Einarsson" Date: Sat, 30 May 2015 13:10:20 +0100 Subject: [PATCH] Basic outgoing proxy & Tor support, adds config.sys.proxy.* Make the gravatar downloader require Tor unless configured otherwise. Make the SMTP client proxy & Tor capable via. the connection broker. Relates to: #1075, #1131, #721 Fixes: #724, #989, #1281 --- mailpile/config.py | 8 ++ mailpile/conn_brokers.py | 200 +++++++++++++++++++++++++---- mailpile/defaults.py | 14 +- mailpile/plugins/setup_magic.py | 10 +- mailpile/plugins/vcard_gravatar.py | 6 +- mailpile/smtp_client.py | 35 ++--- 6 files changed, 216 insertions(+), 57 deletions(-) diff --git a/mailpile/config.py b/mailpile/config.py index af56836cb..08fb39ba8 100644 --- a/mailpile/config.py +++ b/mailpile/config.py @@ -1991,6 +1991,14 @@ def _unlocked_prepare_workers(config, session=None, config.background.ui = BackgroundInteraction(config, log_parent=session.ui) + # Tell conn broker that we exist + from mailpile.conn_brokers import Master as ConnBroker + ConnBroker.set_config(config) + if 'connbroker' in config.sys.debug: + ConnBroker.debug_callback = lambda msg: config.background.ui.debug(msg) + else: + ConnBroker.debug_callback = None + def start_httpd(sspec=None): sspec = sspec or (config.sys.http_host, config.sys.http_port, config.sys.http_path or '') diff --git a/mailpile/conn_brokers.py b/mailpile/conn_brokers.py index 04ad3f6c0..be744685d 100644 --- a/mailpile/conn_brokers.py +++ b/mailpile/conn_brokers.py @@ -35,6 +35,16 @@ import threading import traceback +# Import SOCKS proxy support... +try: + import sockschain as socks +except ImportError: + try: + import socks + except ImportError: + socks = None + + org_cconn = socket.create_connection org_sslwrap = ssl.wrap_socket @@ -51,6 +61,7 @@ class Capability(object): OUTGOING_RAW = 'o:raw' # Request this to avoid meddling brokers OUTGOING_ENCRYPTED = 'o:e' # Request this if sending encrypted data OUTGOING_CLEARTEXT = 'o:c' # Request this if sending clear-text data + OUTGOING_TRACKABLE = 'o:t' # Reject this to require anonymity OUTGOING_SMTP = 'o:smtp' # These inform brokers what protocol is being OUTGOING_IMAP = 'o:imap' # .. used, to allow protocol-specific features OUTGOING_POP3 = 'o:pop3' # .. such as enabling STARTTLS or upgrading @@ -67,6 +78,18 @@ class Capability(object): INCOMING_HTTP = 27 INCOMING_HTTPS = 28 + ALL_OUTGOING = set([OUTGOING_RAW, OUTGOING_ENCRYPTED, OUTGOING_CLEARTEXT, + OUTGOING_TRACKABLE, + OUTGOING_SMTP, OUTGOING_IMAP, OUTGOING_POP3, + OUTGOING_HTTP, OUTGOING_HTTPS]) + + ALL_OUTGOING_ENCRYPTED = set([OUTGOING_RAW, OUTGOING_TRACKABLE, + OUTGOING_ENCRYPTED, OUTGOING_HTTPS]) + + ALL_INCOMING = set([INCOMING_RAW, INCOMING_LOCALNET, INCOMING_INTERNET, + INCOMING_DARKNET, INCOMING_SMTP, INCOMING_IMAP, + INCOMING_POP3, INCOMING_HTTP, INCOMING_HTTPS]) + class CapabilityFailure(IOError): """ @@ -150,13 +173,28 @@ class BaseConnectionBroker(Capability): SUPPORTS = [] def __init__(self, master=None): - self.supports = self.SUPPORTS[:] + self.supports = list(self.SUPPORTS)[:] self.master = master - self._debug = None + self._config = None + self._debug = master._debug if master else None + + def configure(self): + self.supports = list(self.SUPPORTS)[:] - def _raise_or_none(self, exc): + def set_config(self, config): + self._config = config + self.configure() + + def config(self): + if self._config is not None: + return self._config + if self.master is not None: + return self.master.config() + return None + + def _raise_or_none(self, exc, why): if exc is not None: - raise exc() + raise exc(why) return None def _check(self, need, reject, _raise=CapabilityFailure): @@ -164,12 +202,12 @@ def _check(self, need, reject, _raise=CapabilityFailure): if n not in self.supports: if self._debug is not None: self._debug('%s: lacking capabilty %s' % (self, n)) - return self._raise_or_none(_raise) + return self._raise_or_none(_raise, 'Lacking %s' % n) for n in reject or []: if n in self.supports: if self._debug is not None: self._debug('%s: unwanted capabilty %s' % (self, n)) - return self._raise_or_none(_raise) + return self._raise_or_none(_raise, 'Unwanted %s' % n) if self._debug is not None: self._debug('%s: checks passed!' % (self, )) return self @@ -225,35 +263,130 @@ class TcpConnectionBroker(BaseConnectionBroker): The only clever thing this class does, is to avoid trying to connect to .onion addresses, preventing that from leaking over DNS. """ - SUPPORTS = [Capability.OUTGOING_RAW, - Capability.OUTGOING_ENCRYPTED, - Capability.OUTGOING_CLEARTEXT, # In strict mode, omit? - Capability.OUTGOING_SMTP, - Capability.OUTGOING_IMAP, - Capability.OUTGOING_POP3, - Capability.OUTGOING_HTTP, - Capability.OUTGOING_HTTPS, - Capability.INCOMING_RAW, -# Capability.INCOMING_INTERNET, # Only if we have a public IP! - Capability.INCOMING_SMTP, - Capability.INCOMING_IMAP, - Capability.INCOMING_POP3, - Capability.INCOMING_HTTP, - Capability.INCOMING_HTTPS, - Capability.INCOMING_LOCALNET] + SUPPORTS = ( + # Normal TCP/IP is not anonymous, and we do not have incoming + # capability unless we have a public IP. + (Capability.ALL_OUTGOING) | + (Capability.ALL_INCOMING - set([Capability.INCOMING_INTERNET])) + ) + + DEBUG_FMT = '%s: Raw TCP conn to: %s' + + def configure(self): + BaseConnectionBroker.configure(self) + # FIXME: If our config indicates we have a public IP, add the + # INCOMING_INTERNET capability. + # FIXME: If our coonfig indicates that the user does not care + # about anonymity at all, remove OUTGOING_TRACKABLE. + if (self.config().sys.proxy.protocol != 'none' and + not self.config().sys.proxy.fallback): + self.supports = [] def _describe(self, context, conn): context.encryption = None context.is_internet = True return conn - def _create_connection(self, context, address, *args, **kwargs): - if self._debug is not None: - self._debug('%s: Raw TCP conn to: %s' % (self, address)) + def _avoid(self, address): if address[0].endswith('.onion'): raise CapabilityFailure('Cannot connect to .onion addresses') + + def _conn(self, address, *args, **kwargs): return org_cconn(address, *args, **kwargs) + def _create_connection(self, context, address, *args, **kwargs): + self._avoid(address) + if self._debug is not None: + self._debug(self.DEBUG_FMT % (self, address)) + return self._conn(address, *args, **kwargs) + + +class SocksConnBroker(TcpConnectionBroker): + """ + This broker offers the same services as the TcpConnBroker, but over a + SOCKS connection. + """ + SUPPORTS = [] + CONFIGURED = Capability.ALL_OUTGOING + DEBUG_FMT = '%s: Raw SOCKS5 conn to: %s' + PROXY_TYPES = ('socks5', 'http', 'socks4') + + def __init__(self, *args, **kwargs): + TcpConnectionBroker.__init__(self, *args, **kwargs) + self.proxy_config = None + self.typemap = {} + + def configure(self): + BaseConnectionBroker.configure(self) + if self.config().sys.proxy.protocol in self.PROXY_TYPES: + self.proxy_config = self.config().sys.proxy + self.supports = list(self.CONFIGURED)[:] + self.typemap = { + 'socks5': socks.PROXY_TYPE_SOCKS5, + 'socks4': socks.PROXY_TYPE_SOCKS4, + 'http': socks.PROXY_TYPE_HTTP, + 'tor': socks.PROXY_TYPE_SOCKS5 # For TorConnBrokerk + } + else: + self.proxy_config = None + self.supports = [] + + def _conn(self, address, timeout=None, source_address=None): + sock = socks.socksocket() + sock.setproxy(proxytype=self.typemap[self.proxy_config.protocol], + addr=self.proxy_config.host, + port=self.proxy_config.port, + rdns=True, + username=self.proxy_config.username or None, + password=self.proxy_config.password or None) + if timeout and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(float(timeout)) + if source_address: + raise IOError('Cannot bind source address') + try: + address = (str(address[0]), address[1]) + sock.connect(address) + except socks.ProxyError: + self._debug(traceback.format_exc()) + raise IOError('Proxy failed for %s' % address) + return sock + + +class TorConnBroker(SocksConnBroker): + """ + This broker offers the same services as the TcpConnBroker, but over Tor. + + This removes the "trackable" capability, so requests that reject it can + find their way here safely... + """ + SUPPORTS = [] + CONFIGURED = (Capability.ALL_OUTGOING_ENCRYPTED + - set([Capability.OUTGOING_TRACKABLE])) + REJECTS = None + DEBUG_FMT = '%s: Raw Tor conn to: %s' + PROXY_TYPES = ('tor', ) + + def _avoid(self, address): + pass + + +class TorOnionBroker(SocksConnBroker): + """ + This broker offers the same services as the TcpConnBroker, but over Tor. + This removes the "trackable" capability, so requests that reject it can + find their way here safely... + """ + SUPPORTS = [] + CONFIGURED = (Capability.ALL_OUTGOING + - set([Capability.OUTGOING_TRACKABLE])) + REJECTS = None + DEBUG_FMT = '%s: Raw Tor conn to: %s' + PROXY_TYPES = ('tor', ) + + def _avoid(self, address): + if not address[0].endswith('.onion'): + raise CapabilityFailure('Can only connect to .onion addresses') + class BaseConnectionBrokerProxy(TcpConnectionBroker): """ @@ -300,6 +433,8 @@ def _describe(self, context, conn): return conn def _proxy_address(self, address): + if address[0].endswith('.onion'): + raise CapabilityFailure('I do not like .onion addresses') if int(address[1]) == 80: return (address[0], 443) return address @@ -332,6 +467,16 @@ class MasterBroker(BaseConnectionBroker): def __init__(self, *args, **kwargs): BaseConnectionBroker.__init__(self, *args, **kwargs) self.brokers = [] + self._debug = self._debugger + self.debug_callback = None + + def configure(self): + for prio, cb in self.brokers: + cb.configure() + + def _debugger(self, *args, **kwargs): + if self.debug_callback is not None: + self.debug_callback(*args, **kwargs) def register_broker(self, priority, cb): """ @@ -392,6 +537,11 @@ def CreateConnWarning(*args, **kwargs): register(9500, AutoImapStartTLSConnBroker) register(9500, AutoPop3StartTLSConnBroker) + if socks is not None: + register(1500, SocksConnBroker) + register(3500, TorConnBroker) + register(3500, TorOnionBroker) + def SslWrapOnlyOnce(sock, *args, **kwargs): """ Since we like to wrap things our own way, this make ssl.wrap_socket diff --git a/mailpile/defaults.py b/mailpile/defaults.py index e64cf966a..5dd922491 100644 --- a/mailpile/defaults.py +++ b/mailpile/defaults.py @@ -33,6 +33,8 @@ 'sys': p(_('Technical system settings'), False, { 'fd_cache_size': (_('Max files kept open at once'), int, 500), 'history_length': (_('History length (lines, <0=no save)'), int, 100), + 'http_host': p(_('Listening host for web UI'), + 'hostname', 'localhost'), 'http_port': p(_('Listening port for web UI'), int, 33411), 'http_path': p(_('HTTP path of web UI'), 'webroot', ''), 'postinglist_kb': (_('Posting list target size in KB'), int, 64), @@ -43,8 +45,6 @@ str, 'pool.sks-keyservers.net'), 'gpg_home': p(_('Override the home directory of GnuPG'), 'dir', None), - 'http_host': p(_('Listening host for web UI'), - 'hostname', 'localhost'), 'local_mailbox_id': (_('Local read/write Maildir'), 'b36', ''), 'mailindex_file': (_('Metadata index file'), 'file', ''), 'postinglist_dir': (_('Search index directory'), 'dir', ''), @@ -59,6 +59,16 @@ }], 'lockdown': [_('Demo mode, disallow changes'), bool, False], 'login_banner': [_('A custom banner for the login page'), str, ''], + 'proxy': [_('Proxy settings'), False, { + 'protocol': (_('Proxy protocol'), + ["tor", "socks5", "socks4", "http", "none"], + 'none'), + 'fallback': (_('Allow fallback to direct conns'), bool, False), + 'username': (_('User name'), str, ''), + 'password': (_('Password'), str, ''), + 'host': (_('Host'), str, ''), + 'port': (_('Port'), int, 8080) + }], }), 'prefs': p(_("User preferences"), False, { 'num_results': (_('Search results per page'), int, 20), diff --git a/mailpile/plugins/setup_magic.py b/mailpile/plugins/setup_magic.py index 7305c7242..f344f0d32 100644 --- a/mailpile/plugins/setup_magic.py +++ b/mailpile/plugins/setup_magic.py @@ -478,12 +478,16 @@ def _progress(self, message): self.event.message = message self._update_event_state(self.event.RUNNING, log=True) else: - session.ui.mark(message) + self.session.ui.mark(message) def _urlget(self, url): - with ConnBroker.context(need=[ConnBroker.OUTGOING_HTTP]) as context: + if url.lower().startswith('https'): + conn_needs = [ConnBroker.OUTGOING_HTTPS] + else: + conn_needs = [ConnBroker.OUTGOING_HTTP] + with ConnBroker.context(need=conn_needs) as context: self.session.ui.mark('Getting: %s' % url) - return urlopen(url, data=None, timeout=3).read() + return urlopen(url, data=None, timeout=10).read() def _username(self, val, email): lpart = email.split('@')[0] diff --git a/mailpile/plugins/vcard_gravatar.py b/mailpile/plugins/vcard_gravatar.py index d630349a0..d9fb77a2b 100644 --- a/mailpile/plugins/vcard_gravatar.py +++ b/mailpile/plugins/vcard_gravatar.py @@ -32,6 +32,7 @@ class GravatarImporter(VCardImporter): SHORT_NAME = 'gravatar' CONFIG_RULES = { 'active': [_('Enable this importer'), bool, True], + 'anonymous': [_('Require anonymity for use'), bool, True], 'interval': [_('Minimum days between refreshing'), 'int', 7], 'batch': [_('Max batch size per update'), 'int', 30], 'default': [_('Default thumbnail style'), str, 'retro'], @@ -66,7 +67,10 @@ def _jittery_time(): return want def _get(self, url): - with ConnBroker.context(need=[ConnBroker.OUTGOING_HTTP]) as context: + conn_need, conn_reject = [ConnBroker.OUTGOING_HTTP], [] + if self.config.anonymous: + conn_reject += [ConnBroker.OUTGOING_TRACKABLE] + with ConnBroker.context(need=conn_need, reject=conn_reject) as ctx: self.session.ui.mark('Getting: %s' % url) return urlopen(url, data=None, timeout=3).read() diff --git a/mailpile/smtp_client.py b/mailpile/smtp_client.py index eb1a7980f..9f418d46a 100644 --- a/mailpile/smtp_client.py +++ b/mailpile/smtp_client.py @@ -6,6 +6,7 @@ import time import mailpile.util +from mailpile.conn_brokers import Master as ConnBroker from mailpile.util import * from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n @@ -75,32 +76,11 @@ def cb(*args, **kwargs): callback1k=cb)) -def _AddSocksHooks(cls, SSL=False): - - class Socksified(cls): - def _get_socket(self, host, port, timeout): - new_socket = self.socket() - new_socket.connect((host, port)) - - if SSL and ssl is not None: - new_socket = ssl.wrap_socket(new_socket, - self.keyfile, self.certfile) - self.file = smtplib.SSLFakeFile(new_socket) - - return new_socket - - def connect(self, host='localhost', port=0, socket_cls=None): - self.socket = socket_cls or socket.socket - return cls.connect(self, host=host, port=port) - - return Socksified - - -class SMTP(_AddSocksHooks(smtplib.SMTP)): +class SMTP(smtplib.SMTP): pass if ssl is not None: - class SMTP_SSL(_AddSocksHooks(smtplib.SMTP_SSL, SSL=True)): + class SMTP_SSL(smtplib.SMTP_SSL): pass else: SMTP_SSL = SMTP @@ -262,11 +242,13 @@ def sm_close(): def sm_startup(): if 'sendmail' in session.config.sys.debug: server.set_debuglevel(1) - if proto == 'smtorp': - server.connect(host, int(port), - socket_cls=session.config.get_tor_socket()) + if smtp_ssl or sendmail[:7] in ('smtorp', 'smtptls'): + conn_needs = [ConnBroker.OUTGOING_ENCRYPTED] else: + conn_needs = [ConnBroker.OUTGOING_SMTP] + with ConnBroker.context(need=conn_needs) as ctx: server.connect(host, int(port)) + if not smtp_ssl: # We always try to enable TLS, even if the user just # requested plain-text smtp. But we only throw errors @@ -276,6 +258,7 @@ def sm_startup(): except: if sendmail.startswith('smtptls'): raise InsecureSmtpError() + if user and pwd: try: server.login(user.encode('utf-8'), pwd.encode('utf-8'))