From c35efea0d674de469a410c267a418f25dd31f733 Mon Sep 17 00:00:00 2001 From: echudow Date: Wed, 10 Oct 2018 22:35:39 -0400 Subject: [PATCH 01/21] Added ca-file argument for pshtt and added pshtt output fields for Client Authentication and whether the certs are publicly trusted. --- scanners/pshtt.py | 10 +++++++--- utils/scan_utils.py | 8 ++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index 0c4038a4..046f984f 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -92,7 +92,8 @@ def scan(domain, environment, options): { 'timeout': pshtt_timeout, 'user_agent': user_agent, - 'debug': options.get("debug", False) + 'debug': options.get("debug", False), + 'ca_file': options.get("ca_file", True) } ) @@ -120,14 +121,17 @@ def to_rows(data): headers = [ - "Canonical URL", "Live", "Redirect", "Redirect To", + "Canonical URL", "Live", "HTTPS Full Connection", "HTTPS Client Auth Required", + "Redirect", "Redirect To", "Valid HTTPS", "Defaults to HTTPS", "Downgrades HTTPS", "Strictly Forces HTTPS", "HTTPS Bad Chain", "HTTPS Bad Hostname", "HTTPS Expired Cert", "HTTPS Self Signed Cert", "HSTS", "HSTS Header", "HSTS Max Age", "HSTS Entire Domain", "HSTS Preload Ready", "HSTS Preload Pending", "HSTS Preloaded", "Base Domain HSTS Preloaded", "Domain Supports HTTPS", - "Domain Enforces HTTPS", "Domain Uses Strong HSTS", "Unknown Error", + "Domain Enforces HTTPS", "Domain Uses Strong HSTS", + "HTTPS Publicly Trusted", "HTTPS Custom Truststore Trusted", + "Unknown Error", ] diff --git a/utils/scan_utils.py b/utils/scan_utils.py index 4e0d859d..74683eda 100644 --- a/utils/scan_utils.py +++ b/utils/scan_utils.py @@ -426,9 +426,13 @@ def build_scan_options_parser() -> ArgumentParser: # a11y: parser.add_argument("--a11y-config", help="a11y: Location of pa11y config file (used with a11y scanner.") - parser.add_argument("--a11y-redirects", help="a11y: Location of YAML file with redirects to inform the a11y scanner.") + + # pshtt: + parser.add_argument("--ca_file", + help="ca_file: Location of PEM file of trust store to verify certs with.") + # sslyze: parser.add_argument("--sslyze-serial", help="sslyze: If set, will use a synchronous (single-threaded in-process) scanner. Defaults to true.") @@ -625,7 +629,7 @@ def _df_path(arg: Path, domain_suffix: Union[str, None]=None) -> Iterable[str]: if arg.suffix == ".csv": with arg.open(encoding='utf-8', newline='') as csvfile: for row in csv.reader(csvfile): - if (not row[0]) or (row[0].lower() == "domain") or (row[0].lower() == "domain name"): + if (not row) or (not row[0]) or (row[0].lower() == "domain") or (row[0].lower() == "domain name"): continue domain = row[0].lower() if domain_suffix: From 324054bf25d2b8c07bf63da8fd58063a1bcceed5 Mon Sep 17 00:00:00 2001 From: echudow Date: Wed, 10 Oct 2018 23:02:55 -0400 Subject: [PATCH 02/21] Fixed bug in ca-file argument. --- scanners/pshtt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index 046f984f..eb488dc3 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -93,7 +93,7 @@ def scan(domain, environment, options): 'timeout': pshtt_timeout, 'user_agent': user_agent, 'debug': options.get("debug", False), - 'ca_file': options.get("ca_file", True) + 'ca_file': options.get("ca_file") } ) From 03866d593a2c7ad12743d759c5853a128141cb2d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 23 Nov 2018 15:18:48 -0500 Subject: [PATCH 03/21] Remove some whitespace that offends flake8 --- scanners/pshtt.py | 4 ++-- utils/scan_utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index 45493f8d..998fc66d 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -121,7 +121,7 @@ def to_rows(data): headers = [ - "Canonical URL", "Live", "HTTPS Full Connection", "HTTPS Client Auth Required", + "Canonical URL", "Live", "HTTPS Full Connection", "HTTPS Client Auth Required", "Redirect", "Redirect To", "Valid HTTPS", "Defaults to HTTPS", "Downgrades HTTPS", "Strictly Forces HTTPS", "HTTPS Bad Chain", "HTTPS Bad Hostname", @@ -129,7 +129,7 @@ def to_rows(data): "HSTS", "HSTS Header", "HSTS Max Age", "HSTS Entire Domain", "HSTS Preload Ready", "HSTS Preload Pending", "HSTS Preloaded", "Base Domain HSTS Preloaded", "Domain Supports HTTPS", - "Domain Enforces HTTPS", "Domain Uses Strong HSTS", + "Domain Enforces HTTPS", "Domain Uses Strong HSTS", "HTTPS Publicly Trusted", "HTTPS Custom Truststore Trusted", "Unknown Error", ] diff --git a/utils/scan_utils.py b/utils/scan_utils.py index 74683eda..93aa75fa 100644 --- a/utils/scan_utils.py +++ b/utils/scan_utils.py @@ -428,11 +428,11 @@ def build_scan_options_parser() -> ArgumentParser: help="a11y: Location of pa11y config file (used with a11y scanner.") parser.add_argument("--a11y-redirects", help="a11y: Location of YAML file with redirects to inform the a11y scanner.") - + # pshtt: parser.add_argument("--ca_file", help="ca_file: Location of PEM file of trust store to verify certs with.") - + # sslyze: parser.add_argument("--sslyze-serial", help="sslyze: If set, will use a synchronous (single-threaded in-process) scanner. Defaults to true.") From c89e956209a16d5d77e3dcdb9e439a60e5dbac6e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 30 Nov 2018 13:18:39 -0500 Subject: [PATCH 04/21] Reorder some of the values when they are written to CSV --- scanners/pshtt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index 998fc66d..9b720397 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -121,7 +121,7 @@ def to_rows(data): headers = [ - "Canonical URL", "Live", "HTTPS Full Connection", "HTTPS Client Auth Required", + "Canonical URL", "Live", "Redirect", "Redirect To", "Valid HTTPS", "Defaults to HTTPS", "Downgrades HTTPS", "Strictly Forces HTTPS", "HTTPS Bad Chain", "HTTPS Bad Hostname", @@ -130,6 +130,7 @@ def to_rows(data): "HSTS Preload Ready", "HSTS Preload Pending", "HSTS Preloaded", "Base Domain HSTS Preloaded", "Domain Supports HTTPS", "Domain Enforces HTTPS", "Domain Uses Strong HSTS", + "HTTPS Full Connection", "HTTPS Client Auth Required", "HTTPS Publicly Trusted", "HTTPS Custom Truststore Trusted", "Unknown Error", ] From 05b308383db1ad9695e91a56ce0238099c8e19fb Mon Sep 17 00:00:00 2001 From: echudow Date: Tue, 18 Dec 2018 16:26:28 -0500 Subject: [PATCH 05/21] Added specific info to sslyze for weak cipher tasks --- scanners/pshtt.py | 3 +- scanners/sslyze.py | 104 +++++++++++++++++++++++++++++++++++++++++--- utils/scan_utils.py | 2 + utils/utils.py | 4 +- 4 files changed, 105 insertions(+), 8 deletions(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index eb488dc3..7b15f368 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -10,7 +10,7 @@ # Measure a site's HTTP behavior using DHS NCATS' pshtt tool. # Network timeout for each internal pshtt HTTP request. -pshtt_timeout = 20 +pshtt_timeout = 5 # Default to a custom user agent that can be overridden via an environment # variable @@ -131,6 +131,7 @@ def to_rows(data): "Base Domain HSTS Preloaded", "Domain Supports HTTPS", "Domain Enforces HTTPS", "Domain Uses Strong HSTS", "HTTPS Publicly Trusted", "HTTPS Custom Truststore Trusted", + "IP", "Server Header", "Server Version", "Notes", "Unknown Error", ] diff --git a/scanners/sslyze.py b/scanners/sslyze.py index 2e1c46f6..e235f613 100644 --- a/scanners/sslyze.py +++ b/scanners/sslyze.py @@ -13,12 +13,14 @@ ### import logging +import datetime from sslyze.server_connectivity_tester import ServerConnectivityTester, ServerConnectivityError from sslyze.synchronous_scanner import SynchronousScanner from sslyze.concurrent_scanner import ConcurrentScanner, PluginRaisedExceptionScanResult from sslyze.plugins.openssl_cipher_suites_plugin import Tlsv10ScanCommand, Tlsv11ScanCommand, Tlsv12ScanCommand, Tlsv13ScanCommand, Sslv20ScanCommand, Sslv30ScanCommand from sslyze.plugins.certificate_info_plugin import CertificateInfoScanCommand +from sslyze.plugins.session_renegotiation_plugin import SessionRenegotiationScanCommand from sslyze.ssl_settings import TlsWrappedProtocolEnum import idna @@ -153,6 +155,19 @@ def to_rows(data): row['certs'].get('is_symantec_cert'), row['certs'].get('symantec_distrust_date'), + + row['config'].get('any_export'), + row['config'].get('any_NULL'), + row['config'].get('any_MD5'), + row['config'].get('any_less_than_128_bits'), + + row['config'].get('insecure_renegotiation'), + + row['certs'].get('certificate_less_than_2048'), + row['certs'].get('md5_signed_certificate'), + row['certs'].get('sha1_signed_certificate'), + row['certs'].get('expired_certificate'), + str.join(', ', row.get('ciphers', [])), row.get('errors') @@ -182,6 +197,13 @@ def to_rows(data): "EV Trusted OIDs", "EV Trusted Browsers", "Is Symantec Cert", "Symantec Distrust Date", + + "Any Export", "Any NULL", "Any MD5", "Any Less Than 128 Bits", + "Insecure Renegotiation", + "Certificate Less Than 2048", + "MD5 Signed Certificate", "SHA-1 Signed Certificate", + "Expired Certificate", + "Accepted Ciphers", "Errors" @@ -217,9 +239,9 @@ def run_sslyze(data, environment, options): # Whether sync or concurrent, get responses for all scans. if sync: - sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs = scan_serial(scanner, server_info, data, options) + sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg = scan_serial(scanner, server_info, data, options) else: - sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs = scan_parallel(scanner, server_info, data, options) + sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg = scan_parallel(scanner, server_info, data, options) # Only analyze protocols if all the scanners functioned. # Very difficult to draw conclusions if some worked and some did not. @@ -229,6 +251,9 @@ def run_sslyze(data, environment, options): if certs: data['certs'] = analyze_certs(certs) + if reneg: + analyze_reneg(data, reneg) + return data @@ -261,6 +286,10 @@ def analyze_protocols_and_ciphers(data, sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, t all_rc4 = True all_dhe = True any_3des = False + any_export = False + any_NULL = False + any_MD5 = False + any_less_than_128_bits = False for cipher in accepted_ciphers: name = cipher.openssl_name @@ -277,11 +306,29 @@ def analyze_protocols_and_ciphers(data, sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, t else: all_dhe = False + if ("EXPORT" in name): + any_export = True + + if ("NULL" in name): + any_NULL = True + + if ("MD5" in name): + any_MD5 = True + + parts = name.split('_') + for p in parts: + if (p.isdigit()) and (int(p) < 128): + any_less_than_128_bits = True + data['config']['any_rc4'] = any_rc4 data['config']['all_rc4'] = all_rc4 data['config']['any_dhe'] = any_dhe data['config']['all_dhe'] = all_dhe data['config']['any_3des'] = any_3des + data['config']['any_export'] = any_export + data['config']['any_NULL'] = any_NULL + data['config']['any_MD5'] = any_MD5 + data['config']['any_less_than_128_bits'] = any_less_than_128_bits def analyze_certs(certs): @@ -319,6 +366,11 @@ def analyze_certs(certs): else: data['certs']['key_length'] = None + if(data['certs']['key_length'] < 2048): + data['certs']['certificate_less_than_2048'] = True + else: + data['certs']['certificate_less_than_2048'] = False + if isinstance(leaf_key, rsa.RSAPublicKey): leaf_key_type = "RSA" elif isinstance(leaf_key, dsa.DSAPublicKey): @@ -333,10 +385,26 @@ def analyze_certs(certs): # Signature of the leaf certificate only. data['certs']['leaf_signature'] = leaf.signature_hash_algorithm.name + if(leaf.signature_hash_algorithm.name == "MD5"): + data['certs']['md5_signed_certificate'] = True + else: + data['certs']['md5_signed_certificate'] = False + + if(leaf.signature_hash_algorithm.name == "SHA1"): + data['certs']['sha1_signed_certificate'] = True + else: + data['certs']['sha1_signed_certificate'] = False + # Beginning and expiration dates of the leaf certificate data['certs']['not_before'] = leaf.not_valid_before data['certs']['not_after'] = leaf.not_valid_after + now = datetime.datetime.now() + if (now < leaf.not_valid_before) or (now > leaf.not_valid_after): + data['certs']['expired_certificate'] = True + else: + data['certs']['expired_certificate'] = False + any_sha1_served = False for cert in served_chain: if parse_cert(cert).signature_hash_algorithm.name == "sha1": @@ -424,6 +492,13 @@ def cert_issuer_name(parsed): return None return attrs[0].value +# Analyze the results of a renegotiation test +def analyze_reneg(data, reneg): + if (reneg.accepts_client_renegotiation is True) and (reneg.supports_secure_renegotiation is False): + data['config']['insecure_renegotiation'] = True + else: + data['config']['insecure_renegotiation'] = False + # Given CipherSuiteScanResult, whether the protocol is supported def supported_protocol(result): @@ -492,9 +567,21 @@ def scan_serial(scanner, server_info, data, options): else: certs = None + reneg = None + if options.get("sslyze_reneg", True) is True: + logging.debug("\t\tRenegotiation scan.") + try: + reneg = scanner.run_scan_command(server_info, SessionRenegotiationScanCommand()) + except Exception as err: + logging.warning("Error during renegotiation test.") + #logging.debug(utils.format_last_exception()) + logging.debug("Exception: {}".format(err)) + else: + reneg = None + logging.debug("\tDone scanning.") - return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs + return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg # Run each scan in parallel, using multi-processing. @@ -517,7 +604,7 @@ def queue(command): return None, None, None, None, None, None, None # Initialize commands and result containers - sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs = None, None, None, None, None, None + sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg = None, None, None, None, None, None, None, None # Queue them all up queue(Sslv20ScanCommand()) @@ -530,6 +617,9 @@ def queue(command): if options.get("sslyze-certs", True) is True: queue(CertificateInfoScanCommand()) + if options.get("sslyze-reneg", True) is True: + queue(CertificateInfoScanCommand()) + # Reassign them back to predictable places after they're all done was_error = False for result in scanner.get_results(): @@ -554,6 +644,8 @@ def queue(command): tlsv1_3 = result elif type(result.scan_command) == CertificateInfoScanCommand: certs = result + elif type(result.scan_command) == SessionRenegotiationScanCommand: + reneg = result else: error = "Couldn't match scan result with command! %s" % result logging.warning("\t%s" % error) @@ -568,11 +660,11 @@ def queue(command): # There was an error during async processing. if was_error: - return None, None, None, None, None, None, None + return None, None, None, None, None, None, None, None logging.debug("\tDone scanning.") - return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs + return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg # EV Guidelines OID diff --git a/utils/scan_utils.py b/utils/scan_utils.py index 74683eda..09859912 100644 --- a/utils/scan_utils.py +++ b/utils/scan_utils.py @@ -438,6 +438,8 @@ def build_scan_options_parser() -> ArgumentParser: help="sslyze: If set, will use a synchronous (single-threaded in-process) scanner. Defaults to true.") parser.add_argument("--sslyze-certs", help="sslyze: If set, will use the CertificateInfoScanner and return certificate info. Defaults to true.") + parser.add_argument("--sslyze-reneg", + help="sslyze: If set, will use the SessionRenegotiationScanner and return session renegotiation info. Defaults to true.") # trustymail: parser.add_argument("--starttls", help="trustymail: ?") parser.add_argument("--timeout", help="trustymail: ?") diff --git a/utils/utils.py b/utils/utils.py index d30725be..f93803d9 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -553,7 +553,9 @@ def domain_doesnt_support_https(domain, cache_dir="./cache"): httpswww = inspection.get("endpoints").get("httpswww") def endpoint_used(endpoint): - return endpoint.get("live") and (not endpoint.get("https_bad_hostname")) + # commented out next line to remove bad hostname check so we can still get info about weak crypto even if the cert doesn't match + # return endpoint.get("live") and (not endpoint.get("https_bad_hostname")) + return endpoint.get("live") return (not (endpoint_used(https) or endpoint_used(httpswww))) From 652d6a2b315fbf13f9607e3b4c37bb285963dfc5 Mon Sep 17 00:00:00 2001 From: echudow Date: Tue, 18 Dec 2018 16:34:27 -0500 Subject: [PATCH 06/21] Merged changes --- scanners/sslyze.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scanners/sslyze.py b/scanners/sslyze.py index 6144bb8f..96e81764 100644 --- a/scanners/sslyze.py +++ b/scanners/sslyze.py @@ -13,11 +13,8 @@ ### import logging -<<<<<<< HEAD import datetime -======= from typing import Any ->>>>>>> c89e956209a16d5d77e3dcdb9e439a60e5dbac6e from sslyze.server_connectivity_tester import ServerConnectivityTester, ServerConnectivityError from sslyze.synchronous_scanner import SynchronousScanner From 95aa0f57a5e9216ca7e1afab96d9ebcfce9415c7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 Dec 2018 16:52:00 -0500 Subject: [PATCH 07/21] Make flake8 happy --- scanners/sslyze.py | 17 +++++++++-------- utils/utils.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/scanners/sslyze.py b/scanners/sslyze.py index 96e81764..2aaa84a9 100644 --- a/scanners/sslyze.py +++ b/scanners/sslyze.py @@ -232,7 +232,7 @@ def to_rows(data): row['config'].get('any_NULL'), row['config'].get('any_MD5'), row['config'].get('any_less_than_128_bits'), - + row['config'].get('insecure_renegotiation'), row['certs'].get('certificate_less_than_2048'), @@ -271,10 +271,10 @@ def to_rows(data): "Is Symantec Cert", "Symantec Distrust Date", "Any Export", "Any NULL", "Any MD5", "Any Less Than 128 Bits", - "Insecure Renegotiation", - "Certificate Less Than 2048", + "Insecure Renegotiation", + "Certificate Less Than 2048", "MD5 Signed Certificate", "SHA-1 Signed Certificate", - "Expired Certificate", + "Expired Certificate", "Accepted Ciphers", @@ -360,7 +360,7 @@ def analyze_protocols_and_ciphers(data, sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, t any_3des = False any_export = False any_NULL = False - any_MD5 = False + any_MD5 = False any_less_than_128_bits = False for cipher in accepted_ciphers: @@ -399,7 +399,7 @@ def analyze_protocols_and_ciphers(data, sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, t data['config']['any_3des'] = any_3des data['config']['any_export'] = any_export data['config']['any_NULL'] = any_NULL - data['config']['any_MD5'] = any_MD5 + data['config']['any_MD5'] = any_MD5 data['config']['any_less_than_128_bits'] = any_less_than_128_bits @@ -564,11 +564,12 @@ def cert_issuer_name(parsed): return None return attrs[0].value + # Analyze the results of a renegotiation test def analyze_reneg(data, reneg): if (reneg.accepts_client_renegotiation is True) and (reneg.supports_secure_renegotiation is False): data['config']['insecure_renegotiation'] = True - else: + else: data['config']['insecure_renegotiation'] = False @@ -646,7 +647,7 @@ def scan_serial(scanner, server_info, data, options): reneg = scanner.run_scan_command(server_info, SessionRenegotiationScanCommand()) except Exception as err: logging.warning("Error during renegotiation test.") - #logging.debug(utils.format_last_exception()) + # logging.debug(utils.format_last_exception()) logging.debug("Exception: {}".format(err)) else: reneg = None diff --git a/utils/utils.py b/utils/utils.py index f93803d9..422c42a1 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -555,7 +555,7 @@ def domain_doesnt_support_https(domain, cache_dir="./cache"): def endpoint_used(endpoint): # commented out next line to remove bad hostname check so we can still get info about weak crypto even if the cert doesn't match # return endpoint.get("live") and (not endpoint.get("https_bad_hostname")) - return endpoint.get("live") + return endpoint.get("live") return (not (endpoint_used(https) or endpoint_used(httpswww))) From d6e19702e3e99c3d0c904d9460f76489719fbb86 Mon Sep 17 00:00:00 2001 From: echudow Date: Sun, 20 Jan 2019 19:24:42 -0500 Subject: [PATCH 08/21] Refactored to handle some errors --- scanners/sslyze.py | 49 ++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/scanners/sslyze.py b/scanners/sslyze.py index cc06260c..d72ebc29 100644 --- a/scanners/sslyze.py +++ b/scanners/sslyze.py @@ -325,7 +325,7 @@ def run_sslyze(data, environment, options): if reneg: analyze_reneg(data, reneg) - + return data @@ -611,44 +611,47 @@ def init_sslyze(hostname, port, starttls_smtp, options, sync=False): # Run each scan in-process, one at a time. # Takes longer, but no multi-process funny business. def scan_serial(scanner, server_info, data, options): + errors = 0 + def run_scan(scan_type, command, errors): + if(errors >= 2): + return None, errors + logging.debug("\t\t{} scan.".format(scan_type)) + result = None + try: + result = scanner.run_scan_command(server_info, command) + except Exception as err: + logging.warning("{}: Error during {} scan.".format(server_info.hostname, scan_type)) + logging.debug("{}: Exception during {} scan: {}".format(server_info.hostname, scan_type, err)) + errors = errors + 1 + return result, errors + logging.debug("\tRunning scans in serial.") - logging.debug("\t\tSSLv2 scan.") - sslv2 = scanner.run_scan_command(server_info, Sslv20ScanCommand()) - logging.debug("\t\tSSLv3 scan.") - sslv3 = scanner.run_scan_command(server_info, Sslv30ScanCommand()) - logging.debug("\t\tTLSv1.0 scan.") - tlsv1 = scanner.run_scan_command(server_info, Tlsv10ScanCommand()) - logging.debug("\t\tTLSv1.1 scan.") - tlsv1_1 = scanner.run_scan_command(server_info, Tlsv11ScanCommand()) - logging.debug("\t\tTLSv1.2 scan.") - tlsv1_2 = scanner.run_scan_command(server_info, Tlsv12ScanCommand()) - logging.debug("\t\tTLSv1.3 scan.") - tlsv1_3 = scanner.run_scan_command(server_info, Tlsv13ScanCommand()) + sslv2, errors = run_scan("SSLv2", Sslv20ScanCommand(), errors) + sslv3, errors = run_scan("SSLv3", Sslv30ScanCommand(), errors) + tlsv1, errors = run_scan("TLSv1.0", Tlsv10ScanCommand(), errors) + tlsv1_1, errors = run_scan("TLSv1.1", Tlsv11ScanCommand(), errors) + tlsv1_2, errors = run_scan("TLSv1.2", Tlsv12ScanCommand(), errors) + tlsv1_3, errors = run_scan("TLSv1.3", Tlsv13ScanCommand(), errors) certs = None - if options.get("sslyze_certs", True) is True: - + if errors < 2 and options.get("sslyze_certs", True) is True: try: logging.debug("\t\tCertificate information scan.") certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand()) - # Let generic exceptions bubble up. except idna.core.InvalidCodepoint: logging.warning(utils.format_last_exception()) data['errors'].append("Invalid certificate/OCSP for this domain.") certs = None + except Exception as err: + logging.warning("{}: Error during certificate information scan.".format(server_info.hostname)) + logging.debug("{}: Exception during certificate information scan: {}".format(server_info.hostname, err)) else: certs = None reneg = None if options.get("sslyze_reneg", True) is True: - logging.debug("\t\tRenegotiation scan.") - try: - reneg = scanner.run_scan_command(server_info, SessionRenegotiationScanCommand()) - except Exception as err: - logging.warning("Error during renegotiation test.") - # logging.debug(utils.format_last_exception()) - logging.debug("Exception: {}".format(err)) + reneg, errors = run_scan("Renegotiation", SessionRenegotiationScanCommand(), errors) else: reneg = None From 133a5fc85582a716adc2bdda085f16acae885f24 Mon Sep 17 00:00:00 2001 From: echudow Date: Sun, 20 Jan 2019 19:25:11 -0500 Subject: [PATCH 09/21] Added an HTTPS Live field --- scanners/pshtt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index b466822e..acce98f2 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -129,8 +129,8 @@ def to_rows(data): "HSTS", "HSTS Header", "HSTS Max Age", "HSTS Entire Domain", "HSTS Preload Ready", "HSTS Preload Pending", "HSTS Preloaded", "Base Domain HSTS Preloaded", "Domain Supports HTTPS", - "Domain Enforces HTTPS", "Domain Uses Strong HSTS", - "HTTPS Full Connection", "HTTPS Client Auth Required", + "Domain Enforces HTTPS", "Domain Uses Strong HSTS", + "HTTPS Live", "HTTPS Full Connection", "HTTPS Client Auth Required", "HTTPS Publicly Trusted", "HTTPS Custom Truststore Trusted", "IP", "Server Header", "Server Version", "Notes", "Unknown Error", From aeb980d14a8eaa01da6da9de922ee325710443b4 Mon Sep 17 00:00:00 2001 From: echudow Date: Thu, 7 Mar 2019 05:49:56 -0500 Subject: [PATCH 10/21] Added public trust with intermediate certs arg --- scanners/pshtt.py | 6 ++++-- utils/scan_utils.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index acce98f2..df260378 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -93,7 +93,8 @@ def scan(domain, environment, options): 'timeout': pshtt_timeout, 'user_agent': user_agent, 'debug': options.get("debug", False), - 'ca_file': options.get("ca_file") + 'ca_file': options.get("ca_file"), + 'pt_int_ca_file': options.get("pt_int_ca_file") } ) @@ -132,7 +133,8 @@ def to_rows(data): "Domain Enforces HTTPS", "Domain Uses Strong HSTS", "HTTPS Live", "HTTPS Full Connection", "HTTPS Client Auth Required", "HTTPS Publicly Trusted", "HTTPS Custom Truststore Trusted", - "IP", "Server Header", "Server Version", "Notes", + "IP", "Server Header", "Server Version", "HTTPS Cert Chain Length", + "HTTPS Probably Missing Intermediate Cert", "Notes", "Unknown Error", ] diff --git a/utils/scan_utils.py b/utils/scan_utils.py index 4d0d9ae1..a65dc13e 100644 --- a/utils/scan_utils.py +++ b/utils/scan_utils.py @@ -441,6 +441,8 @@ def build_scan_options_parser() -> ArgumentParser: # pshtt: parser.add_argument("--ca_file", help="ca_file: Location of PEM file of trust store to verify certs with.") + parser.add_argument("--pt_int_ca_file", + help="pt_int_ca_file: Location of PEM file of public trust store with any needed intermediate certificates to verify certs with.") # sslyze: parser.add_argument("--sslyze-serial", From 07be17980d3ffa12794b9fefd7011e2a4b083279 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 10 Mar 2019 21:54:10 -0400 Subject: [PATCH 11/21] Make flake8 happy --- scanners/pshtt.py | 4 ++-- scanners/sslyze.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index df260378..5c0f2887 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -130,10 +130,10 @@ def to_rows(data): "HSTS", "HSTS Header", "HSTS Max Age", "HSTS Entire Domain", "HSTS Preload Ready", "HSTS Preload Pending", "HSTS Preloaded", "Base Domain HSTS Preloaded", "Domain Supports HTTPS", - "Domain Enforces HTTPS", "Domain Uses Strong HSTS", + "Domain Enforces HTTPS", "Domain Uses Strong HSTS", "HTTPS Live", "HTTPS Full Connection", "HTTPS Client Auth Required", "HTTPS Publicly Trusted", "HTTPS Custom Truststore Trusted", - "IP", "Server Header", "Server Version", "HTTPS Cert Chain Length", + "IP", "Server Header", "Server Version", "HTTPS Cert Chain Length", "HTTPS Probably Missing Intermediate Cert", "Notes", "Unknown Error", ] diff --git a/scanners/sslyze.py b/scanners/sslyze.py index d72ebc29..bbf1a0b5 100644 --- a/scanners/sslyze.py +++ b/scanners/sslyze.py @@ -325,7 +325,7 @@ def run_sslyze(data, environment, options): if reneg: analyze_reneg(data, reneg) - + return data @@ -625,10 +625,10 @@ def run_scan(scan_type, command, errors): logging.debug("{}: Exception during {} scan: {}".format(server_info.hostname, scan_type, err)) errors = errors + 1 return result, errors - + logging.debug("\tRunning scans in serial.") sslv2, errors = run_scan("SSLv2", Sslv20ScanCommand(), errors) - sslv3, errors = run_scan("SSLv3", Sslv30ScanCommand(), errors) + sslv3, errors = run_scan("SSLv3", Sslv30ScanCommand(), errors) tlsv1, errors = run_scan("TLSv1.0", Tlsv10ScanCommand(), errors) tlsv1_1, errors = run_scan("TLSv1.1", Tlsv11ScanCommand(), errors) tlsv1_2, errors = run_scan("TLSv1.2", Tlsv12ScanCommand(), errors) From 050e26bd2e04895901e8d7342558389ad1f06809 Mon Sep 17 00:00:00 2001 From: echudow Date: Fri, 15 Mar 2019 13:44:39 -0400 Subject: [PATCH 12/21] Added ca_file option to help validate and construct higher trust chains --- scanners/sslyze.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scanners/sslyze.py b/scanners/sslyze.py index d72ebc29..7419fca3 100644 --- a/scanners/sslyze.py +++ b/scanners/sslyze.py @@ -39,6 +39,9 @@ # Advertise Lambda support lambda_support = True +# File with custom root and intermediate certs that should be trusted for +# verifying the cert chain +CA_FILE = None # If we have pshtt data, use it to skip some domains, and to adjust # scan hostnames to canonical URLs where we can. @@ -580,9 +583,11 @@ def supported_protocol(result): # SSlyze initialization boilerplate def init_sslyze(hostname, port, starttls_smtp, options, sync=False): - global network_timeout + global network_timeout, CA_FILE network_timeout = int(options.get("network_timeout", network_timeout)) + if options.get('ca_file'): + CA_FILE = options['ca_file'] tls_wrapped_protocol = TlsWrappedProtocolEnum.PLAIN_TLS if starttls_smtp: @@ -638,7 +643,7 @@ def run_scan(scan_type, command, errors): if errors < 2 and options.get("sslyze_certs", True) is True: try: logging.debug("\t\tCertificate information scan.") - certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand()) + certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand(ca_file=CA_FILE)) except idna.core.InvalidCodepoint: logging.warning(utils.format_last_exception()) data['errors'].append("Invalid certificate/OCSP for this domain.") From 860392e2e13f80266ba3fedbc4ded236b3ecc0a0 Mon Sep 17 00:00:00 2001 From: echudow Date: Fri, 15 Mar 2019 13:47:06 -0400 Subject: [PATCH 13/21] trustymail updates --- scanners/trustymail.py | 12 +++++++----- utils/scan_utils.py | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scanners/trustymail.py b/scanners/trustymail.py index 4e725a7f..3bdac2b7 100644 --- a/scanners/trustymail.py +++ b/scanners/trustymail.py @@ -308,14 +308,16 @@ def to_rows(data): headers = [ "Live", - "MX Record", "Mail Servers", "Mail Server Ports Tested", + "MX Record", "MX Record DNSSEC", "Mail Servers", "Mail Server Ports Tested", "Domain Supports SMTP", "Domain Supports SMTP Results", "Domain Supports STARTTLS", "Domain Supports STARTTLS Results", - "SPF Record", "Valid SPF", "SPF Results", - "DMARC Record", "Valid DMARC", "DMARC Results", - "DMARC Record on Base Domain", "Valid DMARC Record on Base Domain", - "DMARC Results on Base Domain", "DMARC Policy", "DMARC Policy Percentage", + "SPF Record", "SPF Record DNSSEC", "Valid SPF", "SPF Results", + "DMARC Record", "DMARC Record DNSSEC", "Valid DMARC", "DMARC Results", + "DMARC Record on Base Domain", "DMARC Record on Base Domain DNSSEC", + "Valid DMARC Record on Base Domain", "DMARC Results on Base Domain", + "DMARC Policy", "DMARC Subdomain Policy", "DMARC Policy Percentage", "DMARC Aggregate Report URIs", "DMARC Forensic Report URIs", "DMARC Has Aggregate Report URI", "DMARC Has Forensic Report URI", + "DMARC Reporting Address Acceptance Error", "Syntax Errors", "Debug Info" ] diff --git a/utils/scan_utils.py b/utils/scan_utils.py index a65dc13e..eb9840fa 100644 --- a/utils/scan_utils.py +++ b/utils/scan_utils.py @@ -452,7 +452,7 @@ def build_scan_options_parser() -> ArgumentParser: parser.add_argument("--sslyze-reneg", help="sslyze: If set, will use the SessionRenegotiationScanner and return session renegotiation info. Defaults to true.") # trustymail: - parser.add_argument("--starttls", help="".join([ + parser.add_argument("--starttls", action='store_true', help="".join([ "trustymail: Only check mx records and STARTTLS support. ", "(Implies --mx.)" ])) @@ -486,13 +486,13 @@ def build_scan_options_parser() -> ArgumentParser: "may results in slower scans due to testing the ", "same mail servers multiple times." ])) - parser.add_argument("--mx", help="".join([ + parser.add_argument("--mx", action='store_true', help="".join([ "trustymail: Only check MX records" ])) - parser.add_argument("--spf", help="".join([ + parser.add_argument("--spf", action='store_true', help="".join([ "trustymail: Only check SPF records" ])) - parser.add_argument("--dmarc", help="".join([ + parser.add_argument("--dmarc", action='store_true', help="".join([ "trustymail: Only check DMARC records" ])) From 4240b17627492e736c766ab4718607455659023d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 16 Mar 2019 22:34:43 -0400 Subject: [PATCH 14/21] Make a few whitespace changes to make flake8 happy --- scanners/sslyze.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scanners/sslyze.py b/scanners/sslyze.py index c2c06a65..c2d1e573 100644 --- a/scanners/sslyze.py +++ b/scanners/sslyze.py @@ -39,10 +39,11 @@ # Advertise Lambda support lambda_support = True -# File with custom root and intermediate certs that should be trusted for -# verifying the cert chain +# File with custom root and intermediate certs that should be trusted +# for verifying the cert chain CA_FILE = None + # If we have pshtt data, use it to skip some domains, and to adjust # scan hostnames to canonical URLs where we can. # From 369f91b3bbafab648a2c2785731cb416f8e01ed7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 16 Mar 2019 22:39:18 -0400 Subject: [PATCH 15/21] Fix scanners/trustymail.py file Someone had accidentally committed a merge conflict. --- scanners/trustymail.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scanners/trustymail.py b/scanners/trustymail.py index 2808d348..61893155 100644 --- a/scanners/trustymail.py +++ b/scanners/trustymail.py @@ -308,19 +308,11 @@ def to_rows(data): "MX Record", "MX Record DNSSEC", "Mail Servers", "Mail Server Ports Tested", "Domain Supports SMTP", "Domain Supports SMTP Results", "Domain Supports STARTTLS", "Domain Supports STARTTLS Results", -<<<<<<< HEAD "SPF Record", "SPF Record DNSSEC", "Valid SPF", "SPF Results", "DMARC Record", "DMARC Record DNSSEC", "Valid DMARC", "DMARC Results", "DMARC Record on Base Domain", "DMARC Record on Base Domain DNSSEC", - "Valid DMARC Record on Base Domain", "DMARC Results on Base Domain", + "Valid DMARC Record on Base Domain", "DMARC Results on Base Domain", "DMARC Policy", "DMARC Subdomain Policy", "DMARC Policy Percentage", -======= - "SPF Record", "Valid SPF", "SPF Results", - "DMARC Record", "Valid DMARC", "DMARC Results", - "DMARC Record on Base Domain", "Valid DMARC Record on Base Domain", - "DMARC Results on Base Domain", "DMARC Policy", "DMARC Subdomain Policy", - "DMARC Policy Percentage", ->>>>>>> 07be17980d3ffa12794b9fefd7011e2a4b083279 "DMARC Aggregate Report URIs", "DMARC Forensic Report URIs", "DMARC Has Aggregate Report URI", "DMARC Has Forensic Report URI", "DMARC Reporting Address Acceptance Error", From c6763132cc91e0fcad4f4acc907dcf6584e83543 Mon Sep 17 00:00:00 2001 From: echudow Date: Sun, 17 Mar 2019 19:12:02 -0400 Subject: [PATCH 16/21] Adding extra trustymail command-line arguments back for compatibility --- utils/scan_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/scan_utils.py b/utils/scan_utils.py index 348c6b84..d9d884ac 100644 --- a/utils/scan_utils.py +++ b/utils/scan_utils.py @@ -452,7 +452,7 @@ def build_scan_options_parser() -> ArgumentParser: parser.add_argument("--sslyze-reneg", help="sslyze: If set, will use the SessionRenegotiationScanner and return session renegotiation info. Defaults to true.") # trustymail: - parser.add_argument("--starttls", action='store_true', help="".join([ + parser.add_argument("--starttls", help="".join([ "trustymail: Only check mx records and STARTTLS support. ", "(Implies --mx.)" ])) @@ -486,13 +486,13 @@ def build_scan_options_parser() -> ArgumentParser: "may results in slower scans due to testing the ", "same mail servers multiple times." ])) - parser.add_argument("--mx", action='store_true', help="".join([ + parser.add_argument("--mx", help="".join([ "trustymail: Only check MX records" ])) - parser.add_argument("--spf", action='store_true', help="".join([ + parser.add_argument("--spf", help="".join([ "trustymail: Only check SPF records" ])) - parser.add_argument("--dmarc", action='store_true', help="".join([ + parser.add_argument("--dmarc", help="".join([ "trustymail: Only check DMARC records" ])) From cf8a291ad869870ceed74da62d93449944baca2e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 17 Mar 2019 23:10:54 -0400 Subject: [PATCH 17/21] Put back the action='store_true' statements This is for the dmarc, mx, spf, and starttls CLI options. --- utils/scan_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/scan_utils.py b/utils/scan_utils.py index d9d884ac..348c6b84 100644 --- a/utils/scan_utils.py +++ b/utils/scan_utils.py @@ -452,7 +452,7 @@ def build_scan_options_parser() -> ArgumentParser: parser.add_argument("--sslyze-reneg", help="sslyze: If set, will use the SessionRenegotiationScanner and return session renegotiation info. Defaults to true.") # trustymail: - parser.add_argument("--starttls", help="".join([ + parser.add_argument("--starttls", action='store_true', help="".join([ "trustymail: Only check mx records and STARTTLS support. ", "(Implies --mx.)" ])) @@ -486,13 +486,13 @@ def build_scan_options_parser() -> ArgumentParser: "may results in slower scans due to testing the ", "same mail servers multiple times." ])) - parser.add_argument("--mx", help="".join([ + parser.add_argument("--mx", action='store_true', help="".join([ "trustymail: Only check MX records" ])) - parser.add_argument("--spf", help="".join([ + parser.add_argument("--spf", action='store_true', help="".join([ "trustymail: Only check SPF records" ])) - parser.add_argument("--dmarc", help="".join([ + parser.add_argument("--dmarc", action='store_true', help="".join([ "trustymail: Only check DMARC records" ])) From 0848d112a165da080e322a91d9d7bf6e6d56fd33 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 17 Mar 2019 23:18:57 -0400 Subject: [PATCH 18/21] Adjust tests to pass now that some options have action='store_true' These options are the trustymail options --mx, --dmarc, --spf, and --starttls. --- tests/test_scan_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_scan_utils.py b/tests/test_scan_utils.py index 17ec080e..a2c0f4a8 100644 --- a/tests/test_scan_utils.py +++ b/tests/test_scan_utils.py @@ -178,6 +178,10 @@ def test_determine_scan_workers(scanner, options, w_default, w_max, expected): "no_fast_cache": False, "serial": False, "sort": False, + "dmarc": False, + "mx": False, + "starttls": False, + "spf": False, "output": "./", "_": { "cache_dir": "./cache", @@ -201,6 +205,10 @@ def test_determine_scan_workers(scanner, options, w_default, w_max, expected): "no_fast_cache": False, "serial": False, "sort": False, + "dmarc": False, + "mx": False, + "starttls": False, + "spf": False, "output": "./", "_": { "cache_dir": "./cache", From e513aa76565f9d37ebc58dfb593bcd9e8eaa7e13 Mon Sep 17 00:00:00 2001 From: echudow Date: Mon, 18 Mar 2019 06:27:00 -0400 Subject: [PATCH 19/21] Don't convert None to False --- scanners/pshtt.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scanners/pshtt.py b/scanners/pshtt.py index 5c0f2887..bf8e6801 100644 --- a/scanners/pshtt.py +++ b/scanners/pshtt.py @@ -110,12 +110,6 @@ def to_rows(data): row = [] for field in headers: value = data[field] - - # TODO: Fix this upstream - if (field != "HSTS Header") and (field != "HSTS Max Age") and (field != "Redirect To"): - if value is None: - value = False - row.append(value) return [row] From b81292853031d35409881b67977515e9e0b74efa Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 20 Mar 2019 06:24:38 -0400 Subject: [PATCH 20/21] Update to use the latest pshtt and trustymail, which correctly handle websites that require client certs and include DNSSEC data --- requirements-scanners.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-scanners.txt b/requirements-scanners.txt index ac4b1e72..57024cf8 100644 --- a/requirements-scanners.txt +++ b/requirements-scanners.txt @@ -2,10 +2,10 @@ # Requirements used by specific scanners. # pshtt -pshtt>=0.5.4 +pshtt>=0.6.0 # trustymail -trustymail>=0.6.9 +trustymail>=0.7.0 # sslyze sslyze>=2.0.1 From 201962216c05b37fb9ed8232ca7303ec10cf9bfe Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 20 Mar 2019 11:21:24 -0400 Subject: [PATCH 21/21] Update to use pshtt 0.6.1 The differences between 0.6.0 and 0.6.1 are purely cosmetic, but we may as well be current. --- requirements-scanners.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-scanners.txt b/requirements-scanners.txt index 57024cf8..59cb7b84 100644 --- a/requirements-scanners.txt +++ b/requirements-scanners.txt @@ -2,7 +2,7 @@ # Requirements used by specific scanners. # pshtt -pshtt>=0.6.0 +pshtt>=0.6.1 # trustymail trustymail>=0.7.0