From dad654e74af746d7a0f917450751f3da843e342c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diogo=20Os=C3=B3rio?= Date: Fri, 6 Oct 2017 23:47:45 +0100 Subject: [PATCH 1/8] Add some basic tests for the parse_arguments function This function will be responsible from parsing the argv argument list and build an object which encapsulates all the scan command parameters. --- tests/test_input.py | 115 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 tests/test_input.py diff --git a/tests/test_input.py b/tests/test_input.py new file mode 100644 index 0000000..6faba34 --- /dev/null +++ b/tests/test_input.py @@ -0,0 +1,115 @@ +import argparse +import pytest + +from lib.input import cli_argument_parser + +def test_parse_arguments_default_value(tmpdir): + words = ['word1', 'word2', 'word3'] + wordlist = tmpdir.mkdir('test_command').join('default') + wordlist.write('\n'.join(words)) + + argv = ['-t', 'myhost'] + + arguments = cli_argument_parser().parse(argv) + + expected_arguments = { + 'target_hosts': 'myhost', + 'wordlists': None, + 'base_host': False, + 'port': 80, + 'real_port': False, + 'ignore_http_codes': '404', + 'ignore_content_length': 0, + 'unique_depth': 1, + 'fuzzy_logic': False, + 'no_lookup': False, + 'rate_limit': 0, + 'random_agent': False, + 'user_agent': None, + 'add_waf_bypass_headers': False, + 'output_normal': None, + 'output_json': None, + 'stdin': False, + 'ssl': False, + } + + assert vars(arguments) == expected_arguments + + +def test_parse_arguments_custom_arguments(tmpdir): + words = ['some', 'other', 'words'] + wordlist = tmpdir.mkdir('test_command').join('other_words') + wordlist.write('\n'.join(words)) + + argv = [ + '-t', '10.11.1.1', + '-w', str(wordlist), + '-b', 'myhost', + '-p', '8000', + '-r', '8001', + '--ignore-http-codes', '400,500,302', + '--ignore-content-length', '100', + '--unique-depth', '5', + '--ssl', + '--fuzzy-logic', + '--no-lookups', + '--rate-limit', '10', + '--user-agent', 'some-user-agent', + '--waf', + '-oN', '/tmp/on', + '-', + ] + + arguments = cli_argument_parser().parse(argv) + + expected_arguments = { + 'target_hosts': '10.11.1.1', + 'wordlists': str(wordlist), + 'base_host': 'myhost', + 'port': 8000, + 'real_port': 8001, + 'ignore_http_codes': '400,500,302', + 'ignore_content_length': 100, + 'unique_depth': 5, + 'ssl': True, + 'fuzzy_logic': True, + 'no_lookup': True, + 'rate_limit': 10, + 'user_agent': 'some-user-agent', + 'random_agent': False, + 'add_waf_bypass_headers': True, + 'output_normal': '/tmp/on', + 'output_json': None, + 'stdin': True, + } + + assert vars(arguments) == expected_arguments + +def test_parse_arguments_mutually_exclusive_user_agent(): + argv = [ + '-t', '10.11.1.1', + '--user-agent', 'my-user-agent', + '--random-agent', + ] + + with pytest.raises(SystemExit): + cli_argument_parser().parse(argv) + +def test_parse_arguments_mutually_exclusive_output(): + argv = [ + '-t', '10.11.1.1', + '-oJ', + '-oN', + ] + + with pytest.raises(SystemExit): + cli_argument_parser().parse(argv) + +def test_parse_arguments_unknown_argument(): + argv = [ + '-t', '10.11.1.1', + '-i-do-not-exist', + ] + + with pytest.raises(SystemExit): + cli_argument_parser().parse(argv) From 55ded8827e5366b7bf43ce46040e58f6afbfb0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diogo=20Os=C3=B3rio?= Date: Sat, 7 Oct 2017 01:24:55 +0100 Subject: [PATCH 2/8] Remove the lib directory from the gitignore The lib directory was being listed as a git ignore directory - this directory contains the main project files. --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index a896536..f0277b9 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ lib64/ parts/ sdist/ From 7377b6cd8260310abd9a5c3c86d5ccc6e0c60443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diogo=20Os=C3=B3rio?= Date: Sat, 7 Oct 2017 01:26:22 +0100 Subject: [PATCH 3/8] Implement the scanner_argument_parser This class serves as an indirection layer over the argparser library and encapsulates the CLI command definition and the translation of the raw user input into the known set of arguments. The ideia in the near future is for this class to be able to pre-process all the CLI input into some sort of request object for the scanner, in order to decouple this task from the main scanner function which currently is one of the reasons that prevents the function to be used in any other context besides a CLI run. --- lib/input.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 lib/input.py diff --git a/lib/input.py b/lib/input.py new file mode 100644 index 0000000..13f7189 --- /dev/null +++ b/lib/input.py @@ -0,0 +1,106 @@ +from argparse import ArgumentParser + +class cli_argument_parser(object): + def __init__(self): + self._parser = self.setup_parser() + + def parse(self, argv): + return self._parser.parse_args(argv) + + @staticmethod + def setup_parser(): + parser = ArgumentParser() + + parser.add_argument( + '-t', dest='target_hosts', required=True, + help='Set a target range of addresses to target. Ex 10.11.1.1-255' + ), + + parser.add_argument( + '-w', dest='wordlists', + help='Set the wordlists to use (default ./wordlists/virtual-host-scanning.txt)' + ) + + parser.add_argument( + '-b', dest='base_host', default=False, + help='Set host to be used during substitution in wordlist (default to TARGET).' + ) + + parser.add_argument( + '-p', dest='port', default=80, type=int, + help='Set the port to use (default 80).' + ) + + parser.add_argument( + '-r', dest='real_port', type=int, default=False, + help='The real port of the webserver to use in headers when not 80 (see RFC2616 14.23), useful when pivoting through ssh/nc etc (default to PORT).' + ) + + parser.add_argument( + '--ignore-http-codes', dest='ignore_http_codes', default='404', + help='Comma separated list of http codes to ignore with virtual host scans (default 404).' + ) + + parser.add_argument( + '--ignore-content-length', dest='ignore_content_length', type=int, default=0, + help='Ignore content lengths of specificed amount (default 0).' + ) + + parser.add_argument( + '--unique-depth', dest='unique_depth', type=int, default=1, + help='Show likely matches of page content that is found x times (default 1).' + ) + + parser.add_argument( + '--ssl', dest='ssl', action='store_true', default=False, + help='If set then connections will be made over HTTPS instead of HTTP (default http).' + ) + + parser.add_argument( + '--fuzzy-logic', dest='fuzzy_logic', action='store_true', default=False, + help='If set then fuzzy match will be performed against unique hosts (default off).' + ) + + parser.add_argument( + '--no-lookups', dest='no_lookup', action='store_true', default=False, + help='Disable reverse lookups (identifies new targets and appends to wordlist, on by default).' + ) + + parser.add_argument( + '--rate-limit', dest='rate_limit', type=int, default=0, + help='Amount of time in seconds to delay between each scan (default 0).' + ) + + parser.add_argument( + '--waf', dest='add_waf_bypass_headers', action='store_true', default=False, + help='If set then simple WAF bypass headers will be sent.' + ) + + parser.add_argument( + '-', dest='stdin', action='store_true', default=False, + help="By passing a blank '-' you tell VHostScan to expect input from stdin (pipe)." + ) + + output = parser.add_mutually_exclusive_group() + output.add_argument( + '-oN', dest='output_normal', + help='Normal output printed to a file when the -oN option is specified with a filename argument.' + ) + + output.add_argument( + '-oJ', dest='output_json', + help='JSON output printed to a file when the -oJ option is specified with a filename argument.' + ) + + user_agent = parser.add_mutually_exclusive_group() + user_agent.add_argument( + '--random-agent', dest='random_agent', action='store_true', default=False, + help='If set, then each scan will use random user-agent from predefined list.' + ) + + user_agent.add_argument( + '--user-agent', dest='user_agent', + help='Specify a user-agent to use for scans' + ) + + return parser \ No newline at end of file From 21f232b109df4a77b4abb04c109b0e638bdebaf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diogo=20Os=C3=B3rio?= Date: Sun, 8 Oct 2017 17:28:04 +0100 Subject: [PATCH 4/8] Use the new scanner_argument_parser to parse the arguments Instead of build the argparser.ArgumentParser directly, delegate to the new scanner_argument_parser class the responsibility of parsing the CLI arguments. --- VHostScan.py | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/VHostScan.py b/VHostScan.py index 41bc3c5..f02423f 100644 --- a/VHostScan.py +++ b/VHostScan.py @@ -9,6 +9,7 @@ from lib.helpers.output_helper import * from lib.helpers.file_helper import get_combined_word_lists, load_random_user_agents from lib.core.__version__ import __version__ +from lib.input import cli_argument_parser def print_banner(): @@ -19,32 +20,12 @@ def print_banner(): def main(): print_banner() - parser = ArgumentParser() - parser.add_argument("-t", dest="target_hosts", required=True, help="Set a target range of addresses to target. Ex 10.11.1.1-255" ) - parser.add_argument("-w", dest="wordlists", required=False, type=str, help="Set the wordlists to use (default ./wordlists/virtual-host-scanning.txt)", default=False) - parser.add_argument("-b", dest="base_host", required=False, help="Set host to be used during substitution in wordlist (default to TARGET).", default=False) - parser.add_argument("-p", dest="port", required=False, help="Set the port to use (default 80).", default=80) - parser.add_argument("-r", dest="real_port", required=False, help="The real port of the webserver to use in headers when not 80 (see RFC2616 14.23), useful when pivoting through ssh/nc etc (default to PORT).", default=False) - - parser.add_argument('--ignore-http-codes', dest='ignore_http_codes', type=str, help='Comma separated list of http codes to ignore with virtual host scans (default 404).', default='404') - parser.add_argument('--ignore-content-length', dest='ignore_content_length', type=int, help='Ignore content lengths of specificed amount (default 0).', default=0) - parser.add_argument('--unique-depth', dest='unique_depth', type=int, help='Show likely matches of page content that is found x times (default 1).', default=1) - parser.add_argument("--ssl", dest="ssl", action="store_true", help="If set then connections will be made over HTTPS instead of HTTP (default http).", default=False) - parser.add_argument("--fuzzy-logic", dest="fuzzy_logic", action="store_true", help="If set then fuzzy match will be performed against unique hosts (default off).", default=False) - parser.add_argument("--no-lookups", dest="no_lookup", action="store_true", help="Disable reverse lookups (identifies new targets and appends to wordlist, on by default).", default=False) - parser.add_argument("--rate-limit", dest="rate_limit", type=int, help='Amount of time in seconds to delay between each scan (default 0).', default=0) - parser.add_argument('--random-agent', dest='random_agent', action='store_true', help='If set, then each scan will use random user-agent from predefined list.', default=False) - parser.add_argument('--user-agent', dest='user_agent', type=str, help='Specify a user-agent to use for scans') - parser.add_argument("--waf", dest="add_waf_bypass_headers", action="store_true", help="If set then simple WAF bypass headers will be sent.", default=False) - parser.add_argument("-oN", dest="output_normal", help="Normal output printed to a file when the -oN option is specified with a filename argument." ) - parser.add_argument("-oJ", dest="output_json", help="JSON output printed to a file when the -oJ option is specified with a filename argument." ) - parser.add_argument("-", dest="stdin", action="store_true", help="By passing a blank '-' you tell VHostScan to expect input from stdin (pipe).", default=False) - - arguments = parser.parse_args() - wordlist = [] + + parser = cli_argument_parser() + arguments = parser.parse(sys.argv[1:]) + wordlist = [] word_list_types = [] - default_wordlist = "./wordlists/virtual-host-scanning.txt" if not arguments.stdin else None if arguments.stdin: From 0a1dea99eff8e3159100fa01dfc5df7e535a6a35 Mon Sep 17 00:00:00 2001 From: Michael <886344+codingo@users.noreply.github.com> Date: Mon, 9 Oct 2017 09:35:39 +1000 Subject: [PATCH 5/8] Update __version__.py --- lib/core/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/__version__.py b/lib/core/__version__.py index 19f2498..80c68e7 100644 --- a/lib/core/__version__.py +++ b/lib/core/__version__.py @@ -2,5 +2,5 @@ # |V|H|o|s|t|S|c|a|n| Developed by @codingo_ & @__timk # +-+-+-+-+-+-+-+-+-+ https://github.com/codingo/VHostScan -__version__ = '1.5.2' +__version__ = '1.6.1' From 0ccf3d6a1b2103c1c9c1e70255a6b87d943255ee Mon Sep 17 00:00:00 2001 From: Michael <886344+codingo@users.noreply.github.com> Date: Mon, 9 Oct 2017 09:45:29 +1000 Subject: [PATCH 6/8] Re-added first-hit from earlier pull request Added first-hit back into the codebase as it was added into the application after this pr. --- lib/input.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/input.py b/lib/input.py index 13f7189..0ceac2c 100644 --- a/lib/input.py +++ b/lib/input.py @@ -46,6 +46,11 @@ def setup_parser(): help='Ignore content lengths of specificed amount (default 0).' ) + parser.add_argument( + '--first-hit', dest='first_hit', action='store_true', default=False, + help='Return first successful result. Only use in scenarios where you are sure no catch-all is configured (such as a CTF).' + ) + parser.add_argument( '--unique-depth', dest='unique_depth', type=int, default=1, help='Show likely matches of page content that is found x times (default 1).' @@ -103,4 +108,4 @@ def setup_parser(): help='Specify a user-agent to use for scans' ) - return parser \ No newline at end of file + return parser From cf068640e72eccccde8155e7fad508b9b1b9a8d2 Mon Sep 17 00:00:00 2001 From: Michael <886344+codingo@users.noreply.github.com> Date: Mon, 9 Oct 2017 09:46:29 +1000 Subject: [PATCH 7/8] Added first-hit to test case --- tests/test_input.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_input.py b/tests/test_input.py index 6faba34..814e0a1 100644 --- a/tests/test_input.py +++ b/tests/test_input.py @@ -20,6 +20,7 @@ def test_parse_arguments_default_value(tmpdir): 'real_port': False, 'ignore_http_codes': '404', 'ignore_content_length': 0, + 'first_hit': False , 'unique_depth': 1, 'fuzzy_logic': False, 'no_lookup': False, From 4c13b17b3155229bcc4f076b7c88846a4d3f3959 Mon Sep 17 00:00:00 2001 From: Michael <886344+codingo@users.noreply.github.com> Date: Mon, 9 Oct 2017 09:51:01 +1000 Subject: [PATCH 8/8] Update test_input.py --- tests/test_input.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_input.py b/tests/test_input.py index 814e0a1..3b74d1a 100644 --- a/tests/test_input.py +++ b/tests/test_input.py @@ -51,6 +51,7 @@ def test_parse_arguments_custom_arguments(tmpdir): '--ignore-http-codes', '400,500,302', '--ignore-content-length', '100', '--unique-depth', '5', + '--first-hit', '--ssl', '--fuzzy-logic', '--no-lookups', @@ -71,6 +72,7 @@ def test_parse_arguments_custom_arguments(tmpdir): 'real_port': 8001, 'ignore_http_codes': '400,500,302', 'ignore_content_length': 100, + 'first_hit': True, 'unique_depth': 5, 'ssl': True, 'fuzzy_logic': True,