diff --git a/bin/snakebite b/bin/snakebite index 2722047..0a3e347 100755 --- a/bin/snakebite +++ b/bin/snakebite @@ -25,8 +25,7 @@ class SnakebiteCli(object): clparser = CommandLineParser() self.args = clparser.parse() self._setup_logging() - clparser.read_config() - clparser.setup_client() + clparser.init() clparser.execute() def _setup_logging(self): diff --git a/snakebite/commandlineparser.py b/snakebite/commandlineparser.py index d309513..108e700 100644 --- a/snakebite/commandlineparser.py +++ b/snakebite/commandlineparser.py @@ -36,6 +36,13 @@ from snakebite.namenode import Namenode +def print_error_exit(msg, fd=sys.stderr): + print >> fd, "Error: %s" % msg + sys.exit(-1) + +def print_info(msg, fd=sys.stderr): + print >> fd, "Info: %s" % msg + def exitError(error): if isinstance(error, FileNotFoundException) or \ isinstance(error, DirectoryException) or \ @@ -100,7 +107,7 @@ class CommandLineParser(object): "type": float}, 'p': {"short": '-p', "long": '--port', - "help": 'namenode RPC port', + "help": 'namenode RPC port (default: %d)' % Namenode.DEFAULT_PORT, "type": int}, 'h': {"short": '-h', "long": '--help', @@ -129,6 +136,10 @@ class CommandLineParser(object): "help": 'skip the trash (when trash is enabled)', "default": False, "action": 'store_true'}, + 'T': {"short": '-T', + "long": "--usetrash", + "help": "enable the trash", + "action": 'store_true'}, 'z': {"short": '-z', "long": '--zero', "help": 'check for zero length', @@ -230,6 +241,28 @@ def _add_subparsers(self): command_parser = subparsers.add_parser(cmd_name, add_help=False, parents=parents) command_parser.set_defaults(command=cmd_name) + def init(self): + self.read_config() + self._clean_args() + self.setup_client() + + def _clean_args(self): + for path in self.__get_all_directories(): + if path.startswith('hdfs://'): + parse_result = urlparse(path) + if path in self.args.dir: + self.args.dir.remove(path) + self.args.dir.append(parse_result.path) + else: + self.args.single_arg = parse_result.path + + def __usetrash_unset(self): + return not 'usetrash' in self.args or self.args.usetrash == False + + def __use_cl_port_first(self, alt): + # Port provided from CL has the highest priority: + return self.args.port if self.args.port else alt + def read_config(self): # Try to retrieve namenode config from within CL arguments @@ -252,10 +285,10 @@ def read_config(self): if configs: for config in configs: nn = Namenode(config['namenode'], - config['port']) + self.__use_cl_port_first(config['port'])) self.namenodes.append(nn) - - self.args.skiptrash = not any([c['use_trash'] for c in configs]) + if self.__usetrash_unset(): + self.args.usetrash = HDFSConfig.use_trash if len(self.namenodes): return @@ -272,14 +305,15 @@ def read_config(self): print ' "config_version": 2,' print ' "use_trash": true,' print ' "namenodes": [' - print ' {"host": "namenode-ha1", "port": 54310, "version": %d},' % Namenode.DEFAULT_VERSION - print ' {"host": "namenode-ha2", "port": 54310, "version": %d}' % Namenode.DEFAULT_VERSION + print ' {"host": "namenode-ha1", "port": %d, "version": %d},' % (Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) + print ' {"host": "namenode-ha2", "port": %d, "version": %d}' % (Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) print ' ]' print '}' sys.exit(1) def _read_config_snakebiterc(self): + old_version_info = "You're are using snakebite %s with Trash support together with old snakebiterc, please update/remove your ~/.snakebiterc file. By default Trash is %s." % (version(), 'disabled' if not HDFSConfig.use_trash else 'enabled') with open(os.path.join(os.path.expanduser('~'), '.snakebiterc')) as config_file: configs = json.load(config_file) @@ -288,52 +322,81 @@ def _read_config_snakebiterc(self): # config is a list of namenode(s) - possibly HA for config in configs: nn = Namenode(config['namenode'], - config['port'], + self.__use_cl_port_first(config.get('port', Namenode.DEFAULT_PORT)), config.get('version', Namenode.DEFAULT_VERSION)) self.namenodes.append(nn) + if self.__usetrash_unset(): + # commandline setting has higher priority + print_info(old_version_info) + # There's no info about Trash in version 1, use default policy: + self.args.usetrash = HDFSConfig.use_trash elif isinstance(configs, dict): - if configs.get("config_version"): - # Config ersion > 2 + # Version 2: {} + # Can be either new configuration or just one namenode + # which was the very first configuration syntax + if 'config_version' in configs: + # Config version => 2 for nn_config in configs['namenodes']: - nn = Namenode(nn_config['host'], nn_config['port'], nn_config.get('version', Namenode.DEFAULT_VERSION)) + nn = Namenode(nn_config['host'], + self.__use_cl_port_first(nn_config.get('port', Namenode.DEFAULT_PORT)), + nn_config.get('version', Namenode.DEFAULT_VERSION)) self.namenodes.append(nn) - self.args.skiptrash = configs.get("skiptrash") + if self.__usetrash_unset(): + # commandline setting has higher priority + self.args.usetrash = configs.get("use_trash", HDFSConfig.use_trash) else: - # config is a single namenode - no HA + # config is a single namenode - no HA self.namenodes.append(Namenode(configs['namenode'], - configs['port'], + self.__use_cl_port_first(configs.get('port', Namenode.DEFAULT_PORT)), configs.get('version', Namenode.DEFAULT_VERSION))) + if self.__usetrash_unset(): + # commandline setting has higher priority + print_info(old_version_info) + self.args.usetrash = HDFSConfig.use_trash else: - print "Config retrieved from ~/.snakebiterc is corrupted! Remove it!" - sys.exit(1) + print_error_exit("Config retrieved from ~/.snakebiterc is corrupted! Remove it!") - def _read_config_cl(self): - ''' Check if any directory arguments contain hdfs://''' + def __get_all_directories(self): if self.args and 'dir' in self.args: dirs_to_check = list(self.args.dir) if self.args.command == 'mv': dirs_to_check.append(self.args.single_arg) - for directory in dirs_to_check: - if 'hdfs://' in directory: - parse_result = urlparse(directory) - - if not self.args.namenode is None and not self.args.port is None and (self.args.port != parse_result.port or self.args.namenode != parse_result.hostname): - print "error: conflicting nodenames or ports" - sys.exit(-1) - else: - self.args.namenode = parse_result.hostname - self.args.port = parse_result.port - - if directory in self.args.dir: - self.args.dir.remove(directory) - self.args.dir.append(parse_result.path) - else: - self.args.single_arg = parse_result.path - - if self.args.namenode and self.args.port: - # If namenode config found based on arguments, save namenode + return dirs_to_check + else: + return () + + def _read_config_cl(self): + ''' Check if any directory arguments contain hdfs://''' + dirs_to_check = self.__get_all_directories() + hosts, ports = [], [] + for path in dirs_to_check: + if path.startswith('hdfs://'): + parse_result = urlparse(path) + hosts.append(parse_result.hostname) + ports.append(parse_result.port) + + # remove duplicates and None from (hosts + self.args.namenode) + hosts = filter(lambda x: x != None, set(hosts + [self.args.namenode])) + if len(hosts) > 1: + print_error_exit('Conficiting namenode hosts in commandline arguments, hosts: %s' % str(hosts)) + + ports = filter(lambda x: x != None, set(ports + [self.args.port])) + if len(ports) > 1: + print_error_exit('Conflicting namenode ports in commandline arguments, ports: %s' % str(ports)) + + # Store port from CL in arguments - CL port has the highest priority + if len(ports) == 1: + self.args.port = ports[0] + + # do we agree on one namenode? + if len(hosts) == 1 and len(ports) <= 1: + self.args.namenode = hosts[0] + self.args.port = ports[0] if len(ports) == 1 else Namenode.DEFAULT_PORT self.namenodes.append(Namenode(self.args.namenode, self.args.port)) + # we got the info from CL -> check if use_trash is set - if not use default policy: + if self.__usetrash_unset(): + self.args.usetrash = HDFSConfig.use_trash return True else: return False @@ -366,7 +429,10 @@ def parse(self, non_cli_input=None): # Allow input for testing purposes return self.args def setup_client(self): - use_trash = not self.args.skiptrash + if 'skiptrash' in self.args: + use_trash = self.args.usetrash and not self.args.skiptrash + else: + use_trash = self.args.usetrash self.client = HAClient(self.namenodes, use_trash) def execute(self): @@ -504,7 +570,7 @@ def mv(self): for line in format_results(result, json_output=self.args.json): print line - @command(args="[paths]", descr="remove paths", allowed_opts=["R", "S"], req_args=['dir [dirs]']) + @command(args="[paths]", descr="remove paths", allowed_opts=["R", "S", "T"], req_args=['dir [dirs]']) def rm(self): result = self.client.delete(self.args.dir, recurse=self.args.recurse) for line in format_results(result, json_output=self.args.json): diff --git a/snakebite/config.py b/snakebite/config.py index 1243569..be527ed 100644 --- a/snakebite/config.py +++ b/snakebite/config.py @@ -4,6 +4,7 @@ import xml.etree.ElementTree as ET from urlparse import urlparse +from namenode import Namenode log = logging.getLogger(__name__) @@ -48,15 +49,14 @@ def read_core_config(cls, core_site_path): if property.findall('name')[0].text == 'fs.defaultFS': parse_result = urlparse(property.findall('value')[0].text) log.debug("Got namenode '%s' from %s" % (parse_result.geturl(), core_site_path)) + config.append({"namenode": parse_result.hostname, - "port": parse_result.port}) + "port": parse_result.port if parse_result.port + else Namenode.DEFAULT_PORT}) if property.findall('name')[0].text == 'fs.trash.interval': cls.use_trash = True - for c in config: - c["use_trash"] = cls.use_trash - return config @classmethod @@ -67,14 +67,12 @@ def read_hdfs_config(cls, hdfs_site_path): parse_result = urlparse("//" + property.findall('value')[0].text) log.debug("Got namenode '%s' from %s" % (parse_result.geturl(), hdfs_site_path)) configs.append({"namenode": parse_result.hostname, - "port": parse_result.port}) + "port": parse_result.port if parse_result.port + else Namenode.DEFAULT_PORT}) if property.findall('name')[0].text == 'fs.trash.interval': cls.use_trash = True - for c in configs: - c["use_trash"] = cls.use_trash - return configs core_try_paths = ('/etc/hadoop/conf/core-site.xml', diff --git a/snakebite/namenode.py b/snakebite/namenode.py index dbbf40f..cf73bb0 100644 --- a/snakebite/namenode.py +++ b/snakebite/namenode.py @@ -15,10 +15,10 @@ class Namenode(object): '''Namenode class - represents HDFS namenode''' - + DEFAULT_PORT = 8020 DEFAULT_VERSION = 9 - def __init__(self, host, port, version=DEFAULT_VERSION): + def __init__(self, host, port=DEFAULT_PORT, version=DEFAULT_VERSION): self.host = host self.port = port self.version = version @@ -29,4 +29,4 @@ def is_active(self): def toDict(self): return {"namenode": self.host, "port": self.port, - "version": self.version} \ No newline at end of file + "version": self.version} diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/commandlineparser_test.py b/test/commandlineparser_test.py index 46c57d6..ea1e4f6 100644 --- a/test/commandlineparser_test.py +++ b/test/commandlineparser_test.py @@ -19,7 +19,11 @@ from mock import patch, mock_open +from snakebite.config import HDFSConfig from snakebite.commandlineparser import CommandLineParser +from snakebite.namenode import Namenode + +from config_test import ConfigTest class CommandLineParserTest(unittest2.TestCase): @@ -312,6 +316,23 @@ def test_rm(self): output = parser.parse('rm -S some_dir'.split()) self.assertTrue(output.skiptrash) + #skiptrash + output = parser.parse('rm --skiptrash some_dir'.split()) + self.assertTrue(output.skiptrash) + + #usetrash + output = parser.parse('rm -T some_dir'.split()) + self.assertTrue(output.usetrash) + + #usetrash + output =parser.parse('rm --usetrash some_dir'.split()) + self.assertTrue(output.usetrash) + + #usetrash & skiptrash + output = parser.parse('rm --usetrash --skiptrash some_dir'.split()) + self.assertTrue(output.usetrash) + self.assertTrue(output.skiptrash) + def test_touchz(self): parser = self.parser @@ -617,12 +638,14 @@ def __init__(self, dir=[], command=None, namenode=None, port=None, - skiptrash=None): + usetrash=False, + skiptrash=False): self.dir = dir self.single_arg = single_arg self.command = command self.namenode = namenode self.port = port + self.usetrash = usetrash self.skiptrash = skiptrash def __contains__(self, b): @@ -700,7 +723,7 @@ def test_cl_config_reduce_paths(self): "hdfs://foobar:50070/user/rav2"], single_arg="hdfs://foobar:50070/user/rav3", command="mv") - self.parser.read_config() + self.parser.init() self.assert_namenode_spec("foobar", 50070) self.assertIn("/user/rav", self.parser.args.dir) self.assertIn("/user/rav2", self.parser.args.dir) @@ -738,8 +761,8 @@ def test_read_config_snakebiterc_one_valid(self, exists_mock): with patch("snakebite.commandlineparser.open", m, create=True): self.parser.args = MockParseArgs() self.parser.read_config() - self.assertFalse(self.parser.args.skiptrash) self.assert_namenodes_spec("foobar", 54310, 9) + self.assertEquals(self.parser.args.usetrash, HDFSConfig.use_trash) @patch("os.path.exists") def test_read_config_snakebiterc_ha_valid(self, exists_mock): @@ -748,9 +771,9 @@ def test_read_config_snakebiterc_ha_valid(self, exists_mock): with patch("snakebite.commandlineparser.open", m, create=True): self.parser.args = MockParseArgs() self.parser.read_config() - self.assertFalse(self.parser.args.skiptrash) self.assert_namenodes_spec("foobar", 54310, 9) self.assert_namenodes_spec("foobar2", 54310, 9) + self.assertEquals(self.parser.args.usetrash, HDFSConfig.use_trash) @patch("os.path.exists") def test_read_config_snakebiterc_invalid(self, exists_mock): @@ -761,9 +784,74 @@ def test_read_config_snakebiterc_invalid(self, exists_mock): with self.assertRaises(SystemExit): self.parser.read_config() + valid_snake_noport_one_rc = {"namenode": "foobar", "version": 11} + valid_snake_noport_ha_rc = [{"namenode": "foobar", "version": 100}, + {"namenode": "foobar2", "version": 100}] + + @patch("os.path.exists") + def test_read_config_snakebiterc_noport_one_valid(self, exists_mock): + m = mock_open(read_data=json.dumps(self.valid_snake_noport_one_rc)) + + with patch("snakebite.commandlineparser.open", m, create=True): + self.parser.args = MockParseArgs() + self.parser.read_config() + self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, 11) + self.assertEquals(self.parser.args.usetrash, HDFSConfig.use_trash) + + @patch("os.path.exists") + def test_read_config_snakebiterc_noport_ha_valid(self, exists_mock): + m = mock_open(read_data=json.dumps(self.valid_snake_noport_ha_rc)) + + with patch("snakebite.commandlineparser.open", m, create=True): + self.parser.args = MockParseArgs() + self.parser.read_config() + self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, 100) + self.assert_namenodes_spec("foobar2", Namenode.DEFAULT_PORT, 100) + self.assertEquals(self.parser.args.usetrash, HDFSConfig.use_trash) + + + valid_snake_noport_nov_one_rc = {"namenode": "foobar"} + valid_snake_noport_nov_ha_rc = [{"namenode": "foobar"}, + {"namenode": "foobar2"}] + + @patch("os.path.exists") + def test_read_config_snakebiterc_noport_nov_one_valid(self, exists_mock): + m = mock_open(read_data=json.dumps(self.valid_snake_noport_nov_one_rc)) + + with patch("snakebite.commandlineparser.open", m, create=True): + self.parser.args = MockParseArgs() + self.parser.read_config() + self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) + self.assertEquals(self.parser.args.usetrash, HDFSConfig.use_trash) + + @patch("os.path.exists") + def test_read_config_snakebiterc_noport_nov_ha_valid(self, exists_mock): + m = mock_open(read_data=json.dumps(self.valid_snake_noport_nov_ha_rc)) + + with patch("snakebite.commandlineparser.open", m, create=True): + self.parser.args = MockParseArgs() + self.parser.read_config() + self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) + self.assert_namenodes_spec("foobar2", Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION) + self.assertEquals(self.parser.args.usetrash, HDFSConfig.use_trash) + + valid_snake_noport_mix_rc = [{"namenode": "foobar", "version": 100}, + {"namenode": "foobar2", "port": 66}] + + @patch("os.path.exists") + def test_read_config_snakebiterc_noport_mix_valid(self, exists_mock): + m = mock_open(read_data=json.dumps(self.valid_snake_noport_mix_rc)) + + with patch("snakebite.commandlineparser.open", m, create=True): + self.parser.args = MockParseArgs() + self.parser.read_config() + self.assert_namenodes_spec("foobar", Namenode.DEFAULT_PORT, 100) + self.assert_namenodes_spec("foobar2", 66, Namenode.DEFAULT_VERSION) + self.assertEquals(self.parser.args.usetrash, HDFSConfig.use_trash) + valid_snake_one_rc_v2 = { "config_version": 2, - "skiptrash": False, + "use_trash": False, "namenodes": [ {"host": "foobar3", "version": 9, "port": 54310} ] @@ -771,7 +859,7 @@ def test_read_config_snakebiterc_invalid(self, exists_mock): valid_snake_ha_rc_v2 = { "config_version": 2, - "skiptrash": True, + "use_trash": True, "namenodes": [ {"host": "foobar4", "version": 9, "port": 54310}, {"host": "foobar5", "version": 9, "port": 54310} @@ -787,7 +875,7 @@ def test_read_config_snakebiterc_one_valid_v2(self, exists_mock): with patch("snakebite.commandlineparser.open", m, create=True): self.parser.args = MockParseArgs() self.parser.read_config() - self.assertFalse(self.parser.args.skiptrash) + self.assertFalse(self.parser.args.usetrash) self.assert_namenodes_spec("foobar3", 54310, 9) @patch("os.path.exists") @@ -797,7 +885,7 @@ def test_read_config_snakebiterc_ha_valid_v2(self, exists_mock): with patch("snakebite.commandlineparser.open", m, create=True): self.parser.args = MockParseArgs() self.parser.read_config() - self.assertTrue(self.parser.args.skiptrash) + self.assertTrue(self.parser.args.usetrash) self.assert_namenodes_spec("foobar4", 54310, 9) self.assert_namenodes_spec("foobar5", 54310, 9) @@ -810,3 +898,116 @@ def test_read_config_snakebiterc_invalid_v2(self, exists_mock): self.parser.args = MockParseArgs() with self.assertRaises(SystemExit): self.parser.read_config() + + + valid_snake_noport_one_rc_v2 = { + "config_version": 2, + "use_trash": False, + "namenodes": [ + {"host": "foobar3", "version": 9} + ] + } + + valid_snake_mix_ha_rc_v2 = { + "config_version": 2, + "use_trash": True, + "namenodes": [ + {"host": "foobar4", "version": 100}, + {"host": "foobar5", "port": 54310} + ] + } + + @patch("os.path.exists") + def test_read_config_snakebiterc_noport_one_valid_v2(self, exists_mock): + m = mock_open(read_data=json.dumps(self.valid_snake_noport_one_rc_v2)) + + with patch("snakebite.commandlineparser.open", m, create=True): + self.parser.args = MockParseArgs() + self.parser.read_config() + self.assertFalse(self.parser.args.usetrash) + self.assert_namenodes_spec("foobar3", Namenode.DEFAULT_PORT, 9) + + @patch("os.path.exists") + def test_read_config_snakebiterc_mix_ha_valid_v2(self, exists_mock): + m = mock_open(read_data=json.dumps(self.valid_snake_mix_ha_rc_v2)) + + with patch("snakebite.commandlineparser.open", m, create=True): + self.parser.args = MockParseArgs() + self.parser.read_config() + self.assertTrue(self.parser.args.usetrash) + self.assert_namenodes_spec("foobar4", Namenode.DEFAULT_PORT, 100) + self.assert_namenodes_spec("foobar5", 54310, Namenode.DEFAULT_VERSION) + + + def test_cl_default_port(self): + self.parser.args = MockParseArgs(dir=["hdfs://foobar/user/rav"], + single_arg="hdfs://foobar/user/rav", + command="mv") + self.parser.read_config() + self.assert_namenode_spec("foobar", Namenode.DEFAULT_PORT) + + def test_cl_trash_setting_preserved_after_cl_config(self): + # no snakebiterc + # read config from CL + self.parser.args = MockParseArgs(dir=["hdfs://foobar:50070/user/rav"], + skiptrash=True, + command="rm") + self.parser.read_config() + self.assert_namenode_spec("foobar", 50070) + self.assert_namenodes_spec("foobar", 50070) + self.assertEquals(self.parser.args.skiptrash, True) + + def _revert_hdfs_try_paths(self): + # Make sure HDFSConfig is in vanilla state + HDFSConfig.use_trash = False + HDFSConfig.hdfs_try_paths = ConfigTest.original_hdfs_try_path + HDFSConfig.core_try_paths = ConfigTest.original_core_try_path + + @patch("os.path.exists") + def test_cl_trash_setting_preserved_after_snakebiterc_one_valid(self, exists_mock): + m = mock_open(read_data=json.dumps(self.valid_snake_one_rc)) + + with patch("snakebite.commandlineparser.open", m, create=True): + self.parser.args = MockParseArgs(usetrash=True) + self.parser.read_config() + self.assert_namenodes_spec("foobar", 54310, 9) + self.assertTrue(self.parser.args.usetrash) + + + @patch('os.environ.get') + def test_cl_usetrash_setting_preserved_after_external_nontrash_config(self, environ_get): + environ_get.return_value = False + # no snakebiterc + # read external config (hdfs-site, core-site) + self.parser.args = MockParseArgs(dir=["/user/rav/test"], + usetrash=True, + command="rm") + try: + HDFSConfig.core_try_paths = (ConfigTest.get_config_path('ha-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (ConfigTest.get_config_path('ha-noport-hdfs-site.xml'),) + self.parser.init() + + self.assertTrue(self.parser.args.usetrash) + self.assertTrue(self.parser.client.use_trash) + finally: + self._revert_hdfs_try_paths() + + @patch('os.environ.get') + def test_cl_skiptrash_setting_preserved_after_external_nontrash_config(self, environ_get): + environ_get.return_value = False + # no snakebiterc + # read external config (hdfs-site, core-site) + self.parser.args = MockParseArgs(dir=["/user/rav/test"], + skiptrash=True, + usetrash=True, + command="rm") + try: + HDFSConfig.core_try_paths = (ConfigTest.get_config_path('ha-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (ConfigTest.get_config_path('ha-noport-hdfs-site.xml'),) + self.parser.init() + + self.assertTrue(self.parser.args.skiptrash) + self.assertTrue(self.parser.args.usetrash) + self.assertFalse(self.parser.client.use_trash) + finally: + self._revert_hdfs_try_paths() diff --git a/test/config_test.py b/test/config_test.py index 8a72d33..ddf704d 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -2,15 +2,102 @@ import os import time +import os.path +import snakebite from snakebite.config import HDFSConfig +from mock import patch, mock_open class ConfigTest(unittest2.TestCase): + original_hdfs_try_path = set(HDFSConfig.hdfs_try_paths) + original_core_try_path = set(HDFSConfig.core_try_paths) + + def setUp(self): + # Make sure HDFSConfig is in vanilla state + HDFSConfig.use_trash = False + HDFSConfig.hdfs_try_paths = self.original_hdfs_try_path + HDFSConfig.core_try_paths = self.original_core_try_path + + @staticmethod + def get_config_path(config_name): + return os.path.abspath(os.path.join(snakebite.__file__, os.pardir, os.pardir, 'test/testconfig/conf/%s' % config_name)) + + def _verify_hdfs_settings(self, config): + self.assertEquals(len(config), 2) + # assert first NN + self.assertEqual('namenode1.mydomain', config[0]['namenode']) + self.assertEqual(8888, config[0]['port']) + # assert second NN + self.assertEqual('namenode2.mydomain', config[1]['namenode']) + self.assertEqual(8888, config[1]['port']) + + def _verify_hdfs_noport_settings(self, config): + self.assertEquals(len(config), 2) + # assert first NN + self.assertEqual('namenode1.mydomain', config[0]['namenode']) + self.assertEqual(8020, config[0]['port']) + # assert second NN + self.assertEqual('namenode2.mydomain', config[1]['namenode']) + self.assertEqual(8020, config[1]['port']) def test_read_hdfs_config_ha(self): - config = HDFSConfig.read_hdfs_config('test/testconfig/conf/hdfs-site.xml') - # assert first NN - self.assertEqual('namenode1.mydomain', config[0]['namenode']) - self.assertEqual(8020, config[0]['port']) - # assert second NN - self.assertEqual('namenode2.mydomain', config[1]['namenode']) - self.assertEqual(8020, config[1]['port']) + hdfs_site_path = self.get_config_path('ha-port-hdfs-site.xml') + config = HDFSConfig.read_hdfs_config(hdfs_site_path) + self._verify_hdfs_settings(config) + + def test_read_core_config_ha(self): + core_site_path = self.get_config_path('ha-core-site.xml') + config = HDFSConfig.read_core_config(core_site_path) + self.assertEquals(len(config), 1) + self.assertEquals('testha', config[0]['namenode']) + self.assertEquals(8020, config[0]['port']) + self.assertFalse(HDFSConfig.use_trash) + + @patch('os.environ.get') + def test_read_config_ha_with_ports(self, environ_get): + environ_get.return_value = False + HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-port-hdfs-site.xml'),) + config = HDFSConfig.get_external_config() + + self._verify_hdfs_settings(config) + + @patch('os.environ.get') + def test_read_config_non_ha_with_ports(self, environ_get): + environ_get.return_value = False + HDFSConfig.core_try_paths = (self.get_config_path('non-ha-port-core-site.xml'),) + HDFSConfig.hdfs_try_paths = () + config = HDFSConfig.get_external_config() + + self.assertEquals(len(config), 1) + self.assertEquals(config[0]['namenode'], 'testhost.net') + self.assertEquals(config[0]['port'], 8888) + self.assertFalse(HDFSConfig.use_trash) + + @patch('os.environ.get') + def test_ha_without_ports(self, environ_get): + environ_get.return_value = False + HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),) + config = HDFSConfig.get_external_config() + + self._verify_hdfs_noport_settings(config) + + @patch('os.environ.get') + def test_ha_config_trash_in_core(self, environ_get): + environ_get.return_value = False + HDFSConfig.core_try_paths = (self.get_config_path('core-with-trash.xml'),) + HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),) + config = HDFSConfig.get_external_config() + + self._verify_hdfs_noport_settings(config) + self.assertTrue(HDFSConfig.use_trash) + + @patch('os.environ.get') + def test_ha_config_trash_in_hdfs(self, environ_get): + environ_get.return_value = False + HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-trash-hdfs-site.xml'),) + config = HDFSConfig.get_external_config() + + self._verify_hdfs_noport_settings(config) + self.assertTrue(HDFSConfig.use_trash) diff --git a/test/testconfig/conf/core-with-trash.xml b/test/testconfig/conf/core-with-trash.xml new file mode 100644 index 0000000..ba5e77c --- /dev/null +++ b/test/testconfig/conf/core-with-trash.xml @@ -0,0 +1,19 @@ + + + + + + fs.defaultFS + hdfs://testhost.net:8888/ + + + + fs.trash.interval + 1 + + + + fs.trash.checkpoint.interval + 1 + + diff --git a/test/testconfig/conf/core-site.xml b/test/testconfig/conf/ha-core-site.xml similarity index 100% rename from test/testconfig/conf/core-site.xml rename to test/testconfig/conf/ha-core-site.xml diff --git a/test/testconfig/conf/ha-noport-hdfs-site.xml b/test/testconfig/conf/ha-noport-hdfs-site.xml new file mode 100644 index 0000000..ae063d9 --- /dev/null +++ b/test/testconfig/conf/ha-noport-hdfs-site.xml @@ -0,0 +1,35 @@ + + + + + + dfs.nameservices + testha + + + + dfs.ha.namenodes.testha + namenode1-mydomain,namenode2-mydomain + + + + dfs.namenode.rpc-address.testha.namenode1-mydomain + namenode1.mydomain + + + + dfs.namenode.rpc-address.testha.namenode2-mydomain + namenode2.mydomain + + + + dfs.namenode.http-address.testha.namenode1-mydomain + namenode1.mydomain:50070 + + + + dfs.namenode.http-address.testha.namenode2-mydomain + namenode2.mydomain:50070 + + + diff --git a/test/testconfig/conf/ha-noport-trash-hdfs-site.xml b/test/testconfig/conf/ha-noport-trash-hdfs-site.xml new file mode 100644 index 0000000..f049558 --- /dev/null +++ b/test/testconfig/conf/ha-noport-trash-hdfs-site.xml @@ -0,0 +1,44 @@ + + + + + + dfs.nameservices + testha + + + + dfs.ha.namenodes.testha + namenode1-mydomain,namenode2-mydomain + + + + dfs.namenode.rpc-address.testha.namenode1-mydomain + namenode1.mydomain + + + + dfs.namenode.rpc-address.testha.namenode2-mydomain + namenode2.mydomain + + + + dfs.namenode.http-address.testha.namenode1-mydomain + namenode1.mydomain:50070 + + + + dfs.namenode.http-address.testha.namenode2-mydomain + namenode2.mydomain:50070 + + + fs.trash.interval + 1 + + + + fs.trash.checkpoint.interval + 1 + + + diff --git a/test/testconfig/conf/hdfs-site.xml b/test/testconfig/conf/ha-port-hdfs-site.xml similarity index 90% rename from test/testconfig/conf/hdfs-site.xml rename to test/testconfig/conf/ha-port-hdfs-site.xml index cfaf12b..5aa344b 100644 --- a/test/testconfig/conf/hdfs-site.xml +++ b/test/testconfig/conf/ha-port-hdfs-site.xml @@ -14,12 +14,12 @@ dfs.namenode.rpc-address.testha.namenode1-mydomain - namenode1.mydomain:8020 + namenode1.mydomain:8888 dfs.namenode.rpc-address.testha.namenode2-mydomain - namenode2.mydomain:8020 + namenode2.mydomain:8888 diff --git a/test/testconfig/conf/non-ha-port-core-site.xml b/test/testconfig/conf/non-ha-port-core-site.xml new file mode 100644 index 0000000..c686cae --- /dev/null +++ b/test/testconfig/conf/non-ha-port-core-site.xml @@ -0,0 +1,9 @@ + + + + + + fs.defaultFS + hdfs://testhost.net:8888/ + +