gufi_config revamp

added OutputBuffer to Server config
Exec has been renamed to Executable

Server and Client are now classes instead of prefixes
getting values from the config is now done by calling functions instead of passing in hardcoded strings as keys to a dictionary

Author: calccrypto

config/server.example.in

@@ -67,8 +67,11 @@ Threads=@CORES@
 
 # absolute path to gufi_query
 # single path string
-Exec=@CMAKE_INSTALL_PREFIX@/bin/gufi_query
+Executable=@CMAKE_INSTALL_PREFIX@/bin/gufi_query
 
 # directory where all GUFI indicies are placed under
 # single path string
 IndexRoot=@CMAKE_BINARY_DIR@
+
+# size of per-thread print buffers
+OutputBuffer=4096

scripts/completions

@@ -78,7 +78,7 @@ __gufi() {
     fi
 
     root=$(readlink -m $(grep "IndexRoot=" "${CONFIG}"  | sed "s/^.*=//"))
-    bfq=$(readlink -m $(grep "Exec=" ${CONFIG} | sed "s/^.*=//"))
+    bfq=$(readlink -m $(grep "Executable=" ${CONFIG} | sed "s/^.*=//"))
     curr=$(printf "%s" "${COMP_WORDS[@]:${COMP_CWORD}}")
     path="${root}/${curr}"
     name=${path##*/} # basename without expanding

scripts/gufi_client

@@ -80,8 +80,8 @@ def talk_to_server(config, args):
         ssh.load_system_host_keys('/etc/ssh/ssh_known_hosts')
 
         # connect to the server
-        ssh.connect(hostname=config['Server'],
-                    port=config['Port'],
+        ssh.connect(hostname=config.server(),
+                    port=config.port()],
                     username=getpass.getuser(),
                     hostkey=paramiko.opensshkey.load_pubkey_from_file('/etc/ssh/ssh_host_rsa_key.pub'))
 
@@ -100,10 +100,10 @@ def talk_to_server(config, args):
         ssh.close()
 
 if __name__  == '__main__':
-    config = gufi_config.client_config(gufi_config.DEFAULT_CONFIG_PATH)
+    config = gufi_config.Client(gufi_config.DEFAULT_PATH)
 
     # import paramiko from provided path
-    sys.path.append(config['Paramiko'])
+    sys.path.append(config.paramiko())
     import paramiko
 
     talk_to_server(config, sys.argv[1:])

scripts/gufi_config.py

@@ -69,22 +69,9 @@
 import gufi_common
 
 # default configuration file location
-DEFAULT_CONFIG_PATH='/etc/GUFI/config'
-
-# recognized settings
-CLIENT_SETTINGS = [
-    'Server',   # hostname
-    'Port',     # ssh port
-    'Paramiko'  # location of paramiko installation
-]
-
-SERVER_SETTINGS = [
-    'Threads',  # number of threads to use
-    'Exec',     # absolute path of gufi_query
-    'IndexRoot' # absolute path of root directory for GUFI to traverse
-]
-
-def read_config(filename):
+DEFAULT_PATH='/etc/GUFI/config'
+
+def _parse(filename):
     with open(filename, 'r') as f:
         out = {}
         for line in f:
@@ -105,26 +92,67 @@ def read_config(filename):
                 value = gufi_common.get_positive(value)
             elif name in ['Paramiko', 'IndexRoot']:
                 value = os.path.normpath(value)
+            elif name == 'OutputBuffer':
+                value = gufi_common.get_non_negative(value)
             out[name] = value
+
         return out
 
-# command line arguments override configuration file values (logic is inverted)
-def server_config(filename = DEFAULT_CONFIG_PATH):
-    config = read_config(filename)
-    if 'Threads' not in config:
-        raise exceptions.Exception('Missing Threads')
-    if 'Exec' not in config:
-        raise exceptions.Exception('Missing Exec')
-    if 'IndexRoot' not in config:
-        raise exceptions.Exception('Missing IndexRoot')
+def _read(settings, filename = DEFAULT_PATH):
+    config = _parse(filename)
+    for key in settings:
+        if key not in config:
+            raise exceptions.Exception(filename + ' missing ' + key)
     return config
 
-def client_config(filename = DEFAULT_CONFIG_PATH):
-    config = read_config(filename)
-    if 'Server' not in config:
-        raise exceptions.Exception('Missing Server')
-    if 'Port' not in config:
-        raise exceptions.Exception('Missing Port')
-    if 'Paramiko' not in config:
-        raise exceptions.Exception('Missing Paramiko')
-    return config
+class Server:
+    THREADS      = 'Threads'      # number of threads to use
+    EXECUTABLE   = 'Executable'   # absolute path of gufi_query
+    INDEXROOT    = 'IndexRoot'    # absolute path of root directory for GUFI to traverse
+    OUTPUTBUFFER = 'OutputBuffer' # size of per-thread buffers used to buffer prints
+
+    SETTINGS = [
+        THREADS,
+        EXECUTABLE,
+        INDEXROOT,
+        OUTPUTBUFFER
+    ]
+
+    def __init__(self, filename = DEFAULT_PATH):
+        self.config = _read(Server.SETTINGS, filename)
+
+    def threads(self):
+        return self.config[Server.THREADS]
+
+    def executable(self):
+        return self.config[Server.EXECUTABLE]
+
+    def indexroot(self):
+        return self.config[Server.INDEXROOT]
+
+    def outputbuffer(self):
+        return self.config[Server.OUTPUTBUFFER]
+
+class Client:
+    SERVER       = 'Server'       # hostname
+    PORT         = 'Port'         # ssh port
+    PARAMIKO     = 'Paramiko'     # location of paramiko installation
+
+
+    SETTINGS = [
+        SERVER,
+        PORT,
+        PARAMIKO,
+    ]
+
+    def __init__(self, filename = DEFAULT_PATH):
+        self.config = _read(Client.SETTINGS, filename)
+
+    def server(self):
+        return self.config[Client.SERVER]
+
+    def port(self):
+        return self.config[Client.PORT]
+
+    def paramiko(self):
+        return self.config[Client.PARAMIKO]

scripts/gufi_find

@@ -535,15 +535,14 @@ def build_expression_parser():
     parser.add_argument('--num-results',       metavar='n',          dest='num_results',           type=gufi_common.get_non_negative,                  help='first n results')
     parser.add_argument('--smallest',                                dest='smallest',              action='store_true',                                help='top n smallest files')
     parser.add_argument('--largest',                                 dest='largest',               action='store_true',                                help='top n largest files')
-    parser.add_argument('--output-buffer',     metavar='bytes',      dest='output_buffer',         type=gufi_common.get_positive, default=4096,        help='Size of each thread\'s output buffer')
     parser.add_argument('--in-memory-name',    metavar='name',       dest='inmemory_name',         type=str,                     default='out',        help='Name of in-memory database when aggregation is performed')
 
     return parser
 
 # argv[0] should be the command name
 def run(argv, config_path):
     # find and parse the configuration file first
-    config = gufi_config.server_config(config_path)
+    config = gufi_config.Server(config_path)
 
     expression_parser = build_expression_parser()
 
@@ -588,7 +587,7 @@ def run(argv, config_path):
         paths = ['']
 
     # prepend the provided paths with the GUFI root path
-    paths = [os.path.normpath(os.path.sep.join([config['IndexRoot'], path])) for path in paths]
+    paths = [os.path.normpath(os.path.sep.join([config.indexroot(), path])) for path in paths]
 
     # parse expressions without the 'real' and path arguments
     args, unknown = expression_parser.parse_known_args(argv[i:])
@@ -608,8 +607,10 @@ def run(argv, config_path):
     output   = build_output(args)
 
     # create the query command
-    query_cmd = [config['Exec'],
-                 '-n', str(config['Threads'])]
+    query_cmd = [config.executable(),
+                 '-n', str(config.threads()),
+                 '-B', str(config.outputbuffer())
+             ]
 
     S = None
     E = None
@@ -697,13 +698,10 @@ def run(argv, config_path):
     if args.fprint:
         query_cmd += ['-o', args.fprint]
 
-    if args.output_buffer:
-        query_cmd += ['-B', str(args.output_buffer)]
-
     query = subprocess.Popen(query_cmd + paths) # positional arguments must appear after flags
     query.communicate()                         # block until query finishes
 
     return query.returncode
 
 if __name__=='__main__':
-    sys.exit(run(sys.argv, gufi_config.DEFAULT_CONFIG_PATH))
+    sys.exit(run(sys.argv, gufi_config.DEFAULT_PATH))

scripts/gufi_ls

@@ -222,7 +222,7 @@ def build_order_by_recursive(args):
 # argv[0] should be the command name
 def run(argv, config_path):
     # find and parse the configuration file first
-    config = gufi_config.server_config(config_path)
+    config = gufi_config.Server(config_path)
 
     class FullTimeAction(argparse.Action):
         def __init__(self, option_strings, dest, nargs=None, **kwargs):
@@ -259,8 +259,7 @@ def run(argv, config_path):
     parser.add_argument('paths',                                                         type=str, action='append' , nargs='*')
 
     # GUFI specific arguments
-    parser.add_argument('--delim',          metavar='c',          dest='delim',          type=gufi_common.get_char, default=' ',           help='delimiter separating output columns')
-    parser.add_argument('--output-buffer',  metavar='bytes',      dest='output_buffer',  type=gufi_common.get_non_negative, default=4096,  help='Size of each thread\'s output buffer')
+    parser.add_argument('--delim',          metavar='c',          dest='delim',          type=gufi_common.get_char,         default=' ',   help='delimiter separating output columns')
     parser.add_argument('--in-memory-name', metavar='name',       dest='inmemory_name',  type=str,                          default='out', help='Name of in-memory database when -R is used')
     parser.add_argument('--nlink-width',    metavar='chars',      dest='nlink_width',    type=gufi_common.get_non_negative, default=2,     help='Width of nlink column')
     parser.add_argument('--size-width',     metavar='chars',      dest='size_width',     type=gufi_common.get_non_negative, default=10,    help='Width of size column')
@@ -278,10 +277,10 @@ def run(argv, config_path):
 
     for path in args.paths[0]:
         # prepend the provided paths with the GUFI index root
-        fullpath = os.path.normpath(os.path.sep.join([config['IndexRoot'], path]))
+        fullpath = os.path.normpath(os.path.sep.join([config.indexroot(), path]))
 
         # create the base command
-        query_cmd = [config['Exec']]
+        query_cmd = [config.executable()]
 
         # directory and non-directory paths are processed differently
         if os.path.isdir(fullpath):
@@ -340,7 +339,6 @@ def run(argv, config_path):
                                                        None))
 
                 query_cmd += ['-e', '0',
-                              '-n', str(config['Threads']),
                               '-I', I,
                               '-S', S,
                               '-E', E,
@@ -378,7 +376,6 @@ def run(argv, config_path):
 
 
                 query_cmd += ['-e', '0',
-                              '-n', str(config['Threads']),
                               '-I', I,
                               '-S', S,
                               '-E', E,
@@ -388,6 +385,8 @@ def run(argv, config_path):
                               '-z', '1',
                               '-a']
 
+            query_cmd += ['-n', str(config.threads())]
+
         else:
             # if the path is not a directory, remove the last part of the path
             fullpath, match_name = os.path.split(fullpath)
@@ -400,17 +399,15 @@ def run(argv, config_path):
                                         None)
 
             query_cmd += ['-e', '1',
-                          '-n', str(config['Threads']),
                           '-E', E,
                           '-y', '0',
                           '-z', '0']
 
+        query_cmd += ['-B', str(config.outputbuffer())]
+
         if args.delim:
             query_cmd += ['-d', args.delim]
 
-        if args.output_buffer:
-            query_cmd += ['-B', str(args.output_buffer)]
-
         query = subprocess.Popen(query_cmd + [fullpath])
         query.communicate() # block until query finishes
 
@@ -420,4 +417,4 @@ def run(argv, config_path):
     return rc
 
 if __name__=='__main__':
-    sys.exit(run(sys.argv, gufi_config.DEFAULT_CONFIG_PATH))
+    sys.exit(run(sys.argv, gufi_config.DEFAULT_PATH))