mismanager

#!/usr/bin/python
    
import sys,os
import thread

import lxml
import llfuse
import logging

import mis

def test(manifest_path):
    manifest = mis.manifest.serializer.fromXML(manifest_path)
    

def merge(manifest_path_a, manifest_path_b, manifest_path_dest):
    manifest_a = mis.manifest.serializer.fromXML(manifest_path_a)
    manifest_b = mis.manifest.serializer.fromXML(manifest_path_b)

    manifest_dest = manifest_a + manifest_b
    xml = manifest_dest.toXML()
    xml.write(manifest_path_dest, pretty_print=True)

def export_files(manifest_path, destination_path, export_type, config):
    """creates all files form the manifest in the provided directory"""

    datastore = mis.datastore.Datastore(
        config.datastore.url,
        config.datastore.type)
    manifest = mis.manifest.serializer.fromXML(manifest_path)
    manifest.export(destination_path, datastore, export_type)

def import_files(path, dest):
    """traveres through given path and creates a manifest"""
    print "reading files from %s" % path
    datastore = mis.datastore.Datastore(
        config.datastore.url,
        config.datastore.type)
    manifest = mis.manifest.serializer.fromPath(path, datastore)
    print "generating xml"
    xml = manifest.toXML()
    print "saving xml to %s" % dest
    xml.write(dest, pretty_print=True)

def datastore_cleanup(manifests):
    """removes all files that are not in the manifest files provied as an argument"""

    hashes = list()

    for manifest_path in manifests:
        manifest = mis.manifest.serializer.fromXML(manifest_path)
        hashes += manifest.get_hashes()

    datastore = mis.datastore.Datastore(
        config.datastore.url,
        config.datastore.type)


    todel = set(datastore.contents()) - set(hashes)
    for node in todel:
        print "removing " + datastore.getPath(node)
        datastore.remove(filehash)

	
def datastore_du(manifests):
    """sums up the real data comsumed by the given manifests"""
	
    sizes = dict()
    size = 0
    datastore = mis.datastore.Datastore(
        config.datastore.url,
        config.datastore.type)

    for manifest_path in manifests:
        print "reading %s" % manifest_path
        manifest = mis.manifest.serializer.fromXML(manifest_path)
        
        for hash in manifest.get_hashes():
            path = datastore.getPath2(hash[0], str(hash[1]))
            try:
                if hash[0] not in sizes:
                    sizes[hash[0]] = os.path.getsize(path)
            except Exception:
                print "error in " + path
           
        print "size : %s" % sum(sizes.values())
        print "count: %s" % len(sizes)
	

def mount_manifest(manifest_path, mountpoint_path):
    #init logging
    
    formatter = logging.Formatter('%(message)s') 
    handler = logging.StreamHandler()
    handler.setFormatter(formatter)
    handler.setLevel(logging.DEBUG)
    log = logging.getLogger()
    log.setLevel(logging.INFO)    
    log.addHandler(handler)

    manifest = mis.manifest.serializer.fromXML(manifest_path)
    datastore = mis.datastore.Datastore(
        config.datastore.url,
        config.datastore.type)
 
    operations = mis.fs.Operations(
        manifest,
        datastore)

    llfuse.init(operations, mountpoint_path, [ b'fsname=marvinfs' ])
    llfuse.main(single=True)
    llfuse.close()

def diff(file_a, file_b):
    manifest_a = mis.manifest.serializer.fromXML(file_a)
    manifest_b = mis.manifest.serializer.fromXML(file_b)

    for line in manifest_a.diff(manifest_b):
        print line


def datastore_store(source, manifest_path):
    """stores all files specified in the manifest in the provided datastore location"""
    manifest = mis.manifest.serializer.fromXML(manifest_path)
    datastore = mis.datastore.Datastore(
        config.datastore.url,
        config.datastore.type)
 

    for node in manifest:
        if (isinstance (node, mis.manifest.nodes.File)
            and not datastore.contains(node)):
            print "storing {0}".format(source + node.path)
            datastore.saveData(node, source + node.path)

def datastore_fsck():
    """checks the files in the datastore for consistency"""
    datastore = mis.datastore.Datastore(
        config.datastore.url,
        config.datastore.type)

    for filehash, filesize in datastore.contents():
        try:
		if not datastore.check(filehash, filesize):
            		print "error in " + datastore.getPath2(filehash, filesize)
	except Exception:
		print "error in " + datastore.getPath2(filehash, filesize)

if __name__ == "__main__":
    
    help_message = """usage: {0} <config> <command> [<args>]""".format(sys.argv[0])
    help_message += """\navailable commands are:"""
    help_message += """\n   import : create a manifest from a directory"""
    help_message += """\n   export : store contents of a manifest to a directory"""
    help_message += """\n   store  : store files to datastore"""
    help_message += """\n   clean  : remove unneded files form datastore"""
    help_message += """\n   du     : print the used size in bytes"""
    help_message += """\n   fsck   : check the datastore for file corruption"""
    help_message += """\n   diff   : compare two manifest files"""
    help_message += """\n   mount  : mount manifest to mountpoint"""
    help_message += """\n   merge  : merge two manifests"""

    config    = None
    action    = None
    arguments = None

    try:
        with open(sys.argv[1]) as configfile:
            config = mis.config.Config(configfile)
        action = sys.argv[2]
        arguments = sys.argv[3:]
    except IndexError:
        print help_message
        print sys.argv[0] + " too few arguments"
        sys.exit(1)


    if (action == "export"):
        try:
            manifest_path    = arguments[0]
            destination_path = arguments[1]
            export_type      = None
    
            if len(arguments) == 3:
                export_type = arguments[2]
                if not (export_type == 'aufs' or export_type == 'unionfs'):
                    raise IndexError
        except IndexError as e:
            print "usage: {0} export MANIFEST DESTINATION [aufs,unionfs]".format(sys.argv[0])
            sys.exit(1)

        if not (os.path.exists(destination_path) and os.path.isdir(destination_path)):
            print ("ERROR: %s not a directory" % dir)
            sys.exit(1)
        export_files(manifest_path, destination_path, export_type, config)

    if (action == "import"):
        try:
            import_files(arguments[0], arguments[1])
        except IndexError:
            print "usage: {0} import PATH XML".format(sys.argv[0])
            sys.exit(1)

    if (action == "store"):
        try:
            datastore_store(arguments[0], arguments[1])
        except IndexError:
            print "usage {0} store SOURCE XML".format(sys.argv[0])
            sys.exit(1)

    if (action == "fsck"):
        try:
            datastore_fsck()
        except IndexError:
            print "usage {0} fsck".format(sys.argv[0])
            sys.exit(1)

    if (action == "diff"):
        try:
            diff(arguments[0], arguments[1])
        except IndexError:
            print "usage {0} diff FILE_A FILE_B".format(sys.argv[0])
            sys.exit(1)

    if (action == "cleanup"):
        try:
            datastore_cleanup(arguments)
        except IndexError:
            print "usage {0} cleanup MANIFEST [MANIFEST [MANIFEST ...".format(sys.argv[0])
	
    if (action == "du"):
        try:
            datastore_du(arguments)
        except IndexError:
            print "usage {0} du MANIFEST [MANIFEST [MANIFEST ...".format(sys.argv[0])


    if (action == "mount"):
        try:
            mount_manifest(
                manifest_path = arguments[0],
                mountpoint_path = arguments[1])
        except IndexError:
            print "usage {0} mount MANIFEST MOUNTPOINT".format(sys.argv[0])

    if (action == "merge"):
        try:
            merge(
                manifest_path_a = arguments[0],
                manifest_path_b = arguments[1],
                manifest_path_dest = arguments[2])
        except IndexError:
            print "usage {0} merge MANIFEST_A MANITEST_B OUTPUT".format(sys.argv[0])

    if (action == "test"):
        try:
            test(
                manifest_path = arguments[0]
            )
        except IndexError:
            print "usage {0} test MANIFEST".format(sys.argv[0])