list-urgent-package-dependencies

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# A script to find which packages are needed to merge the given package(s)
# This is usually used for new distribution roadmaps
#
# usage examples:
# list-urgent-package-dependencies 2009/devel/pisi-index.xml 2011/devel/pisi-index.xml.bz2 mplayer kdebase kdenetwork
#
#   or
#
# list-urgent-package-dependencies 2009/devel/pisi-index.xml 2011/devel/pisi-index.xml.bz2 -user "Name Surname" "Name2 Surname2"
#
#   or
#
# list-urgent-package-dependencies 2009/devel/pisi-index.xml 2011/devel/pisi-index.xml.bz2
# (search for all packages)



import piksemel
import bz2
import sys
import os


def loadFile(_file):
    try:
        f = file(_file)
        d = [a.lstrip().rstrip("\n") for a in f]
        d = filter(lambda x: not (x.startswith("#") or x == ""), d)
        f.close()
        return d
    except:
        return []

def getIndexFile(_file):
    # TODO: add support for remote fetching
    if os.path.isfile(_file):
        result = _file
    else:
        weguess = os.path.join(_file, "pisi-index.xml")
        if os.path.exists(weguess):
            result = weguess
        else:
            print "%s is not a valid file" % _file
            sys.exit(1)

    return result

def uniq(seq, loopback=None):
    seen = {}
    result = []

    if loopback is None:
        def loopback(x): return x

    for item in seq:
        marker = loopback(item)
        if marker in seen:
            continue

        seen[marker] = 1
        result.append(item)

    return result

def getXmlData(_file):
    if os.path.exists(_file):
        if _file.endswith(".bz2"):
            indexdata = bz2.decompress(file(_file).read())
            return piksemel.parseString(indexdata)
        else:
            return piksemel.parse(_file)
    else:
        print "%s not found" % _file
        sys.exit(1)

def getPackageProperties(parent, pkgDict):
    dependencies = []
    providedPackages = []

    sourceTag = parent.getTag("Source")
    sourcePartOf = sourceTag.getTagData("PartOf").replace(".", "/")
    sourceName = sourceTag.getTagData("Name")
    maintainerName = sourceTag.getTag("Packager").getTagData("Name")

    buildDependencies = sourceTag.getTag("BuildDependencies")
    if buildDependencies:
        for dep in buildDependencies.tags("Dependency"):
            dependencies.append(dep.firstChild().data())

    packageTag = parent.tags("Package")
    for node in packageTag:
        providedPackages.append(node.getTagData("Name"))
        runtimeDependencies = node.getTag("RuntimeDependencies")
        if runtimeDependencies:
            for dep in runtimeDependencies.tags("Dependency"):
                dependencies.append(dep.firstChild().data())

    # some packages provide packages that don't have the original package name
    if sourceName not in providedPackages:
        providedPackages.append(sourceName)

    for i in providedPackages:
        pkgDict[i] = maintainerName, sourcePartOf, sourceName, uniq(dependencies)

def parseOldRepo(_file):
    pkgDict = {}
    ix = getXmlData(_file)

    distroInfo = "%s %s" % (ix.getTag("Distribution").getTagData("SourceName"), ix.getTag("Distribution").getTagData("Version"))

    for i in ix.tags("SpecFile"):
        getPackageProperties(i, pkgDict)

    return pkgDict, distroInfo

def parseNewRepo(_file):
    pkglist = []
    obsoletelist = []
    ix = getXmlData(_file)

    for node in ix.tags("SpecFile"):
        # Some packages have source name that does not match package name, see gfxtheme-pardus, icontheme-milky etc.
        pkglist.append(node.getTag("Source").getTagData("Name"))

        for pkgnode in node.tags("Package"):
            pkglist.append(pkgnode.getTagData("Name"))

            # replaced packages are also obsoleted packages, and the package is in target repo
            replacedPackages = pkgnode.getTag("Replaces")
            if replacedPackages:
                for replaced in replacedPackages.tags("Package"):
                    obsoletelist.append(replaced.firstChild().data())

    distroInfo = "%s %s" % (ix.getTag("Distribution").getTagData("SourceName"), ix.getTag("Distribution").getTagData("Version"))

    obsoleteParent = ix.getTag("Distribution").getTag("Obsoletes")
    for node in obsoleteParent.tags("Package"):
        obsoletelist.append(node.firstChild().data())

    return uniq(pkglist), uniq(obsoletelist), distroInfo

def hasDeps(package):
    global neededPackages

    depsOfPackage = set(repoIndexFull[package][3])
    depsToMerge = set(neededPackages)

    return ",".join(depsOfPackage.intersection(depsToMerge))

def printOwners(ownerDict, pkgowner=False):
    ownerlist = ownerDict.keys()
    ownerlist.sort()

    for i in ownerlist:
        srcpackages = []
        packages = ownerDict[i]
        pkglist = {}

        for src in packages:
            fullpath = "%s/%s" % repoIndexFull[src][1:3]
            srcpackages.append(fullpath)
            pkglist[fullpath] = src

        srcpackages = uniq(srcpackages)
        srcpackages.sort()

        if pkgowner:
            for packager in pkgowner:
                if i == packager:
                    print
                    print "%s (%d)" % (i, len(srcpackages))
                    for k in srcpackages:
                        print "    %s (%s)" % (k, hasDeps(pkglist[k]))
                else:
                    pass
        else:
            print
            print "%s (%d)" % (i, len(srcpackages))
            for k in srcpackages:
                print "    %s (%s)" % (k, hasDeps(pkglist[k]))

    print

def printRepoInfo(sourceRepo, targetRepo, targetPackages, ignoredPackages, singlePackages=False):
    print
    print "Base Repository    : %s" % sourceRepo
    print "Target Repository  : %s" % targetRepo
    print "Obsoleted Packages : %s" % " ".join(ignoredPackages)
    if singlePackages:
        print "Target Packages    : %s" % " ".join(targetPackages)
    else:
        print "Target Packages    : All packages"
    print


class Dependency:
    def __init__(self):
        self.depList = []
        self.depPosition = 0
        self.indexMissing = []
        self.obsoletes = []
        self.indexFull = {}
        self.ownerDict = {}

    def _addNewDep(self, newdeps):
        for i in newdeps:
            if i not in self.indexMissing and i not in self.depList and i not in self.obsoletes:
                self.depList.append(i)

    def add(self, newPackages):
        for i in newPackages:
            if i not in self.indexMissing:
                self.depList.append(i)

    def solve(self):
        while self.depPosition != len(self.depList):
            newdeps = self.indexFull[self.depList[self.depPosition]][3]

            self._addNewDep(newdeps)
            self.depPosition += 1

    def setOwners(self):
        for pkgname in self.depList:
            pkgowner = self.indexFull[pkgname][0]

            if pkgowner not in self.ownerDict:
                self.ownerDict[pkgowner] = [pkgname]
            else:
                self.ownerDict[pkgowner].append(pkgname)


if __name__ == "__main__":
    if len(sys.argv) < 3:
        print " usages:"
        print " %s <full repo indexfile> <missing repo indexfile>" % sys.argv[0]
        print "     all packages are searched\n"
        print " %s <full repo indexfile> <missing repo indexfile> <required packages ...>" % sys.argv[0]
        print "     only the written packages are searched\n"
        print " %s <full repo indexfile> <missing repo indexfile> --user \"Name1 Surname2\" \"Name2 Surname2\" ..."  % sys.argv[0]
        print "     only packages that belongs to the packager are searched"
    else:
        indexFileFull = getIndexFile(sys.argv[1])
        indexFileMissing = getIndexFile(sys.argv[2])

        # repoIndexFull[PackageName] = { Maintainer, SourcePartOf, SourceName, Dependencies }
        repoIndexFull, repoInfoFull = parseOldRepo(indexFileFull)

        # repoIndexMissing = [pkname1, pkgname2, ...]
        repoIndexMissing, obsoleteList, repoInfoMissing = parseNewRepo(indexFileMissing)
        repoIndexMissingOld, obsoleteListOld, repoInfoMissingOld = parseNewRepo(indexFileFull)

        # list of target packages, silently ignore obsoleted packages
        packagesToSearch = []
        ignoredPackages = []

        pkgowner = False
        singlePackages = False
        if len(sys.argv) < 4:
            for i in repoIndexMissingOld:
                if i in obsoleteList:
                    ignoredPackages.append(i)
                else:
                    packagesToSearch.append(i)
        else:
            if sys.argv[3] == "--user":
                pkgowner = sys.argv[4:]
                for i in repoIndexMissingOld:
                    if i in obsoleteList:
                        ignoredPackages.append(i)
                    else:
                        packagesToSearch.append(i)
            else:
                for i in sys.argv[3:]:
                    singlePackages = True
                    if i in obsoleteList:
                        ignoredPackages.append(i)
                    else:
                        packagesToSearch.append(i)

        printRepoInfo(repoInfoFull, repoInfoMissing, packagesToSearch, ignoredPackages, singlePackages)

        deptree = Dependency()
        deptree.indexMissing = repoIndexMissing
        deptree.indexFull = repoIndexFull
        deptree.obsoletes = obsoleteList

        deptree.add(packagesToSearch)
        deptree.solve()
        neededPackages = deptree.depList
        deptree.setOwners()

        printOwners(deptree.ownerDict, pkgowner)