fgmeta/maintain_catalog.py

#!/usr/bin/python

import os, sys, re, glob
import hashlib # for MD5
import subprocess
import shutil # for copy2
import catalogTags
import sgprops
#from multiprocessing import Pool
import argparse
import urllib2

import svn_catalog_repository
import git_catalog_repository
import git_discrete_repository

parser = argparse.ArgumentParser()
parser.add_argument("--clean", help="Regenerate every package", type=bool)
parser.add_argument("dir", help="Catalog directory")
args = parser.parse_args()

standardTagSet = frozenset(catalogTags.tags)
def isNonstandardTag(t):
    return t not in standardTagSet

thumbnailNames = ["thumbnail.png", "thumbnail.jpg"]
includePaths = []

class VariantData:
    def __init__(self, path, node):
        #self._primary = primary
        self._path = path
        self._name = node.getValue("sim/description")

        # ratings

        # seperate thumbnails

    @property
    def catalogNode(self):
        n = sgprops.Node("variant")
        n.addChild("id").value = self._path
        n.addChild("name").value = self._name

class PackageData:
    def __init__(self, path):
        self._path = path
        self._previousSCMRevision = None
        self._previousRevision = 0
        self._thumbnails = []
        self._variants = {}
        self._revision = 0
        self._md5 = None
        self._fileSize = 0

        self._node = sgprops.Node("package")
        self._node.addChild("id").value = self.id

    def setPreviousData(self, node):
        self._previousRevision = node.getValue("revision")
        self._previousMD5 = node.getValue("md5")
        self._previousSCMRevision = node.getValue("scm-revision")
        self._fileSize = int(node.getValue("file-size-bytes"))

    @property
    def id(self):
        return os.path.basename(self._path)

    @property
    def thumbnails(self):
        return self._thumbnails

    @property
    def path(self):
        return self._path

    @property
    def variants(self):
        return self._variants

    @property
    def scmRevision(self):
        currentRev = scmRepo.scmRevisionForPath(self._path)
        if (currentRev is None):
            raise RuntimeError("Unable to query SCM revision of files")

        return currentRev

    def isSourceModified(self, scmRepo):
        if (self._previousSCMRevision == None):
            return True

        if (self._previousSCMRevision == self.scmRevision):
            return False

        return True

    def scanSetXmlFiles(self, includes):
        foundPrimary = False
        foundMultiple = False

        for f in os.listdir(self._path):
            if not f.endswith("-set.xml"):
                continue

            p = os.path.join(self._path, f)
            node = sgprops.readProps(p, includePaths = includes)
            if not node.hasChild("sim"):
                continue

            simNode = node.getChild("sim")
            if (simNode.getValue("exclude", False)):
                continue

            primary = simNode.getValue("variant-of", None)
            if primary:
                if not primary in self.variants:
                    self._variants[primary] = []
                self._variants[primary].append(VariantData(self, node))
                continue

            if foundPrimary:
                if not foundMultiple:
                    print "Multiple primary -set.xml files at:" + self._path
                    foundMultiple = True
                continue
            else:
                foundPrimary = True;

            self.parsePrimarySetNode(simNode)

            for n in thumbnailNames:
                if os.path.exists(os.path.join(self._path, n)):
                    self._thumbnails.append(n)

        if not foundPrimary:
            raise RuntimeError("No primary -set.xml found at:" + self._path)



    def parsePrimarySetNode(self, sim):

        # basic / mandatory values
        self._node.addChild('name').value = sim.getValue('description')

        longDesc = sim.getValue('long-description')
        if longDesc is not None:
            self._node.addChild('description').value = longDesc

        # copy all the standard values
        for p in ['status', 'author', 'license']:
            v = sim.getValue(p)
            if v is not None:
                self._node.addChild(p).value = v

        # ratings
        if sim.hasChild('rating'):
            pkgRatings = self._node.addChild('rating')
            for r in ['FDM', 'systems', 'cockpit', 'model']:
                pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)

        # copy tags
        if sim.hasChild('tags'):
            for c in sim.getChild('tags').getChildren('tag'):
                if isNonstandardTag(c.value):
                    print "Skipping non-standard tag:", c.value, self.path
                else:
                    self._node.addChild('tag').value = c.value

        for t in sim.getChildren("thumbnail"):
            self._thumbnails.append(t.value)

    def validate(self):
        for t in self._thumbnails:
            if not os.path.exists(os.path.join(self._path, t)):
                raise RuntimeError("missing thumbnail:" + t);

    def generateZip(self, outDir):
        self._revision = self._previousRevision + 1

        zipName = self.id + ".zip"
        zipFilePath = os.path.join(outDir, zipName)

        os.chdir(os.path.dirname(self.path))

        print "Creating zip", zipFilePath
        # TODO: exclude certain files
        # anything we can do to make this faster?
        subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id])

        zipFile = open(zipFilePath, 'r')
        self._md5 = hashlib.md5(zipFile.read()).hexdigest()
        self._fileSize = os.path.getsize(zipFilePath)

    def useExistingCatalogData(self):
        self._md5 = self._previousMD5

    def packageNode(self, mirrorUrls, thumbnailUrl):
        self._node.getChild("md5", create = True).value = self._md5
        self._node.getChild("file-size-bytes", create = True).value = self._fileSize
        self._node.getChild("revision", create = True).value = int(self._revision)
        self._node.getChild("scm-revision", create = True).value = self.scmRevision

        for m in mirrorUrls:
            self._node.addChild("url").value = m + "/" + self.id + ".zip"

        for t in self._thumbnails:
            self._node.addChild("thumbnail").value = thumbnailUrl + "/" + self.id + "_" + t

        for pr in self._variants:
            for vr in self._variants[pr]:
                self._node.addChild(vr.catalogNode)

        return self._node

    def extractThumbnails(self, thumbnailDir):
        for t in self._thumbnails:
            fullName = self.id + "_" + t
            shutil.copy2(os.path.join(self._path, t),
                         os.path.join(thumbnailDir, fullName)
                         )
            # TODO : verify image format, size and so on

def scanPackages(globPath):
    result = []
    print "Scanning", globPath
    print os.getcwd()
    for d in glob.glob(globPath):
        # check dir contains at least one -set.xml file
        if len(glob.glob(os.path.join(d, "*-set.xml"))) == 0:
            print "no -set.xml in", d
            continue

        result.append(PackageData(d))

    return result

def initScmRepository(node):
    scmType = node.getValue("type")
    if (scmType == "svn"):
        svnPath = node.getValue("path")
        return svn_catalog_repository.SVNCatalogRepository(svnPath)
    elif (scmType == "git"):
        gitPath = node.getValue("path")
        usesSubmodules = node.getValue("uses-submodules", False)
        return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
    elif (scmType == "git-discrete"):
        return git_discrete_repository.GitDiscreteSCM(node)
    elif (scmType == None):
        raise RuntimeError("No scm/type defined in catalog configuration")
    else:
        raise RuntimeError("Unspported SCM type:" + scmType)

def processUpload(node, outputPath):
    if not node.getValue("enabled", True):
        print "Upload disabled"
        return

    uploadType = node.getValue("type")
    if (uploadType == "rsync"):
        subprocess.call(["rsync", node.getValue("args", "-az"), ".",
            node.getValue("remote")],
        cwd = outputPath)
    elif (uploadType == "rsync-ssh"):
        subprocess.call(["rsync", node.getValue("args", "-azve"),
            "ssh", ".",
            node.getValue("remote")],
            cwd = outputPath)
    elif (uploadType == "scp"):
        subprocess.call(["scp", node.getValue("args", "-r"), ".",
            node.getValue("remote")],
            cwd = outputPath)
    else:
        raise RuntimeError("Unsupported upload type:" + uploadType)

# dictionary
packages = {}

rootDir = args.dir
if not os.path.isabs(rootDir):
    rootDir = os.path.abspath(rootDir)
os.chdir(rootDir)

configPath = 'catalog.config.xml'
if not os.path.exists(configPath):
    raise RuntimeError("no config file found at:" + configPath)

config = sgprops.readProps(configPath)

# out path
outPath = config.getValue('output-dir')
if outPath is None:
    # default out path
    outPath = os.path.join(rootDir, "output")
elif not os.path.isabs(outPath):
    outPath = os.path.join(rootDir, "output")

if args.clean:
    print "Cleaning output"
    shutil.rmtree(outPath)

if not os.path.exists(outPath):
    os.mkdir(outPath)

thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
if not os.path.exists(thumbnailPath):
    os.mkdir(thumbnailPath)

thumbnailUrl = config.getValue('thumbnail-url')

for i in config.getChildren("include-dir"):
    if not os.path.exists(i.value):
        print "Skipping missing include path:", i.value
        continue
    includePaths.append(i.value)

mirrorUrls = []

# contains existing catalog
existingCatalogPath = os.path.join(outPath, 'catalog.xml')

scmRepo = initScmRepository(config.getChild('scm'))

# scan the directories in the aircraft paths
for g in config.getChildren("aircraft-dir"):
    for p in scanPackages(g.value):
        packages[p.id] = p

if not os.path.exists(existingCatalogPath):
    try:
    # can happen on new or from clean, try to pull current
    # catalog from the upload location
        response = urllib2.urlopen(config.getValue("template/url"), timeout = 5)
        content = response.read()
        f = open(existingCatalogPath, 'w' )
        f.write( content )
        f.close()
    except urllib2.URLError as e:
        print "Downloading current catalog failed", e


if os.path.exists(existingCatalogPath):
    try:
        previousCatalog = sgprops.readProps(existingCatalogPath)
    except:
        print "Previous catalog is malformed"
        previousCatalog = sgprops.Node()

    for p in previousCatalog.getChildren("package"):
        pkgId = p.getValue("id")
        if not pkgId in packages.keys():
            print "Orphaned old package:", pkgId
            continue

        packages[pkgId].setPreviousData(p)
else:
    print "No previous catalog"

catalogNode = sgprops.Node("catalog")
sgprops.copy(config.getChild("template"), catalogNode)

mirrorUrls = (m.value for m in config.getChildren("mirror"))

packagesToGenerate = []
for p in packages.values():
    p.scanSetXmlFiles(includePaths)

    if (p.isSourceModified(scmRepo)):
        packagesToGenerate.append(p)
    else:
        p.useExistingCatalogData()


# def f(x):
#     x.generateZip(outPath)
#     x.extractThumbnails(thumbnailPath)
#     return True
#
# p = Pool(8)
# print(p.map(f,packagesToGenerate))

for p in packagesToGenerate:
   p.generateZip(outPath)
   p.extractThumbnails(thumbnailPath)

print "Creating catalog"
for p in packages.values():
    catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl))

catalogNode.write(os.path.join(outPath, "catalog.xml"))

print "Uploading"
if config.hasChild("upload"):
    processUpload(config.getChild("upload"), outPath)