diff --git a/git_catalog_repository.py b/git_catalog_repository.py new file mode 100644 index 0000000..789c468 --- /dev/null +++ b/git_catalog_repository.py @@ -0,0 +1,40 @@ +# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/ + +import subprocess +import os + +class GITCatalogRepository: + def __init__(self, path, usesSubmodules = False, singleAircraft = False): + self._path = path + + if !os.path.exists(os.path.join(path, ".git")): + raise RuntimeError("not a Git directory:" + path) + + self._usesSubmodules = usesSubmodules + self._singleAircraft = singleAircraft + + self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"], + cwd = self._path) + + def hasPathChanged(self, path, oldRev): + diffArgs = ["git", "diff", "--quiet", oldRev, "--"] + if not (self._usesSubmodules and self._singleAircraft): + diffArgs.append(path) + + return subprocess.call(diffArgs, cwd = self._path) + + def update(self): + subprocess.call(["git", "pull"]) + self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"], + cwd = self._path) + + if self._usesSubmodules: + subprocess.call(["git", "submodule", "update"], cwd = self._path) + + def scmRevisionForPath(self, path): + if self._usesSubmodules: + return subprocess.catch_output(["git", "rev-parse", "HEAD"], cwd = self._path) + + return self._currentRevision + + \ No newline at end of file diff --git a/git_discrete_repository.py b/git_discrete_repository.py new file mode 100644 index 0000000..33dc496 --- /dev/null +++ b/git_discrete_repository.py @@ -0,0 +1,34 @@ +# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/ + +import subprocess +import os +import sgprops + +import GITCatalogRepository + +class GitDiscreteSCM: + def __init__(self, node): + + configNode = node.parent + + self._repos = {} + + # iterate over aicraft paths finding repositories + for g in config.getChildren("aircraft-dir"): + repo = GITCatalogRepository(g, useSubmodules = False, + singleAircraft = True) + + + def hasPathChanged(self, path, oldRev): + + return self._repos[path].hasPathChanged(path, oldRev) + + def update(self): + for r in self._repos: + r.update() + + def scmRevisionForPath(self, path): + return self._repos[path].scmRevisionForPath(path) + + + \ No newline at end of file diff --git a/maintain_catalog.py b/maintain_catalog.py new file mode 100755 index 0000000..e32ce4f --- /dev/null +++ b/maintain_catalog.py @@ -0,0 +1,276 @@ +#!/usr/bin/python + +import os, sys, re, glob +import hashlib # for MD5 +import subprocess + +import catalogTags +import sgprops + +import svn_catalog_repository +import git_catalog_repository +import git_discrete_repository + +# TODO +# uploading / rsyncing + +class VariantData: + def __init__(self, primary, path, node): + self._primary = primary + self._path = path + self._name = node.getValue("sim/description") + + # ratings + + # seperate thumbnails + + @property + def catalogNode(self): + n = Node("variant") + n.addChild("id") = path + m.addChild("name") = self._name + +class PackageData: + def __init__(path): + self._path = path + self._previousSCMRevision = None + self._previousRevision = 0 + self._thumbnails = [] + self._variants = {} + + self._node = sgprops.Node() + self._node.addChild("id").value = self.id + + def setPreviousData(node): + self._previousRevision = node.getValue("revision") + self._previousMD5 = node.getValue("md5") + self._previousSCMRevision = node.getValue("scm-revision") + + @property + def id(self): + return os.path.basename(self._path) + + @property + def thumbnails(self): + return self._thumbnails + + def isSourceModified(self, scmRepo): + if (self._previousSCMRevision == None): + return True + + currentRev = scmRepo.scmRevisionForPath(self._path) + if (currentRev is None): + raise RuntimeError("Unable to query SCM revision of files") + + if (self._previousSCMRevision == currentRev): + self._scm = self._previousSCMRevision + return False + + self._scm = currentRev + return True + + def scanSetXmlFiles(self): + foundPrimary = False + + for f in os.listdir(self._path): + if !f.endswith("-set.xml"): + continue + + p = os.path.join(self._path, f) + node = readProps(p) + simNode = node.getChild("sim") + if (simNode.getValue("exclude")): + continue + + if primary = simNode.getValue("variant-of", None): + if not primary in variants: + self._variants[primary] = [] + self._variants[primary].append(VariantData(self, node)) + continue + + if foundPrimary: + print "Multiple primary -set.xml files at:" + self._path + continue + else: + foundPrimary = True; + + parsePrimarySetNode(simNode) + + if os.path.exists(os.path.join(self._path, "thumbnail.png")): + self._thumbnails.append("thumbnail.png") + + if not foundPrimary: + raise RuntimeError("No primary -set.xml found at:" + self._path) + + + + def parsePrimarySetNode(self, sim): + + # basic / mandatory values + self._node.addChild('id').value = d + self._node.addChild('name').value = sim.getValue('description') + + longDesc = sim.getValue('long-description') + if longDesc is not None: + self._node.addChild('description').value = longDesc + + # copy all the standard values + for p in ['status', 'author', 'license']: + v = sim.getValue(p) + if v is not None: + self._node.addChild(p).value = v + + # ratings + if sim.hasChild('rating'): + pkgRatings = self._node.addChild('rating') + for r in ['FDM', 'systems', 'cockpit', 'model']: + pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0) + + # copy tags + if sim.hasChild('tags'): + for c in sim.getChild('tags').getChildren('tag'): + if isNonstandardTag(c.value): + print "Skipping non-standard tag:", c.value + else: + self._node.addChild('tag').value = c.value + + self._thumbnails = (t.value for t in self.getChildren("thumbnail")) + + def validate(self): + for t in self._thumbnails: + if not os.path.exists(os.path.join(self._path, t)): + raise RuntimeError("missing thumbnail:" + t); + + def generateZip(self, outDir): + self._revision = self._previousRevision + 1 + + zipName = self.id + zipFilePath = os.path.join(outDir, zipName) + + # TODO: exclude certain files + subprocess.call(['zip', '-r', self.path, zipFilePath]) + + zipFile = open(zipFilePath + ".zip", 'r') + self._md5 = hashlib.md5(zipFile.read()).hexdigest() + self._fileSize = os.path.getsize(zipFile) + + @property + def catalogNode(self, mirrorUrls, thumbnailUrl): + self._node.getChild("md5", create = True).value = self._md5 + self._node.getChild("file-size-bytes", create = True).value = self._fileSize + self._node.addChild("revision", create = True).value = self._revision + self._node.addChild("scm-revision", create = True).value = self._scm + + for m in mirrorUrls: + self._node.addChild("url", m + "/" + self.id + ".zip") + + for t in self._thumbnails: + self._node.addChild("thumbnail", thumbnailUrl + "/" + self.id + "_" + t) + + for pr in self._variants: + for vr in self._variants[pr]: + self._node.addChild(vr.catalogNode) + + return self._node + + def extractThumnbails(self, thumbnailDir): + for t in self._thumbnails: + fullName = self.id + "_" + t + os.file.copy(os.path.join(self._path, t), + os.path.join(thumbnailDir, fullName) + ) + # TODO : verify image format, size and so on + +def scanPackages(globPath): + result = [] + for d = in glob.glob(globPath): + result.append(PackageData(d)) + + return result + +def initScmRepository(node): + scmType = node.getValue("type") + if (scmType == "svn"): + svnPath = node.getValue("path") + return SVNCatalogRepository(svnPath) + else if (scmType == "git"): + gitPath = node.getValue("path") + usesSubmodules = node.getValue("uses-submodules", False) + return GitCatalogRepository(gitPath, usesSubmodules) + else if (scmType == "git-discrete") + return GitDiscreteSCM(node) + else if (scmType == None): + raise RuntimeError("No scm/type defined in catalog configuration") + else: + raise RuntimeError("Unspported SCM type:" + scmType) + +def processUpload(node, outputPath): + uploadType = node.getValue("type") + if (type == "rsync"): + subprocess.call(["rsync", node.getValue("args", "-az"), ".", + node.getValue("remote")], + cwd = outputPath) + else if (type == "scp"): + subprocess.call(["scp", node.getValue("args", "-r"), outputPath, + node.getValue("remote")]) + else: + raise RuntimeError("Unsupported upload type:" + uploadType) + +# dictionary +packages = {} + +rootDir = sys.argv[1] +os.path.chdir(rootDir) + +configPath = 'catalog.config.xml' +if !os.path.exists(configPath): + raise RuntimeError("no config file found at:" + configPath) + +config = readProps(configPath) + +# out path +outPath = config.getValue('output-dir') +if outPath is None: + # default out path + outPath = "output" + +print "Output path is:" + outPath + +thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails")) + +# contains existing catalog +existingCatalogPath = os.path.join(outPath, 'catalog.xml') + +scmRepo = initScmRepository(config.getChild('scm')) + +# scan the directories in the aircraft paths +for g in config.getChildren("aircraft-dir"): + for p in scanPackages(g): + packages[p.id] = p + +previousCatalog = readProps(existingCatalogPath) +for p in previousCatalog.getChildren("package"): + pkgId = p.getValue("id") + if !packages.contains(pkgId): + print "Orphaned old package:", pkgId + continue + + packages[pkgId].setPreviousData(p) + + +catalogNode = sgprops.Node() + +sgprops.copy(config.getChild("template"), catalogNode) + +packagesToGenerate = [] +for p in packages: + if (p.isSourceModified(scmRepo)): + packagesToGenerate.append(p) + +for p in packagesToGenerate: + p.generateZip(outPath) + p.extractThumbnails(thumbnailPath) + catalogNode.addChild(p.catalogNode) + +if config.hasChild("upload"): + processUpload(config.getChild("upload"), outPath) \ No newline at end of file diff --git a/sgprops.py b/sgprops.py index ee280d0..b528b4a 100644 --- a/sgprops.py +++ b/sgprops.py @@ -56,17 +56,24 @@ class Node(object): raise IndexError("no such child:" + str(n) + " index=" + str(i)) def addChild(self, n): - i = 0 - - # find first free index + # adding an existing instance + if isinstance(n, Node): + n._parent = self + n._index = self.firstUnusedIndex(n.name) + self._children.append(n) + return n + + i = self.firstUnusedIndex(n) + # create it via getChild + return self.getChild(n, i, create=True) + + def firstUnusedIndex(self, n): usedIndices = frozenset(c.index for c in self.getChildren(n)) while i < 1000: if i not in usedIndices: - break + return i i += 1 - - # create it via getChild - return self.getChild(n, i, create=True) + raise RuntimeException("too many children with name:" + n) def hasChild(self, nm): for c in self._children: @@ -221,4 +228,12 @@ def readProps(path, root = None, dataDirPath = None): h.setDocumentLocator(locator) parser.setContentHandler(h) parser.parse(path) - return h.root \ No newline at end of file + return h.root + +def copy(src, dest): + dest.value = src.value + + # recurse over children + for c in src.children: + dc = dest.getChild(c.name, i = c.index, create = True) + copy(c, dc) diff --git a/svn_catalog_repository.py b/svn_catalog_repository.py new file mode 100644 index 0000000..5e5f56d --- /dev/null +++ b/svn_catalog_repository.py @@ -0,0 +1,24 @@ + +import subprocess +import xml.etree.cElementTree as ET + +class SVNCatalogRepository: + def __init__(self, path): + self._path = path + xml = subprocess.check_output(["svn", "info", "--xml", path]) + root = ET.fromstring(xml) + if (root.find("repository/root") == None): + raise RuntimeError("Not an SVN repository:" + path) + + def hasPathChanged(self, path, oldRevision): + return self.scmRevisionForPath(path) != oldRevision + + def scmRevisionForPath(self, path): + xml = subprocess.check_output(["svn", "info", "--xml", path]) + root = ET.fromstring(xml) + commit = root.find("entry/commit") + return commit.get('revision', 0) + + def update(self): + subprocess.call(["svn", "update"]) + \ No newline at end of file diff --git a/template_catalog.xml b/template_catalog.xml new file mode 100644 index 0000000..ed315fe --- /dev/null +++ b/template_catalog.xml @@ -0,0 +1,52 @@ + + + + + + + 3.4.* + 3.5.* + 3.6.* + + org.myorganisation.hangar + GPL + http://some.stable.url.com/foo/bar/catalog.xml + + A collection of interesting aircraft with some features + + + + Au Deutsch + + + + Francais + + + http://some.url/ + + + + http://some.url/images + + git://some.git.repo/ + + Aircraft + + + + \ No newline at end of file