Work on catalog creation.

2015-06-04 22:09:46 +01:00 · 2015-06-04 22:09:46 +01:00 · 287380d98d
commit 287380d98d
parent 355a88d0cd
6 changed files with 449 additions and 8 deletions
--- a/git_catalog_repository.py
+++ b/git_catalog_repository.py
@ -0,0 +1,40 @@
+# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
+ 
+import subprocess
+import os
+
+class GITCatalogRepository:
+    def __init__(self, path, usesSubmodules = False, singleAircraft = False):
+        self._path = path
+        
+        if !os.path.exists(os.path.join(path, ".git")):
+            raise RuntimeError("not a Git directory:" + path)
+        
+        self._usesSubmodules = usesSubmodules
+        self._singleAircraft = singleAircraft
+        
+        self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"],
+            cwd = self._path)
+
+    def hasPathChanged(self, path, oldRev):
+        diffArgs = ["git", "diff", "--quiet", oldRev, "--"]
+        if not (self._usesSubmodules and self._singleAircraft):
+            diffArgs.append(path)
+            
+        return subprocess.call(diffArgs, cwd = self._path)
+
+    def update(self):
+        subprocess.call(["git", "pull"])
+        self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"],
+                cwd = self._path)
+        
+        if self._usesSubmodules:
+            subprocess.call(["git", "submodule", "update"], cwd = self._path)    
+
+    def scmRevisionForPath(self, path):
+        if self._usesSubmodules:
+            return subprocess.catch_output(["git", "rev-parse", "HEAD"], cwd = self._path)
+            
+        return self._currentRevision
+
+ 
--- a/git_discrete_repository.py
+++ b/git_discrete_repository.py
@ -0,0 +1,34 @@
+# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
+ 
+import subprocess
+import os
+import sgprops
+
+import GITCatalogRepository
+
+class GitDiscreteSCM:
+    def __init__(self, node):
+        
+        configNode = node.parent
+        
+        self._repos = {}
+        
+        # iterate over aicraft paths finding repositories
+        for g in config.getChildren("aircraft-dir"):     
+            repo = GITCatalogRepository(g, useSubmodules = False, 
+                singleAircraft = True)
+        
+
+    def hasPathChanged(self, path, oldRev):
+        
+        return self._repos[path].hasPathChanged(path, oldRev)
+
+    def update(self):
+        for r in self._repos:
+            r.update()
+
+    def scmRevisionForPath(self, path):
+        return self._repos[path].scmRevisionForPath(path)
+        
+
+ 
--- a/maintain_catalog.py
+++ b/maintain_catalog.py
@ -0,0 +1,276 @@
+#!/usr/bin/python
+
+import os, sys, re, glob
+import hashlib # for MD5
+import subprocess
+
+import catalogTags
+import sgprops
+
+import svn_catalog_repository
+import git_catalog_repository
+import git_discrete_repository
+
+# TODO
+# uploading / rsyncing 
+
+class VariantData:
+    def __init__(self, primary, path, node):
+        self._primary = primary
+        self._path = path
+        self._name = node.getValue("sim/description")
+        
+        # ratings
+        
+        # seperate thumbnails
+        
+    @property
+    def catalogNode(self):
+        n = Node("variant")
+        n.addChild("id") = path
+        m.addChild("name") = self._name
+        
+class PackageData:
+    def __init__(path):
+        self._path = path
+        self._previousSCMRevision = None
+        self._previousRevision = 0
+        self._thumbnails = []
+        self._variants = {}
+        
+        self._node = sgprops.Node()
+        self._node.addChild("id").value = self.id
+        
+    def setPreviousData(node):
+        self._previousRevision = node.getValue("revision")
+        self._previousMD5 = node.getValue("md5")
+        self._previousSCMRevision = node.getValue("scm-revision")
+    
+    @property
+    def id(self):
+        return os.path.basename(self._path)
+    
+    @property
+    def thumbnails(self):
+        return self._thumbnails
+        
+    def isSourceModified(self, scmRepo):
+        if (self._previousSCMRevision == None):
+            return True
+            
+        currentRev = scmRepo.scmRevisionForPath(self._path)
+        if (currentRev is None):
+            raise RuntimeError("Unable to query SCM revision of files")
+            
+        if (self._previousSCMRevision == currentRev):
+            self._scm = self._previousSCMRevision
+            return False
+            
+        self._scm = currentRev
+        return True
+        
+    def scanSetXmlFiles(self):
+        foundPrimary = False
+        
+        for f in os.listdir(self._path):
+            if !f.endswith("-set.xml"):
+                continue
+                
+            p = os.path.join(self._path, f)
+            node = readProps(p)
+            simNode = node.getChild("sim")
+            if (simNode.getValue("exclude")):
+                continue
+                
+            if primary = simNode.getValue("variant-of", None):
+                if not primary in variants:
+                    self._variants[primary] = []
+                self._variants[primary].append(VariantData(self, node))
+                continue
+            
+            if foundPrimary:
+                print "Multiple primary -set.xml files at:" + self._path
+                continue
+            else:
+                foundPrimary = True;
+                
+            parsePrimarySetNode(simNode)
+            
+            if os.path.exists(os.path.join(self._path, "thumbnail.png")):
+                self._thumbnails.append("thumbnail.png")
+            
+        if not foundPrimary:
+            raise RuntimeError("No primary -set.xml found at:" + self._path)
+            
+        
+        
+    def parsePrimarySetNode(self, sim):
+        
+        # basic / mandatory values
+        self._node.addChild('id').value = d
+        self._node.addChild('name').value = sim.getValue('description')
+        
+        longDesc = sim.getValue('long-description')
+        if longDesc is not None:
+            self._node.addChild('description').value = longDesc
+            
+        # copy all the standard values
+        for p in ['status', 'author', 'license']:
+            v = sim.getValue(p)
+            if v is not None:
+                self._node.addChild(p).value = v
+            
+        # ratings
+        if sim.hasChild('rating'):
+            pkgRatings = self._node.addChild('rating')
+            for r in ['FDM', 'systems', 'cockpit', 'model']:
+                pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)
+            
+        # copy tags
+        if sim.hasChild('tags'):
+            for c in sim.getChild('tags').getChildren('tag'):
+                if isNonstandardTag(c.value):
+                    print "Skipping non-standard tag:", c.value
+                else:
+                    self._node.addChild('tag').value = c.value
+                    
+        self._thumbnails = (t.value for t in self.getChildren("thumbnail"))
+        
+    def validate(self):
+        for t in self._thumbnails:
+            if not os.path.exists(os.path.join(self._path, t)):
+                raise RuntimeError("missing thumbnail:" + t);
+        
+    def generateZip(self, outDir):
+        self._revision = self._previousRevision + 1
+        
+        zipName = self.id
+        zipFilePath = os.path.join(outDir, zipName)
+        
+        # TODO: exclude certain files
+        subprocess.call(['zip', '-r', self.path, zipFilePath])
+        
+        zipFile = open(zipFilePath + ".zip", 'r')
+        self._md5 = hashlib.md5(zipFile.read()).hexdigest()
+        self._fileSize = os.path.getsize(zipFile)
+        
+    @property
+    def catalogNode(self, mirrorUrls, thumbnailUrl):
+        self._node.getChild("md5", create = True).value = self._md5
+        self._node.getChild("file-size-bytes", create = True).value = self._fileSize
+        self._node.addChild("revision", create = True).value = self._revision
+        self._node.addChild("scm-revision", create = True).value = self._scm
+        
+        for m in mirrorUrls:
+            self._node.addChild("url", m + "/" + self.id + ".zip")
+            
+        for t in self._thumbnails:
+            self._node.addChild("thumbnail", thumbnailUrl + "/" + self.id + "_" + t)
+        
+        for pr in self._variants:
+            for vr in self._variants[pr]:
+                self._node.addChild(vr.catalogNode)
+        
+        return self._node
+        
+    def extractThumnbails(self, thumbnailDir):
+        for t in self._thumbnails:
+            fullName = self.id + "_" + t
+            os.file.copy(os.path.join(self._path, t),
+                         os.path.join(thumbnailDir, fullName)
+                         )
+            # TODO : verify image format, size and so on
+        
+def scanPackages(globPath):
+    result = []
+    for d = in glob.glob(globPath):
+        result.append(PackageData(d))
+    
+    return result
+
+def initScmRepository(node):
+    scmType = node.getValue("type")
+    if (scmType == "svn"):
+        svnPath = node.getValue("path")
+        return SVNCatalogRepository(svnPath)
+    else if (scmType == "git"):
+        gitPath = node.getValue("path")
+        usesSubmodules = node.getValue("uses-submodules", False)
+        return GitCatalogRepository(gitPath, usesSubmodules)
+    else if (scmType == "git-discrete")
+        return GitDiscreteSCM(node)
+    else if (scmType == None):
+        raise RuntimeError("No scm/type defined in catalog configuration")
+    else:
+        raise RuntimeError("Unspported SCM type:" + scmType)
+    
+def processUpload(node, outputPath):
+    uploadType = node.getValue("type")
+    if (type == "rsync"):
+        subprocess.call(["rsync", node.getValue("args", "-az"), ".", 
+            node.getValue("remote")],
+        cwd = outputPath)
+    else if (type == "scp"):
+        subprocess.call(["scp", node.getValue("args", "-r"), outputPath,
+            node.getValue("remote")])
+    else:
+        raise RuntimeError("Unsupported upload type:" + uploadType)
+
+# dictionary    
+packages = {}
+
+rootDir = sys.argv[1]
+os.path.chdir(rootDir)
+
+configPath = 'catalog.config.xml'
+if !os.path.exists(configPath):
+    raise RuntimeError("no config file found at:" + configPath)
+    
+config = readProps(configPath)
+
+# out path
+outPath = config.getValue('output-dir')
+if outPath is None:
+    # default out path
+    outPath = "output"
+    
+print "Output path is:" + outPath
+
+thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
+
+# contains existing catalog
+existingCatalogPath = os.path.join(outPath, 'catalog.xml')
+
+scmRepo = initScmRepository(config.getChild('scm'))
+
+# scan the directories in the aircraft paths
+for g in config.getChildren("aircraft-dir"):     
+    for p in scanPackages(g):
+        packages[p.id] = p
+
+previousCatalog = readProps(existingCatalogPath)
+for p in previousCatalog.getChildren("package"):
+    pkgId = p.getValue("id")
+    if !packages.contains(pkgId):
+        print "Orphaned old package:", pkgId
+        continue
+        
+    packages[pkgId].setPreviousData(p)
+    
+
+catalogNode = sgprops.Node()
+
+sgprops.copy(config.getChild("template"), catalogNode)
+
+packagesToGenerate = []
+for p in packages:
+    if (p.isSourceModified(scmRepo)):
+        packagesToGenerate.append(p)
+    
+for p in packagesToGenerate:
+    p.generateZip(outPath)
+    p.extractThumbnails(thumbnailPath)
+    catalogNode.addChild(p.catalogNode)
+    
+if config.hasChild("upload"):
+    processUpload(config.getChild("upload"), outPath)
--- a/sgprops.py
+++ b/sgprops.py
@ -56,17 +56,24 @@ class Node(object):
            raise IndexError("no such child:" + str(n) + " index=" + str(i))
            
    def addChild(self, n):
-        i = 0
-        
-        # find first free index
+        # adding an existing instance
+        if isinstance(n, Node):
+            n._parent = self
+            n._index = self.firstUnusedIndex(n.name)
+            self._children.append(n)
+            return n
+            
+        i = self.firstUnusedIndex(n)        
+        # create it via getChild
+        return self.getChild(n, i, create=True)
+            
+    def firstUnusedIndex(self, n):
        usedIndices = frozenset(c.index for c in self.getChildren(n))
        while i < 1000:
            if i not in usedIndices:
-                 break
+                 return i
            i += 1
-        
-        # create it via getChild
-        return self.getChild(n, i, create=True)
+        raise RuntimeException("too many children with name:" + n)
            
    def hasChild(self, nm):
        for c in self._children:
@ -221,4 +228,12 @@ def readProps(path, root = None, dataDirPath = None):
    h.setDocumentLocator(locator)
    parser.setContentHandler(h)
    parser.parse(path)
-    return h.root
+    return h.root
+    
+def copy(src, dest):
+    dest.value = src.value
+    
+    # recurse over children
+    for c in src.children:
+        dc = dest.getChild(c.name, i = c.index, create = True)
+        copy(c, dc)
--- a/svn_catalog_repository.py
+++ b/svn_catalog_repository.py
@ -0,0 +1,24 @@
+
+import subprocess
+import xml.etree.cElementTree as ET
+
+class SVNCatalogRepository:
+    def __init__(self, path):
+        self._path = path
+        xml = subprocess.check_output(["svn", "info", "--xml", path])
+        root = ET.fromstring(xml)
+        if (root.find("repository/root") == None):
+            raise RuntimeError("Not an SVN repository:" + path)
+    
+    def hasPathChanged(self, path, oldRevision):
+        return self.scmRevisionForPath(path) != oldRevision
+        
+    def scmRevisionForPath(self, path):
+        xml = subprocess.check_output(["svn", "info", "--xml", path])
+        root = ET.fromstring(xml)
+        commit = root.find("entry/commit")
+        return commit.get('revision', 0)
+        
+    def update(self):
+        subprocess.call(["svn", "update"])
+        
--- a/template_catalog.xml
+++ b/template_catalog.xml
@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<!--
+Template catalog - copy and modify for your site as required
+-->
+<PropertyList>
+    
+    <!--
+    catalogProps.addChild('version').value = '3.1.0'
+    catalogProps.addChild('id').value = 'org.flightgear.default'
+    catalogProps.addChild('license').value = 'GPL'
+    catalogProps.addChild('url').value = "http://fgfs.goneabitbursar.com/pkg/3.1.0/default-catalog.xml"
+
+    catalogProps.addChild('description').value = "Aircraft developed and maintained by the FlightGear project"
+
+    de = catalogProps.addChild('de')
+    # de.addChild('description').value = "<German translation of catalog description>"
+
+    fr = catalogProps.addChild('fr')
+    -->
+        
+        <version>3.4.*</version>
+        <version>3.5.*</version>
+        <version>3.6.*</version>
+        
+        <id>org.myorganisation.hangar</id>
+        <license>GPL</license>
+        <url>http://some.stable.url.com/foo/bar/catalog.xml</url>
+        
+        <description>A collection of interesting aircraft with some features
+        </description>
+        
+        <de>
+            <description>Au Deutsch</description>
+        </de>
+        
+        <fr>
+            <description>Francais</description>
+        </fr>
+        
+        <mirror>http://some.url/</mirror>
+        <!--  <mirror>another mirror</mirror> -->
+        
+        
+        <thumbnails>http://some.url/images</thumbnails>
+        
+        <git-repository>git://some.git.repo/</git-repository>
+        
+        <repository-prefix>Aircraft</repository-prefix>
+        
+</PropertyList>
+
+