Work on catalog creation.
This commit is contained in:
James Turner
parent
355a88d0cd
commit
287380d98d
40
git_catalog_repository.py
Normal file
40
git_catalog_repository.py
Normal file
@ -0,0 +1,40 @@
|
||||
# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
class GITCatalogRepository:
|
||||
def __init__(self, path, usesSubmodules = False, singleAircraft = False):
|
||||
self._path = path
|
||||
|
||||
if !os.path.exists(os.path.join(path, ".git")):
|
||||
raise RuntimeError("not a Git directory:" + path)
|
||||
|
||||
self._usesSubmodules = usesSubmodules
|
||||
self._singleAircraft = singleAircraft
|
||||
|
||||
self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"],
|
||||
cwd = self._path)
|
||||
|
||||
def hasPathChanged(self, path, oldRev):
|
||||
diffArgs = ["git", "diff", "--quiet", oldRev, "--"]
|
||||
if not (self._usesSubmodules and self._singleAircraft):
|
||||
diffArgs.append(path)
|
||||
|
||||
return subprocess.call(diffArgs, cwd = self._path)
|
||||
|
||||
def update(self):
|
||||
subprocess.call(["git", "pull"])
|
||||
self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"],
|
||||
cwd = self._path)
|
||||
|
||||
if self._usesSubmodules:
|
||||
subprocess.call(["git", "submodule", "update"], cwd = self._path)
|
||||
|
||||
def scmRevisionForPath(self, path):
|
||||
if self._usesSubmodules:
|
||||
return subprocess.catch_output(["git", "rev-parse", "HEAD"], cwd = self._path)
|
||||
|
||||
return self._currentRevision
|
||||
|
||||
|
||||
34
git_discrete_repository.py
Normal file
34
git_discrete_repository.py
Normal file
@ -0,0 +1,34 @@
|
||||
# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import sgprops
|
||||
|
||||
import GITCatalogRepository
|
||||
|
||||
class GitDiscreteSCM:
|
||||
def __init__(self, node):
|
||||
|
||||
configNode = node.parent
|
||||
|
||||
self._repos = {}
|
||||
|
||||
# iterate over aicraft paths finding repositories
|
||||
for g in config.getChildren("aircraft-dir"):
|
||||
repo = GITCatalogRepository(g, useSubmodules = False,
|
||||
singleAircraft = True)
|
||||
|
||||
|
||||
def hasPathChanged(self, path, oldRev):
|
||||
|
||||
return self._repos[path].hasPathChanged(path, oldRev)
|
||||
|
||||
def update(self):
|
||||
for r in self._repos:
|
||||
r.update()
|
||||
|
||||
def scmRevisionForPath(self, path):
|
||||
return self._repos[path].scmRevisionForPath(path)
|
||||
|
||||
|
||||
|
||||
276
maintain_catalog.py
Executable file
276
maintain_catalog.py
Executable file
@ -0,0 +1,276 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import os, sys, re, glob
|
||||
import hashlib # for MD5
|
||||
import subprocess
|
||||
|
||||
import catalogTags
|
||||
import sgprops
|
||||
|
||||
import svn_catalog_repository
|
||||
import git_catalog_repository
|
||||
import git_discrete_repository
|
||||
|
||||
# TODO
|
||||
# uploading / rsyncing
|
||||
|
||||
class VariantData:
|
||||
def __init__(self, primary, path, node):
|
||||
self._primary = primary
|
||||
self._path = path
|
||||
self._name = node.getValue("sim/description")
|
||||
|
||||
# ratings
|
||||
|
||||
# seperate thumbnails
|
||||
|
||||
@property
|
||||
def catalogNode(self):
|
||||
n = Node("variant")
|
||||
n.addChild("id") = path
|
||||
m.addChild("name") = self._name
|
||||
|
||||
class PackageData:
|
||||
def __init__(path):
|
||||
self._path = path
|
||||
self._previousSCMRevision = None
|
||||
self._previousRevision = 0
|
||||
self._thumbnails = []
|
||||
self._variants = {}
|
||||
|
||||
self._node = sgprops.Node()
|
||||
self._node.addChild("id").value = self.id
|
||||
|
||||
def setPreviousData(node):
|
||||
self._previousRevision = node.getValue("revision")
|
||||
self._previousMD5 = node.getValue("md5")
|
||||
self._previousSCMRevision = node.getValue("scm-revision")
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return os.path.basename(self._path)
|
||||
|
||||
@property
|
||||
def thumbnails(self):
|
||||
return self._thumbnails
|
||||
|
||||
def isSourceModified(self, scmRepo):
|
||||
if (self._previousSCMRevision == None):
|
||||
return True
|
||||
|
||||
currentRev = scmRepo.scmRevisionForPath(self._path)
|
||||
if (currentRev is None):
|
||||
raise RuntimeError("Unable to query SCM revision of files")
|
||||
|
||||
if (self._previousSCMRevision == currentRev):
|
||||
self._scm = self._previousSCMRevision
|
||||
return False
|
||||
|
||||
self._scm = currentRev
|
||||
return True
|
||||
|
||||
def scanSetXmlFiles(self):
|
||||
foundPrimary = False
|
||||
|
||||
for f in os.listdir(self._path):
|
||||
if !f.endswith("-set.xml"):
|
||||
continue
|
||||
|
||||
p = os.path.join(self._path, f)
|
||||
node = readProps(p)
|
||||
simNode = node.getChild("sim")
|
||||
if (simNode.getValue("exclude")):
|
||||
continue
|
||||
|
||||
if primary = simNode.getValue("variant-of", None):
|
||||
if not primary in variants:
|
||||
self._variants[primary] = []
|
||||
self._variants[primary].append(VariantData(self, node))
|
||||
continue
|
||||
|
||||
if foundPrimary:
|
||||
print "Multiple primary -set.xml files at:" + self._path
|
||||
continue
|
||||
else:
|
||||
foundPrimary = True;
|
||||
|
||||
parsePrimarySetNode(simNode)
|
||||
|
||||
if os.path.exists(os.path.join(self._path, "thumbnail.png")):
|
||||
self._thumbnails.append("thumbnail.png")
|
||||
|
||||
if not foundPrimary:
|
||||
raise RuntimeError("No primary -set.xml found at:" + self._path)
|
||||
|
||||
|
||||
|
||||
def parsePrimarySetNode(self, sim):
|
||||
|
||||
# basic / mandatory values
|
||||
self._node.addChild('id').value = d
|
||||
self._node.addChild('name').value = sim.getValue('description')
|
||||
|
||||
longDesc = sim.getValue('long-description')
|
||||
if longDesc is not None:
|
||||
self._node.addChild('description').value = longDesc
|
||||
|
||||
# copy all the standard values
|
||||
for p in ['status', 'author', 'license']:
|
||||
v = sim.getValue(p)
|
||||
if v is not None:
|
||||
self._node.addChild(p).value = v
|
||||
|
||||
# ratings
|
||||
if sim.hasChild('rating'):
|
||||
pkgRatings = self._node.addChild('rating')
|
||||
for r in ['FDM', 'systems', 'cockpit', 'model']:
|
||||
pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)
|
||||
|
||||
# copy tags
|
||||
if sim.hasChild('tags'):
|
||||
for c in sim.getChild('tags').getChildren('tag'):
|
||||
if isNonstandardTag(c.value):
|
||||
print "Skipping non-standard tag:", c.value
|
||||
else:
|
||||
self._node.addChild('tag').value = c.value
|
||||
|
||||
self._thumbnails = (t.value for t in self.getChildren("thumbnail"))
|
||||
|
||||
def validate(self):
|
||||
for t in self._thumbnails:
|
||||
if not os.path.exists(os.path.join(self._path, t)):
|
||||
raise RuntimeError("missing thumbnail:" + t);
|
||||
|
||||
def generateZip(self, outDir):
|
||||
self._revision = self._previousRevision + 1
|
||||
|
||||
zipName = self.id
|
||||
zipFilePath = os.path.join(outDir, zipName)
|
||||
|
||||
# TODO: exclude certain files
|
||||
subprocess.call(['zip', '-r', self.path, zipFilePath])
|
||||
|
||||
zipFile = open(zipFilePath + ".zip", 'r')
|
||||
self._md5 = hashlib.md5(zipFile.read()).hexdigest()
|
||||
self._fileSize = os.path.getsize(zipFile)
|
||||
|
||||
@property
|
||||
def catalogNode(self, mirrorUrls, thumbnailUrl):
|
||||
self._node.getChild("md5", create = True).value = self._md5
|
||||
self._node.getChild("file-size-bytes", create = True).value = self._fileSize
|
||||
self._node.addChild("revision", create = True).value = self._revision
|
||||
self._node.addChild("scm-revision", create = True).value = self._scm
|
||||
|
||||
for m in mirrorUrls:
|
||||
self._node.addChild("url", m + "/" + self.id + ".zip")
|
||||
|
||||
for t in self._thumbnails:
|
||||
self._node.addChild("thumbnail", thumbnailUrl + "/" + self.id + "_" + t)
|
||||
|
||||
for pr in self._variants:
|
||||
for vr in self._variants[pr]:
|
||||
self._node.addChild(vr.catalogNode)
|
||||
|
||||
return self._node
|
||||
|
||||
def extractThumnbails(self, thumbnailDir):
|
||||
for t in self._thumbnails:
|
||||
fullName = self.id + "_" + t
|
||||
os.file.copy(os.path.join(self._path, t),
|
||||
os.path.join(thumbnailDir, fullName)
|
||||
)
|
||||
# TODO : verify image format, size and so on
|
||||
|
||||
def scanPackages(globPath):
|
||||
result = []
|
||||
for d = in glob.glob(globPath):
|
||||
result.append(PackageData(d))
|
||||
|
||||
return result
|
||||
|
||||
def initScmRepository(node):
|
||||
scmType = node.getValue("type")
|
||||
if (scmType == "svn"):
|
||||
svnPath = node.getValue("path")
|
||||
return SVNCatalogRepository(svnPath)
|
||||
else if (scmType == "git"):
|
||||
gitPath = node.getValue("path")
|
||||
usesSubmodules = node.getValue("uses-submodules", False)
|
||||
return GitCatalogRepository(gitPath, usesSubmodules)
|
||||
else if (scmType == "git-discrete")
|
||||
return GitDiscreteSCM(node)
|
||||
else if (scmType == None):
|
||||
raise RuntimeError("No scm/type defined in catalog configuration")
|
||||
else:
|
||||
raise RuntimeError("Unspported SCM type:" + scmType)
|
||||
|
||||
def processUpload(node, outputPath):
|
||||
uploadType = node.getValue("type")
|
||||
if (type == "rsync"):
|
||||
subprocess.call(["rsync", node.getValue("args", "-az"), ".",
|
||||
node.getValue("remote")],
|
||||
cwd = outputPath)
|
||||
else if (type == "scp"):
|
||||
subprocess.call(["scp", node.getValue("args", "-r"), outputPath,
|
||||
node.getValue("remote")])
|
||||
else:
|
||||
raise RuntimeError("Unsupported upload type:" + uploadType)
|
||||
|
||||
# dictionary
|
||||
packages = {}
|
||||
|
||||
rootDir = sys.argv[1]
|
||||
os.path.chdir(rootDir)
|
||||
|
||||
configPath = 'catalog.config.xml'
|
||||
if !os.path.exists(configPath):
|
||||
raise RuntimeError("no config file found at:" + configPath)
|
||||
|
||||
config = readProps(configPath)
|
||||
|
||||
# out path
|
||||
outPath = config.getValue('output-dir')
|
||||
if outPath is None:
|
||||
# default out path
|
||||
outPath = "output"
|
||||
|
||||
print "Output path is:" + outPath
|
||||
|
||||
thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
|
||||
|
||||
# contains existing catalog
|
||||
existingCatalogPath = os.path.join(outPath, 'catalog.xml')
|
||||
|
||||
scmRepo = initScmRepository(config.getChild('scm'))
|
||||
|
||||
# scan the directories in the aircraft paths
|
||||
for g in config.getChildren("aircraft-dir"):
|
||||
for p in scanPackages(g):
|
||||
packages[p.id] = p
|
||||
|
||||
previousCatalog = readProps(existingCatalogPath)
|
||||
for p in previousCatalog.getChildren("package"):
|
||||
pkgId = p.getValue("id")
|
||||
if !packages.contains(pkgId):
|
||||
print "Orphaned old package:", pkgId
|
||||
continue
|
||||
|
||||
packages[pkgId].setPreviousData(p)
|
||||
|
||||
|
||||
catalogNode = sgprops.Node()
|
||||
|
||||
sgprops.copy(config.getChild("template"), catalogNode)
|
||||
|
||||
packagesToGenerate = []
|
||||
for p in packages:
|
||||
if (p.isSourceModified(scmRepo)):
|
||||
packagesToGenerate.append(p)
|
||||
|
||||
for p in packagesToGenerate:
|
||||
p.generateZip(outPath)
|
||||
p.extractThumbnails(thumbnailPath)
|
||||
catalogNode.addChild(p.catalogNode)
|
||||
|
||||
if config.hasChild("upload"):
|
||||
processUpload(config.getChild("upload"), outPath)
|
||||
27
sgprops.py
27
sgprops.py
@ -56,17 +56,24 @@ class Node(object):
|
||||
raise IndexError("no such child:" + str(n) + " index=" + str(i))
|
||||
|
||||
def addChild(self, n):
|
||||
i = 0
|
||||
# adding an existing instance
|
||||
if isinstance(n, Node):
|
||||
n._parent = self
|
||||
n._index = self.firstUnusedIndex(n.name)
|
||||
self._children.append(n)
|
||||
return n
|
||||
|
||||
# find first free index
|
||||
i = self.firstUnusedIndex(n)
|
||||
# create it via getChild
|
||||
return self.getChild(n, i, create=True)
|
||||
|
||||
def firstUnusedIndex(self, n):
|
||||
usedIndices = frozenset(c.index for c in self.getChildren(n))
|
||||
while i < 1000:
|
||||
if i not in usedIndices:
|
||||
break
|
||||
return i
|
||||
i += 1
|
||||
|
||||
# create it via getChild
|
||||
return self.getChild(n, i, create=True)
|
||||
raise RuntimeException("too many children with name:" + n)
|
||||
|
||||
def hasChild(self, nm):
|
||||
for c in self._children:
|
||||
@ -222,3 +229,11 @@ def readProps(path, root = None, dataDirPath = None):
|
||||
parser.setContentHandler(h)
|
||||
parser.parse(path)
|
||||
return h.root
|
||||
|
||||
def copy(src, dest):
|
||||
dest.value = src.value
|
||||
|
||||
# recurse over children
|
||||
for c in src.children:
|
||||
dc = dest.getChild(c.name, i = c.index, create = True)
|
||||
copy(c, dc)
|
||||
|
||||
24
svn_catalog_repository.py
Normal file
24
svn_catalog_repository.py
Normal file
@ -0,0 +1,24 @@
|
||||
|
||||
import subprocess
|
||||
import xml.etree.cElementTree as ET
|
||||
|
||||
class SVNCatalogRepository:
|
||||
def __init__(self, path):
|
||||
self._path = path
|
||||
xml = subprocess.check_output(["svn", "info", "--xml", path])
|
||||
root = ET.fromstring(xml)
|
||||
if (root.find("repository/root") == None):
|
||||
raise RuntimeError("Not an SVN repository:" + path)
|
||||
|
||||
def hasPathChanged(self, path, oldRevision):
|
||||
return self.scmRevisionForPath(path) != oldRevision
|
||||
|
||||
def scmRevisionForPath(self, path):
|
||||
xml = subprocess.check_output(["svn", "info", "--xml", path])
|
||||
root = ET.fromstring(xml)
|
||||
commit = root.find("entry/commit")
|
||||
return commit.get('revision', 0)
|
||||
|
||||
def update(self):
|
||||
subprocess.call(["svn", "update"])
|
||||
|
||||
52
template_catalog.xml
Normal file
52
template_catalog.xml
Normal file
@ -0,0 +1,52 @@
|
||||
<?xml version="1.0"?>
|
||||
<!--
|
||||
Template catalog - copy and modify for your site as required
|
||||
-->
|
||||
<PropertyList>
|
||||
|
||||
<!--
|
||||
catalogProps.addChild('version').value = '3.1.0'
|
||||
catalogProps.addChild('id').value = 'org.flightgear.default'
|
||||
catalogProps.addChild('license').value = 'GPL'
|
||||
catalogProps.addChild('url').value = "http://fgfs.goneabitbursar.com/pkg/3.1.0/default-catalog.xml"
|
||||
|
||||
catalogProps.addChild('description').value = "Aircraft developed and maintained by the FlightGear project"
|
||||
|
||||
de = catalogProps.addChild('de')
|
||||
# de.addChild('description').value = "<German translation of catalog description>"
|
||||
|
||||
fr = catalogProps.addChild('fr')
|
||||
-->
|
||||
|
||||
<version>3.4.*</version>
|
||||
<version>3.5.*</version>
|
||||
<version>3.6.*</version>
|
||||
|
||||
<id>org.myorganisation.hangar</id>
|
||||
<license>GPL</license>
|
||||
<url>http://some.stable.url.com/foo/bar/catalog.xml</url>
|
||||
|
||||
<description>A collection of interesting aircraft with some features
|
||||
</description>
|
||||
|
||||
<de>
|
||||
<description>Au Deutsch</description>
|
||||
</de>
|
||||
|
||||
<fr>
|
||||
<description>Francais</description>
|
||||
</fr>
|
||||
|
||||
<mirror>http://some.url/</mirror>
|
||||
<!-- <mirror>another mirror</mirror> -->
|
||||
|
||||
|
||||
<thumbnails>http://some.url/images</thumbnails>
|
||||
|
||||
<git-repository>git://some.git.repo/</git-repository>
|
||||
|
||||
<repository-prefix>Aircraft</repository-prefix>
|
||||
|
||||
</PropertyList>
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user