FGFS-4.1/fgmeta
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Catalog creation work

Browse Source 
- XML scanning works
- thumbnail extraction progressing
This commit is contained in:
James Turner 2015-07-18 15:30:20 -05:00
parent 8d1e9abcc9
commit f0fff6d16e
3 changed files with 72 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
catalogTags.py
View File

@ -1,12 +1,13 @@
aircraftTypeTags = [
"ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle",
"ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle",
"tanker", "cargo", "transport", "bizjet", "trainer", "airship", "balloon"
]
manufacturerTags = [
"boeing", "cessna", "diamond", "douglas", "bell", "piper",
"boeing", "cessna", "diamond", "douglas", "bell", "piper",
"airbus", "vickers", "lockheed", "fokker",
"embrarer", "bombardier", "pilatus", "robin"
"embrarer", "bombardier", "pilatus", "robin",
"eurocopter"
]
eraTags = [
@ -21,6 +22,8 @@ eraTags = [
"1960s",
"1970s",
"1980s",
"1990s",
"2000s",
"gulfwar1",
"gulfwar2"
]
@ -69,7 +72,8 @@ propulsionTags = [
simFeatureTags = [
"tow",
"dual-controls"
"dual-controls",
"rembrandt"
]
tags = aircraftTypeTags + manufacturerTags + eraTags + simFeatureTags + propulsionTags + featureTags

47
maintain_catalog.py
View File

@ -3,7 +3,7 @@
import os, sys, re, glob
import hashlib # for MD5
import subprocess
import shutil # for copy2
import catalogTags
import sgprops
@ -11,10 +11,12 @@ import svn_catalog_repository
import git_catalog_repository
import git_discrete_repository
# TODO
# uploading / rsyncing
standardTagSet = frozenset(catalogTags.tags)
def isNonstandardTag(t):
return t not in standardTagSet
thumbnailNames = ["thumbnail.png", "thumbnail.jpg"]
includePaths = []
class VariantData:
def __init__(self, path, node):
@ -28,8 +30,8 @@ class VariantData:
@property
def catalogNode(self):
n = Node("variant")
n.addChild("id").value = path
n = sgprops.Node("variant")
n.addChild("id").value = self._path
n.addChild("name").value = self._name
class PackageData:
@ -85,15 +87,19 @@ class PackageData:
return True
def scanSetXmlFiles(self):
def scanSetXmlFiles(self, includes):
foundPrimary = False
foundMultiple = False
for f in os.listdir(self._path):
if not f.endswith("-set.xml"):
continue
p = os.path.join(self._path, f)
node = sgprops.readProps(p)
node = sgprops.readProps(p, includePaths = includes)
if not node.hasChild("sim"):
continue
simNode = node.getChild("sim")
if (simNode.getValue("exclude", False)):
continue
@ -106,7 +112,9 @@ class PackageData:
continue
if foundPrimary:
print "Multiple primary -set.xml files at:" + self._path
if not foundMultiple:
print "Multiple primary -set.xml files at:" + self._path
foundMultiple = True
continue
else:
foundPrimary = True;
@ -147,11 +155,12 @@ class PackageData:
if sim.hasChild('tags'):
for c in sim.getChild('tags').getChildren('tag'):
if isNonstandardTag(c.value):
print "Skipping non-standard tag:", c.value
print "Skipping non-standard tag:", c.value, self.path
else:
self._node.addChild('tag').value = c.value
self._thumbnails.append(t.value for t in sim.getChildren("thumbnail"))
for t in sim.getChildren("thumbnail"):
self._thumbnails.append(t.value)
def validate(self):
for t in self._thumbnails:
@ -199,7 +208,7 @@ class PackageData:
def extractThumbnails(self, thumbnailDir):
for t in self._thumbnails:
fullName = self.id + "_" + t
os.file.copy(os.path.join(self._path, t),
shutil.copy2(os.path.join(self._path, t),
os.path.join(thumbnailDir, fullName)
)
# TODO : verify image format, size and so on
@ -209,6 +218,11 @@ def scanPackages(globPath):
print "Scanning", globPath
print os.getcwd()
for d in glob.glob(globPath):
# check dir contains at least one -set.xml file
if len(glob.glob(os.path.join(d, "*-set.xml"))) == 0:
print "no -set.xml in", d
continue
result.append(PackageData(d))
return result
@ -278,10 +292,19 @@ if not os.path.exists(outPath):
print "Output path is:" + outPath
thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
if not os.path.exists(thumbnailPath):
os.mkdir(thumbnailPath)
thumbnailUrl = config.getValue('thumbnail-url')
print "Thumbnail url is:", thumbnailUrl
for i in config.getChildren("include-dir"):
if not os.path.exists(i.value):
print "Skipping missing include path:", i.value
continue
includePaths.append(i.value)
mirrorUrls = []
# contains existing catalog
@ -318,7 +341,7 @@ mirrorUrls = (m.value for m in config.getChildren("mirror"))
packagesToGenerate = []
for p in packages.values():
p.scanSetXmlFiles()
p.scanSetXmlFiles(includePaths)
if (p.isSourceModified(scmRepo)):
packagesToGenerate.append(p)

40
sgprops.py
View File

@ -146,11 +146,11 @@ class Node(object):
class PropsHandler(handler.ContentHandler):
def __init__(self, root = None, path = None, dataDirPath = None):
def __init__(self, root = None, path = None, includePaths = []):
self._root = root
self._path = path
self._basePath = os.path.dirname(path)
self._dataDirPath = dataDirPath
self._includes = includePaths
self._locator = None
if root is None:
@ -167,7 +167,13 @@ class PropsHandler(handler.ContentHandler):
return
if 'n' in attrs.keys():
index = int(attrs['n'])
try:
index = int(attrs['n'])
except:
print "Invalid index at line:", self._locator.getLineNumber(), "of", self._path
self._current = self._current.addChild(name)
return
self._current = self._current.getChild(name, index, create=True)
else:
self._current = self._current.addChild(name)
@ -186,11 +192,17 @@ class PropsHandler(handler.ContentHandler):
p = os.path.join(self._basePath, includePath)
if not os.path.exists(p):
p = os.path.join(self._dataDirPath, includePath)
if not os.path.exists(p):
found = False
for i in self._includes:
p = os.path.join(i, includePath)
if os.path.exists(p):
found = True
break
if not found:
raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
readProps(p, self._current, self._dataDirPath)
readProps(p, self._current, self._includes)
def endElement(self, name):
if (name == 'PropertyList'):
@ -200,16 +212,23 @@ class PropsHandler(handler.ContentHandler):
# convert and store value
self._current.value = self._content
if self._currentTy == "int":
self._current.value = int(self._content)
self._current.value = int(self._content) if self._content is not None else 0
if self._currentTy == "bool":
self._current.value = self.parsePropsBool(self._content)
if self._currentTy == "double":
self._current.value = float(self._content)
if self._content is None:
self._current.value = 0.0
else:
if self._content.endswith('f'):
self._content = self._content[:-1]
self._current.value = float(self._content)
except:
print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
self._current = self._current.parent
self._content = None
self._currentTy = None
def parsePropsBool(self, content):
if content == "True" or content == "true":
@ -240,11 +259,10 @@ class PropsHandler(handler.ContentHandler):
def root(self):
return self._root
def readProps(path, root = None, dataDirPath = None):
def readProps(path, root = None, includePaths = []):
parser = make_parser()
locator = expatreader.ExpatLocator( parser )
h = PropsHandler(root, path, dataDirPath)
h = PropsHandler(root, path, includePaths)
h.setDocumentLocator(locator)
parser.setContentHandler(h)
parser.parse(path)