From Sukender, new ExternalFileWriter helper class that helps the management of writing out external files to disk

avoid writing out of duplicates.

include/osgDB/ExternalFileWriter

@@ -0,0 +1,106 @@
+/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield 
+ *
+ * This library is open source and may be redistributed and/or modified under  
+ * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or 
+ * (at your option) any later version.  The full license is in LICENSE file
+ * included with this distribution, and on the openscenegraph.org website.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
+ * OpenSceneGraph Public License for more details.
+*/
+
+#ifndef OSGDB_PLUGIN_IMAGE_WRITER
+#define OSGDB_PLUGIN_IMAGE_WRITER 1
+
+#include <osgDB/Export>
+#include <string>
+#include <map>
+
+namespace osg
+{
+    class Object;
+}
+
+namespace osgDB
+{
+
+    class Options;
+
+    /// Helper allowing 'intelligent' writing of external files (images, shaders, etc.), regarding to a main file (a scene), especially in plugins.
+    /// Goals are:
+    ///   - Enable writing out objects only once (even if referenced multiple times)
+    ///   - Handle duplicates (avoid writing two different objects at the same place, renaming files as needed)
+    ///   - Handle directory creation when paths don't just exist
+    ///   - Generate writing paths which may keep original directory structure (depending on user wishes). Ex:
+    ///       Reading: model.osg and images/img1.jpg
+    ///          Writing with    'keepRelativePaths': /somePath/newmodel.osg and /somePath/images/img1.jpg
+    ///          Writing without 'keepRelativePaths': /somePath/newmodel.osg and /somePath/img1.jpg
+    ///\author Sukender
+    ///\todo Handling of naming constraints (such as "8.3" names in 3DS)
+    class OSGDB_EXPORT ExternalFileWriter
+    {
+    public:
+        /// Builds the helper class with all options.
+        ///\param srcDirectory Directory of the initial main file (if any), used as a base when relativising objects names. Not used if keepRelativePaths==false.
+        ///\param destDirectory Directory where to write the main file.
+        ///\param keepRelativePaths If true, then relative paths of source objects are kept if possible (ex: If an image is initially "imageDir/image.jpg" relatively to the source dir, then we'd like to get "destDir/imageDir/image.jpg"). If false, then only the simple file name is used to write the object file.
+        ///\param allowUpDirs When relativising objects paths, sets the maximum number of directories the objects can be written "up" the destination directory. Not used if keepRelativePaths==false. Examples: If an image is initially "../image.jpg" relatively to the source dir *AND* if we allow one dir level up, then we'd like to get "destDirParent/destDir/../image.jpg" (= "destDirParent/image.jpg"). If we *DO NOT* allow one dir level up, then we'd like to get "destDir/image.jpg".
+        ExternalFileWriter(const std::string & srcDirectory, const std::string & destDirectory, bool keepRelativePaths, unsigned int allowUpDirs=0);
+
+        /// Short constructor used when not relativising objects paths, or when having no initial model file (which is pretty the same here).
+        ExternalFileWriter(const std::string & destDirectory);
+
+        /// Writes the current object if not already done.
+        ///\param obj Object to write, using corresponding osgDB::write method.
+        ///\param options Writing options to pass to corresponding osgDB::write method.
+        ///\param [out] out_absolutePath Pointer to a string to be filled with absolute writing path, or NULL.
+        ///\param [out] out_relativePath Pointer to a string to be filled with write path relative to the destination directory if possible (absolute path if not), or NULL.
+        ///\return true on success, false otherwise.
+        bool write(const osg::Object & obj, const osgDB::Options * options, std::string * out_absolutePath=NULL, std::string * out_relativePath=NULL);
+
+        struct ObjectData
+        {
+            ObjectData() : written(false) {}
+            ObjectData(const std::string & absolutePath, const std::string & relativePath, bool written) : absolutePath(absolutePath), relativePath(relativePath), written(written) {}
+            std::string absolutePath;
+            std::string relativePath;
+            bool written;        ///< Says if write succeded or not.
+        };
+
+        /// Set of written objects, with their absolute writing path.
+        /// Objects being passed to the write() method but which have failed to be effectively written are also included.
+        typedef std::map<const osg::Object*, ObjectData> ObjectsSet;
+
+        /// Returns the written objects.
+        const ObjectsSet & getObjects() const { return _objects; }
+
+    protected:
+        // Dev note:
+        // A multi-indexed structure would be more efficient for ObjectsSet (such as boost::multi_index, indexed on object pointer (unique), and hashed indexed on absolute path (unique)).
+        // In order to get a correct search time, SearchMap "replaces" the multi-index structure for hashed indexes on absolute paths.
+        typedef std::multimap<unsigned int, const osg::Object*> SearchMap;
+        typedef unsigned int ObjectIndex;       ///< Integer type used for indices of unnamed objects
+        ObjectsSet _objects;
+        SearchMap _searchMap;                    ///< Map used to search by absolute file path.
+        ObjectIndex _lastGeneratedObjectIndex;
+        const std::string _srcDirectory;
+        const std::string _destDirectory;
+        bool _keepRelativePaths;
+        const unsigned int _allowUpDirs;
+
+        /// Generates a unique name for an object to be written on disk.
+        /// Side effect: updates _lastGeneratedObjectIndex to the index associated withe the returned name.
+        void generateObjectName(std::string & out_relativePath, std::string & out_absolutePath, int type);
+
+        bool absoluteObjectPathExists(const std::string & path);
+
+    private:
+        // Prevent copy
+        ExternalFileWriter & operator=(const ExternalFileWriter &);
+        ExternalFileWriter(const ExternalFileWriter &);
+    };
+}
+
+#endif    // OSGDB_PLUGIN_IMAGE_WRITER

include/osgDB/FileNameUtils

@@ -17,6 +17,7 @@
 #include <osgDB/Export>
 
 #include <string>
+#include <vector>
 
 namespace osgDB {
 
@@ -77,6 +78,8 @@ extern OSGDB_EXPORT std::string concatPaths(const std::string& left, const std::
 /** Removes .. and . dirs in a path */
 extern OSGDB_EXPORT std::string getRealPath(const std::string& path);
 
+/** Splits a path into elements between separators (including Windows' root, if any). */
+extern OSGDB_EXPORT void getPathElements(const std::string& path, std::vector<std::string> & out_elements);
 
 }
 

src/osgDB/CMakeLists.txt

@@ -60,6 +60,7 @@ SET(TARGET_H
     ${HEADER_PATH}/DotOsgWrapper
     ${HEADER_PATH}/DynamicLibrary
     ${HEADER_PATH}/Export
+    ${HEADER_PATH}/ExternalFileWriter
     ${HEADER_PATH}/FileCache
     ${HEADER_PATH}/FileNameUtils
     ${HEADER_PATH}/FileUtils
@@ -94,6 +95,7 @@ SET(TARGET_SRC
     DatabaseRevisions.cpp
     DotOsgWrapper.cpp
     DynamicLibrary.cpp
+    ExternalFileWriter.cpp
     Field.cpp
     FieldReader.cpp
     FieldReaderIterator.cpp

src/osgDB/ExternalFileWriter.cpp

@@ -0,0 +1,258 @@
+/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield 
+ *
+ * This library is open source and may be redistributed and/or modified under  
+ * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or 
+ * (at your option) any later version.  The full license is in LICENSE file
+ * included with this distribution, and on the openscenegraph.org website.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
+ * OpenSceneGraph Public License for more details.
+*/
+
+#include <osgDB/ExternalFileWriter>
+#include <osgDB/FileNameUtils>
+#include <osgDB/FileUtils>
+#include <osgDB/WriteFile>
+#include <stdexcept>
+
+namespace osgDB
+{
+
+/// Counts the number of directories the given relative path goes "up" the current dir, or 0 if not.
+/// This returns 0 for absolute paths.
+/// Examples:
+///    - "../a" goes 1 level up
+///    - "../../a/b/c/d/e" goes 2
+///    - "../a/../b/../.." goes 2
+///    - "a/b/../c" goes 0
+unsigned int countNbDirsUp(const std::string & path)
+{
+    // Algorithm:
+    //    - For each path component, count +1 for "..", 0 for ".", and -1 for anything else
+    //    - Ignore everything after  the last ".." of the path.
+    if (osgDB::isAbsolutePath(path)) return 0;
+    int result(0), tempResult(0);
+    //for(osgDB::PathIterator it(path); it.valid(); ++it)
+    std::vector<std::string> pathElems;
+    getPathElements(path, pathElems);
+    for(std::vector<std::string>::const_iterator it(pathElems.begin()), itEnd(pathElems.end()); it!=itEnd; ++it)
+    {
+        if (*it == "..")
+        {
+            // Count +1, and "validates" temporary result
+            ++tempResult;
+            result = tempResult;
+        }
+        else if (*it != ".") --tempResult;
+    }
+    return result<=0 ? 0 : static_cast<unsigned int>(result);
+}
+
+
+/// Local hash function for a path.
+/// Does not canonize the given path, but is not confused with mixed separators.
+unsigned int pathHash(const std::string & s)
+{
+    // This is based on the DJB hash algorithm
+    // Note: SDBM Hash initializes at 0 and is
+    //        hash = c + (hash << 6) + (hash << 16) - hash;
+    unsigned int hash = 5381;
+    for(std::string::const_iterator it=s.begin(), itEnd=s.end(); it!=itEnd; ++it)
+    {
+        std::string::value_type c = *it;
+        if (c == '\\') c = '/';        // We're processing a path and don't want to be affected by differences in separators
+        hash = ((hash << 5) + hash) + c;
+    }
+    return hash;
+}
+
+
+//virtual ReaderWriter::WriteResult writeObject(const osg::Object& obj, const std::string& fileName,const Options* options);
+//virtual ReaderWriter::WriteResult writeImage(const osg::Image& obj, const std::string& fileName,const Options* options);
+//virtual ReaderWriter::WriteResult writeHeightField(const osg::HeightField& obj, const std::string& fileName,const Options* options);
+//virtual ReaderWriter::WriteResult writeNode(const osg::Node& obj, const std::string& fileName,const Options* options);
+//virtual ReaderWriter::WriteResult writeShader(const osg::Shader& obj, const std::string& fileName,const Options* options);
+
+enum WriteType {
+    WRITE_TYPE_OBJECT,
+    WRITE_TYPE_IMAGE,
+    WRITE_TYPE_HEIGHT_FIELD,
+    WRITE_TYPE_NODE,
+    WRITE_TYPE_SHADER,
+
+    MAX_WRITE_TYPE
+};
+
+/// Default prefixes for unnamed objects.
+const char * const PREFIX[/*MAX_WRITE_TYPE*/] = {
+    "Object_",
+    "Image_",
+    "HF_",
+    "Node_",
+    "Shader_"
+};
+
+
+//.frag
+//.vert
+
+inline WriteType getType(const osg::Object & obj)
+{
+    // Is there something faster than a dynamic_cast<>?
+    if (dynamic_cast<const osg::Image       *>(&obj)) return WRITE_TYPE_IMAGE;
+    if (dynamic_cast<const osg::HeightField *>(&obj)) return WRITE_TYPE_HEIGHT_FIELD;
+    if (dynamic_cast<const osg::Node        *>(&obj)) return WRITE_TYPE_NODE;
+    if (dynamic_cast<const osg::Shader      *>(&obj)) return WRITE_TYPE_SHADER;
+    return WRITE_TYPE_OBJECT;
+}
+
+/// Returns the object filename if available, or its name otherwise.
+inline const std::string & getFileName(const osg::Object & obj, WriteType type)
+{
+    switch(type) {
+        case WRITE_TYPE_IMAGE:        return static_cast<const osg::Image       &>(obj).getFileName();
+        case WRITE_TYPE_SHADER:       return static_cast<const osg::Shader      &>(obj).getFileName();
+        default:        // WRITE_TYPE_OBJECT, WRITE_TYPE_NODE, WRITE_TYPE_HEIGHT_FIELD
+            return obj.getName();
+    }
+}
+
+
+inline bool doWrite(const osg::Object & obj, WriteType type, const std::string& fileName, const Options * options)
+{
+    switch(type) {
+        case WRITE_TYPE_IMAGE:        return osgDB::writeImageFile      (static_cast<const osg::Image       &>(obj), fileName, options);
+        case WRITE_TYPE_HEIGHT_FIELD: return osgDB::writeHeightFieldFile(static_cast<const osg::HeightField &>(obj), fileName, options);
+        case WRITE_TYPE_NODE:         return osgDB::writeNodeFile       (static_cast<const osg::Node        &>(obj), fileName, options);
+        case WRITE_TYPE_SHADER:       return osgDB::writeShaderFile     (static_cast<const osg::Shader      &>(obj), fileName, options);
+        default:        // WRITE_TYPE_OBJECT
+            return osgDB::writeObjectFile(obj, fileName, options);
+    }
+}
+
+
+
+// --------------------------------------------------------------------------------
+
+
+ExternalFileWriter::ExternalFileWriter(const std::string & srcDirectory, const std::string & destDirectory, bool keepRelativePaths, unsigned int allowUpDirs)
+    : _lastGeneratedObjectIndex(0), _srcDirectory(srcDirectory), _destDirectory(destDirectory), _keepRelativePaths(keepRelativePaths), _allowUpDirs(allowUpDirs)
+{}
+
+ExternalFileWriter::ExternalFileWriter(const std::string & destDirectory)
+    : _lastGeneratedObjectIndex(0), _destDirectory(destDirectory), _keepRelativePaths(false), _allowUpDirs(0)
+{}
+
+
+bool ExternalFileWriter::write(const osg::Object & obj, const osgDB::Options * options, std::string * out_absolutePath, std::string * out_relativePath)
+{
+    ObjectsSet::iterator it( _objects.find(&obj) );
+    if (it != _objects.end())
+    {
+        // Object has already been passed to this method
+        if (out_absolutePath) *out_absolutePath = it->second.absolutePath;
+        if (out_relativePath) *out_relativePath = it->second.relativePath;
+        return it->second.written;
+    }
+
+    // Object is a new entry
+
+    // Get absolute source path
+    WriteType type( getType(obj) );
+    std::string originalFileName( getFileName(obj, type) );
+    std::string absoluteSourcePath;
+    if (_keepRelativePaths && !originalFileName.empty())        // if keepRelativePaths is false, absoluteSourcePath is not used, then we can skip this part
+    {
+        if (osgDB::isAbsolutePath(originalFileName)) absoluteSourcePath = originalFileName;
+        else absoluteSourcePath = osgDB::concatPaths(_srcDirectory, originalFileName);
+        absoluteSourcePath = osgDB::getRealPath(osgDB::convertFileNameToNativeStyle(absoluteSourcePath));      // getRealPath() here is only used to canonize the path, not to add current directory in front of relative paths, hence the "concatPaths(_srcDirectory, ...)" just above
+    }
+
+    // Compute destination paths from the source path
+    std::string relativeDestinationPath;
+    std::string absoluteDestinationPath;
+    if (absoluteSourcePath.empty())
+    {
+        // We have no name. Generate one.
+        generateObjectName(relativeDestinationPath, absoluteDestinationPath, type);
+    }
+    else
+    {
+        // We have a name.
+        if (_keepRelativePaths)
+        {
+            // We'll try to keep images relative path.
+            relativeDestinationPath = osgDB::getPathRelative(_srcDirectory, absoluteSourcePath);
+            unsigned int nbDirsUp = countNbDirsUp(relativeDestinationPath);
+            // TODO if nbDirsUp>nb dirs in _destDirectory, then issue a warning, and use simple file name
+            if (nbDirsUp > _allowUpDirs) relativeDestinationPath = osgDB::getSimpleFileName(absoluteSourcePath);
+        }
+        else
+        {
+            // We keep only the simple file name.
+            relativeDestinationPath = osgDB::getSimpleFileName(absoluteSourcePath);
+        }
+        absoluteDestinationPath = osgDB::getRealPath(osgDB::convertFileNameToNativeStyle( osgDB::concatPaths(_destDirectory, relativeDestinationPath) ));
+        // TODO Check for absolute paths collisions between multiple objects
+    }
+
+    // Write object
+    bool written(false);
+    if (!osgDB::makeDirectoryForFile(absoluteDestinationPath))
+    {
+        OSG_NOTICE << "Can't create directory for file '" << absoluteDestinationPath << "'. May fail creating the image file." << std::endl;
+    }
+    if (!doWrite(obj, type, absoluteDestinationPath, options))
+    {
+        OSG_WARN << "Can't write file '" << absoluteDestinationPath << "'." << std::endl;
+    }
+    else written = true;
+
+    // Add entry
+    _objects.insert(ObjectsSet::value_type(&obj, ObjectData(relativeDestinationPath, absoluteDestinationPath, written))).first;
+    _searchMap.insert(SearchMap::value_type(pathHash(absoluteDestinationPath), &obj));
+
+    // Fill output strings
+    if (out_absolutePath) *out_absolutePath = absoluteDestinationPath;
+    if (out_relativePath) *out_relativePath = relativeDestinationPath;
+
+    return written;
+}
+
+
+bool ExternalFileWriter::absoluteObjectPathExists(const std::string & path)
+{
+    // For all paths in the search map having the same hash as 'path', check if paths correspond
+    std::pair<SearchMap::iterator, SearchMap::iterator> bounds( _searchMap.equal_range(pathHash(path)) );
+    for(SearchMap::iterator it=bounds.first; it!=bounds.second; ++it)
+    {
+        const osg::Object * img( it->second );
+        if (_objects[img].absolutePath == path) return true;
+    }
+    return false;
+}
+
+void ExternalFileWriter::generateObjectName(std::string & out_relativePath, std::string & out_absolutePath, int type)
+{
+    static const ObjectIndex MAX_NUMBER = UINT_MAX-1;        // -1 to allow doing +1 without an overflow
+    static const char * const IMAGE_EXT  = ".tga";     // Default extension (PNG would be preferable since widely used, but it requires libpng)
+    static const char * const SHADER_EXT = ".frag";    // Default extension
+    for (ObjectIndex number=_lastGeneratedObjectIndex+1; number<MAX_NUMBER; ++number)
+    {
+        std::ostringstream oss;
+        oss << PREFIX[type] << number << IMAGE_EXT;
+        out_relativePath = oss.str();
+        out_absolutePath = osgDB::concatPaths(_destDirectory, out_relativePath);
+
+        if (!absoluteObjectPathExists(out_absolutePath))
+        {
+            _lastGeneratedObjectIndex = number;
+            return;
+        }
+    }
+    throw std::runtime_error("Could not get a free index to write image.");
+}
+
+}

src/osgDB/FileNameUtils.cpp

@@ -413,6 +413,12 @@ std::string::const_iterator osgDB::PathIterator::next(std::string::const_iterato
     return std::find_first_of(it, end, PATH_SEPARATORS, PATH_SEPARATORS+PATH_SEPARATORS_LEN);
 }
 
+void osgDB::getPathElements(const std::string& path, std::vector<std::string> & out_elements)
+{
+    out_elements.clear();
+    for(osgDB::PathIterator it(path); it.valid(); ++it) out_elements.push_back(*it);
+}
+
 
 std::string osgDB::getPathRoot(const std::string& path) {
     // Test for unix root