From 066a1f6e72c80aa6d4bf4887ac541d4a3b90f304 Mon Sep 17 00:00:00 2001 From: Robert Osfield Date: Thu, 12 May 2011 13:27:14 +0000 Subject: [PATCH] From Sukender, new ExternalFileWriter helper class that helps the management of writing out external files to disk avoid writing out of duplicates. --- include/osgDB/ExternalFileWriter | 106 +++++++++++++ include/osgDB/FileNameUtils | 3 + src/osgDB/CMakeLists.txt | 2 + src/osgDB/ExternalFileWriter.cpp | 258 +++++++++++++++++++++++++++++++ src/osgDB/FileNameUtils.cpp | 6 + 5 files changed, 375 insertions(+) create mode 100644 include/osgDB/ExternalFileWriter create mode 100644 src/osgDB/ExternalFileWriter.cpp diff --git a/include/osgDB/ExternalFileWriter b/include/osgDB/ExternalFileWriter new file mode 100644 index 000000000..6cd0b314f --- /dev/null +++ b/include/osgDB/ExternalFileWriter @@ -0,0 +1,106 @@ +/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield + * + * This library is open source and may be redistributed and/or modified under + * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or + * (at your option) any later version. The full license is in LICENSE file + * included with this distribution, and on the openscenegraph.org website. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * OpenSceneGraph Public License for more details. +*/ + +#ifndef OSGDB_PLUGIN_IMAGE_WRITER +#define OSGDB_PLUGIN_IMAGE_WRITER 1 + +#include +#include +#include + +namespace osg +{ + class Object; +} + +namespace osgDB +{ + + class Options; + + /// Helper allowing 'intelligent' writing of external files (images, shaders, etc.), regarding to a main file (a scene), especially in plugins. + /// Goals are: + /// - Enable writing out objects only once (even if referenced multiple times) + /// - Handle duplicates (avoid writing two different objects at the same place, renaming files as needed) + /// - Handle directory creation when paths don't just exist + /// - Generate writing paths which may keep original directory structure (depending on user wishes). Ex: + /// Reading: model.osg and images/img1.jpg + /// Writing with 'keepRelativePaths': /somePath/newmodel.osg and /somePath/images/img1.jpg + /// Writing without 'keepRelativePaths': /somePath/newmodel.osg and /somePath/img1.jpg + ///\author Sukender + ///\todo Handling of naming constraints (such as "8.3" names in 3DS) + class OSGDB_EXPORT ExternalFileWriter + { + public: + /// Builds the helper class with all options. + ///\param srcDirectory Directory of the initial main file (if any), used as a base when relativising objects names. Not used if keepRelativePaths==false. + ///\param destDirectory Directory where to write the main file. + ///\param keepRelativePaths If true, then relative paths of source objects are kept if possible (ex: If an image is initially "imageDir/image.jpg" relatively to the source dir, then we'd like to get "destDir/imageDir/image.jpg"). If false, then only the simple file name is used to write the object file. + ///\param allowUpDirs When relativising objects paths, sets the maximum number of directories the objects can be written "up" the destination directory. Not used if keepRelativePaths==false. Examples: If an image is initially "../image.jpg" relatively to the source dir *AND* if we allow one dir level up, then we'd like to get "destDirParent/destDir/../image.jpg" (= "destDirParent/image.jpg"). If we *DO NOT* allow one dir level up, then we'd like to get "destDir/image.jpg". + ExternalFileWriter(const std::string & srcDirectory, const std::string & destDirectory, bool keepRelativePaths, unsigned int allowUpDirs=0); + + /// Short constructor used when not relativising objects paths, or when having no initial model file (which is pretty the same here). + ExternalFileWriter(const std::string & destDirectory); + + /// Writes the current object if not already done. + ///\param obj Object to write, using corresponding osgDB::write method. + ///\param options Writing options to pass to corresponding osgDB::write method. + ///\param [out] out_absolutePath Pointer to a string to be filled with absolute writing path, or NULL. + ///\param [out] out_relativePath Pointer to a string to be filled with write path relative to the destination directory if possible (absolute path if not), or NULL. + ///\return true on success, false otherwise. + bool write(const osg::Object & obj, const osgDB::Options * options, std::string * out_absolutePath=NULL, std::string * out_relativePath=NULL); + + struct ObjectData + { + ObjectData() : written(false) {} + ObjectData(const std::string & absolutePath, const std::string & relativePath, bool written) : absolutePath(absolutePath), relativePath(relativePath), written(written) {} + std::string absolutePath; + std::string relativePath; + bool written; ///< Says if write succeded or not. + }; + + /// Set of written objects, with their absolute writing path. + /// Objects being passed to the write() method but which have failed to be effectively written are also included. + typedef std::map ObjectsSet; + + /// Returns the written objects. + const ObjectsSet & getObjects() const { return _objects; } + + protected: + // Dev note: + // A multi-indexed structure would be more efficient for ObjectsSet (such as boost::multi_index, indexed on object pointer (unique), and hashed indexed on absolute path (unique)). + // In order to get a correct search time, SearchMap "replaces" the multi-index structure for hashed indexes on absolute paths. + typedef std::multimap SearchMap; + typedef unsigned int ObjectIndex; ///< Integer type used for indices of unnamed objects + ObjectsSet _objects; + SearchMap _searchMap; ///< Map used to search by absolute file path. + ObjectIndex _lastGeneratedObjectIndex; + const std::string _srcDirectory; + const std::string _destDirectory; + bool _keepRelativePaths; + const unsigned int _allowUpDirs; + + /// Generates a unique name for an object to be written on disk. + /// Side effect: updates _lastGeneratedObjectIndex to the index associated withe the returned name. + void generateObjectName(std::string & out_relativePath, std::string & out_absolutePath, int type); + + bool absoluteObjectPathExists(const std::string & path); + + private: + // Prevent copy + ExternalFileWriter & operator=(const ExternalFileWriter &); + ExternalFileWriter(const ExternalFileWriter &); + }; +} + +#endif // OSGDB_PLUGIN_IMAGE_WRITER diff --git a/include/osgDB/FileNameUtils b/include/osgDB/FileNameUtils index 553a9c282..6f30620fc 100644 --- a/include/osgDB/FileNameUtils +++ b/include/osgDB/FileNameUtils @@ -17,6 +17,7 @@ #include #include +#include namespace osgDB { @@ -77,6 +78,8 @@ extern OSGDB_EXPORT std::string concatPaths(const std::string& left, const std:: /** Removes .. and . dirs in a path */ extern OSGDB_EXPORT std::string getRealPath(const std::string& path); +/** Splits a path into elements between separators (including Windows' root, if any). */ +extern OSGDB_EXPORT void getPathElements(const std::string& path, std::vector & out_elements); } diff --git a/src/osgDB/CMakeLists.txt b/src/osgDB/CMakeLists.txt index df8d6441e..e79f67148 100644 --- a/src/osgDB/CMakeLists.txt +++ b/src/osgDB/CMakeLists.txt @@ -60,6 +60,7 @@ SET(TARGET_H ${HEADER_PATH}/DotOsgWrapper ${HEADER_PATH}/DynamicLibrary ${HEADER_PATH}/Export + ${HEADER_PATH}/ExternalFileWriter ${HEADER_PATH}/FileCache ${HEADER_PATH}/FileNameUtils ${HEADER_PATH}/FileUtils @@ -94,6 +95,7 @@ SET(TARGET_SRC DatabaseRevisions.cpp DotOsgWrapper.cpp DynamicLibrary.cpp + ExternalFileWriter.cpp Field.cpp FieldReader.cpp FieldReaderIterator.cpp diff --git a/src/osgDB/ExternalFileWriter.cpp b/src/osgDB/ExternalFileWriter.cpp new file mode 100644 index 000000000..c895caf6f --- /dev/null +++ b/src/osgDB/ExternalFileWriter.cpp @@ -0,0 +1,258 @@ +/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield + * + * This library is open source and may be redistributed and/or modified under + * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or + * (at your option) any later version. The full license is in LICENSE file + * included with this distribution, and on the openscenegraph.org website. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * OpenSceneGraph Public License for more details. +*/ + +#include +#include +#include +#include +#include + +namespace osgDB +{ + +/// Counts the number of directories the given relative path goes "up" the current dir, or 0 if not. +/// This returns 0 for absolute paths. +/// Examples: +/// - "../a" goes 1 level up +/// - "../../a/b/c/d/e" goes 2 +/// - "../a/../b/../.." goes 2 +/// - "a/b/../c" goes 0 +unsigned int countNbDirsUp(const std::string & path) +{ + // Algorithm: + // - For each path component, count +1 for "..", 0 for ".", and -1 for anything else + // - Ignore everything after the last ".." of the path. + if (osgDB::isAbsolutePath(path)) return 0; + int result(0), tempResult(0); + //for(osgDB::PathIterator it(path); it.valid(); ++it) + std::vector pathElems; + getPathElements(path, pathElems); + for(std::vector::const_iterator it(pathElems.begin()), itEnd(pathElems.end()); it!=itEnd; ++it) + { + if (*it == "..") + { + // Count +1, and "validates" temporary result + ++tempResult; + result = tempResult; + } + else if (*it != ".") --tempResult; + } + return result<=0 ? 0 : static_cast(result); +} + + +/// Local hash function for a path. +/// Does not canonize the given path, but is not confused with mixed separators. +unsigned int pathHash(const std::string & s) +{ + // This is based on the DJB hash algorithm + // Note: SDBM Hash initializes at 0 and is + // hash = c + (hash << 6) + (hash << 16) - hash; + unsigned int hash = 5381; + for(std::string::const_iterator it=s.begin(), itEnd=s.end(); it!=itEnd; ++it) + { + std::string::value_type c = *it; + if (c == '\\') c = '/'; // We're processing a path and don't want to be affected by differences in separators + hash = ((hash << 5) + hash) + c; + } + return hash; +} + + +//virtual ReaderWriter::WriteResult writeObject(const osg::Object& obj, const std::string& fileName,const Options* options); +//virtual ReaderWriter::WriteResult writeImage(const osg::Image& obj, const std::string& fileName,const Options* options); +//virtual ReaderWriter::WriteResult writeHeightField(const osg::HeightField& obj, const std::string& fileName,const Options* options); +//virtual ReaderWriter::WriteResult writeNode(const osg::Node& obj, const std::string& fileName,const Options* options); +//virtual ReaderWriter::WriteResult writeShader(const osg::Shader& obj, const std::string& fileName,const Options* options); + +enum WriteType { + WRITE_TYPE_OBJECT, + WRITE_TYPE_IMAGE, + WRITE_TYPE_HEIGHT_FIELD, + WRITE_TYPE_NODE, + WRITE_TYPE_SHADER, + + MAX_WRITE_TYPE +}; + +/// Default prefixes for unnamed objects. +const char * const PREFIX[/*MAX_WRITE_TYPE*/] = { + "Object_", + "Image_", + "HF_", + "Node_", + "Shader_" +}; + + +//.frag +//.vert + +inline WriteType getType(const osg::Object & obj) +{ + // Is there something faster than a dynamic_cast<>? + if (dynamic_cast(&obj)) return WRITE_TYPE_IMAGE; + if (dynamic_cast(&obj)) return WRITE_TYPE_HEIGHT_FIELD; + if (dynamic_cast(&obj)) return WRITE_TYPE_NODE; + if (dynamic_cast(&obj)) return WRITE_TYPE_SHADER; + return WRITE_TYPE_OBJECT; +} + +/// Returns the object filename if available, or its name otherwise. +inline const std::string & getFileName(const osg::Object & obj, WriteType type) +{ + switch(type) { + case WRITE_TYPE_IMAGE: return static_cast(obj).getFileName(); + case WRITE_TYPE_SHADER: return static_cast(obj).getFileName(); + default: // WRITE_TYPE_OBJECT, WRITE_TYPE_NODE, WRITE_TYPE_HEIGHT_FIELD + return obj.getName(); + } +} + + +inline bool doWrite(const osg::Object & obj, WriteType type, const std::string& fileName, const Options * options) +{ + switch(type) { + case WRITE_TYPE_IMAGE: return osgDB::writeImageFile (static_cast(obj), fileName, options); + case WRITE_TYPE_HEIGHT_FIELD: return osgDB::writeHeightFieldFile(static_cast(obj), fileName, options); + case WRITE_TYPE_NODE: return osgDB::writeNodeFile (static_cast(obj), fileName, options); + case WRITE_TYPE_SHADER: return osgDB::writeShaderFile (static_cast(obj), fileName, options); + default: // WRITE_TYPE_OBJECT + return osgDB::writeObjectFile(obj, fileName, options); + } +} + + + +// -------------------------------------------------------------------------------- + + +ExternalFileWriter::ExternalFileWriter(const std::string & srcDirectory, const std::string & destDirectory, bool keepRelativePaths, unsigned int allowUpDirs) + : _lastGeneratedObjectIndex(0), _srcDirectory(srcDirectory), _destDirectory(destDirectory), _keepRelativePaths(keepRelativePaths), _allowUpDirs(allowUpDirs) +{} + +ExternalFileWriter::ExternalFileWriter(const std::string & destDirectory) + : _lastGeneratedObjectIndex(0), _destDirectory(destDirectory), _keepRelativePaths(false), _allowUpDirs(0) +{} + + +bool ExternalFileWriter::write(const osg::Object & obj, const osgDB::Options * options, std::string * out_absolutePath, std::string * out_relativePath) +{ + ObjectsSet::iterator it( _objects.find(&obj) ); + if (it != _objects.end()) + { + // Object has already been passed to this method + if (out_absolutePath) *out_absolutePath = it->second.absolutePath; + if (out_relativePath) *out_relativePath = it->second.relativePath; + return it->second.written; + } + + // Object is a new entry + + // Get absolute source path + WriteType type( getType(obj) ); + std::string originalFileName( getFileName(obj, type) ); + std::string absoluteSourcePath; + if (_keepRelativePaths && !originalFileName.empty()) // if keepRelativePaths is false, absoluteSourcePath is not used, then we can skip this part + { + if (osgDB::isAbsolutePath(originalFileName)) absoluteSourcePath = originalFileName; + else absoluteSourcePath = osgDB::concatPaths(_srcDirectory, originalFileName); + absoluteSourcePath = osgDB::getRealPath(osgDB::convertFileNameToNativeStyle(absoluteSourcePath)); // getRealPath() here is only used to canonize the path, not to add current directory in front of relative paths, hence the "concatPaths(_srcDirectory, ...)" just above + } + + // Compute destination paths from the source path + std::string relativeDestinationPath; + std::string absoluteDestinationPath; + if (absoluteSourcePath.empty()) + { + // We have no name. Generate one. + generateObjectName(relativeDestinationPath, absoluteDestinationPath, type); + } + else + { + // We have a name. + if (_keepRelativePaths) + { + // We'll try to keep images relative path. + relativeDestinationPath = osgDB::getPathRelative(_srcDirectory, absoluteSourcePath); + unsigned int nbDirsUp = countNbDirsUp(relativeDestinationPath); + // TODO if nbDirsUp>nb dirs in _destDirectory, then issue a warning, and use simple file name + if (nbDirsUp > _allowUpDirs) relativeDestinationPath = osgDB::getSimpleFileName(absoluteSourcePath); + } + else + { + // We keep only the simple file name. + relativeDestinationPath = osgDB::getSimpleFileName(absoluteSourcePath); + } + absoluteDestinationPath = osgDB::getRealPath(osgDB::convertFileNameToNativeStyle( osgDB::concatPaths(_destDirectory, relativeDestinationPath) )); + // TODO Check for absolute paths collisions between multiple objects + } + + // Write object + bool written(false); + if (!osgDB::makeDirectoryForFile(absoluteDestinationPath)) + { + OSG_NOTICE << "Can't create directory for file '" << absoluteDestinationPath << "'. May fail creating the image file." << std::endl; + } + if (!doWrite(obj, type, absoluteDestinationPath, options)) + { + OSG_WARN << "Can't write file '" << absoluteDestinationPath << "'." << std::endl; + } + else written = true; + + // Add entry + _objects.insert(ObjectsSet::value_type(&obj, ObjectData(relativeDestinationPath, absoluteDestinationPath, written))).first; + _searchMap.insert(SearchMap::value_type(pathHash(absoluteDestinationPath), &obj)); + + // Fill output strings + if (out_absolutePath) *out_absolutePath = absoluteDestinationPath; + if (out_relativePath) *out_relativePath = relativeDestinationPath; + + return written; +} + + +bool ExternalFileWriter::absoluteObjectPathExists(const std::string & path) +{ + // For all paths in the search map having the same hash as 'path', check if paths correspond + std::pair bounds( _searchMap.equal_range(pathHash(path)) ); + for(SearchMap::iterator it=bounds.first; it!=bounds.second; ++it) + { + const osg::Object * img( it->second ); + if (_objects[img].absolutePath == path) return true; + } + return false; +} + +void ExternalFileWriter::generateObjectName(std::string & out_relativePath, std::string & out_absolutePath, int type) +{ + static const ObjectIndex MAX_NUMBER = UINT_MAX-1; // -1 to allow doing +1 without an overflow + static const char * const IMAGE_EXT = ".tga"; // Default extension (PNG would be preferable since widely used, but it requires libpng) + static const char * const SHADER_EXT = ".frag"; // Default extension + for (ObjectIndex number=_lastGeneratedObjectIndex+1; number & out_elements) +{ + out_elements.clear(); + for(osgDB::PathIterator it(path); it.valid(); ++it) out_elements.push_back(*it); +} + std::string osgDB::getPathRoot(const std::string& path) { // Test for unix root