From 24eb2f6c43151e99e0560d017821f860eedc6d0b Mon Sep 17 00:00:00 2001 From: Robert Osfield Date: Mon, 20 Oct 2008 16:24:57 +0000 Subject: [PATCH] Introduce osgDB::FileCache, and updated osgfilecache and DatabasePager to use it. --- applications/osgfilecache/osgfilecache.cpp | 87 ++++++++++++++--- include/osgDB/DatabasePager | 2 - include/osgDB/FileCache | 49 ++++++++++ include/osgDB/Registry | 22 ++++- src/osgDB/CMakeLists.txt | 4 +- src/osgDB/DatabasePager.cpp | 108 ++++----------------- src/osgDB/FileCache.cpp | 80 +++++++++++++++ src/osgDB/ReadFile.cpp | 5 +- src/osgDB/Registry.cpp | 10 ++ 9 files changed, 252 insertions(+), 115 deletions(-) create mode 100644 include/osgDB/FileCache create mode 100644 src/osgDB/FileCache.cpp diff --git a/applications/osgfilecache/osgfilecache.cpp b/applications/osgfilecache/osgfilecache.cpp index 2c4ebbb3c..1be395215 100644 --- a/applications/osgfilecache/osgfilecache.cpp +++ b/applications/osgfilecache/osgfilecache.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -134,6 +136,8 @@ public: LoadDataVisitor(): osg::NodeVisitor(osg::NodeVisitor::TRAVERSE_ALL_CHILDREN), _currentLevel(0) {} + + void setFileCache(osgDB::FileCache* fileCache) { _fileCache = fileCache; } void addExtents(unsigned int maxLevel, double minX, double minY, double maxX, double maxY) { @@ -242,9 +246,8 @@ public: filename = plod.getFileName(i); } - osg::notify(osg::NOTICE)<<"reading "< node = osgDB::readNodeFile(filename); + osg::ref_ptr node = readNodeFileAndWriteToCache(filename); if (!s_ExitApplication && node.valid()) node->accept(*this); } @@ -267,6 +270,37 @@ public: } } + osg::Node* readNodeFileAndWriteToCache(const std::string& filename) + { + + if (_fileCache.valid() && osgDB::containsServerAddress(filename)) + { + if (_fileCache->existsInCache(filename)) + { + osg::notify(osg::NOTICE)<<"reading from FileCache: "<readNode(filename, osgDB::Registry::instance()->getOptions()).takeNode(); + } + else + { + osg::notify(osg::NOTICE)<<"reading : "<writeNode(*node, filename, osgDB::Registry::instance()->getOptions()); + } + return node; + } + } + else + { + osg::notify(osg::NOTICE)<<"reading : "< ExtentsList; typedef std::vector MatrixStack; typedef std::vector CSNStack; - + + osg::ref_ptr _fileCache; ExtentsList _extentsList; unsigned int _currentLevel; @@ -384,6 +419,26 @@ int main( int argc, char **argv ) LoadDataVisitor ldv; + std::string fileCachePath; + while(arguments.read("--file-cache",fileCachePath) || arguments.read("-c",fileCachePath)) {} + + if (fileCachePath.empty()) + { + const char* env_fileCachePath = getenv("OSG_FILE_CACHE"); + if (env_fileCachePath) + { + fileCachePath = env_fileCachePath; + } + } + + if (fileCachePath.empty()) + { + std::cout<<"No path to the file cache defined, please set OSG_FILE_PATH env var, or use --file-cache to set a suitable directory for the file cache."< loadedModel = ldv.readNodeFileAndWriteToCache(filename); if (!loadedModel) { std::cout<<"No data loaded, please specify a database to load"< dpReadRefNodeFile(const std::string& fileName,const ReaderWriter::Options* options); - protected: virtual ~DatabaseThread(); diff --git a/include/osgDB/FileCache b/include/osgDB/FileCache new file mode 100644 index 000000000..19ce3c29b --- /dev/null +++ b/include/osgDB/FileCache @@ -0,0 +1,49 @@ +/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield + * + * This library is open source and may be redistributed and/or modified under + * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or + * (at your option) any later version. The full license is in LICENSE file + * included with this distribution, and on the openscenegraph.org website. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * OpenSceneGraph Public License for more details. +*/ + +#ifndef OSGDB_FILECACHE +#define OSGDB_FILECACHE 1 + +#include + +#include + +namespace osgDB { + +class OSGDB_EXPORT FileCache : public osg::Referenced +{ + public: + + FileCache(const std::string& path); + + const std::string& getFileCachePath() const { return _fileCachePath; } + + virtual std::string createCacheFileName(const std::string& originalFileName) const; + + virtual bool existsInCache(const std::string& originalFileName) const; + + virtual ReaderWriter::ReadResult readNode(const std::string& originalFileName, const osgDB::ReaderWriter::Options* options, bool buildKdTreeIfRequired=true) const; + + virtual ReaderWriter::WriteResult writeNode(const osg::Node& node, const std::string& originalFileName, const osgDB::ReaderWriter::Options* options) const; + + protected: + + virtual ~FileCache(); + + std::string _fileCachePath; + +}; + +} + +#endif diff --git a/include/osgDB/Registry b/include/osgDB/Registry index 003dd23bc..eacd33926 100644 --- a/include/osgDB/Registry +++ b/include/osgDB/Registry @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -203,13 +204,13 @@ class OSGDB_EXPORT Registry : public osg::Referenced } ReaderWriter::ReadResult openArchiveImplementation(const std::string& fileName, ReaderWriter::ArchiveStatus status, unsigned int indexBlockSizeHint, const ReaderWriter::Options* options); - ReaderWriter::ReadResult readObject(const std::string& fileName,const ReaderWriter::Options* options) + ReaderWriter::ReadResult readObject(const std::string& fileName,const ReaderWriter::Options* options, bool buildKdTreeIfRequired=true) { ReaderWriter::ReadResult result = _readFileCallback.valid() ? _readFileCallback->readObject(fileName,options) : readObjectImplementation(fileName,options); - buildKdTreeIfRequired(result, options); + if (buildKdTreeIfRequired) _buildKdTreeIfRequired(result, options); return result; } @@ -229,13 +230,13 @@ class OSGDB_EXPORT Registry : public osg::Referenced } ReaderWriter::ReadResult readHeightFieldImplementation(const std::string& fileName,const ReaderWriter::Options* options); - ReaderWriter::ReadResult readNode(const std::string& fileName,const ReaderWriter::Options* options) + ReaderWriter::ReadResult readNode(const std::string& fileName,const ReaderWriter::Options* options, bool buildKdTreeIfRequired=true) { ReaderWriter::ReadResult result = _readFileCallback.valid() ? _readFileCallback->readNode(fileName,options) : readNodeImplementation(fileName,options); - buildKdTreeIfRequired(result, options); + if (buildKdTreeIfRequired) _buildKdTreeIfRequired(result, options); return result; } @@ -329,7 +330,7 @@ class OSGDB_EXPORT Registry : public osg::Referenced ReaderWriter::WriteResult writeShaderImplementation(const osg::Shader& obj, const std::string& fileName,const ReaderWriter::Options* options); - inline void buildKdTreeIfRequired(ReaderWriter::ReadResult& result, const ReaderWriter::Options* options) + inline void _buildKdTreeIfRequired(ReaderWriter::ReadResult& result, const ReaderWriter::Options* options) { bool doKdTreeBuilder = (options && options->getBuildKdTreesHint()!=ReaderWriter::Options::NO_PREFERENCE) ? options->getBuildKdTreesHint() == ReaderWriter::Options::BUILD_KDTREES : @@ -355,6 +356,15 @@ class OSGDB_EXPORT Registry : public osg::Referenced /** Get the KdTreeBuilder visitor that is used to build KdTree on loaded models.*/ osg::KdTreeBuilder* getKdTreeBuilder() { return _kdTreeBuilder.get(); } + /** Set the FileCache that is used to manage local storage of files downloaded from the internet.*/ + void setFileCache(FileCache* fileCache) { _fileCache = fileCache; } + + /** Get the FileCache that is used to manage local storage of files downloaded from the internet.*/ + FileCache* getFileCache() { return _fileCache.get(); } + + /** Get the const FileCache that is used to manage local storage of files downloaded from the internet.*/ + const FileCache* getFileCache() const { return _fileCache.get(); } + /** Set the password map to be used by plugins when access files from secure locations.*/ void setAuthenticationMap(AuthenticationMap* authenticationMap) { _authenticationMap = authenticationMap; } @@ -503,6 +513,8 @@ class OSGDB_EXPORT Registry : public osg::Referenced ReaderWriter::Options::BuildKdTreesHint _buildKdTreesHint; osg::ref_ptr _kdTreeBuilder; + osg::ref_ptr _fileCache; + osg::ref_ptr _authenticationMap; bool _createNodeFromImage; diff --git a/src/osgDB/CMakeLists.txt b/src/osgDB/CMakeLists.txt index df3ca645c..431838c47 100644 --- a/src/osgDB/CMakeLists.txt +++ b/src/osgDB/CMakeLists.txt @@ -17,6 +17,7 @@ SET(LIB_PUBLIC_HEADERS ${HEADER_PATH}/Field ${HEADER_PATH}/FieldReader ${HEADER_PATH}/FieldReaderIterator + ${HEADER_PATH}/FileCache ${HEADER_PATH}/FileNameUtils ${HEADER_PATH}/FileUtils ${HEADER_PATH}/ImageOptions @@ -28,8 +29,8 @@ SET(LIB_PUBLIC_HEADERS ${HEADER_PATH}/ReaderWriter ${HEADER_PATH}/ReadFile ${HEADER_PATH}/Registry - ${HEADER_PATH}/SharedStateManager ${HEADER_PATH}/Serializer + ${HEADER_PATH}/SharedStateManager ${HEADER_PATH}/Version ${HEADER_PATH}/WriteFile ) @@ -46,6 +47,7 @@ ADD_LIBRARY(${LIB_NAME} Field.cpp FieldReader.cpp FieldReaderIterator.cpp + FileCache.cpp FileNameUtils.cpp FileUtils.cpp ImageOptions.cpp diff --git a/src/osgDB/DatabasePager.cpp b/src/osgDB/DatabasePager.cpp index 5953ea62a..b4c26fd1e 100644 --- a/src/osgDB/DatabasePager.cpp +++ b/src/osgDB/DatabasePager.cpp @@ -398,17 +398,6 @@ int DatabasePager::DatabaseThread::cancel() } -osg::ref_ptr DatabasePager::DatabaseThread::dpReadRefNodeFile(const std::string& fileName,const ReaderWriter::Options* options) -{ - ReaderWriter::ReadResult rr = Registry::instance()->getReadFileCallback() ? - Registry::instance()->getReadFileCallback()->readNode(fileName,options) : - Registry::instance()->readNodeImplementation(fileName,options); - - if (rr.validNode()) return rr.getNode(); - if (rr.error()) osg::notify(osg::WARN) << rr.message() << std::endl; - return 0; -} - void DatabasePager::DatabaseThread::run() { osg::notify(osg::INFO)<<_name<<": DatabasePager::DatabaseThread::run"< fileCache = osgDB::Registry::instance()->getFileCache(); do { @@ -480,6 +465,8 @@ void DatabasePager::DatabaseThread::run() // osg::ref_ptr databaseRequest; read_queue->takeFirst(databaseRequest); + + bool readFromFileCache = false; if (databaseRequest.valid()) { @@ -494,57 +481,24 @@ void DatabasePager::DatabaseThread::run() // do nothing as this thread can handle the load if (osgDB::containsServerAddress(databaseRequest->_fileName)) { - std::string cacheFileName; - if (!cacheFilePath.empty()) + if (fileCache.valid() && fileCache->existsInCache(databaseRequest->_fileName)) { - cacheFileName = cacheFilePath + "/" + - osgDB::getServerAddress(databaseRequest->_fileName) + "/" + - osgDB::getServerFileName(databaseRequest->_fileName); - - std::string path = osgDB::getFilePath(cacheFileName); - - if (!osgDB::fileExists(path)) - { - cacheFileName.clear(); - } - } - - if (!cacheFilePath.empty() && osgDB::fileExists(cacheFileName)) - { - osg::notify(osg::INFO)<<_name<<": Reading cache file " << cacheFileName <<", previous path "<_fileName)<_fileName = cacheFileName; + readFromFileCache = true; } } - break; case(HANDLE_NON_HTTP): // check the cache first if (osgDB::containsServerAddress(databaseRequest->_fileName)) { - std::string cacheFileName; - if (!cacheFilePath.empty()) + if (fileCache.valid() && fileCache->existsInCache(databaseRequest->_fileName)) { - cacheFileName = cacheFilePath + "/" + - osgDB::getServerAddress(databaseRequest->_fileName) + "/" + - osgDB::getServerFileName(databaseRequest->_fileName); - - std::string path = osgDB::getFilePath(cacheFileName); - - if (!osgDB::fileExists(path)) - { - cacheFileName.clear(); - } - } - - if (!cacheFilePath.empty() && osgDB::fileExists(cacheFileName)) - { - osg::notify(osg::INFO)<<_name<<": Reading cache file " << cacheFileName <<", previous path "<_fileName)<_fileName = cacheFileName; + readFromFileCache = true; } else { - osg::notify(osg::INFO)<<_name<<": Passing http requests over "<_fileName<<" cacheFileName="<_fileName<add(databaseRequest.get()); databaseRequest = 0; } @@ -577,46 +531,20 @@ void DatabasePager::DatabaseThread::run() //osg::Timer_t before = osg::Timer::instance()->tick(); - bool serialize_readNodeFile = false; - if (serialize_readNodeFile) - { - // do *not* assume that we only have one DatabasePager, or that reaNodeFile is thread safe... - static OpenThreads::Mutex s_serialize_readNodeFile_mutex; - OpenThreads::ScopedLock lock(s_serialize_readNodeFile_mutex); - databaseRequest->_loadedModel = dpReadRefNodeFile(databaseRequest->_fileName, - databaseRequest->_loadOptions.get()); - } - else - { - // assume that we only have one DatabasePager, or that readNodeFile is thread safe... - databaseRequest->_loadedModel = dpReadRefNodeFile(databaseRequest->_fileName, - databaseRequest->_loadOptions.get()); - } + // assume that readNode is thread safe... + ReaderWriter::ReadResult rr = readFromFileCache ? + fileCache->readNode(databaseRequest->_fileName, databaseRequest->_loadOptions.get(), false) : + Registry::instance()->readNode(databaseRequest->_fileName, databaseRequest->_loadOptions.get(), false); + + if (rr.validNode()) databaseRequest->_loadedModel = rr.getNode(); + if (rr.error()) osg::notify(osg::WARN) << rr.message() << std::endl; if (databaseRequest->_loadedModel.valid() && osgDB::containsServerAddress(databaseRequest->_fileName) && - !cacheFilePath.empty()) + fileCache.valid() && + !readFromFileCache) { - std::string cacheFileName = cacheFilePath + "/" + - osgDB::getServerAddress(databaseRequest->_fileName) + "/" + - osgDB::getServerFileName(databaseRequest->_fileName); - - std::string path = osgDB::getFilePath(cacheFileName); - - if (!osgDB::fileExists(path) && !osgDB::makeDirectory(path)) - { - cacheFileName.clear(); - } - - if (!cacheFileName.empty() && osgDB::fileExists(cacheFileName)) - { - osg::notify(osg::NOTICE)<<_name<<": Warning, file already in cache file, shouldn't have needed to be reloaded. cache file=" << cacheFileName <<", previous path "<_fileName)<_loadedModel), cacheFileName, databaseRequest->_loadOptions.get()); - } + fileCache->writeNode(*(databaseRequest->_loadedModel), databaseRequest->_fileName, databaseRequest->_loadOptions.get()); } if ((_pager->_frameNumber-databaseRequest->_frameNumberLastRequest)>1) diff --git a/src/osgDB/FileCache.cpp b/src/osgDB/FileCache.cpp new file mode 100644 index 000000000..1dc85a861 --- /dev/null +++ b/src/osgDB/FileCache.cpp @@ -0,0 +1,80 @@ +/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield + * + * This library is open source and may be redistributed and/or modified under + * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or + * (at your option) any later version. The full license is in LICENSE file + * included with this distribution, and on the openscenegraph.org website. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * OpenSceneGraph Public License for more details. +*/ + +#include +#include +#include +#include +#include + +using namespace osgDB; + +FileCache::FileCache(const std::string& path): + _fileCachePath(path) +{ + osg::notify(osg::INFO)<<"Constructed FileCache : "<readNode(cacheFileName, options, buildKdTreeIfRequired); + } + else + { + return 0; + } +} + +ReaderWriter::WriteResult FileCache::writeNode(const osg::Node& node, const std::string& originalFileName, const osgDB::ReaderWriter::Options* options) const +{ + std::string cacheFileName = createCacheFileName(originalFileName); + if (!cacheFileName.empty()) + { + std::string path = osgDB::getFilePath(cacheFileName); + + if (!osgDB::fileExists(path) && !osgDB::makeDirectory(path)) + { + osg::notify(osg::NOTICE)<<"Could not create cache directory: "<writeNode(node, cacheFileName, options); + } + return ReaderWriter::WriteResult::FILE_NOT_HANDLED; +} diff --git a/src/osgDB/ReadFile.cpp b/src/osgDB/ReadFile.cpp index 4391fda8d..5a93fd035 100644 --- a/src/osgDB/ReadFile.cpp +++ b/src/osgDB/ReadFile.cpp @@ -131,10 +131,7 @@ Node* osgDB::readNodeFiles(osg::ArgumentParser& arguments,const ReaderWriter::Op while (arguments.read("--file-cache",filename)) { - std::string str("OSG_FILE_CACHE="); - str += filename; - - putenv(strdup((char*)str.c_str())); + osgDB::Registry::instance()->setFileCache(new osgDB::FileCache(filename)); } while (arguments.read("--image",filename)) diff --git a/src/osgDB/Registry.cpp b/src/osgDB/Registry.cpp index 80e954bcb..9937fd600 100644 --- a/src/osgDB/Registry.cpp +++ b/src/osgDB/Registry.cpp @@ -178,6 +178,12 @@ Registry::Registry() else _buildKdTreesHint = ReaderWriter::Options::BUILD_KDTREES; } + const char* fileCachePath = getenv("OSG_FILE_CACHE"); + if (fileCachePath) + { + _fileCache = new FileCache(fileCachePath); + } + _createNodeFromImage = false; _openingLibrary = false; @@ -326,6 +332,10 @@ void Registry::destruct() _sharedStateManager = 0; + // clean up the FileCache + _fileCache = 0; + + // object cache clear needed here to prevent crash in unref() of // the objects it contains when running the TXP plugin. // Not sure why, but perhaps there is is something in a TXP plugin