FGFS-4.1/OpenSceneGraph
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Introduce osgDB::FileCache, and updated osgfilecache and DatabasePager to use it.

Browse Source
This commit is contained in:
Robert Osfield 2008-10-20 16:24:57 +00:00
parent 10186190f6
commit 24eb2f6c43
9 changed files with 252 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
applications/osgfilecache/osgfilecache.cpp
View File

@ -22,6 +22,8 @@
#include <osgDB/ReadFile>
#include <osgDB/WriteFile>
#include <osgDB/FileUtils>
#include <osgDB/FileCache>
#include <osgDB/FileNameUtils>
#include <iostream>
#include <algorithm>
@ -135,6 +137,8 @@ public:
osg::NodeVisitor(osg::NodeVisitor::TRAVERSE_ALL_CHILDREN),
_currentLevel(0) {}
void setFileCache(osgDB::FileCache* fileCache) { _fileCache = fileCache; }
void addExtents(unsigned int maxLevel, double minX, double minY, double maxX, double maxY)
{
_extentsList.push_back(Extents(maxLevel, minX, minY, maxX, maxY));
@ -242,9 +246,8 @@ public:
filename = plod.getFileName(i);
}
osg::notify(osg::NOTICE)<<"reading "<<filename<<std::endl;
osg::ref_ptr<osg::Node> node = osgDB::readNodeFile(filename);
osg::ref_ptr<osg::Node> node = readNodeFileAndWriteToCache(filename);
if (!s_ExitApplication && node.valid()) node->accept(*this);
}
@ -267,6 +270,37 @@ public:
}
}
osg::Node* readNodeFileAndWriteToCache(const std::string& filename)
{
if (_fileCache.valid() && osgDB::containsServerAddress(filename))
{
if (_fileCache->existsInCache(filename))
{
osg::notify(osg::NOTICE)<<"reading from FileCache: "<<filename<<std::endl;
return _fileCache->readNode(filename, osgDB::Registry::instance()->getOptions()).takeNode();
}
else
{
osg::notify(osg::NOTICE)<<"reading : "<<filename<<std::endl;
osg::Node* node = osgDB::readNodeFile(filename);
if (node)
{
osg::notify(osg::NOTICE)<<"write to FileCache : "<<filename<<std::endl;
_fileCache->writeNode(*node, filename, osgDB::Registry::instance()->getOptions());
}
return node;
}
}
else
{
osg::notify(osg::NOTICE)<<"reading : "<<filename<<std::endl;
return osgDB::readNodeFile(filename);
}
}
protected:
inline void pushMatrix(osg::Matrix& matrix) { _matrixStack.push_back(matrix); }
@ -333,6 +367,7 @@ protected:
typedef std::vector<osg::Matrix> MatrixStack;
typedef std::vector<osg::CoordinateSystemNode*> CSNStack;
osg::ref_ptr<osgDB::FileCache> _fileCache;
ExtentsList _extentsList;
unsigned int _currentLevel;
@ -384,6 +419,26 @@ int main( int argc, char **argv )
LoadDataVisitor ldv;
std::string fileCachePath;
while(arguments.read("--file-cache",fileCachePath) || arguments.read("-c",fileCachePath)) {}
if (fileCachePath.empty())
{
const char* env_fileCachePath = getenv("OSG_FILE_CACHE");
if (env_fileCachePath)
{
fileCachePath = env_fileCachePath;
}
}
if (fileCachePath.empty())
{
std::cout<<"No path to the file cache defined, please set OSG_FILE_PATH env var, or use --file-cache <directory> to set a suitable directory for the file cache."<<std::endl;
return 1;
}
ldv.setFileCache(new osgDB::FileCache(fileCachePath));
unsigned int maxLevels = 0;
while(arguments.read("-l",maxLevels))
{
@ -396,18 +451,24 @@ int main( int argc, char **argv )
ldv.addExtents(maxLevels, minX, minY, maxX, maxY);
}
std::string fileCache;
while(arguments.read("--file-cache",fileCache) || arguments.read("-c",fileCache)) {}
if (!fileCache.empty())
std::string filename;
for(int i=1; i<arguments.argc(); ++i)
{
std::string str("OSG_FILE_CACHE=");
str += fileCache;
putenv(strdup((char*)str.c_str()));
if (!arguments.isOption(i))
{
filename = arguments[i];
break;
}
}
osg::ref_ptr<osg::Node> loadedModel = osgDB::readNodeFiles(arguments);
if (filename.empty())
{
std::cout<<"No file to load specified."<<std::endl;
return 1;
}
osg::ref_ptr<osg::Node> loadedModel = ldv.readNodeFileAndWriteToCache(filename);
if (!loadedModel)
{
std::cout<<"No data loaded, please specify a database to load"<<std::endl;

2
include/osgDB/DatabasePager
View File

@ -105,8 +105,6 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
virtual void run();
osg::ref_ptr<osg::Node> dpReadRefNodeFile(const std::string& fileName,const ReaderWriter::Options* options);
protected:
virtual ~DatabaseThread();

49
include/osgDB/FileCache Normal file
View File

@ -0,0 +1,49 @@
/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
*
* This library is open source and may be redistributed and/or modified under
* the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
* (at your option) any later version. The full license is in LICENSE file
* included with this distribution, and on the openscenegraph.org website.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* OpenSceneGraph Public License for more details.
*/
#ifndef OSGDB_FILECACHE
#define OSGDB_FILECACHE 1
#include <osg/Node>
#include <osgDB/ReaderWriter>
namespace osgDB {
class OSGDB_EXPORT FileCache : public osg::Referenced
{
public:
FileCache(const std::string& path);
const std::string& getFileCachePath() const { return _fileCachePath; }
virtual std::string createCacheFileName(const std::string& originalFileName) const;
virtual bool existsInCache(const std::string& originalFileName) const;
virtual ReaderWriter::ReadResult readNode(const std::string& originalFileName, const osgDB::ReaderWriter::Options* options, bool buildKdTreeIfRequired=true) const;
virtual ReaderWriter::WriteResult writeNode(const osg::Node& node, const std::string& originalFileName, const osgDB::ReaderWriter::Options* options) const;
protected:
virtual ~FileCache();
std::string _fileCachePath;
};
}
#endif

22
include/osgDB/Registry
View File

@ -24,6 +24,7 @@
#include <osgDB/ReaderWriter>
#include <osgDB/DotOsgWrapper>
#include <osgDB/DatabasePager>
#include <osgDB/FileCache>
#include <vector>
#include <map>
@ -203,13 +204,13 @@ class OSGDB_EXPORT Registry : public osg::Referenced
}
ReaderWriter::ReadResult openArchiveImplementation(const std::string& fileName, ReaderWriter::ArchiveStatus status, unsigned int indexBlockSizeHint, const ReaderWriter::Options* options);
ReaderWriter::ReadResult readObject(const std::string& fileName,const ReaderWriter::Options* options)
ReaderWriter::ReadResult readObject(const std::string& fileName,const ReaderWriter::Options* options, bool buildKdTreeIfRequired=true)
{
ReaderWriter::ReadResult result = _readFileCallback.valid() ?
_readFileCallback->readObject(fileName,options) :
readObjectImplementation(fileName,options);
buildKdTreeIfRequired(result, options);
if (buildKdTreeIfRequired) _buildKdTreeIfRequired(result, options);
return result;
}
@ -229,13 +230,13 @@ class OSGDB_EXPORT Registry : public osg::Referenced
}
ReaderWriter::ReadResult readHeightFieldImplementation(const std::string& fileName,const ReaderWriter::Options* options);
ReaderWriter::ReadResult readNode(const std::string& fileName,const ReaderWriter::Options* options)
ReaderWriter::ReadResult readNode(const std::string& fileName,const ReaderWriter::Options* options, bool buildKdTreeIfRequired=true)
{
ReaderWriter::ReadResult result = _readFileCallback.valid() ?
_readFileCallback->readNode(fileName,options) :
readNodeImplementation(fileName,options);
buildKdTreeIfRequired(result, options);
if (buildKdTreeIfRequired) _buildKdTreeIfRequired(result, options);
return result;
}
@ -329,7 +330,7 @@ class OSGDB_EXPORT Registry : public osg::Referenced
ReaderWriter::WriteResult writeShaderImplementation(const osg::Shader& obj, const std::string& fileName,const ReaderWriter::Options* options);
inline void buildKdTreeIfRequired(ReaderWriter::ReadResult& result, const ReaderWriter::Options* options)
inline void _buildKdTreeIfRequired(ReaderWriter::ReadResult& result, const ReaderWriter::Options* options)
{
bool doKdTreeBuilder = (options && options->getBuildKdTreesHint()!=ReaderWriter::Options::NO_PREFERENCE) ?
options->getBuildKdTreesHint() == ReaderWriter::Options::BUILD_KDTREES :
@ -355,6 +356,15 @@ class OSGDB_EXPORT Registry : public osg::Referenced
/** Get the KdTreeBuilder visitor that is used to build KdTree on loaded models.*/
osg::KdTreeBuilder* getKdTreeBuilder() { return _kdTreeBuilder.get(); }
/** Set the FileCache that is used to manage local storage of files downloaded from the internet.*/
void setFileCache(FileCache* fileCache) { _fileCache = fileCache; }
/** Get the FileCache that is used to manage local storage of files downloaded from the internet.*/
FileCache* getFileCache() { return _fileCache.get(); }
/** Get the const FileCache that is used to manage local storage of files downloaded from the internet.*/
const FileCache* getFileCache() const { return _fileCache.get(); }
/** Set the password map to be used by plugins when access files from secure locations.*/
void setAuthenticationMap(AuthenticationMap* authenticationMap) { _authenticationMap = authenticationMap; }
@ -503,6 +513,8 @@ class OSGDB_EXPORT Registry : public osg::Referenced
ReaderWriter::Options::BuildKdTreesHint _buildKdTreesHint;
osg::ref_ptr<osg::KdTreeBuilder> _kdTreeBuilder;
osg::ref_ptr<FileCache> _fileCache;
osg::ref_ptr<AuthenticationMap> _authenticationMap;
bool _createNodeFromImage;

4
src/osgDB/CMakeLists.txt
View File

@ -17,6 +17,7 @@ SET(LIB_PUBLIC_HEADERS
${HEADER_PATH}/Field
${HEADER_PATH}/FieldReader
${HEADER_PATH}/FieldReaderIterator
${HEADER_PATH}/FileCache
${HEADER_PATH}/FileNameUtils
${HEADER_PATH}/FileUtils
${HEADER_PATH}/ImageOptions
@ -28,8 +29,8 @@ SET(LIB_PUBLIC_HEADERS
${HEADER_PATH}/ReaderWriter
${HEADER_PATH}/ReadFile
${HEADER_PATH}/Registry
${HEADER_PATH}/SharedStateManager
${HEADER_PATH}/Serializer
${HEADER_PATH}/SharedStateManager
${HEADER_PATH}/Version
${HEADER_PATH}/WriteFile
)
@ -46,6 +47,7 @@ ADD_LIBRARY(${LIB_NAME}
Field.cpp
FieldReader.cpp
FieldReaderIterator.cpp
FileCache.cpp
FileNameUtils.cpp
FileUtils.cpp
ImageOptions.cpp

108
src/osgDB/DatabasePager.cpp
View File

@ -398,17 +398,6 @@ int DatabasePager::DatabaseThread::cancel()
}
osg::ref_ptr<osg::Node> DatabasePager::DatabaseThread::dpReadRefNodeFile(const std::string& fileName,const ReaderWriter::Options* options)
{
ReaderWriter::ReadResult rr = Registry::instance()->getReadFileCallback() ?
Registry::instance()->getReadFileCallback()->readNode(fileName,options) :
Registry::instance()->readNodeImplementation(fileName,options);
if (rr.validNode()) return rr.getNode();
if (rr.error()) osg::notify(osg::WARN) << rr.message() << std::endl;
return 0;
}
void DatabasePager::DatabaseThread::run()
{
osg::notify(osg::INFO)<<_name<<": DatabasePager::DatabaseThread::run"<<std::endl;
@ -447,11 +436,7 @@ void DatabasePager::DatabaseThread::run()
}
//Getting CURL Environment Variables (If found rewrite OSG Options)
std::string cacheFilePath;
const char* fileCachePath = getenv("OSG_FILE_CACHE");
if (fileCachePath) //Env Cache Directory
cacheFilePath = std::string(fileCachePath);
osg::ref_ptr<FileCache> fileCache = osgDB::Registry::instance()->getFileCache();
do
{
@ -481,6 +466,8 @@ void DatabasePager::DatabaseThread::run()
osg::ref_ptr<DatabaseRequest> databaseRequest;
read_queue->takeFirst(databaseRequest);
bool readFromFileCache = false;
if (databaseRequest.valid())
{
// check if databaseRequest is still relevant
@ -494,57 +481,24 @@ void DatabasePager::DatabaseThread::run()
// do nothing as this thread can handle the load
if (osgDB::containsServerAddress(databaseRequest->_fileName))
{
std::string cacheFileName;
if (!cacheFilePath.empty())
if (fileCache.valid() && fileCache->existsInCache(databaseRequest->_fileName))
{
cacheFileName = cacheFilePath + "/" +
osgDB::getServerAddress(databaseRequest->_fileName) + "/" +
osgDB::getServerFileName(databaseRequest->_fileName);
std::string path = osgDB::getFilePath(cacheFileName);
if (!osgDB::fileExists(path))
{
cacheFileName.clear();
}
}
if (!cacheFilePath.empty() && osgDB::fileExists(cacheFileName))
{
osg::notify(osg::INFO)<<_name<<": Reading cache file " << cacheFileName <<", previous path "<<osgDB::getFilePath(databaseRequest->_fileName)<<std::endl;
databaseRequest->_fileName = cacheFileName;
readFromFileCache = true;
}
}
break;
case(HANDLE_NON_HTTP):
// check the cache first
if (osgDB::containsServerAddress(databaseRequest->_fileName))
{
std::string cacheFileName;
if (!cacheFilePath.empty())
if (fileCache.valid() && fileCache->existsInCache(databaseRequest->_fileName))
{
cacheFileName = cacheFilePath + "/" +
osgDB::getServerAddress(databaseRequest->_fileName) + "/" +
osgDB::getServerFileName(databaseRequest->_fileName);
std::string path = osgDB::getFilePath(cacheFileName);
if (!osgDB::fileExists(path))
{
cacheFileName.clear();
}
}
if (!cacheFilePath.empty() && osgDB::fileExists(cacheFileName))
{
osg::notify(osg::INFO)<<_name<<": Reading cache file " << cacheFileName <<", previous path "<<osgDB::getFilePath(databaseRequest->_fileName)<<std::endl;
databaseRequest->_fileName = cacheFileName;
readFromFileCache = true;
}
else
{
osg::notify(osg::INFO)<<_name<<": Passing http requests over "<<databaseRequest->_fileName<<" cacheFileName="<<cacheFileName<<std::endl;
osg::notify(osg::INFO)<<_name<<": Passing http requests over "<<databaseRequest->_fileName<<std::endl;
out_queue->add(databaseRequest.get());
databaseRequest = 0;
}
@ -577,46 +531,20 @@ void DatabasePager::DatabaseThread::run()
//osg::Timer_t before = osg::Timer::instance()->tick();
bool serialize_readNodeFile = false;
if (serialize_readNodeFile)
{
// do *not* assume that we only have one DatabasePager, or that reaNodeFile is thread safe...
static OpenThreads::Mutex s_serialize_readNodeFile_mutex;
OpenThreads::ScopedLock<OpenThreads::Mutex> lock(s_serialize_readNodeFile_mutex);
databaseRequest->_loadedModel = dpReadRefNodeFile(databaseRequest->_fileName,
databaseRequest->_loadOptions.get());
}
else
{
// assume that we only have one DatabasePager, or that readNodeFile is thread safe...
databaseRequest->_loadedModel = dpReadRefNodeFile(databaseRequest->_fileName,
databaseRequest->_loadOptions.get());
}
// assume that readNode is thread safe...
ReaderWriter::ReadResult rr = readFromFileCache ?
fileCache->readNode(databaseRequest->_fileName, databaseRequest->_loadOptions.get(), false) :
Registry::instance()->readNode(databaseRequest->_fileName, databaseRequest->_loadOptions.get(), false);
if (rr.validNode()) databaseRequest->_loadedModel = rr.getNode();
if (rr.error()) osg::notify(osg::WARN) << rr.message() << std::endl;
if (databaseRequest->_loadedModel.valid() &&
osgDB::containsServerAddress(databaseRequest->_fileName) &&
!cacheFilePath.empty())
fileCache.valid() &&
!readFromFileCache)
{
std::string cacheFileName = cacheFilePath + "/" +
osgDB::getServerAddress(databaseRequest->_fileName) + "/" +
osgDB::getServerFileName(databaseRequest->_fileName);
std::string path = osgDB::getFilePath(cacheFileName);
if (!osgDB::fileExists(path) && !osgDB::makeDirectory(path))
{
cacheFileName.clear();
}
if (!cacheFileName.empty() && osgDB::fileExists(cacheFileName))
{
osg::notify(osg::NOTICE)<<_name<<": Warning, file already in cache file, shouldn't have needed to be reloaded. cache file=" << cacheFileName <<", previous path "<<osgDB::getFilePath(databaseRequest->_fileName)<<std::endl;
}
else
{
osg::notify(osg::INFO)<<_name<<": Write to cache "<<cacheFileName<<std::endl;
osgDB::writeNodeFile(*(databaseRequest->_loadedModel), cacheFileName, databaseRequest->_loadOptions.get());
}
fileCache->writeNode(*(databaseRequest->_loadedModel), databaseRequest->_fileName, databaseRequest->_loadOptions.get());
}
if ((_pager->_frameNumber-databaseRequest->_frameNumberLastRequest)>1)

80
src/osgDB/FileCache.cpp Normal file
View File

@ -0,0 +1,80 @@
/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
*
* This library is open source and may be redistributed and/or modified under
* the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
* (at your option) any later version. The full license is in LICENSE file
* included with this distribution, and on the openscenegraph.org website.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* OpenSceneGraph Public License for more details.
*/
#include <osgDB/FileCache>
#include <osgDB/FileUtils>
#include <osgDB/FileNameUtils>
#include <osgDB/ReadFile>
#include <osgDB/WriteFile>
using namespace osgDB;
FileCache::FileCache(const std::string& path):
_fileCachePath(path)
{
osg::notify(osg::INFO)<<"Constructed FileCache : "<<path<<std::endl;
}
FileCache::~FileCache()
{
osg::notify(osg::INFO)<<"Destructed FileCache "<<std::endl;
}
std::string FileCache::createCacheFileName(const std::string& originalFileName) const
{
std::string cacheFileName = _fileCachePath + "/" +
osgDB::getServerAddress(originalFileName) + "/" +
osgDB::getServerFileName(originalFileName);
osg::notify(osg::INFO)<<"FileCache::createCacheFileName("<<originalFileName<<") = "<<cacheFileName<<std::endl;
return cacheFileName;
}
bool FileCache::existsInCache(const std::string& originalFileName) const
{
return osgDB::fileExists(createCacheFileName(originalFileName));
}
ReaderWriter::ReadResult FileCache::readNode(const std::string& originalFileName, const osgDB::ReaderWriter::Options* options, bool buildKdTreeIfRequired) const
{
std::string cacheFileName = createCacheFileName(originalFileName);
if (!cacheFileName.empty() && osgDB::fileExists(cacheFileName))
{
osg::notify(osg::INFO)<<"FileCache::readNodeFromCache("<<originalFileName<<") as "<<cacheFileName<<std::endl;
return osgDB::Registry::instance()->readNode(cacheFileName, options, buildKdTreeIfRequired);
}
else
{
return 0;
}
}
ReaderWriter::WriteResult FileCache::writeNode(const osg::Node& node, const std::string& originalFileName, const osgDB::ReaderWriter::Options* options) const
{
std::string cacheFileName = createCacheFileName(originalFileName);
if (!cacheFileName.empty())
{
std::string path = osgDB::getFilePath(cacheFileName);
if (!osgDB::fileExists(path) && !osgDB::makeDirectory(path))
{
osg::notify(osg::NOTICE)<<"Could not create cache directory: "<<path<<std::endl;
return ReaderWriter::WriteResult::ERROR_IN_WRITING_FILE;
}
osg::notify(osg::INFO)<<"FileCache::writeNodeToCache("<<originalFileName<<") as "<<cacheFileName<<std::endl;
return osgDB::Registry::instance()->writeNode(node, cacheFileName, options);
}
return ReaderWriter::WriteResult::FILE_NOT_HANDLED;
}

5
src/osgDB/ReadFile.cpp
View File

@ -131,10 +131,7 @@ Node* osgDB::readNodeFiles(osg::ArgumentParser& arguments,const ReaderWriter::Op
while (arguments.read("--file-cache",filename))
{
std::string str("OSG_FILE_CACHE=");
str += filename;
putenv(strdup((char*)str.c_str()));
osgDB::Registry::instance()->setFileCache(new osgDB::FileCache(filename));
}
while (arguments.read("--image",filename))

10
src/osgDB/Registry.cpp
View File

@ -178,6 +178,12 @@ Registry::Registry()
else _buildKdTreesHint = ReaderWriter::Options::BUILD_KDTREES;
}
const char* fileCachePath = getenv("OSG_FILE_CACHE");
if (fileCachePath)
{
_fileCache = new FileCache(fileCachePath);
}
_createNodeFromImage = false;
_openingLibrary = false;
@ -326,6 +332,10 @@ void Registry::destruct()
_sharedStateManager = 0;
// clean up the FileCache
_fileCache = 0;
// object cache clear needed here to prevent crash in unref() of
// the objects it contains when running the TXP plugin.
// Not sure why, but perhaps there is is something in a TXP plugin