From 4044fd5a74c3f48661877b43baf70a77917f7981 Mon Sep 17 00:00:00 2001 From: Robert Osfield Date: Mon, 24 Jun 2013 08:48:55 +0000 Subject: [PATCH] From Wang Rui, "The file attached includes two new features for the serialization IO functionality. First, custom serializer version control should work now, just by defining a new REGISTER_CUSTOM_OBJECT_WRAPPER macro. For example: // A custom class namespace CustomDomain { class MyGroup : public osg::Group { public: META_Node( CustomDomain, MyGroup ); void setMyName( const std::string& n ); const std::string& getMyName() const; void setMyID( int id ); int getMyID() const; ... }; } // The serialization wrapper using a custom domain name REGISTER_CUSTOM_OBJECT_WRAPPER( MyDomain, CustomDomain_MyGroup, new CustomDomain::MyGroup, CustomDomain::MyGroup, "osg::Object osg::Node osg::Group CustomDomain::MyGroup" ) { ADD_STRING_SERIALIZER( MyName, std::string() ); { UPDATE_TO_VERSION_SCOPED( 1 ); // Updated for a new domain version ADD_INT_SERIALIZER( MyID, 0 ); } } Save the class instance as follows: osgDB::writeNodeFile( *myGroup, "serializer_test.osgt", new osgDB::Options("CustomDomains=MyDomain:1") ); The output file will include the domain version definition and all the class data, and can be read back. We can also force setting the domain version by the CustomDomains option while reading the saved files. If we save the class instance without any options, MyID will be ignored because the default domain version is 0. This may help third-party libraries like osgEarth to maintain their own serializers without regarding to the OSG soversion changes. Another feature added is a more robust binary format, which in fact adds a size-offset at each block's beginning. When there are problems or unsupported data types while reading, we can now directly jump to the block end indicated by the offset value. So a .osgb file will automatically ignore bad data and read remains as normal (at present it will fail at all). This feature will not break the backward compatibility, and can be disabled by setting "RobustBinaryFormat=false" while writing out. Hope these changes can work smoothly with present and future community projects. Maybe we should also consider have an osgserializer example to test and demonstrate all things we can do now." --- include/osgDB/InputStream | 5 +- include/osgDB/ObjectWrapper | 35 ++++++++++++- include/osgDB/OutputStream | 6 ++- include/osgDB/StreamOperator | 12 ++++- src/osgDB/InputStream.cpp | 44 ++++++++++++++-- src/osgDB/ObjectWrapper.cpp | 43 ++++++++++++++-- src/osgDB/OutputStream.cpp | 57 +++++++++++++++++++-- src/osgPlugins/osg/BinaryStreamOperator.h | 61 ++++++++++++++++++++++- 8 files changed, 245 insertions(+), 18 deletions(-) diff --git a/include/osgDB/InputStream b/include/osgDB/InputStream index 93f44684a..68875944d 100644 --- a/include/osgDB/InputStream +++ b/include/osgDB/InputStream @@ -75,8 +75,10 @@ public: InputStream( const osgDB::Options* options ); virtual ~InputStream(); + void setFileVersion( const std::string& d, int v ) { _domainVersionMap[d] = v; } + int getFileVersion( const std::string& d=std::string() ) const; + bool isBinary() const { return _in->isBinary(); } - int getFileVersion() const { return _fileVersion; } const osgDB::Options* getOptions() const { return _options.get(); } // Serialization related functions @@ -184,6 +186,7 @@ protected: ArrayMap _arrayMap; IdentifierMap _identifierMap; + std::map _domainVersionMap; int _fileVersion; bool _useSchemaData; bool _forceReadingImage; diff --git a/include/osgDB/ObjectWrapper b/include/osgDB/ObjectWrapper index 99f0b36e9..d43c5aa73 100644 --- a/include/osgDB/ObjectWrapper +++ b/include/osgDB/ObjectWrapper @@ -50,7 +50,9 @@ public: ObjectWrapper( osg::Object* proto, const std::string& name, const std::string& associates ); - + ObjectWrapper( osg::Object* proto, const std::string& domain, const std::string& name, + const std::string& associates ); + void setUpdatedVersion( int ver ) { _version = ver; } int getUpdatedVersion() const { return _version; } @@ -76,6 +78,7 @@ protected: virtual ~ObjectWrapper() {} osg::ref_ptr _proto; + std::string _domain; std::string _name; StringList _associates; SerializerList _serializers; @@ -165,6 +168,20 @@ protected: osg::ref_ptr _wrapper; }; +class OSGDB_EXPORT RegisterCustomWrapperProxy +{ +public: + typedef void (*AddPropFunc)( const char*, ObjectWrapper* ); + + RegisterCustomWrapperProxy( osg::Object* proto, const std::string& domain, const std::string& name, + const std::string& associates, AddPropFunc func ); + + virtual ~RegisterCustomWrapperProxy(); + +protected: + osg::ref_ptr _wrapper; +}; + #define REGISTER_OBJECT_WRAPPER(NAME, PROTO, CLASS, ASSOCIATES) \ extern "C" void wrapper_serializer_##NAME(void) {} \ extern void wrapper_propfunc_##NAME(osgDB::ObjectWrapper*); \ @@ -181,6 +198,22 @@ protected: typedef CLASS MyClass; \ void wrapper_propfunc_##NAME(osgDB::ObjectWrapper* wrapper) +#define REGISTER_CUSTOM_OBJECT_WRAPPER(DOMAIN, NAME, PROTO, CLASS, ASSOCIATES) \ + extern "C" void wrapper_serializer_##NAME(void) {} \ + extern void wrapper_propfunc_##NAME(const char*, osgDB::ObjectWrapper*); \ + static osgDB::RegisterCustomWrapperProxy wrapper_proxy_##NAME( \ + PROTO, #DOMAIN, #CLASS, ASSOCIATES, &wrapper_propfunc_##NAME); \ + typedef CLASS MyClass; \ + void wrapper_propfunc_##NAME(const char* domain, osgDB::ObjectWrapper* wrapper) + +#define REGISTER_CUSTOM_OBJECT_WRAPPER2(DOMAIN, NAME, PROTO, CLASS, CLASSNAME, ASSOCIATES) \ + extern "C" void wrapper_serializer_##NAME(void) {} \ + extern void wrapper_propfunc_##NAME(const char*, osgDB::ObjectWrapper*); \ + static osgDB::RegisterCustomWrapperProxy wrapper_proxy_##NAME( \ + PROTO, #DOMAIN, CLASSNAME, ASSOCIATES, &wrapper_propfunc_##NAME); \ + typedef CLASS MyClass; \ + void wrapper_propfunc_##NAME(const char* domain, osgDB::ObjectWrapper* wrapper) + class OSGDB_EXPORT RegisterCompressorProxy { public: diff --git a/include/osgDB/OutputStream b/include/osgDB/OutputStream index 2a9be1303..f9bf597f6 100644 --- a/include/osgDB/OutputStream +++ b/include/osgDB/OutputStream @@ -76,6 +76,9 @@ public: OutputStream( const osgDB::Options* options ); virtual ~OutputStream(); + void setFileVersion( const std::string& d, int v ) { _domainVersionMap[d] = v; } + int getFileVersion( const std::string& d=std::string() ) const; + bool isBinary() const { return _out->isBinary(); } const std::string& getSchemaName() const { return _schemaName; } const osgDB::Options* getOptions() const { return _options.get(); } @@ -185,8 +188,9 @@ protected: ArrayMap _arrayMap; ObjectMap _objectMap; + std::map _domainVersionMap; WriteImageHint _writeImageHint; - bool _useSchemaData; + bool _useSchemaData, _useRobustBinaryFormat; std::map _inbuiltSchemaMap; std::vector _fields; std::string _schemaName; diff --git a/include/osgDB/StreamOperator b/include/osgDB/StreamOperator index 406df4c50..3c352e219 100644 --- a/include/osgDB/StreamOperator +++ b/include/osgDB/StreamOperator @@ -16,13 +16,16 @@ class InputStream; class OSGDB_EXPORT OutputIterator : public osg::Referenced { public: - OutputIterator() : _out(0) {} + OutputIterator() : _out(0), _supportBinaryBrackets(false) {} virtual ~OutputIterator() {} void setStream( std::ostream* ostream ) { _out = ostream; } std::ostream* getStream() { return _out; } const std::ostream* getStream() const { return _out; } + void setSupportBinaryBrackets( bool b ) { _supportBinaryBrackets = b; } + bool getSupportBinaryBrackets() const { return _supportBinaryBrackets; } + virtual bool isBinary() const = 0; virtual void writeBool( bool b ) = 0; @@ -64,12 +67,13 @@ protected: } std::ostream* _out; + bool _supportBinaryBrackets; }; class OSGDB_EXPORT InputIterator : public osg::Referenced { public: - InputIterator() : _in(0), _inputStream(0), _byteSwap(0), _failed(false) {} + InputIterator() : _in(0), _inputStream(0), _byteSwap(0), _supportBinaryBrackets(false), _failed(false) {} virtual ~InputIterator() {} void setStream( std::istream* istream ) { _in = istream; } @@ -83,6 +87,9 @@ public: void setByteSwap(int byteSwap) { _byteSwap = byteSwap; } int getByteSwap() const { return _byteSwap; } + void setSupportBinaryBrackets( bool b ) { _supportBinaryBrackets = b; } + bool getSupportBinaryBrackets() const { return _supportBinaryBrackets; } + void checkStream() const { if (_in->rdstate()&_in->failbit) _failed = true; } bool isFailed() const { return _failed; } @@ -121,6 +128,7 @@ protected: std::istream* _in; osgDB::InputStream* _inputStream; int _byteSwap; + bool _supportBinaryBrackets; mutable bool _failed; }; diff --git a/src/osgDB/InputStream.cpp b/src/osgDB/InputStream.cpp index aa23a6164..859665b53 100644 --- a/src/osgDB/InputStream.cpp +++ b/src/osgDB/InputStream.cpp @@ -32,9 +32,22 @@ InputStream::InputStream( const osgDB::Options* options ) if ( !options ) return; _options = options; - std::string schema; if ( options->getPluginStringData("ForceReadingImage")=="true" ) _forceReadingImage = true; + + if ( !options->getPluginStringData("CustomDomains").empty() ) + { + StringList domains, keyAndValue; + split( options->getPluginStringData("CustomDomains"), domains, ';' ); + for ( unsigned int i=0; i1 ) + _domainVersionMap[keyAndValue.front()] = atoi(keyAndValue.back().c_str()); + } + } + + std::string schema; if ( !options->getPluginStringData("SchemaFile").empty() ) { schema = options->getPluginStringData("SchemaFile"); @@ -60,6 +73,13 @@ InputStream::~InputStream() delete _dataDecompress; } +int InputStream::getFileVersion( const std::string& d ) const +{ + if ( d.empty() ) return _fileVersion; + std::map::const_iterator itr = _domainVersionMap.find(d); + return itr==_domainVersionMap.end() ? 0 : itr->second; +} + InputStream& InputStream::operator>>( osg::Vec2b& v ) { char x, y; *this >> x >> y; @@ -668,7 +688,6 @@ osg::Object* InputStream::readObjectFields( const std::string& className, unsign << className << std::endl; return NULL; } - _fields.push_back( className ); osg::ref_ptr obj = existingObj ? existingObj : wrapper->getProto()->cloneType(); _identifierMap[id] = obj; @@ -691,7 +710,6 @@ osg::Object* InputStream::readObjectFields( const std::string& className, unsign _fields.pop_back(); } } - _fields.pop_back(); return obj.release(); } @@ -734,7 +752,20 @@ InputStream::ReadType InputStream::start( InputIterator* inIterator ) type = static_cast(typeValue); unsigned int attributes; *this >> attributes; + if ( attributes&0x4 ) inIterator->setSupportBinaryBrackets( true ); if ( attributes&0x2 ) _useSchemaData = true; + + // Record custom domains + if ( attributes&0x1 ) + { + unsigned int numDomains; *this >> numDomains; + for ( unsigned int i=0; i> domainName; + int domainVersion; *this >> domainVersion; + _domainVersionMap[domainName] = domainVersion; + } + } } if ( !isBinary() ) { @@ -746,6 +777,13 @@ InputStream::ReadType InputStream::start( InputIterator* inIterator ) std::string osgName, osgVersion; *this >> PROPERTY("#Version") >> version; *this >> PROPERTY("#Generator") >> osgName >> osgVersion; + + while ( matchString("#CustomDomain") ) + { + std::string domainName; *this >> domainName; + int domainVersion; *this >> domainVersion; + _domainVersionMap[domainName] = domainVersion; + } } // Record file version for back-compatibility checking of wrappers diff --git a/src/osgDB/ObjectWrapper.cpp b/src/osgDB/ObjectWrapper.cpp index d6d694ab3..356f9511e 100644 --- a/src/osgDB/ObjectWrapper.cpp +++ b/src/osgDB/ObjectWrapper.cpp @@ -92,6 +92,14 @@ ObjectWrapper::ObjectWrapper( osg::Object* proto, const std::string& name, split( associates, _associates ); } +ObjectWrapper::ObjectWrapper( osg::Object* proto, const std::string& domain, const std::string& name, + const std::string& associates ) +: osg::Referenced(), + _proto(proto), _domain(domain), _name(name), _version(0) +{ + split( associates, _associates ); +} + void ObjectWrapper::addSerializer( BaseSerializer* s, BaseSerializer::Type t ) { s->_firstVersion = _version; @@ -143,12 +151,13 @@ BaseSerializer* ObjectWrapper::getSerializer( const std::string& name ) bool ObjectWrapper::read( InputStream& is, osg::Object& obj ) { bool readOK = true; + int inputVersion = is.getFileVersion(_domain); for ( SerializerList::iterator itr=_serializers.begin(); itr!=_serializers.end(); ++itr ) { BaseSerializer* serializer = itr->get(); - if ( serializer->_firstVersion <= is.getFileVersion() && - is.getFileVersion() <= serializer->_lastVersion) + if ( serializer->_firstVersion <= inputVersion && + inputVersion <= serializer->_lastVersion ) { if ( !serializer->read(is, obj) ) { @@ -176,12 +185,13 @@ bool ObjectWrapper::read( InputStream& is, osg::Object& obj ) bool ObjectWrapper::write( OutputStream& os, const osg::Object& obj ) { bool writeOK = true; + int outputVersion = os.getFileVersion(_domain); for ( SerializerList::iterator itr=_serializers.begin(); itr!=_serializers.end(); ++itr ) { BaseSerializer* serializer = itr->get(); - if ( serializer->_firstVersion <= OPENSCENEGRAPH_SOVERSION && - OPENSCENEGRAPH_SOVERSION <= serializer->_lastVersion) + if ( serializer->_firstVersion <= outputVersion && + outputVersion <= serializer->_lastVersion ) { if ( !serializer->write(os, obj) ) { @@ -280,6 +290,31 @@ RegisterWrapperProxy::~RegisterWrapperProxy() } } +//////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// RegisterCustomWrapperProxy +// +RegisterCustomWrapperProxy::RegisterCustomWrapperProxy( + osg::Object* proto, const std::string& domain, const std::string& name, + const std::string& associates, AddPropFunc func ) +{ + _wrapper = new ObjectWrapper( proto, domain, name, associates ); + if ( func ) (*func)( domain.c_str(), _wrapper.get() ); + + if (Registry::instance()) + { + Registry::instance()->getObjectWrapperManager()->addWrapper( _wrapper.get() ); + } +} + +RegisterCustomWrapperProxy::~RegisterCustomWrapperProxy() +{ + if (Registry::instance()) + { + Registry::instance()->getObjectWrapperManager()->removeWrapper( _wrapper.get() ); + } +} + //////////////////////////////////////////////////////////////////////////////////////////////////////// // // ObjectWrapperManager diff --git a/src/osgDB/OutputStream.cpp b/src/osgDB/OutputStream.cpp index b7b2ab004..e5afe5613 100644 --- a/src/osgDB/OutputStream.cpp +++ b/src/osgDB/OutputStream.cpp @@ -23,7 +23,7 @@ using namespace osgDB; OutputStream::OutputStream( const osgDB::Options* options ) -: _writeImageHint(WRITE_USE_IMAGE_HINT), _useSchemaData(false) +: _writeImageHint(WRITE_USE_IMAGE_HINT), _useSchemaData(false), _useRobustBinaryFormat(true) { BEGIN_BRACKET.set( "{", +INDENT_VALUE ); END_BRACKET.set( "}", -INDENT_VALUE ); @@ -31,6 +31,8 @@ OutputStream::OutputStream( const osgDB::Options* options ) if ( !options ) return; _options = options; + if ( options->getPluginStringData("RobustBinaryFormat")=="false" ) + _useRobustBinaryFormat = false; if ( options->getPluginStringData("SchemaData")=="true" ) _useSchemaData = true; if ( !options->getPluginStringData("SchemaFile").empty() ) @@ -45,12 +47,31 @@ OutputStream::OutputStream( const osgDB::Options* options ) else if ( hintString=="UseExternal" ) _writeImageHint = WRITE_USE_EXTERNAL; else if ( hintString=="WriteOut" ) _writeImageHint = WRITE_EXTERNAL_FILE; } + + if ( !options->getPluginStringData("CustomDomains").empty() ) + { + StringList domains, keyAndValue; + split( options->getPluginStringData("CustomDomains"), domains, ';' ); + for ( unsigned int i=0; i1 ) + _domainVersionMap[keyAndValue.front()] = atoi(keyAndValue.back().c_str()); + } + } } OutputStream::~OutputStream() { } +int OutputStream::getFileVersion( const std::string& d ) const +{ + if ( d.empty() ) return OPENSCENEGRAPH_SOVERSION; + std::map::const_iterator itr = _domainVersionMap.find(d); + return itr==_domainVersionMap.end() ? 0 : itr->second; +} + OutputStream& OutputStream::operator<<( const osg::Vec2b& v ) { *this << v.x() << v.y(); return *this; } @@ -483,7 +504,6 @@ void OutputStream::writeObjectFields( const osg::Object* obj ) << name << std::endl; return; } - _fields.push_back( name ); const StringList& associates = wrapper->getAssociates(); for ( StringList::const_iterator itr=associates.begin(); itr!=associates.end(); ++itr ) @@ -523,8 +543,6 @@ void OutputStream::writeObjectFields( const osg::Object* obj ) _fields.pop_back(); } - _fields.pop_back(); - } void OutputStream::start( OutputIterator* outIterator, OutputStream::WriteType type ) @@ -544,13 +562,36 @@ void OutputStream::start( OutputIterator* outIterator, OutputStream::WriteType t bool useCompressSource = false; unsigned int attributes = 0; + // From SOVERSION 98, start to support custom wrapper domains, enabling the attribute bit + if ( _domainVersionMap.size()>0 ) attributes |= 0x1; + if ( _useSchemaData ) { attributes |= 0x2; // Record if we use inbuilt schema data or not useCompressSource = true; } + + // From SOVERSION 98, start to support binary begin/end brackets so we can easily ignore + // errors and unsupport classes, enabling the attribute bit + if ( _useRobustBinaryFormat ) + { + outIterator->setSupportBinaryBrackets( true ); + attributes |= 0x4; + } *this << attributes; + // Record all custom versions + if ( _domainVersionMap.size()>0 ) + { + unsigned int numDomains = _domainVersionMap.size(); + *this << numDomains; + for ( std::map::iterator itr=_domainVersionMap.begin(); + itr!=_domainVersionMap.end(); ++itr ) + { + *this << itr->first << itr->second; + } + } + if ( !_compressorName.empty() ) { BaseCompressor* compressor = Registry::instance()->getObjectWrapperManager()->findCompressor(_compressorName); @@ -590,6 +631,14 @@ void OutputStream::start( OutputIterator* outIterator, OutputStream::WriteType t *this << PROPERTY("#Version") << (unsigned int)OPENSCENEGRAPH_SOVERSION << std::endl; *this << PROPERTY("#Generator") << std::string("OpenSceneGraph") << std::string(osgGetVersion()) << std::endl; + if ( _domainVersionMap.size()>0 ) + { + for ( std::map::iterator itr=_domainVersionMap.begin(); + itr!=_domainVersionMap.end(); ++itr ) + { + *this << PROPERTY("#CustomDomain") << itr->first << itr->second << std::endl; + } + } *this << std::endl; } _fields.pop_back(); diff --git a/src/osgPlugins/osg/BinaryStreamOperator.h b/src/osgPlugins/osg/BinaryStreamOperator.h index 329ec579a..5f3c3ce1e 100644 --- a/src/osgPlugins/osg/BinaryStreamOperator.h +++ b/src/osgPlugins/osg/BinaryStreamOperator.h @@ -2,6 +2,7 @@ #define OSG2_BINARYSTREAMOPERATOR #include +#include #if defined(_MSC_VER) typedef unsigned __int32 uint32_t; @@ -77,13 +78,37 @@ public: virtual void writeProperty( const osgDB::ObjectProperty& prop ) { if (prop._mapProperty) _out->write((char*)&(prop._value), osgDB::INT_SIZE); } - virtual void writeMark( const osgDB::ObjectMark& mark ) {} + virtual void writeMark( const osgDB::ObjectMark& mark ) + { + if ( _supportBinaryBrackets ) + { + if ( mark._name=="{" ) + { + int size = 0; + _beginPositions.push_back( _out->tellp() ); + _out->write( (char*)&size, osgDB::INT_SIZE ); + } + else if ( mark._name=="}" && _beginPositions.size()>0 ) + { + int pos = _out->tellp(), beginPos = _beginPositions.back(); + _beginPositions.pop_back(); + _out->seekp( beginPos, std::ios_base::beg ); + + int size = pos - beginPos; + _out->write( (char*)&size, osgDB::INT_SIZE ); + _out->seekp( pos, std::ios_base::beg ); + } + } + } virtual void writeCharArray( const char* s, unsigned int size ) { if ( size>0 ) _out->write( s, size ); } virtual void writeWrappedString( const std::string& str ) { writeString( str ); } + +protected: + std::vector _beginPositions; }; class BinaryInputIterator : public osgDB::InputIterator @@ -206,15 +231,47 @@ public: prop.set( value ); } - virtual void readMark( osgDB::ObjectMark& mark ) {} + virtual void readMark( osgDB::ObjectMark& mark ) + { + if ( _supportBinaryBrackets ) + { + if ( mark._name=="{" ) + { + int size = 0; + _beginPositions.push_back( _in->tellg() ); + + _in->read( (char*)&size, osgDB::INT_SIZE ); + if ( _byteSwap ) osg::swapBytes( (char*)&size, osgDB::INT_SIZE ); + _blockSizes.push_back( size ); + } + else if ( mark._name=="}" && _beginPositions.size()>0 ) + { + _beginPositions.pop_back(); + _blockSizes.pop_back(); + } + } + } virtual void readCharArray( char* s, unsigned int size ) { if ( size>0 ) _in->read( s, size ); } virtual void readWrappedString( std::string& str ) { readString( str ); } + + virtual void advanceToCurrentEndBracket() + { + if ( _supportBinaryBrackets && _beginPositions.size()>0 ) + { + int pos = _beginPositions.back() + _blockSizes.back(); + _in->seekg( pos, std::ios_base::beg ); + _beginPositions.pop_back(); + _blockSizes.pop_back(); + } + } protected: + std::vector _beginPositions; + std::vector _blockSizes; }; #endif