From Wang Rui, "The file attached includes two new features for the serialization IO functionality. First, custom serializer version control should work now, just by defining a new REGISTER_CUSTOM_OBJECT_WRAPPER macro. For example:

// A custom class
namespace CustomDomain {

class MyGroup : public osg::Group
{
public:
    META_Node( CustomDomain, MyGroup );

    void setMyName( const std::string& n );
    const std::string& getMyName() const;

    void setMyID( int id );
    int getMyID() const;

    ...
};

}

// The serialization wrapper using a custom domain name
REGISTER_CUSTOM_OBJECT_WRAPPER( MyDomain,
                                CustomDomain_MyGroup,
                                new CustomDomain::MyGroup,
                                CustomDomain::MyGroup,
                                "osg::Object osg::Node osg::Group CustomDomain::MyGroup" )
{
    ADD_STRING_SERIALIZER( MyName, std::string() );
    {
        UPDATE_TO_VERSION_SCOPED( 1 );  // Updated for a new domain version
        ADD_INT_SERIALIZER( MyID, 0 );
    }
}

Save the class instance as follows:
osgDB::writeNodeFile( *myGroup, "serializer_test.osgt", new osgDB::Options("CustomDomains=MyDomain:1") );

The output file will include the domain version definition and all the class data, and can be read back. We can also force setting the domain version by the CustomDomains option while reading the saved files. If we save the class instance without any options, MyID will be ignored because the default domain version is 0.

This may help third-party libraries like osgEarth to maintain their own serializers without regarding to the OSG soversion changes.

Another feature added is a more robust binary format, which in fact adds a size-offset at each block's beginning. When there are problems or unsupported data types while reading, we can now directly jump to the block end indicated by the offset value. So a .osgb file will automatically ignore bad data and read remains as normal (at present it will fail at all). This feature will not break the backward compatibility, and can be disabled by setting "RobustBinaryFormat=false" while writing out.

Hope these changes can work smoothly with present and future community projects. Maybe we should also consider have an osgserializer example to test and demonstrate all things we can do now."

include/osgDB/InputStream

@@ -75,8 +75,10 @@ public:
     InputStream( const osgDB::Options* options );
     virtual ~InputStream();
 
+    void setFileVersion( const std::string& d, int v ) { _domainVersionMap[d] = v; }
+    int getFileVersion( const std::string& d=std::string() ) const;
+
     bool isBinary() const { return _in->isBinary(); }
-    int getFileVersion() const { return _fileVersion; }
     const osgDB::Options* getOptions() const { return _options.get(); }
 
     // Serialization related functions
@@ -184,6 +186,7 @@ protected:
     ArrayMap _arrayMap;
     IdentifierMap _identifierMap;
 
+    std::map<std::string, int> _domainVersionMap;
     int _fileVersion;
     bool _useSchemaData;
     bool _forceReadingImage;

include/osgDB/ObjectWrapper

@@ -50,6 +50,8 @@ public:
 
     ObjectWrapper( osg::Object* proto, const std::string& name,
                    const std::string& associates );
+    ObjectWrapper( osg::Object* proto, const std::string& domain, const std::string& name,
+                   const std::string& associates );
 
     void setUpdatedVersion( int ver ) { _version = ver; }
     int getUpdatedVersion() const { return _version; }
@@ -76,6 +78,7 @@ protected:
     virtual ~ObjectWrapper() {}
 
     osg::ref_ptr<osg::Object> _proto;
+    std::string _domain;
     std::string _name;
     StringList _associates;
     SerializerList _serializers;
@@ -165,6 +168,20 @@ protected:
     osg::ref_ptr<ObjectWrapper> _wrapper;
 };
 
+class OSGDB_EXPORT RegisterCustomWrapperProxy
+{
+public:
+    typedef void (*AddPropFunc)( const char*, ObjectWrapper* );
+
+    RegisterCustomWrapperProxy( osg::Object* proto, const std::string& domain, const std::string& name,
+                                const std::string& associates, AddPropFunc func );
+
+    virtual ~RegisterCustomWrapperProxy();
+
+protected:
+    osg::ref_ptr<ObjectWrapper> _wrapper;
+};
+
 #define REGISTER_OBJECT_WRAPPER(NAME, PROTO, CLASS, ASSOCIATES) \
     extern "C" void wrapper_serializer_##NAME(void) {} \
     extern void wrapper_propfunc_##NAME(osgDB::ObjectWrapper*); \
@@ -181,6 +198,22 @@ protected:
     typedef CLASS MyClass; \
     void wrapper_propfunc_##NAME(osgDB::ObjectWrapper* wrapper)
 
+#define REGISTER_CUSTOM_OBJECT_WRAPPER(DOMAIN, NAME, PROTO, CLASS, ASSOCIATES) \
+    extern "C" void wrapper_serializer_##NAME(void) {} \
+    extern void wrapper_propfunc_##NAME(const char*, osgDB::ObjectWrapper*); \
+    static osgDB::RegisterCustomWrapperProxy wrapper_proxy_##NAME( \
+        PROTO, #DOMAIN, #CLASS, ASSOCIATES, &wrapper_propfunc_##NAME); \
+    typedef CLASS MyClass; \
+    void wrapper_propfunc_##NAME(const char* domain, osgDB::ObjectWrapper* wrapper)
+
+#define REGISTER_CUSTOM_OBJECT_WRAPPER2(DOMAIN, NAME, PROTO, CLASS, CLASSNAME, ASSOCIATES) \
+    extern "C" void wrapper_serializer_##NAME(void) {} \
+    extern void wrapper_propfunc_##NAME(const char*, osgDB::ObjectWrapper*); \
+    static osgDB::RegisterCustomWrapperProxy wrapper_proxy_##NAME( \
+        PROTO, #DOMAIN, CLASSNAME, ASSOCIATES, &wrapper_propfunc_##NAME); \
+    typedef CLASS MyClass; \
+    void wrapper_propfunc_##NAME(const char* domain, osgDB::ObjectWrapper* wrapper)
+
 class OSGDB_EXPORT RegisterCompressorProxy
 {
 public:

include/osgDB/OutputStream

@@ -76,6 +76,9 @@ public:
     OutputStream( const osgDB::Options* options );
     virtual ~OutputStream();
 
+    void setFileVersion( const std::string& d, int v ) { _domainVersionMap[d] = v; }
+    int getFileVersion( const std::string& d=std::string() ) const;
+
     bool isBinary() const { return _out->isBinary(); }
     const std::string& getSchemaName() const { return _schemaName; }
     const osgDB::Options* getOptions() const { return _options.get(); }
@@ -185,8 +188,9 @@ protected:
     ArrayMap _arrayMap;
     ObjectMap _objectMap;
 
+    std::map<std::string, int> _domainVersionMap;
     WriteImageHint _writeImageHint;
-    bool _useSchemaData;
+    bool _useSchemaData, _useRobustBinaryFormat;
     std::map<std::string, std::string> _inbuiltSchemaMap;
     std::vector<std::string> _fields;
     std::string _schemaName;

include/osgDB/StreamOperator

@@ -16,13 +16,16 @@ class InputStream;
 class OSGDB_EXPORT OutputIterator : public osg::Referenced
 {
 public:
-    OutputIterator() : _out(0) {}
+    OutputIterator() : _out(0), _supportBinaryBrackets(false) {}
     virtual ~OutputIterator() {}
 
     void setStream( std::ostream* ostream ) { _out = ostream; }
     std::ostream* getStream() { return _out; }
     const std::ostream* getStream() const { return _out; }
 
+    void setSupportBinaryBrackets( bool b ) { _supportBinaryBrackets = b; }
+    bool getSupportBinaryBrackets() const { return _supportBinaryBrackets; }
+
     virtual bool isBinary() const = 0;
 
     virtual void writeBool( bool b ) = 0;
@@ -64,12 +67,13 @@ protected:
     }
 
     std::ostream* _out;
+    bool _supportBinaryBrackets;
 };
 
 class OSGDB_EXPORT InputIterator : public osg::Referenced
 {
 public:
-    InputIterator() : _in(0), _inputStream(0), _byteSwap(0), _failed(false) {}
+    InputIterator() : _in(0), _inputStream(0), _byteSwap(0), _supportBinaryBrackets(false), _failed(false) {}
     virtual ~InputIterator() {}
 
     void setStream( std::istream* istream ) { _in = istream; }
@@ -83,6 +87,9 @@ public:
     void setByteSwap(int byteSwap) { _byteSwap = byteSwap; }
     int getByteSwap() const { return _byteSwap; }
 
+    void setSupportBinaryBrackets( bool b ) { _supportBinaryBrackets = b; }
+    bool getSupportBinaryBrackets() const { return _supportBinaryBrackets; }
+
     void checkStream() const { if (_in->rdstate()&_in->failbit) _failed = true; }
     bool isFailed() const { return _failed; }
 
@@ -121,6 +128,7 @@ protected:
     std::istream*       _in;
     osgDB::InputStream* _inputStream;
     int                 _byteSwap;
+    bool                _supportBinaryBrackets;
     mutable bool        _failed;
 };
 

src/osgDB/InputStream.cpp

@@ -32,9 +32,22 @@ InputStream::InputStream( const osgDB::Options* options )
     if ( !options ) return;
     _options = options;
 
-    std::string schema;
     if ( options->getPluginStringData("ForceReadingImage")=="true" )
         _forceReadingImage = true;
+
+    if ( !options->getPluginStringData("CustomDomains").empty() )
+    {
+        StringList domains, keyAndValue;
+        split( options->getPluginStringData("CustomDomains"), domains, ';' );
+        for ( unsigned int i=0; i<domains.size(); ++i )
+        {
+            split( domains[i], keyAndValue, ':' );
+            if ( keyAndValue.size()>1 )
+                _domainVersionMap[keyAndValue.front()] = atoi(keyAndValue.back().c_str());
+        }
+    }
+
+    std::string schema;
     if ( !options->getPluginStringData("SchemaFile").empty() )
     {
         schema = options->getPluginStringData("SchemaFile");
@@ -60,6 +73,13 @@ InputStream::~InputStream()
         delete _dataDecompress;
 }
 
+int InputStream::getFileVersion( const std::string& d ) const
+{
+    if ( d.empty() ) return _fileVersion;
+    std::map<std::string, int>::const_iterator itr = _domainVersionMap.find(d);
+    return itr==_domainVersionMap.end() ? 0 : itr->second;
+}
+
 InputStream& InputStream::operator>>( osg::Vec2b& v )
 {
     char x, y; *this >> x >> y;
@@ -668,7 +688,6 @@ osg::Object* InputStream::readObjectFields( const std::string& className, unsign
                                << className << std::endl;
         return NULL;
     }
-    _fields.push_back( className );
 
     osg::ref_ptr<osg::Object> obj = existingObj ? existingObj : wrapper->getProto()->cloneType();
     _identifierMap[id] = obj;
@@ -691,7 +710,6 @@ osg::Object* InputStream::readObjectFields( const std::string& className, unsign
             _fields.pop_back();
         }
     }
-    _fields.pop_back();
     return obj.release();
 }
 
@@ -734,7 +752,20 @@ InputStream::ReadType InputStream::start( InputIterator* inIterator )
         type = static_cast<ReadType>(typeValue);
 
         unsigned int attributes; *this >> attributes;
+        if ( attributes&0x4 ) inIterator->setSupportBinaryBrackets( true );
         if ( attributes&0x2 ) _useSchemaData = true;
+
+        // Record custom domains
+        if ( attributes&0x1 )
+        {
+            unsigned int numDomains; *this >> numDomains;
+            for ( unsigned int i=0; i<numDomains; ++i )
+            {
+                std::string domainName; *this >> domainName;
+                int domainVersion; *this >> domainVersion;
+                _domainVersionMap[domainName] = domainVersion;
+            }
+        }
     }
     if ( !isBinary() )
     {
@@ -746,6 +777,13 @@ InputStream::ReadType InputStream::start( InputIterator* inIterator )
         std::string osgName, osgVersion;
         *this >> PROPERTY("#Version") >> version;
         *this >> PROPERTY("#Generator") >> osgName >> osgVersion;
+
+        while ( matchString("#CustomDomain") )
+        {
+            std::string domainName; *this >> domainName;
+            int domainVersion; *this >> domainVersion;
+            _domainVersionMap[domainName] = domainVersion;
+        }
     }
 
     // Record file version for back-compatibility checking of wrappers

src/osgDB/ObjectWrapper.cpp

@@ -92,6 +92,14 @@ ObjectWrapper::ObjectWrapper( osg::Object* proto, const std::string& name,
     split( associates, _associates );
 }
 
+ObjectWrapper::ObjectWrapper( osg::Object* proto, const std::string& domain, const std::string& name,
+                              const std::string& associates )
+:   osg::Referenced(),
+    _proto(proto), _domain(domain), _name(name), _version(0)
+{
+    split( associates, _associates );
+}
+
 void ObjectWrapper::addSerializer( BaseSerializer* s, BaseSerializer::Type t )
 {
     s->_firstVersion = _version;
@@ -143,12 +151,13 @@ BaseSerializer* ObjectWrapper::getSerializer( const std::string& name )
 bool ObjectWrapper::read( InputStream& is, osg::Object& obj )
 {
     bool readOK = true;
+    int inputVersion = is.getFileVersion(_domain);
     for ( SerializerList::iterator itr=_serializers.begin();
           itr!=_serializers.end(); ++itr )
     {
         BaseSerializer* serializer = itr->get();
-        if ( serializer->_firstVersion <= is.getFileVersion() &&
+        if ( serializer->_firstVersion <= inputVersion &&
-             is.getFileVersion() <= serializer->_lastVersion)
+             inputVersion <= serializer->_lastVersion )
         {
             if ( !serializer->read(is, obj) )
             {
@@ -176,12 +185,13 @@ bool ObjectWrapper::read( InputStream& is, osg::Object& obj )
 bool ObjectWrapper::write( OutputStream& os, const osg::Object& obj )
 {
     bool writeOK = true;
+    int outputVersion = os.getFileVersion(_domain);
     for ( SerializerList::iterator itr=_serializers.begin();
           itr!=_serializers.end(); ++itr )
     {
         BaseSerializer* serializer = itr->get();
-        if ( serializer->_firstVersion <= OPENSCENEGRAPH_SOVERSION &&
+        if ( serializer->_firstVersion <= outputVersion &&
-             OPENSCENEGRAPH_SOVERSION <= serializer->_lastVersion)
+             outputVersion <= serializer->_lastVersion )
         {
             if ( !serializer->write(os, obj) )
             {
@@ -280,6 +290,31 @@ RegisterWrapperProxy::~RegisterWrapperProxy()
     }
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// RegisterCustomWrapperProxy
+//
+RegisterCustomWrapperProxy::RegisterCustomWrapperProxy(
+        osg::Object* proto, const std::string& domain, const std::string& name,
+        const std::string& associates, AddPropFunc func )
+{
+    _wrapper = new ObjectWrapper( proto, domain, name, associates );
+    if ( func ) (*func)( domain.c_str(), _wrapper.get() );
+
+    if (Registry::instance())
+    {
+        Registry::instance()->getObjectWrapperManager()->addWrapper( _wrapper.get() );
+    }
+}
+
+RegisterCustomWrapperProxy::~RegisterCustomWrapperProxy()
+{
+    if (Registry::instance())
+    {
+        Registry::instance()->getObjectWrapperManager()->removeWrapper( _wrapper.get() );
+    }
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////
 //
 // ObjectWrapperManager

src/osgDB/OutputStream.cpp

@@ -23,7 +23,7 @@
 using namespace osgDB;
 
 OutputStream::OutputStream( const osgDB::Options* options )
-:   _writeImageHint(WRITE_USE_IMAGE_HINT), _useSchemaData(false)
+:   _writeImageHint(WRITE_USE_IMAGE_HINT), _useSchemaData(false), _useRobustBinaryFormat(true)
 {
     BEGIN_BRACKET.set( "{", +INDENT_VALUE );
     END_BRACKET.set( "}", -INDENT_VALUE );
@@ -31,6 +31,8 @@ OutputStream::OutputStream( const osgDB::Options* options )
     if ( !options ) return;
     _options = options;
 
+    if ( options->getPluginStringData("RobustBinaryFormat")=="false" )
+        _useRobustBinaryFormat = false;
     if ( options->getPluginStringData("SchemaData")=="true" )
         _useSchemaData = true;
     if ( !options->getPluginStringData("SchemaFile").empty() )
@@ -45,12 +47,31 @@ OutputStream::OutputStream( const osgDB::Options* options )
         else if ( hintString=="UseExternal" ) _writeImageHint = WRITE_USE_EXTERNAL;
         else if ( hintString=="WriteOut" ) _writeImageHint = WRITE_EXTERNAL_FILE;
     }
+
+    if ( !options->getPluginStringData("CustomDomains").empty() )
+    {
+        StringList domains, keyAndValue;
+        split( options->getPluginStringData("CustomDomains"), domains, ';' );
+        for ( unsigned int i=0; i<domains.size(); ++i )
+        {
+            split( domains[i], keyAndValue, ':' );
+            if ( keyAndValue.size()>1 )
+                _domainVersionMap[keyAndValue.front()] = atoi(keyAndValue.back().c_str());
+        }
+    }
 }
 
 OutputStream::~OutputStream()
 {
 }
 
+int OutputStream::getFileVersion( const std::string& d ) const
+{
+    if ( d.empty() ) return OPENSCENEGRAPH_SOVERSION;
+    std::map<std::string, int>::const_iterator itr = _domainVersionMap.find(d);
+    return itr==_domainVersionMap.end() ? 0 : itr->second;
+}
+
 OutputStream& OutputStream::operator<<( const osg::Vec2b& v )
 { *this << v.x() << v.y(); return *this; }
 
@@ -483,7 +504,6 @@ void OutputStream::writeObjectFields( const osg::Object* obj )
                                 << name << std::endl;
         return;
     }
-    _fields.push_back( name );
 
     const StringList& associates = wrapper->getAssociates();
     for ( StringList::const_iterator itr=associates.begin(); itr!=associates.end(); ++itr )
@@ -523,8 +543,6 @@ void OutputStream::writeObjectFields( const osg::Object* obj )
 
         _fields.pop_back();
     }
-    _fields.pop_back();
-
 }
 
 void OutputStream::start( OutputIterator* outIterator, OutputStream::WriteType type )
@@ -544,13 +562,36 @@ void OutputStream::start( OutputIterator* outIterator, OutputStream::WriteType t
         bool useCompressSource = false;
         unsigned int attributes = 0;
 
+        // From SOVERSION 98, start to support custom wrapper domains, enabling the attribute bit
+        if ( _domainVersionMap.size()>0 ) attributes |= 0x1; 
+
         if ( _useSchemaData )
         {
             attributes |= 0x2;  // Record if we use inbuilt schema data or not
             useCompressSource = true;
         }
+        
+        // From SOVERSION 98, start to support binary begin/end brackets so we can easily ignore
+        // errors and unsupport classes, enabling the attribute bit
+        if ( _useRobustBinaryFormat )
+        {
+            outIterator->setSupportBinaryBrackets( true );
+            attributes |= 0x4;
+        }
         *this << attributes;
 
+        // Record all custom versions
+        if ( _domainVersionMap.size()>0 )
+        {
+            unsigned int numDomains = _domainVersionMap.size();
+            *this << numDomains;
+            for ( std::map<std::string, int>::iterator itr=_domainVersionMap.begin();
+                  itr!=_domainVersionMap.end(); ++itr )
+            {
+                *this << itr->first << itr->second;
+            }
+        }
+
         if ( !_compressorName.empty() )
         {
             BaseCompressor* compressor = Registry::instance()->getObjectWrapperManager()->findCompressor(_compressorName);
@@ -590,6 +631,14 @@ void OutputStream::start( OutputIterator* outIterator, OutputStream::WriteType t
         *this << PROPERTY("#Version") << (unsigned int)OPENSCENEGRAPH_SOVERSION << std::endl;
         *this << PROPERTY("#Generator") << std::string("OpenSceneGraph")
               << std::string(osgGetVersion()) << std::endl;
+        if ( _domainVersionMap.size()>0 )
+        {
+            for ( std::map<std::string, int>::iterator itr=_domainVersionMap.begin();
+                  itr!=_domainVersionMap.end(); ++itr )
+            {
+                *this << PROPERTY("#CustomDomain") << itr->first << itr->second << std::endl;
+            }
+        }
         *this << std::endl;
     }
     _fields.pop_back();

src/osgPlugins/osg/BinaryStreamOperator.h

@@ -2,6 +2,7 @@
 #define OSG2_BINARYSTREAMOPERATOR
 
 #include <osgDB/StreamOperator>
+#include <vector>
 
 #if defined(_MSC_VER)
 typedef unsigned __int32 uint32_t;
@@ -77,13 +78,37 @@ public:
     virtual void writeProperty( const osgDB::ObjectProperty& prop )
     { if (prop._mapProperty) _out->write((char*)&(prop._value), osgDB::INT_SIZE); }
 
-    virtual void writeMark( const osgDB::ObjectMark& mark ) {}
+    virtual void writeMark( const osgDB::ObjectMark& mark )
+    {
+        if ( _supportBinaryBrackets )
+        {
+            if ( mark._name=="{" )
+            {
+                int size = 0;
+                _beginPositions.push_back( _out->tellp() );
+                _out->write( (char*)&size, osgDB::INT_SIZE );
+            }
+            else if ( mark._name=="}" && _beginPositions.size()>0 )
+            {
+                int pos = _out->tellp(), beginPos = _beginPositions.back();
+                _beginPositions.pop_back();
+                _out->seekp( beginPos, std::ios_base::beg );
+
+                int size = pos - beginPos;
+                _out->write( (char*)&size, osgDB::INT_SIZE );
+                _out->seekp( pos, std::ios_base::beg );
+            }
+        }
+    }
 
     virtual void writeCharArray( const char* s, unsigned int size )
     { if ( size>0 ) _out->write( s, size ); }
 
     virtual void writeWrappedString( const std::string& str )
     { writeString( str ); }
+    
+protected:
+    std::vector<int> _beginPositions;
 };
 
 class BinaryInputIterator : public osgDB::InputIterator
@@ -206,7 +231,26 @@ public:
         prop.set( value );
     }
 
-    virtual void readMark( osgDB::ObjectMark& mark ) {}
+    virtual void readMark( osgDB::ObjectMark& mark )
+    {
+        if ( _supportBinaryBrackets )
+        {
+            if ( mark._name=="{" )
+            {
+                int size = 0;
+                _beginPositions.push_back( _in->tellg() );
+
+                _in->read( (char*)&size, osgDB::INT_SIZE );
+                if ( _byteSwap ) osg::swapBytes( (char*)&size, osgDB::INT_SIZE );
+                _blockSizes.push_back( size );
+            }
+            else if ( mark._name=="}" && _beginPositions.size()>0 )
+            {
+                _beginPositions.pop_back();
+                _blockSizes.pop_back();
+            }
+        }
+    }
 
     virtual void readCharArray( char* s, unsigned int size )
     { if ( size>0 ) _in->read( s, size ); }
@@ -214,7 +258,20 @@ public:
     virtual void readWrappedString( std::string& str )
     { readString( str ); }
     
+    virtual void advanceToCurrentEndBracket()
+    {
+        if ( _supportBinaryBrackets && _beginPositions.size()>0 )
+        {
+            int pos = _beginPositions.back() + _blockSizes.back();
+            _in->seekg( pos, std::ios_base::beg );
+            _beginPositions.pop_back();
+            _blockSizes.pop_back();
+        }
+    }
+
 protected:
+    std::vector<int> _beginPositions;
+    std::vector<int> _blockSizes;
 };
 
 #endif