From Ulrich Hertlein, "attached is a stencil buffer-based constant-width outline f/x with example. I've also modified osgfxbrowser to setup the stencil buffer accordingly."

examples/CMakeLists.txt

@@ -41,6 +41,7 @@ IF(DYNAMIC_OPENSCENEGRAPH)
     ADD_SUBDIRECTORY(osgfont)
     ADD_SUBDIRECTORY(osgforest)
     ADD_SUBDIRECTORY(osgfxbrowser)
+    ADD_SUBDIRECTORY(osgoutline)
     ADD_SUBDIRECTORY(osggameoflife)
     ADD_SUBDIRECTORY(osggeodemo)
     ADD_SUBDIRECTORY(osggeometry)

examples/osgfxbrowser/osgfxbrowser.cpp

@@ -320,6 +320,12 @@ int main(int argc, char *argv[])
 
     viewer.setThreadingModel(threading);
 
+    // setup stencil buffer for Outline f/x.
+    osg::DisplaySettings::instance()->setMinimumNumStencilBits(1);
+    unsigned int clearMask = viewer.getCamera()->getClearMask();
+    viewer.getCamera()->setClearMask(clearMask | GL_STENCIL_BUFFER_BIT);
+    viewer.getCamera()->setClearStencil(0);
+ 
     // any option left unread are converted into errors to write out later.
     arguments.reportRemainingOptionsAsUnrecognized();
 

examples/osgoutline/CMakeLists.txt

@@ -0,0 +1,6 @@
+#this file is automatically generated 
+
+SET(TARGET_SRC osgoutline.cpp )
+SET(TARGET_ADDED_LIBRARIES osgFX osgGA )
+#### end var setup  ###
+SETUP_EXAMPLE(osgoutline)

examples/osgoutline/osgoutline.cpp

@@ -0,0 +1,46 @@
+// -*-c++-*-
+
+/*
+ * Draw an outline around a model.
+ */
+
+#include <osg/Group>
+
+#include <osgDB/ReadFile>
+#include <osgViewer/Viewer>
+
+#include <osgFX/Outline>
+
+
+int main(int argc, char** argv)
+{
+    osg::ArgumentParser arguments(&argc,argv);
+    arguments.getApplicationUsage()->setCommandLineUsage(arguments.getApplicationName()+" [options] <file>");
+    arguments.getApplicationUsage()->addCommandLineOption("-h or --help","Display this information");
+   
+    // create outline effect
+    osg::ref_ptr<osgFX::Outline> outline = new osgFX::Outline;
+    outline->setWidth(8);
+    outline->setColor(osg::Vec4(1,1,0,1));
+
+    // create scene
+    osg::ref_ptr<osg::Group> root = new osg::Group;
+    root->addChild(outline);
+
+    osg::ref_ptr<osg::Node> model0 = osgDB::readNodeFile(arguments.argc() > 1 ? arguments[1] : "al.obj");
+    outline->addChild(model0);
+
+    // must have stencil buffer...
+    osg::DisplaySettings::instance()->setMinimumNumStencilBits(1);
+
+    // construct the viewer
+    osgViewer::Viewer viewer;
+    viewer.setSceneData(root);
+
+    // must clear stencil buffer...
+    unsigned int clearMask = viewer.getCamera()->getClearMask();
+    viewer.getCamera()->setClearMask(clearMask | GL_STENCIL_BUFFER_BIT);
+    viewer.getCamera()->setClearStencil(0);
+
+    return viewer.run();
+}

include/osgFX/Outline

@@ -0,0 +1,97 @@
+// -*-c++-*-
+
+/*
+ * OpenSceneGraph - Copyright (C) 1998-2009 Robert Osfield
+ *
+ * This library is open source and may be redistributed and/or modified under
+ * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
+ * (at your option) any later version.  The full license is in LICENSE file
+ * included with this distribution, and on the openscenegraph.org website.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * OpenSceneGraph Public License for more details.
+ */
+
+/*
+ * osgFX::Outline - Copyright (C) 2004,2009 Ulrich Hertlein
+ */
+
+#ifndef OSGFX_OUTLINE_
+#define OSGFX_OUTLINE_
+
+#include <osgFX/Export>
+#include <osgFX/Effect>
+
+namespace osgFX
+{
+    /**
+     * Outline effect.
+     * This effect draws a stencil buffer-based outline around an object.
+     * Color and width of the outline can be modified.
+     * To function correctly the context must be setup with a stencil buffer
+     * and the stencil buffer must be cleared to zero before each render.
+     *
+     * osg::DisplaySettings::instance()->setMinimumNumStencilBits(1);
+     * camera->setClearMask(clearMask | GL_STENCIL_BUFFER_BIT);
+     * camera->setClearStencil(0);
+     */
+    class Outline : public Effect
+    {
+    public:
+        /// Constructor.
+        Outline();
+
+        /// Copy constructor.
+        Outline(const Outline& copy, const osg::CopyOp& op = osg::CopyOp::SHALLOW_COPY) : Effect(copy, op) {
+            _width = copy._width;
+            _color = copy._color;
+            _technique = copy._technique;
+        }
+
+        // Effect class info
+        META_Effect(osgFX, Outline, "Outline",
+                    "Stencil buffer based object outline effect.\n"
+                    "This effect needs a properly setup stencil buffer.",
+                    "Ulrich Hertlein <u.hertlein@sandbox.de>");
+
+        /// Set outline width.
+        void setWidth(float w);
+
+        /// Get outline width.
+        float getWidth() const {
+            return _width;
+        }
+
+        /// Set outline color.
+        void setColor(const osg::Vec4& color);
+
+        /// Get outline color.
+        const osg::Vec4& getColor() const {
+            return _color;
+        }
+
+    protected:
+        /// Destructor.
+        virtual ~Outline() {
+        }
+
+        /// Define available techniques.
+        bool define_techniques();
+
+    private:
+        /// Outline width.
+        float _width;
+
+        /// Outline color.
+        osg::Vec4 _color;
+
+        /// Technique.
+        class OutlineTechnique;
+        OutlineTechnique* _technique;
+    };
+
+}
+
+#endif

runexamples.bat

@@ -42,6 +42,9 @@ osgforest
 echo osgfxbrowser
 osgfxbrowser
 
+echo osgoutline cow.osg
+osgoutline cow.osg
+
 echo osggeometry
 osggeometry
 

src/osgFX/CMakeLists.txt

@@ -14,6 +14,7 @@ SET(LIB_PUBLIC_HEADERS
     ${HEADER_PATH}/Effect
     ${HEADER_PATH}/Export
     ${HEADER_PATH}/MultiTextureControl
+    ${HEADER_PATH}/Outline
     ${HEADER_PATH}/Registry
     ${HEADER_PATH}/Scribe
     ${HEADER_PATH}/SpecularHighlights
@@ -31,6 +32,7 @@ ADD_LIBRARY(${LIB_NAME}
     Cartoon.cpp
     Effect.cpp
     MultiTextureControl.cpp
+    Outline.cpp
     Registry.cpp
     Scribe.cpp
     SpecularHighlights.cpp

src/osgFX/Outline.cpp

@@ -0,0 +1,196 @@
+// -*-c++-*-
+
+/*
+ * OpenSceneGraph - Copyright (C) 1998-2009 Robert Osfield
+ *
+ * This library is open source and may be redistributed and/or modified under
+ * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
+ * (at your option) any later version.  The full license is in LICENSE file
+ * included with this distribution, and on the openscenegraph.org website.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * OpenSceneGraph Public License for more details.
+ */
+
+/*
+ * osgFX::Outline - Copyright (C) 2004,2009 Ulrich Hertlein
+ */
+
+#include <osgFX/Outline>
+#include <osgFX/Registry>
+
+#include <osg/Group>
+#include <osg/Stencil>
+#include <osg/CullFace>
+#include <osg/PolygonMode>
+#include <osg/LineWidth>
+#include <osg/Material>
+
+
+namespace {
+    const unsigned int Override_On = osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE;
+    const unsigned int Override_Off = osg::StateAttribute::OFF|osg::StateAttribute::OVERRIDE;
+}
+
+
+namespace osgFX
+{
+    /// Register prototype.
+    Registry::Proxy proxy(new Outline);
+
+    /**
+     * Outline technique.
+     */
+    class Outline::OutlineTechnique : public Technique
+    {
+    public:
+        /// Constructor.
+        OutlineTechnique() : Technique(),
+            _lineWidth(), _width(2),
+            _material(), _color(1,1,1,1) {
+        }
+
+        /// Validate.
+        bool validate(osg::State&) const {
+            return true;
+        }
+
+        /// Set outline width.
+        void setWidth(float w) {
+            _width = w;
+            if (_lineWidth.valid()) {
+                _lineWidth->setWidth(w);
+            }
+        }
+
+        /// Set outline color.
+        void setColor(const osg::Vec4& color) {
+            _color = color;
+            if (_material.valid()) {
+                const osg::Material::Face face = osg::Material::FRONT_AND_BACK;
+                _material->setAmbient(face, color);
+                _material->setDiffuse(face, color);
+                _material->setEmission(face, color);
+            }
+        }
+
+    protected:
+        /// Define render passes.
+        void define_passes() {
+
+            /*
+             * draw
+             * - set stencil buffer to ref=1 where draw occurs
+             * - clear stencil buffer to 0 where test fails
+             */
+            {
+                osg::StateSet* state = new osg::StateSet;
+
+                // stencil op
+                osg::Stencil* stencil  = new osg::Stencil;
+                stencil->setFunction(osg::Stencil::ALWAYS, 1, ~0);
+                stencil->setOperation(osg::Stencil::KEEP,
+                                      osg::Stencil::KEEP,
+                                      osg::Stencil::REPLACE);
+                state->setAttributeAndModes(stencil, Override_On);
+
+                addPass(state);
+            }
+
+            /*
+             * post-draw
+             * - only draw where draw didn't set the stencil buffer
+             * - draw only back-facing polygons
+             * - draw back-facing polys as lines
+             * - disable depth-test, lighting & texture
+             */
+            {
+                osg::StateSet* state = new osg::StateSet;
+
+                // stencil op
+                osg::Stencil* stencil  = new osg::Stencil;
+                stencil->setFunction(osg::Stencil::NOTEQUAL, 1, ~0);
+                stencil->setOperation(osg::Stencil::KEEP,
+                                      osg::Stencil::KEEP,
+                                      osg::Stencil::REPLACE);
+                state->setAttributeAndModes(stencil, Override_On);
+
+                // cull front-facing polys
+                osg::CullFace* cf = new osg::CullFace;
+                cf->setMode(osg::CullFace::FRONT);
+                state->setAttributeAndModes(cf, Override_On);
+
+                // draw back-facing polygon lines
+                osg::PolygonMode* pm = new osg::PolygonMode;
+                pm->setMode(osg::PolygonMode::BACK, osg::PolygonMode::LINE);
+                state->setAttributeAndModes(pm, Override_On);
+
+                // outline width
+                _lineWidth = new osg::LineWidth;
+                setWidth(_width);
+                state->setAttributeAndModes(_lineWidth, Override_On);
+
+                // outline color/material
+                _material = new osg::Material;
+                _material->setColorMode(osg::Material::AMBIENT_AND_DIFFUSE);
+                setColor(_color);
+                state->setAttributeAndModes(_material, Override_On);
+
+                // disable modes
+                state->setMode(GL_BLEND, Override_Off);
+                state->setMode(GL_DEPTH_TEST, Override_Off);
+                state->setTextureMode(0, GL_TEXTURE_1D, Override_Off);
+                state->setTextureMode(0, GL_TEXTURE_2D, Override_Off);
+                state->setTextureMode(0, GL_TEXTURE_3D, Override_Off);
+
+                addPass(state);
+            }
+        }
+
+    private:
+        /// Outline width.
+        osg::ref_ptr<osg::LineWidth> _lineWidth;
+        float _width;
+
+        /// Outline Material.
+        osg::ref_ptr<osg::Material> _material;
+        osg::Vec4 _color;
+    };
+
+
+    /**
+     * Outline effect.
+     */
+    Outline::Outline() : Effect(), _width(2), _color(1,1,1,1), _technique(0)
+    {
+    }
+
+    void Outline::setWidth(float w)
+    {
+        _width = w;
+        if (_technique) {
+            _technique->setWidth(w);
+        }
+    }
+
+    void Outline::setColor(const osg::Vec4& color)
+    {
+        _color = color;
+        if (_technique) {
+            _technique->setColor(color);
+        }
+    }
+
+    bool Outline::define_techniques()
+    {
+        _technique = new OutlineTechnique;
+        addTechnique(_technique);
+
+        setWidth(_width);
+        setColor(_color);
+
+        return true;
+    }
+};