From dc2f142480d8671e030680d3317ac48e8310d322 Mon Sep 17 00:00:00 2001 From: ThomasS Date: Thu, 3 May 2018 09:55:17 +0200 Subject: [PATCH 01/25] Feature for requesting canvas images per HTTP as described in http://wiki.flightgear.org/Read_canvas_image_by_HTTP --- simgear/canvas/Canvas.cxx | 119 ++++++++++++++++++++++++++++++++++++++ simgear/canvas/Canvas.hxx | 18 +++++- 2 files changed, 136 insertions(+), 1 deletion(-) diff --git a/simgear/canvas/Canvas.cxx b/simgear/canvas/Canvas.cxx index 74ae92b0..4650e099 100644 --- a/simgear/canvas/Canvas.cxx +++ b/simgear/canvas/Canvas.cxx @@ -39,6 +39,75 @@ namespace simgear { namespace canvas { + static int globalinstanceid = 1; + /** + * Camera Callback for moving completed canvas images to subscribed listener. + */ + class CanvasImageCallback : public osg::Camera::DrawCallback { + public: + osg::Image *_rawImage; + + CanvasImageCallback(osg::Image *rawImage) + : _min_delta_tick(1.0 / 8.0) { + _previousFrameTick = osg::Timer::instance()->tick(); + _rawImage = rawImage; + SG_LOG(SG_GENERAL,SG_INFO,"CanvasImageCallback created. instance is " << instanceid); + } + + virtual void operator()(osg::RenderInfo& renderInfo) const { + osg::Timer_t n = osg::Timer::instance()->tick(); + double dt = osg::Timer::instance()->delta_s(_previousFrameTick, n); + if (dt < _min_delta_tick) + return; + _previousFrameTick = n; + SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImageCallback " << instanceid << ": image available for " << _subscribers.size() << " subscribers. camera is " << renderInfo.getCurrentCamera()); + + bool hasSubscribers = false; + { + OpenThreads::ScopedLock lock(_lock); + hasSubscribers = !_subscribers.empty(); + } + if (hasSubscribers) { + //Make sure image can be overwritten by next frame while it is still returned to the client + osg::Image* image = new osg::Image(*_rawImage, osg::CopyOp::DEEP_COPY_ALL); + { + OpenThreads::ScopedLock lock(_lock); + while (!_subscribers.empty()) { + try { + CanvasImageReadyListener *subs = _subscribers.back(); + if (subs){ + subs->imageReady(image); + }else{ + SG_LOG(SG_GENERAL,SG_WARN,"CanvasImageCallback subscriber null"); + } + } catch (...) { } + _subscribers.pop_back(); + } + } + } + } + + void subscribe(CanvasImageReadyListener * subscriber) { + OpenThreads::ScopedLock lock(_lock); + _subscribers.push_back(subscriber); + } + + void unsubscribe(CanvasImageReadyListener * subscriber) { + OpenThreads::ScopedLock lock(_lock); + _subscribers.remove(subscriber); + } + + int getSubscriberCount() { + return _subscribers.size(); + } + + private: + mutable list _subscribers; + mutable OpenThreads::Mutex _lock; + mutable double _previousFrameTick; + double _min_delta_tick; + int instanceid = globalinstanceid++; + }; //---------------------------------------------------------------------------- Canvas::CullCallback::CullCallback(const CanvasWeakPtr& canvas): @@ -248,6 +317,21 @@ namespace canvas osg::Camera* camera = _texture.getCamera(); + string canvasname = _node->getStringValue("name"); + int renderToImage = _node->getBoolValue("render-to-image"); + + if (renderToImage){ + CanvasImageCallback *_screenshotCallback = dynamic_cast (camera->getFinalDrawCallback()); + if (!_screenshotCallback) { + // no draw callback yet + osg::Image* shot = new osg::Image(); + shot->allocateImage(getSizeX(), getSizeY(), 24, GL_RGB, GL_UNSIGNED_BYTE); + camera->attach(osg::Camera::COLOR_BUFFER, shot); + camera->setFinalDrawCallback(new CanvasImageCallback(shot)); + SG_LOG(SG_GENERAL,SG_INFO,"CanvasImage: attached image and draw callback to camera " << camera << " for canvas " << canvasname << ". Ready for subscriber now."); + } + } + // TODO Allow custom render order? For now just keep in order with // property tree. camera->setRenderOrder(osg::Camera::PRE_RENDER, _node->getIndex()); @@ -347,6 +431,41 @@ namespace canvas } } + int Canvas::subscribe(CanvasImageReadyListener * subscriber) { + osg::Camera* camera = _texture.getCamera(); + const string canvasname = _node->getStringValue("name"); + + SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImage: subscribe to canvas " << canvasname.c_str() << ", camera ="<< camera); + + if (!_node->getBoolValue("render-to-image")) { + SG_LOG(SG_GENERAL,SG_INFO,"CanvasImage: Setting render-to-image"); + _node->addChild("render-to-image", 0)->setBoolValue(1); + setStatusFlags(STATUS_DIRTY, true); + } + + CanvasImageCallback *_screenshotCallback = dynamic_cast (camera->getFinalDrawCallback()); + if (_screenshotCallback) { + // Camera ready for subscriber. Otherwise, draw callback is created by canvas thread later. + SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImage: adding subscriber to camera draw callback"); + _screenshotCallback->subscribe(subscriber); + // TODO: check: Is this the correct way to ensure the canvas will be available? + enableRendering(true); + return 1; + } + return 0; + } + + int Canvas::unsubscribe(CanvasImageReadyListener * subscriber) { + osg::Camera* camera = _texture.getCamera(); + SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImage: unsubscribe"); + CanvasImageCallback *cb = dynamic_cast (camera->getFinalDrawCallback()); + if (cb) { + SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImage: unsubscribe from camera " << camera); + cb->unsubscribe(subscriber); + } + return 0; + } + //---------------------------------------------------------------------------- bool Canvas::addEventListener( const std::string& type, const EventListener& cb ) diff --git a/simgear/canvas/Canvas.hxx b/simgear/canvas/Canvas.hxx index 7e22275c..10b7403c 100644 --- a/simgear/canvas/Canvas.hxx +++ b/simgear/canvas/Canvas.hxx @@ -44,6 +44,17 @@ namespace canvas class CanvasMgr; class MouseEvent; + /** + * A listener interested in completed canvas drawing. + */ + class CanvasImageReadyListener { + public: + virtual void imageReady(osg::ref_ptr) = 0; + virtual ~CanvasImageReadyListener() + { + } + }; + /** * Canvas to draw onto (to an off-screen render target). */ @@ -160,7 +171,12 @@ namespace canvas */ void enableRendering(bool force = false); - void update(double delta_time_sec); + void update(double delta_time_sec) override; + + osg::Camera* getCamera(); + int subscribe(CanvasImageReadyListener * subscriber); + int unsubscribe(CanvasImageReadyListener * subscriber); + int getSubscriberCount(); bool addEventListener(const std::string& type, const EventListener& cb); bool dispatchEvent(const EventPtr& event); From 7dfe705717eb8a39df47db84daafdb2cc16ac55e Mon Sep 17 00:00:00 2001 From: James Turner Date: Mon, 28 May 2018 21:57:31 +0200 Subject: [PATCH 02/25] Resource manager clean-up code --- simgear/misc/ResourceManager.cxx | 34 +++++++++++++++++++++++++++----- simgear/misc/ResourceManager.hxx | 12 ++++++++--- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/simgear/misc/ResourceManager.cxx b/simgear/misc/ResourceManager.cxx index 178cc96e..71d83928 100644 --- a/simgear/misc/ResourceManager.cxx +++ b/simgear/misc/ResourceManager.cxx @@ -20,11 +20,17 @@ #include #include +#include namespace simgear { -static ResourceManager* static_manager = NULL; +static ResourceManager* static_manager = nullptr; + +ResourceProvider::~ResourceProvider() +{ + // pin to this compilation unit +} ResourceManager::ResourceManager() { @@ -40,13 +46,20 @@ ResourceManager* ResourceManager::instance() return static_manager; } +ResourceManager::~ResourceManager() +{ + assert(this == static_manager); + static_manager = nullptr; + std::for_each(_providers.begin(), _providers.end(), + [](ResourceProvider* p) { delete p; }); +} /** * trivial provider using a fixed base path */ class BasePathProvider : public ResourceProvider { public: - BasePathProvider(const SGPath& aBase, int aPriority) : + BasePathProvider(const SGPath& aBase, ResourceManager::Priority aPriority) : ResourceProvider(aPriority), _base(aBase) { @@ -69,6 +82,8 @@ void ResourceManager::addBasePath(const SGPath& aPath, Priority aPriority) void ResourceManager::addProvider(ResourceProvider* aProvider) { + assert(aProvider); + ProviderVec::iterator it = _providers.begin(); for (; it != _providers.end(); ++it) { if (aProvider->priority() > (*it)->priority()) { @@ -81,6 +96,16 @@ void ResourceManager::addProvider(ResourceProvider* aProvider) _providers.push_back(aProvider); } +void ResourceManager::removeProvider(ResourceProvider* aProvider) +{ + assert(aProvider); + auto it = std::find(_providers.begin(), _providers.end(), aProvider); + if (it == _providers.end()) { + SG_LOG(SG_GENERAL, SG_DEV_ALERT, "unknown provider doing remove"); + return; + } +} + SGPath ResourceManager::findPath(const std::string& aResource, SGPath aContext) { if (!aContext.isNull()) { @@ -90,9 +115,8 @@ SGPath ResourceManager::findPath(const std::string& aResource, SGPath aContext) } } - ProviderVec::iterator it = _providers.begin(); - for (; it != _providers.end(); ++it) { - SGPath path = (*it)->resolve(aResource, aContext); + for (auto provider : _providers) { + SGPath path = provider->resolve(aResource, aContext); if (!path.isNull()) { return path; } diff --git a/simgear/misc/ResourceManager.hxx b/simgear/misc/ResourceManager.hxx index bfaa5310..a15aaf15 100644 --- a/simgear/misc/ResourceManager.hxx +++ b/simgear/misc/ResourceManager.hxx @@ -36,6 +36,8 @@ class ResourceProvider; class ResourceManager { public: + ~ResourceManager(); + typedef enum { PRIORITY_DEFAULT = 0, PRIORITY_FALLBACK = -100, @@ -55,6 +57,8 @@ public: */ void addProvider(ResourceProvider* aProvider); + void removeProvider(ResourceProvider* aProvider); + /** * given a resource name (or path), find the appropriate real resource * path. @@ -75,17 +79,19 @@ class ResourceProvider public: virtual SGPath resolve(const std::string& aResource, SGPath& aContext) const = 0; - virtual int priority() const + virtual ~ResourceProvider(); + + virtual ResourceManager::Priority priority() const { return _priority; } protected: - ResourceProvider(int aPriority) : + ResourceProvider(ResourceManager::Priority aPriority) : _priority(aPriority) {} - int _priority; + ResourceManager::Priority _priority = ResourceManager::PRIORITY_DEFAULT; }; } // of simgear namespace From 856473ca43aa9f285b397522817a57bad02beedc Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Tue, 29 May 2018 09:44:37 +0200 Subject: [PATCH 03/25] Add missing headers --- simgear/misc/ResourceManager.cxx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simgear/misc/ResourceManager.cxx b/simgear/misc/ResourceManager.cxx index 71d83928..11b67846 100644 --- a/simgear/misc/ResourceManager.cxx +++ b/simgear/misc/ResourceManager.cxx @@ -17,6 +17,9 @@ // // $Id$ +#include +#include + #include #include From 3dceaf7a0b796fe0bb08bfd802faecf412bcb704 Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Sat, 2 Jun 2018 10:53:03 +0200 Subject: [PATCH 04/25] Update to the latest version --- CMakeModules/FindAAX.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeModules/FindAAX.cmake b/CMakeModules/FindAAX.cmake index 4e0bfdcd..edbef409 100644 --- a/CMakeModules/FindAAX.cmake +++ b/CMakeModules/FindAAX.cmake @@ -27,14 +27,14 @@ FIND_PATH(AAX_INCLUDE_DIR aax/aax.h ) FIND_LIBRARY(AAX_LIBRARY - NAMES AAX aax AAX32 + NAMES AAX aax libAAX HINTS $ENV{AAXDIR} $ENV{ProgramFiles}/AAX $ENV{ProgramFiles}/AeonWave $ENV{ProgramFiles}/Adalin/AeonWave ${CMAKE_BUILD_DIR}/aax - PATH_SUFFIXES bin lib lib/${CMAKE_LIBRARY_ARCHITECTURE} lib64 libs64 libs libs/Win32 libs/Win64 + PATH_SUFFIXES lib64 lib lib/${CMAKE_LIBRARY_ARCHITECTURE} libs64 libs libs/Win32 libs/Win64 bin PATHS ~/Library/Frameworks /Library/Frameworks @@ -52,7 +52,7 @@ ELSE(AAX_LIBRARY AND AAX_INCLUDE_DIR) ENDIF(NOT AAX_INCLUDE_DIR) IF(NOT AAX_LIBRARY) IF(SINGLE_PACKAGE) - SET(AAX_LIBRARY "${aax_BUILD_DIR}/aax/AAX32.dll") + SET(AAX_LIBRARY "${aax_BUILD_DIR}/aax/AAX.lib") SET(AAX_FOUND "YES") ELSE(SINGLE_PACKAGE) ENDIF(SINGLE_PACKAGE) From 08258ee4b3c65745485905a44552874dfeed5927 Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Sat, 2 Jun 2018 14:06:15 +0200 Subject: [PATCH 05/25] Detect AeonWave and if it is installed use it, otherwise fall back to OpenAL. Also let get_available_devices() use C++ strings instead of const char* --- CMakeLists.txt | 14 ++++-- CMakeModules/FindAAX.cmake | 15 ++++--- README.OpenAL | 20 --------- README.sound | 39 +++++++++++++++++ simgear/simgear_config_cmake.h.in | 1 + simgear/sound/soundmgr.hxx | 2 +- simgear/sound/soundmgr_aeonwave.cxx | 66 +++++++++++++++-------------- simgear/sound/soundmgr_openal.cxx | 4 +- 8 files changed, 97 insertions(+), 64 deletions(-) delete mode 100644 README.OpenAL create mode 100644 README.sound diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b21730c..6f95a39b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,7 +126,7 @@ option(ENABLE_RTI "Set to ON to build SimGear with RTI support" OFF) option(ENABLE_GDAL "Set to ON to build SimGear with GDAL support" OFF) option(ENABLE_TESTS "Set to OFF to disable building SimGear's test applications" ON) option(ENABLE_SOUND "Set to OFF to disable building SimGear's sound support" ON) -option(USE_AEONWAVE "Set to ON to use AeonWave instead of OpenAL" OFF) +option(USE_AEONWAVE "Set to ON to use AeonWave instead of OpenAL" ON) option(ENABLE_PKGUTIL "Set to ON to build the sg_pkgutil application (default)" ON) option(ENABLE_DNS "Set to ON to use udns library and DNS service resolver" ON) option(ENABLE_SIMD "Enable SSE/SSE2 support for x86 compilers" ON) @@ -222,12 +222,18 @@ else() if (ENABLE_SOUND) if (USE_AEONWAVE) - find_package(AAX COMPONENTS aax REQUIRED) - else() + find_package(AAX) + endif() + + if(NOT AAX_FOUND) find_package(OpenAL REQUIRED) endif() - message(STATUS "Sound support: ENABLED") + if(AAX_FOUND) + message(STATUS "Sound support: AeonWave") + else() + message(STATUS "Sound support: OpenAL") + endif() endif(ENABLE_SOUND) find_package(OpenSceneGraph 3.2.0 REQUIRED osgText osgSim osgDB osgParticle osgGA osgViewer osgUtil) diff --git a/CMakeModules/FindAAX.cmake b/CMakeModules/FindAAX.cmake index edbef409..0e21386f 100644 --- a/CMakeModules/FindAAX.cmake +++ b/CMakeModules/FindAAX.cmake @@ -1,14 +1,19 @@ -# Locate AAX +# Try to find AAX (AeonWave) # This module defines -# AAX_LIBRARIES -# AAX_FOUND, if false, do not try to link to AAX -# AAX_INCLUDE_DIR, where to find the headers +# +# AAX_FOUND - if false, do not try to link to AAX +# AAX_INCLUDE_DIR - where to find the headers +# AAX_LIBRARIES - Link these to use AAX +# +# Copyright (C) 2016-2018 by Erik Hofman. +# Copyright (C) 2016-2018 by Adalin B.V. # # $AAXDIR is an environment variable that would # correspond to the ./configure --prefix=$AAXDIR # used in building AAX. # -# Created by Erik Hofman. +# This file is Public Domain (www.unlicense.org) +# This is free and unencumbered software released into the public domain. FIND_PATH(AAX_INCLUDE_DIR aax/aax.h HINTS diff --git a/README.OpenAL b/README.OpenAL deleted file mode 100644 index f21c656a..00000000 --- a/README.OpenAL +++ /dev/null @@ -1,20 +0,0 @@ -[This file is mirrored in both the FlightGear and SimGear packages.] - -You *must* have the development components of OpenAL installed on your system -to build FlightGear!" You can get a copy here: - - http://connect.creativelabs.com/openal/default.aspx - -Build notes: - -You can download a versioned release of the openal library from -http://www.openal.org/downloads.html. Download the openal source, -release 0.0.8 (dated February 11, 2006) and run: -tar xjvf openal-soft-1.5.304.tar.bz2 -cd openal-soft-1.5.304/ -ccmake . - -[ While running ccmake: press 'c' to configure, press 'c' once more, and - then press 'g' to generate and exit ] - - diff --git a/README.sound b/README.sound new file mode 100644 index 00000000..bee72ffe --- /dev/null +++ b/README.sound @@ -0,0 +1,39 @@ +[This file is mirrored in both the FlightGear and SimGear packages.] + +For Sound support FlightGear requires one of the two following packages: +- OpenAL +- AeonWave + +== OpenAL === + +You *must* have the development components of OpenAL installed on your system +to build FlightGear!" You can get a copy here: + + http://connect.creativelabs.com/openal/default.aspx + +Build notes: + +You can download a versioned release of the openal library from +http://www.openal.org/downloads.html. Download the openal source, +release 0.0.8 (dated February 11, 2006) and run: +tar xjvf openal-soft-1.5.304.tar.bz2 +cd openal-soft-1.5.304/ +ccmake . + +[ While running ccmake: press 'c' to configure, press 'c' once more, and + then press 'g' to generate and exit ] + + +== AeonWave === + +For FlightGear AeonWave has a number of advantages over OpenAL: +* Correct Doppler effect behavior +* Default distance attenuation frequency filtering +* Native support for 29 types of audio formats. +* Native support for wav, mp3, vorbis and raw file formats. + +The source code of AeonWave can be found on GitHub: +https://github.com/adalinbv + +Optimized binary packages are available at: +http://www.adalin.com/ diff --git a/simgear/simgear_config_cmake.h.in b/simgear/simgear_config_cmake.h.in index c3668638..de1b69c6 100644 --- a/simgear/simgear_config_cmake.h.in +++ b/simgear/simgear_config_cmake.h.in @@ -25,5 +25,6 @@ #cmakedefine SYSTEM_EXPAT #cmakedefine ENABLE_SOUND +#cmakedefine USE_AEONWAVE #cmakedefine ENABLE_SIMD #cmakedefine ENABLE_GDAL diff --git a/simgear/sound/soundmgr.hxx b/simgear/sound/soundmgr.hxx index 21d87d8d..89b8cda4 100644 --- a/simgear/sound/soundmgr.hxx +++ b/simgear/sound/soundmgr.hxx @@ -314,7 +314,7 @@ public: /** * Get a list of available playback devices. */ - std::vector get_available_devices(); + std::vector get_available_devices(); /** * Get the current OpenAL vendor or rendering backend. diff --git a/simgear/sound/soundmgr_aeonwave.cxx b/simgear/sound/soundmgr_aeonwave.cxx index 383c6039..b25fe008 100644 --- a/simgear/sound/soundmgr_aeonwave.cxx +++ b/simgear/sound/soundmgr_aeonwave.cxx @@ -75,10 +75,6 @@ public: ~SoundManagerPrivate() { - for (auto it = _devices.begin(); it != _devices.end(); ++it) { - free((void*)*it); - } - _devices.clear(); _sample_groups.clear(); } @@ -125,8 +121,6 @@ public: } sample_group_map _sample_groups; - - std::vector _devices; }; @@ -444,7 +438,7 @@ unsigned int SGSoundMgr::request_buffer(SGSoundSample *sample) std::string sample_name = sample->get_sample_name(); aax::Buffer& buf = d->_aax.buffer(sample_name); - if (!buf) { + if (sample->is_file() && !buf) { SG_LOG(SG_SOUND, SG_ALERT, "Unable to create buffer: " << sample_name); sample->set_buffer( SGSoundMgr::FAILED_BUFFER ); @@ -561,6 +555,7 @@ void SGSoundMgr::sample_play( SGSoundSample *sample ) if (bufid == SGSoundMgr::FAILED_BUFFER || bufid == SGSoundMgr::NO_BUFFER) { +printf("A: release: %i, bufid: %i (%i, %i)\n", sample->get_source(), bufid, SGSoundMgr::FAILED_BUFFER, SGSoundMgr::NO_BUFFER); release_source(sample->get_source()); return; } @@ -646,39 +641,46 @@ void SGSoundMgr::update_sample_config( SGSoundSample *sample, SGVec3d& position, aax::Emitter& emitter = d->get_emitter(sample->get_source()); aax::dsp dsp; - aax::Vector64 pos = position.data(); - aax::Vector ori = orientation.data(); - aax::Vector vel = velocity.data(); + if (emitter != d->nullEmitter) + { + aax::Vector64 pos = position.data(); + aax::Vector ori = orientation.data(); + aax::Vector vel = velocity.data(); - aax::Matrix64 mtx(pos, ori); - TRY( emitter.matrix(mtx) ); - TRY( emitter.velocity(vel) ); + aax::Matrix64 mtx(pos, ori); + TRY( emitter.matrix(mtx) ); + TRY( emitter.velocity(vel) ); - dsp = emitter.get(AAX_VOLUME_FILTER); - TRY( dsp.set(AAX_GAIN, sample->get_volume()) ); - TRY( emitter.set(dsp) ); - - dsp = emitter.get(AAX_PITCH_EFFECT); - TRY( dsp.set(AAX_PITCH, sample->get_pitch()) ); - TRY( emitter.set(dsp) ); - - if ( sample->has_static_data_changed() ) { - dsp = emitter.get(AAX_ANGULAR_FILTER); - TRY( dsp.set(AAX_INNER_ANGLE, sample->get_innerangle()) ); - TRY( dsp.set(AAX_OUTER_ANGLE, sample->get_outerangle()) ); - TRY( dsp.set(AAX_OUTER_GAIN, sample->get_outergain()) ); + dsp = emitter.get(AAX_VOLUME_FILTER); + TRY( dsp.set(AAX_GAIN, sample->get_volume()) ); TRY( emitter.set(dsp) ); - dsp = emitter.get(AAX_DISTANCE_FILTER); - TRY( dsp.set(AAX_REF_DISTANCE, sample->get_reference_dist()) ); - TRY( dsp.set(AAX_MAX_DISTANCE, sample->get_max_dist()) ); + dsp = emitter.get(AAX_PITCH_EFFECT); + TRY( dsp.set(AAX_PITCH, sample->get_pitch()) ); TRY( emitter.set(dsp) ); + + if ( sample->has_static_data_changed() ) { + dsp = emitter.get(AAX_ANGULAR_FILTER); + TRY( dsp.set(AAX_INNER_ANGLE, sample->get_innerangle()) ); + TRY( dsp.set(AAX_OUTER_ANGLE, sample->get_outerangle()) ); + TRY( dsp.set(AAX_OUTER_GAIN, sample->get_outergain()) ); + TRY( emitter.set(dsp) ); + + dsp = emitter.get(AAX_DISTANCE_FILTER); + TRY( dsp.set(AAX_REF_DISTANCE, sample->get_reference_dist()) ); + TRY( dsp.set(AAX_MAX_DISTANCE, sample->get_max_dist()) ); + TRY( emitter.set(dsp) ); + } + } + else { + SG_LOG( SG_SOUND, SG_ALERT, "Error: source: " << sample->get_source() << " not found"); } } -vector SGSoundMgr::get_available_devices() +vector SGSoundMgr::get_available_devices() { + vector devices; #ifdef ENABLE_SOUND std::string on = " on "; std::string colon = ": "; @@ -690,12 +692,12 @@ vector SGSoundMgr::get_available_devices() else if (*r) name += on + r; else if (*i) name += colon + i; - d->_devices.push_back( strdup(name.c_str()) ); + devices.push_back( name ); } } } #endif - return d->_devices; + return devices; } diff --git a/simgear/sound/soundmgr_openal.cxx b/simgear/sound/soundmgr_openal.cxx index 64a706df..cc3b420a 100644 --- a/simgear/sound/soundmgr_openal.cxx +++ b/simgear/sound/soundmgr_openal.cxx @@ -821,9 +821,9 @@ bool SGSoundMgr::load( const std::string &samplepath, return true; } -vector SGSoundMgr::get_available_devices() +vector SGSoundMgr::get_available_devices() { - vector devices; + vector devices; #ifdef ENABLE_SOUND const ALCchar *s; From 03156a6d94d7e2aa80b46f0370a3f8445f42652f Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Tue, 5 Jun 2018 09:39:51 +0200 Subject: [PATCH 06/25] Better AeoNWave detection --- CMakeLists.txt | 1 + CMakeModules/FindAAX.cmake | 100 ++++++++++++++++++++----------------- 2 files changed, 55 insertions(+), 46 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f95a39b..b728f990 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -226,6 +226,7 @@ else() endif() if(NOT AAX_FOUND) + set(USE_AEONWAVE FALSE) find_package(OpenAL REQUIRED) endif() diff --git a/CMakeModules/FindAAX.cmake b/CMakeModules/FindAAX.cmake index 0e21386f..3a2da47d 100644 --- a/CMakeModules/FindAAX.cmake +++ b/CMakeModules/FindAAX.cmake @@ -15,52 +15,60 @@ # This file is Public Domain (www.unlicense.org) # This is free and unencumbered software released into the public domain. -FIND_PATH(AAX_INCLUDE_DIR aax/aax.h - HINTS - $ENV{AAXDIR} - $ENV{ProgramFiles}/aax - $ENV{ProgramFiles}/AeonWave - $ENV{ProgramFiles}/Adalin/AeonWave - ${CMAKE_SOURCE_DIR}/aax - PATH_SUFFIXES include - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt -) +if (AAX_LIBRARY AND AAX_INCLUDE_DIR) + # in cache already + set(AAX_FOUND TRUE) +else() + find_path(AAX_INCLUDE_DIR aax/aax.h + HINTS + $ENV{AAXDIR} + $ENV{ProgramFiles}/aax + $ENV{ProgramFiles}/AeonWave + $ENV{ProgramFiles}/Adalin/AeonWave + ${CMAKE_SOURCE_DIR}/aax + PATH_SUFFIXES include + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /opt + ) -FIND_LIBRARY(AAX_LIBRARY - NAMES AAX aax libAAX - HINTS - $ENV{AAXDIR} - $ENV{ProgramFiles}/AAX - $ENV{ProgramFiles}/AeonWave - $ENV{ProgramFiles}/Adalin/AeonWave - ${CMAKE_BUILD_DIR}/aax - PATH_SUFFIXES lib64 lib lib/${CMAKE_LIBRARY_ARCHITECTURE} libs64 libs libs/Win32 libs/Win64 bin - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt -) + find_library(AAX_LIBRARY + NAMES AAX aax libAAX + HINTS + $ENV{AAXDIR} + $ENV{ProgramFiles}/AAX + $ENV{ProgramFiles}/AeonWave + $ENV{ProgramFiles}/Adalin/AeonWave + ${CMAKE_BUILD_DIR}/aax + PATH_SUFFIXES lib64 lib lib/${CMAKE_LIBRARY_ARCHITECTURE} libs64 libs libs/Win32 libs/Win64 bin + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /opt + ) -IF(AAX_LIBRARY AND AAX_INCLUDE_DIR) - SET(AAX_FOUND "YES") -ELSE(AAX_LIBRARY AND AAX_INCLUDE_DIR) - IF(NOT AAX_INCLUDE_DIR) - MESSAGE(FATAL_ERROR "Unable to find the AAX library development files.") - SET(AAX_FOUND "NO") - ENDIF(NOT AAX_INCLUDE_DIR) - IF(NOT AAX_LIBRARY) - IF(SINGLE_PACKAGE) - SET(AAX_LIBRARY "${aax_BUILD_DIR}/aax/AAX.lib") - SET(AAX_FOUND "YES") - ELSE(SINGLE_PACKAGE) - ENDIF(SINGLE_PACKAGE) - ENDIF(NOT AAX_LIBRARY) -ENDIF(AAX_LIBRARY AND AAX_INCLUDE_DIR) + set(AAX_DEFINITIONS "") + if (AAX_LIBRARY AND AAX_INCLUDE_DIR) + set(AAX_FOUND TRUE) + endif() + + if (AAX_FOUND) + if (NOT Udns_FIND_QUIETLY) + message(STATUS "Found AeonWave: ${AAX_LIBRARIES}") + endif () + else () + if (Udns_FIND_REQUIRED) + message(FATAL_ERROR "Could not find AeonWave") + endif () + endif () + + # show the AAX_INCLUDE_DIRS and AAX_LIBRARIES variables only in the advanced view + mark_as_advanced(AAX_INCLUDE_DIRS AAX_LIBRARIES) + +endif() From 1d7c3984ca2e9a1299c8c546b0d9194073dac556 Mon Sep 17 00:00:00 2001 From: Stuart Buchanan Date: Tue, 5 Jun 2018 21:55:23 +0100 Subject: [PATCH 07/25] Fix minor warning for no return value. --- simgear/io/HTTPRepository.cxx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/simgear/io/HTTPRepository.cxx b/simgear/io/HTTPRepository.cxx index 690d5622..2c1c61c6 100644 --- a/simgear/io/HTTPRepository.cxx +++ b/simgear/io/HTTPRepository.cxx @@ -90,8 +90,10 @@ namespace simgear case HTTPRepository::REPO_ERROR_CANCELLED: return "cancelled"; case HTTPRepository::REPO_PARTIAL_UPDATE: return "partial update (incomplete)"; } + + return "Unknown response code"; } - + class HTTPRepoPrivate { public: @@ -291,7 +293,7 @@ public: if (!cp.exists()) { continue; } - + SGBinaryFile src(cp); SGBinaryFile dst(p); src.open(SG_IO_IN); @@ -677,7 +679,7 @@ std::string HTTPRepository::resultCodeAsString(ResultCode code) { return innerResultCodeAsString(code); } - + HTTPRepository::ResultCode HTTPRepository::failure() const { @@ -746,7 +748,7 @@ HTTPRepository::failure() const if (responseCode() == -1) { code = HTTPRepository::REPO_ERROR_CANCELLED; } - + file.reset(); if (pathInRepo.exists()) { pathInRepo.remove(); @@ -1084,7 +1086,7 @@ HTTPRepository::failure() const } else { // we encounter this code path when deleting an orphaned directory } - + Dir dir(absPath); bool result = dir.remove(true); @@ -1152,7 +1154,7 @@ HTTPRepository::failure() const } SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update repository:" << baseUrl - << "\n\tchecksum failure for: " << relativePath + << "\n\tchecksum failure for: " << relativePath << "\n\tthis typically indicates the remote repository is corrupt or was being updated during the sync"); } else if (fileStatus == HTTPRepository::REPO_ERROR_CANCELLED) { // if we were cancelled, don't report or log From 7fb89e4d45bbf1d5b24416d80afa058c166a26f0 Mon Sep 17 00:00:00 2001 From: Stuart Buchanan Date: Tue, 5 Jun 2018 21:56:36 +0100 Subject: [PATCH 08/25] Multiple LoD levels of MP Aircraft Support loading multiple models into PagedLOD. --- simgear/scene/model/modellib.cxx | 59 +++++++++++++++++++++++++------- simgear/scene/model/modellib.hxx | 51 ++++++++++++++++++++++++--- 2 files changed, 93 insertions(+), 17 deletions(-) diff --git a/simgear/scene/model/modellib.cxx b/simgear/scene/model/modellib.cxx index 3d931f2b..34707957 100644 --- a/simgear/scene/model/modellib.cxx +++ b/simgear/scene/model/modellib.cxx @@ -161,33 +161,68 @@ SGModelLib::loadDeferredModel(const string &path, SGPropertyNode *prop_root, return proxyNode; } + +/* + * Load a set of models at different LOD range_nearest + * + */ osg::PagedLOD* -SGModelLib::loadPagedModel(const string &path, SGPropertyNode *prop_root, - SGModelData *data) +SGModelLib::loadPagedModel(SGPropertyNode *prop_root, SGModelData *data, SGModelLOD model_lods) { osg::PagedLOD *plod = new osg::PagedLOD; - plod->setName("Paged LOD for \"" + path + "\""); - plod->setFileName(0, path); - plod->setRange(0, 0.0, 50.0*SG_NM_TO_METER); - plod->setMinimumExpiryTime( 0, prop_root->getDoubleValue("/sim/rendering/plod-minimum-expiry-time-secs", 180.0 ) ); osg::ref_ptr opt; opt = SGReaderWriterOptions::copyOrCreate(osgDB::Registry::instance()->getOptions()); opt->setPropertyNode(prop_root ? prop_root: static_propRoot.get()); opt->setModelData(data); opt->setLoadPanel(static_panelFunc); - std::string lext = SGPath(path).lower_extension(); - - if ((lext == "ac") || (lext == "obj")) { - opt->setInstantiateEffects(true); - } - if (!prop_root || prop_root->getBoolValue("/sim/rendering/cache", true)) opt->setObjectCacheHint(osgDB::Options::CACHE_ALL); else opt->setObjectCacheHint(osgDB::Options::CACHE_NONE); + + for(unsigned int i = 0; i < model_lods.getNumLODs(); i++) { + SGModelLOD::ModelLOD lod = model_lods.getModelLOD(i); + plod->setName("Paged LOD for \"" + lod.path + "\""); + plod->setFileName(i, lod.path); + plod->setRange(i, lod.min_range, lod.max_range); + plod->setMinimumExpiryTime(i, prop_root->getDoubleValue("/sim/rendering/plod-minimum-expiry-time-secs", 180.0 ) ); + + std::string lext = SGPath(lod.path).lower_extension(); + + // We can only have one set of ReaderWriterOptions for the PagedLOD, so + // we will just have to assume that if one of the defined models can + // handle instantiated effects, then the other can as well. + if ((lext == "ac") || (lext == "obj")) { + opt->setInstantiateEffects(true); + } + } + plod->setDatabaseOptions(opt.get()); + return plod; } +osg::PagedLOD* +SGModelLib::loadPagedModel(const string &path, SGPropertyNode *prop_root, + SGModelData *data) +{ + SGModelLOD model_lods; + model_lods.insert(path, 0.0, 50.0*SG_NM_TO_METER); + return SGModelLib::loadPagedModel(prop_root, data, model_lods); +} + +osg::PagedLOD* +SGModelLib::loadPagedModel(std::vector paths, SGPropertyNode *prop_root, + SGModelData *data) +{ + SGModelLOD model_lods; + for(unsigned int i = 0; i < paths.size(); i++) { + // We don't have any range data, so simply set them all up to full range. + // Some other code will update the LoD ranges later. (AIBase::updateLOD) + model_lods.insert(paths[i], 0.0, 50.0*SG_NM_TO_METER); + } + return SGModelLib::loadPagedModel(prop_root, data, model_lods); +} + // end of modellib.cxx diff --git a/simgear/scene/model/modellib.hxx b/simgear/scene/model/modellib.hxx index cdd6d820..b9be0311 100644 --- a/simgear/scene/model/modellib.hxx +++ b/simgear/scene/model/modellib.hxx @@ -39,6 +39,7 @@ namespace osg { namespace simgear { class SGModelData; // defined below +class SGModelLOD; // defined below /** * Class for loading and managing models with XML wrappers. @@ -51,9 +52,9 @@ public: static void init(const std::string &root_dir, SGPropertyNode* root); static void resetPropertyRoot(); - + static void setPanelFunc(panel_func pf); - + // Load a 3D model (any format) // data->modelLoaded() will be called after the model is loaded static osg::Node* loadModel(const std::string &path, @@ -72,17 +73,24 @@ public: // the model file. Once the viewer steps onto that node the // model will be loaded. When the viewer does no longer reference this // node for a long time the node is unloaded again. + static osg::PagedLOD* loadPagedModel(SGPropertyNode *prop_root, + SGModelData *data, + SGModelLOD model_lods); static osg::PagedLOD* loadPagedModel(const std::string &path, SGPropertyNode *prop_root = NULL, SGModelData *data=0); - static std::string findDataFile(const std::string& file, + static osg::PagedLOD* loadPagedModel(std::vector paths, + SGPropertyNode *prop_root = NULL, + SGModelData *data=0); + + static std::string findDataFile(const std::string& file, const osgDB::Options* opts = NULL, - SGPath currentDir = SGPath()); + SGPath currentDir = SGPath()); protected: SGModelLib(); ~SGModelLib (); - + private: static SGPropertyNode_ptr static_propRoot; static panel_func static_panelFunc; @@ -102,6 +110,39 @@ public: virtual SGModelData* clone() const = 0; }; +/* + * Data for a model with multiple LoD versions + */ + +class SGModelLOD { +public: + struct ModelLOD { + ModelLOD(const string &p, float minrange, float maxrange) : + path(p), min_range(minrange), max_range(maxrange) + { } + const string &path; + float min_range; + float max_range; + }; +typedef std::vector ModelLODList; + +void insert(const ModelLOD& model) +{ + _models.push_back(model); +} + +void insert(const string &p, float minrange, float maxrange) +{ + insert(ModelLOD(p, minrange, maxrange)); +} + +unsigned getNumLODs() const { return _models.size(); } +const ModelLOD& getModelLOD(unsigned i) const { return _models[i]; } + +private: + ModelLODList _models; +}; + } #endif // _SG_MODEL_LIB_HXX From 100bb3a571aeefb51e986a4ce49acea746f4ba78 Mon Sep 17 00:00:00 2001 From: James Turner Date: Fri, 15 Jun 2018 11:08:53 +0100 Subject: [PATCH 09/25] Fix flags passed to SHGetKnownFolderPath Custom folder locations were not being picked up. Fixes: https://sourceforge.net/p/flightgear/codetickets/2019/ --- simgear/misc/sg_path.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simgear/misc/sg_path.cxx b/simgear/misc/sg_path.cxx index bbb6a894..3d0e41b1 100644 --- a/simgear/misc/sg_path.cxx +++ b/simgear/misc/sg_path.cxx @@ -107,7 +107,7 @@ static SGPath pathForKnownFolder(REFKNOWNFOLDERID folderId, const SGPath& def) // system call will allocate dynamic memory... which we must release when done wchar_t* localFolder = 0; - if (pSHGetKnownFolderPath(folderId, KF_FLAG_DEFAULT_PATH, NULL, &localFolder) == S_OK) { + if (pSHGetKnownFolderPath(folderId, KF_FLAG_DONT_VERIFY, NULL, &localFolder) == S_OK) { SGPath folder_path = SGPath(localFolder, def.getPermissionChecker()); // release dynamic memory CoTaskMemFree(static_cast(localFolder)); From 99d30d5bb742eedd96d66a353fea84625ae80a51 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 20 Jun 2018 22:00:03 +0100 Subject: [PATCH 10/25] CMake tweak: include headers in targets This improves navigation in XCode (any maybe some other tools) --- CMakeModules/SimGearComponent.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeModules/SimGearComponent.cmake b/CMakeModules/SimGearComponent.cmake index 00944de3..bf7a550f 100644 --- a/CMakeModules/SimGearComponent.cmake +++ b/CMakeModules/SimGearComponent.cmake @@ -12,6 +12,11 @@ macro(simgear_component_common name includePath sourcesList sources headers) set_property(GLOBAL APPEND PROPERTY PUBLIC_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/${h}") set(fh${sourcesList} "${fh${sourcesList}}#${CMAKE_CURRENT_SOURCE_DIR}/${h}") + + # also append headers to the sources list, so that IDEs find the files + # correctly (otherwise they are not in the project) + set_property(GLOBAL + APPEND PROPERTY ${sourcesList} "${CMAKE_CURRENT_SOURCE_DIR}/${h}") endforeach() set_property(GLOBAL APPEND PROPERTY FG_GROUPS_${sourcesList}_C "${fc${sourcesList}}@") From 9b1444deb505be11ac32b8b179571b49e989be14 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 20 Jun 2018 22:00:58 +0100 Subject: [PATCH 11/25] Strutils to format and parse SGGeod/lat/lon Format portion is taken from code in fg_props, parsing code is my own horrible concoction. --- simgear/misc/strutils.cxx | 212 ++++++++++++++++++++++++++++++++- simgear/misc/strutils.hxx | 41 +++++++ simgear/misc/strutils_test.cxx | 33 +++++ 3 files changed, 285 insertions(+), 1 deletion(-) diff --git a/simgear/misc/strutils.cxx b/simgear/misc/strutils.cxx index e8117e6b..b0a75cbf 100644 --- a/simgear/misc/strutils.cxx +++ b/simgear/misc/strutils.cxx @@ -37,7 +37,7 @@ #include #include // SG_WINDOWS #include - +#include #if defined(SG_WINDOWS) #include @@ -1133,6 +1133,216 @@ bool matchPropPathToTemplate(const std::string& path, const std::string& templat // unreachable } + +bool parseStringAsLatLonValue(const std::string& s, double& degrees) +{ + string ss = simplify(s); + auto spacePos = ss.find(' '); + + if (spacePos == std::string::npos) { + degrees = std::stof(ss); + } else { + degrees = std::stof(ss.substr(0, spacePos)); + + double minutes = 0.0, seconds = 0.0; + + // check for minutes marker + auto quotePos = ss.find('\''); + if (quotePos == std::string::npos) { + minutes = std::stof(ss.substr(spacePos)); + } else { + minutes = std::stof(ss.substr(spacePos, quotePos - spacePos)); + seconds = std::stof(ss.substr(quotePos+1)); + } + + if ((seconds < 0.0) || (minutes < 0.0)) { + // don't allow sign information in minutes or seconds + return false; + } + + double offset = (minutes / 60.0) + (seconds / 3600.0); + degrees += (degrees >= 0.0) ? offset : -offset; + } + + // since we simplified, any trailing N/S/E/W must be the last char + const char lastChar = ss.back(); + if ((lastChar == 'W') || (lastChar == 'S')) { + degrees = -degrees; + } + + return true; +} + +bool parseStringAsGeod(const std::string& s, SGGeod* result) +{ + if (s.empty()) + return false; + + const auto commaPos = s.find(','); + if (commaPos == string::npos) { + return false; + } + + double lat, lon; + if (!parseStringAsLatLonValue(s.substr(0, commaPos), lat) || + !parseStringAsLatLonValue(s.substr(commaPos+1), lon)) + { + return false; + } + + if (result) { + *result = SGGeod::fromDeg(lon, lat); + } + return true; +} + +std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) +{ + double min, sec; + const int sign = deg < 0.0 ? -1 : 1; + deg = fabs(deg); + char buf[128]; + + switch (format) { + case LatLonFormat::DECIMAL_DEGREES: + ::snprintf(buf, sizeof(buf), "%3.6f%c", deg, c); + break; + + case LatLonFormat::DEGREES_MINUTES: + // d mm.mmm' (DMM format) -- uses a round-off factor tailored to the + // required precision of the minutes field (three decimal places), + // preventing minute values of 60. + min = (deg - int(deg)) * 60.0; + if (min >= 59.9995) { + min -= 60.0; + deg += 1.0; + } + snprintf(buf, sizeof(buf), "%d*%06.3f'%c", int(deg), fabs(min), c); + break; + + case LatLonFormat::DEGREES_MINUTES_SECONDS: + // d mm'ss.s" (DMS format) -- uses a round-off factor tailored to the + // required precision of the seconds field (one decimal place), + // preventing second values of 60. + min = (deg - int(deg)) * 60.0; + sec = (min - int(min)) * 60.0; + if (sec >= 59.95) { + sec -= 60.0; + min += 1.0; + if (min >= 60.0) { + min -= 60.0; + deg += 1.0; + } + } + ::snprintf(buf, sizeof(buf), "%d*%02d'%04.1f\"%c", int(deg), int(min), fabs(sec), c); + break; + + case LatLonFormat::SIGNED_DECIMAL_DEGREES: + // d.dddddd' (signed DDD format). + ::snprintf(buf, sizeof(buf), "%3.6f", sign*deg); + break; + + case LatLonFormat::SIGNED_DEGREES_MINUTES: + // d mm.mmm' (signed DMM format). + min = (deg - int(deg)) * 60.0; + if (min >= 59.9995) { + min -= 60.0; + deg += 1.0; + } + if (sign == 1) { + snprintf(buf, sizeof(buf), "%d*%06.3f'", int(deg), fabs(min)); + } else { + snprintf(buf, sizeof(buf), "-%d*%06.3f'", int(deg), fabs(min)); + } + break; + + + case LatLonFormat::SIGNED_DEGREES_MINUTES_SECONDS: + // d mm'ss.s" (signed DMS format). + min = (deg - int(deg)) * 60.0; + sec = (min - int(min)) * 60.0; + if (sec >= 59.95) { + sec -= 60.0; + min += 1.0; + if (min >= 60.0) { + min -= 60.0; + deg += 1.0; + } + } + if (sign == 1) { + snprintf(buf, sizeof(buf), "%d*%02d'%04.1f\"", int(deg), int(min), fabs(sec)); + } else { + snprintf(buf, sizeof(buf), "-%d*%02d'%04.1f\"", int(deg), int(min), fabs(sec)); + } + break; + + case LatLonFormat::ZERO_PAD_DECIMAL_DEGRESS: + // dd.dddddd X, ddd.dddddd X (zero padded DDD format). + if (c == 'N' || c == 'S') { + snprintf(buf, sizeof(buf), "%09.6f%c", deg, c); + } else { + snprintf(buf, sizeof(buf), "%010.6f%c", deg, c); + } + break; + + case LatLonFormat::ZERO_PAD_DEGREES_MINUTES: + // dd mm.mmm' X, ddd mm.mmm' X (zero padded DMM format). + min = (deg - int(deg)) * 60.0; + if (min >= 59.9995) { + min -= 60.0; + deg += 1.0; + } + if (c == 'N' || c == 'S') { + snprintf(buf, sizeof(buf), "%02d*%06.3f'%c", int(deg), fabs(min), c); + } else { + snprintf(buf, sizeof(buf), "%03d*%06.3f'%c", int(deg), fabs(min), c); + } + break; + + case LatLonFormat::ZERO_PAD_DEGREES_MINUTES_SECONDS: + // dd mm'ss.s" X, dd mm'ss.s" X (zero padded DMS format). + min = (deg - int(deg)) * 60.0; + sec = (min - int(min)) * 60.0; + if (sec >= 59.95) { + sec -= 60.0; + min += 1.0; + if (min >= 60.0) { + min -= 60.0; + deg += 1.0; + } + } + if (c == 'N' || c == 'S') { + snprintf(buf, sizeof(buf), "%02d*%02d'%04.1f\"%c", int(deg), int(min), fabs(sec), c); + } else { + snprintf(buf, sizeof(buf), "%03d*%02d'%04.1f\"%c", int(deg), int(min), fabs(sec), c); + } + break; + + case LatLonFormat::TRINITY_HOUSE: + // dd* mm'.mmm X, ddd* mm'.mmm X (Trinity House Navigation standard). + min = (deg - int(deg)) * 60.0; + if (min >= 59.9995) { + min -= 60.0; + deg += 1.0; + } + if (c == 'N' || c == 'S') { + snprintf(buf, sizeof(buf), "%02d* %02d'.%03d%c", int(deg), int(min), int(SGMisc::round((min-int(min))*1000)), c); + } else { + snprintf(buf, sizeof(buf), "%03d* %02d'.%03d%c", int(deg), int(min), int(SGMisc::round((min-int(min))*1000)), c); + } + break; + } + + return std::string(buf); +} + +std::string formatGeodAsString(const SGGeod& geod, LatLonFormat format) +{ + const char ns = (geod.getLatitudeDeg() > 0.0) ? 'N' : 'S'; + const char ew = (geod.getLongitudeDeg() > 0.0) ? 'E' : 'W'; + + return formatLatLonValueAsString(geod.getLatitudeDeg(), format, ns) + "," + formatLatLonValueAsString(geod.getLongitudeDeg(), format, ew); +} } // end namespace strutils diff --git a/simgear/misc/strutils.hxx b/simgear/misc/strutils.hxx index ec464a5e..e17244fe 100644 --- a/simgear/misc/strutils.hxx +++ b/simgear/misc/strutils.hxx @@ -36,6 +36,9 @@ typedef std::vector < std::string > string_list; +// forward decls +class SGGeod; + namespace simgear { namespace strutils { @@ -354,6 +357,44 @@ namespace simgear { * /views[0]/view[4]/fig, /views[0]/view[1000]/flight */ bool matchPropPathToTemplate(const std::string& path, const std::string& templatePath); + + bool parseStringAsLatLonValue(const std::string& s, double& result); + + /** + * Attempt to parse a string as a latitude,longitude input. Returns true + * or false based on success, and returns the SGGeod by pointer. Leading, + * trailing and internal white-space is skipped / ignored. + * + * Supported formats: + * , + */ + bool parseStringAsGeod(const std::string& string, SGGeod* result = nullptr); + + // enum values here correspond to existing lon-lat format codes inside + // FlightGear (property: /sim/lon-lat-format ) + // Don't re-order, just add new ones, or things may break + enum class LatLonFormat + { + DECIMAL_DEGREES = 0, ///< 88.4N,4.54W, + DEGREES_MINUTES, ///< 88 24.6'N, 4 30.5'W + DEGREES_MINUTES_SECONDS, + SIGNED_DECIMAL_DEGREES, ///< 88.4,-4.54 + SIGNED_DEGREES_MINUTES, + SIGNED_DEGREES_MINUTES_SECONDS, + ZERO_PAD_DECIMAL_DEGRESS, + ZERO_PAD_DEGREES_MINUTES, + ZERO_PAD_DEGREES_MINUTES_SECONDS, + TRINITY_HOUSE, ///< dd* mm'.mmm X, ddd* mm'.mmm X (Trinity House Navigation standard). + }; + + std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c); + + /** + * Format an SGGeod as a string according to the provided rule. + * if the SGGeod is invalid (default constructed), will return an empty string + */ + std::string formatGeodAsString(const SGGeod& geod, + LatLonFormat format = LatLonFormat::DECIMAL_DEGREES); } // end namespace strutils } // end namespace simgear diff --git a/simgear/misc/strutils_test.cxx b/simgear/misc/strutils_test.cxx index 3a5f2e72..c7c13dd2 100644 --- a/simgear/misc/strutils_test.cxx +++ b/simgear/misc/strutils_test.cxx @@ -20,6 +20,7 @@ #include #include #include +#include using std::string; using std::vector; @@ -619,6 +620,36 @@ void test_utf8Convert() SG_VERIFY(a == aRoundTrip); } +void test_parseGeod() +{ + SGGeod a; + SG_VERIFY(strutils::parseStringAsGeod("56.12,-3.0", &a)); + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.0); + SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), 56.12, 1e-4); + + // embedded whitepace, DMS notation, NSEW notation + SG_VERIFY(strutils::parseStringAsGeod("\t40 30'50\"S, 12 34'56\"W ", &a)); + SG_CHECK_EQUAL_EP2(a.getLongitudeDeg(), -12.58222222, 1e-4); + SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), -40.5138888, 1e-4); + + // signed degrees-minutes + SG_VERIFY(strutils::parseStringAsGeod("-45 27.89,-12 34.56", &a)); + SG_CHECK_EQUAL_EP2(a.getLongitudeDeg(), -12.576, 1e-4); + SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), -45.464833, 1e-4); + + SG_VERIFY(strutils::parseStringAsGeod("") == false); + SG_VERIFY(strutils::parseStringAsGeod("aaaaaaaa") == false); +} + +void test_formatGeod() +{ + SGGeod a = SGGeod::fromDeg(-3.46, 55.45); + SG_CHECK_EQUAL(strutils::formatGeodAsString(a, strutils::LatLonFormat::SIGNED_DECIMAL_DEGREES), "55.450000,-3.460000"); + SG_CHECK_EQUAL(strutils::formatGeodAsString(a, strutils::LatLonFormat::DEGREES_MINUTES_SECONDS), + "55*27'00.0\"N,3*27'36.0\"W"); + +} + int main(int argc, char* argv[]) { test_strip(); @@ -639,6 +670,8 @@ int main(int argc, char* argv[]) test_propPathMatch(); test_readTime(); test_utf8Convert(); + test_parseGeod(); + test_formatGeod(); return EXIT_SUCCESS; } From 05e3c29ee43d56bcb0e5eaa2a1752cfbf78c51ec Mon Sep 17 00:00:00 2001 From: Scott Giese Date: Fri, 22 Jun 2018 22:19:49 -0500 Subject: [PATCH 12/25] Support Visual Studio 2017 --- CMakeLists.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b728f990..02c7cd2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,14 +167,22 @@ endif (MSVC) if (MSVC AND MSVC_3RDPARTY_ROOT) message(STATUS "3rdparty files located in ${MSVC_3RDPARTY_ROOT}") + string(SUBSTRING ${MSVC_VERSION} 0 2 MSVC_VERSION_MAJOR) + string(SUBSTRING ${MSVC_VERSION} 2 2 MSVC_VERSION_MINOR) + set( OSG_MSVC "msvc" ) - if (${MSVC_VERSION} EQUAL 1900) + if (${MSVC_VERSION_MAJOR} EQUAL "19") + if (${MSVC_VERSION_MINOR} EQUAL "00") set( OSG_MSVC ${OSG_MSVC}140 ) - elseif (${MSVC_VERSION} EQUAL 1800) - set( OSG_MSVC ${OSG_MSVC}120 ) + else () + set( OSG_MSVC ${OSG_MSVC}141 ) + endif () + elseif (${MSVC_VERSION_MAJOR} EQUAL "18") + set( OSG_MSVC ${OSG_MSVC}120 ) else () - message(FATAL_ERROR "Visual Studio 2013/2015 is required now") + message(FATAL_ERROR "Visual Studio 2013/15/17 is required") endif () + if (CMAKE_CL_64) set( OSG_MSVC ${OSG_MSVC}-64 ) set( MSVC_3RDPARTY_DIR 3rdParty.x64 ) From 123c597e016f337498751516430eb28efcb6d811 Mon Sep 17 00:00:00 2001 From: Scott Giese Date: Sat, 23 Jun 2018 14:44:48 -0500 Subject: [PATCH 13/25] msvc has no support plans beyond OpenMP 2.0 We must use a signed integral data type --- simgear/scene/dem/SGMesh.cxx | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/simgear/scene/dem/SGMesh.cxx b/simgear/scene/dem/SGMesh.cxx index 50e1eca8..2c4fef81 100644 --- a/simgear/scene/dem/SGMesh.cxx +++ b/simgear/scene/dem/SGMesh.cxx @@ -125,7 +125,7 @@ SGMesh::SGMesh( const SGDemPtr dem, ::std::vector geodes(grid_width*grid_height); - // session can't be paralell yet - save alts in geode array + // session can't be parallel yet - save alts in geode array fprintf( stderr, "SGMesh::SGMesh - create session - num dem roots is %d\n", dem->getNumRoots() ); SGDemSession s = dem->openSession( wo, so, eo, no, lvl, true ); s.getGeods( wo, so, eo, no, grid_width, grid_height, skipx, skipy, geodes, Debug1, Debug2 ); @@ -159,10 +159,10 @@ SGMesh::SGMesh( const SGDemPtr dem, index.push_back( src_idx ); } - // we can convert to cartesian in paralell - unsigned int nv = geodes.size(); + // we can convert to cartesian in parallel + long nv = geodes.size(); #pragma omp parallel for - for (unsigned int i = 0; i < nv; i++) { + for (long i = 0; i < nv; i++) { (*vertices)[i].set( toOsg( SGVec3f::fromGeod( geodes[i] ) ) ); } @@ -195,7 +195,7 @@ SGMesh::SGMesh( const SGDemPtr dem, // translate pos after normals computed #pragma omp parallel for - for ( unsigned int i=0; i < nv; i++ ) { + for ( long i=0; i < nv; i++ ) { (*vertices)[i].set( transform.preMult( (*vertices)[i]) ); } @@ -245,7 +245,7 @@ SGMesh::SGMesh( const SGDemPtr dem, osg::Geometry* geometry = new osg::Geometry; char geoName[64]; - snprintf( geoName, sizeof(geoName), "tilemesh (%u,%u)-(%u,%u):level%d,%d", + snprintf( geoName, sizeof(geoName), "tilemesh (%u,%u)-(%u,%u):level%u,%u", wo, so, eo, no, widthLevel, heightLevel ); geometry->setName(geoName); @@ -364,9 +364,9 @@ void SGMesh::need_normals() // need_faces(); if ( !faces.empty() ) { // Compute from faces - int nf = faces.size(); + long nf = faces.size(); #pragma omp parallel for - for (int i = 0; i < nf; i++) { + for (long i = 0; i < nf; i++) { const osg::Vec3 &p0 = (*vertices)[faces[i][0]]; const osg::Vec3 &p1 = (*vertices)[faces[i][1]]; const osg::Vec3 &p2 = (*vertices)[faces[i][2]]; @@ -384,9 +384,9 @@ void SGMesh::need_normals() } // Make them all unit-length - unsigned int nn = normals->size(); + long nn = normals->size(); #pragma omp parallel for - for (unsigned int i = 0; i < nn; i++) + for (long i = 0; i < nn; i++) (*normals)[i].normalize(); } } From 22c2971c3c0e8adaca544de78f78613fdc2231e7 Mon Sep 17 00:00:00 2001 From: James Turner Date: Sun, 24 Jun 2018 10:30:50 +0100 Subject: [PATCH 14/25] Lat-lon formatting: support nice degree symbols Allow Latin-1 (for PUI fnt) and UTF-8 (for Qt, osgText and everybody else) degree symbols as well as the use of * --- simgear/misc/strutils.cxx | 45 ++++++++++++++++++++++++---------- simgear/misc/strutils.hxx | 16 ++++++++++-- simgear/misc/strutils_test.cxx | 13 +++++++++- 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/simgear/misc/strutils.cxx b/simgear/misc/strutils.cxx index b0a75cbf..54b129d3 100644 --- a/simgear/misc/strutils.cxx +++ b/simgear/misc/strutils.cxx @@ -1196,12 +1196,24 @@ bool parseStringAsGeod(const std::string& s, SGGeod* result) return true; } -std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) +namespace { + const char* static_degreeSymbols[] = { + "*", + " ", + "\xB0", // Latin-1 B0 codepoint + "\xC2\xB0" // UTF-8 equivalent + }; +} // of anonymous namespace + +std::string formatLatLonValueAsString(double deg, LatLonFormat format, + char c, + DegreeSymbol degreeSymbol) { double min, sec; const int sign = deg < 0.0 ? -1 : 1; deg = fabs(deg); char buf[128]; + const char* degSym = static_degreeSymbols[static_cast(degreeSymbol)]; switch (format) { case LatLonFormat::DECIMAL_DEGREES: @@ -1217,7 +1229,7 @@ std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) min -= 60.0; deg += 1.0; } - snprintf(buf, sizeof(buf), "%d*%06.3f'%c", int(deg), fabs(min), c); + snprintf(buf, sizeof(buf), "%d%s%06.3f'%c", int(deg), degSym, fabs(min), c); break; case LatLonFormat::DEGREES_MINUTES_SECONDS: @@ -1234,7 +1246,8 @@ std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) deg += 1.0; } } - ::snprintf(buf, sizeof(buf), "%d*%02d'%04.1f\"%c", int(deg), int(min), fabs(sec), c); + ::snprintf(buf, sizeof(buf), "%d%s%02d'%04.1f\"%c", int(deg), degSym, + int(min), fabs(sec), c); break; case LatLonFormat::SIGNED_DECIMAL_DEGREES: @@ -1250,9 +1263,9 @@ std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) deg += 1.0; } if (sign == 1) { - snprintf(buf, sizeof(buf), "%d*%06.3f'", int(deg), fabs(min)); + snprintf(buf, sizeof(buf), "%d%s%06.3f'", int(deg), degSym, fabs(min)); } else { - snprintf(buf, sizeof(buf), "-%d*%06.3f'", int(deg), fabs(min)); + snprintf(buf, sizeof(buf), "-%d%s%06.3f'", int(deg), degSym, fabs(min)); } break; @@ -1270,9 +1283,9 @@ std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) } } if (sign == 1) { - snprintf(buf, sizeof(buf), "%d*%02d'%04.1f\"", int(deg), int(min), fabs(sec)); + snprintf(buf, sizeof(buf), "%d%s%02d'%04.1f\"", int(deg), degSym, int(min), fabs(sec)); } else { - snprintf(buf, sizeof(buf), "-%d*%02d'%04.1f\"", int(deg), int(min), fabs(sec)); + snprintf(buf, sizeof(buf), "-%d%s%02d'%04.1f\"", int(deg), degSym, int(min), fabs(sec)); } break; @@ -1293,9 +1306,9 @@ std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) deg += 1.0; } if (c == 'N' || c == 'S') { - snprintf(buf, sizeof(buf), "%02d*%06.3f'%c", int(deg), fabs(min), c); + snprintf(buf, sizeof(buf), "%02d%s%06.3f'%c", int(deg), degSym, fabs(min), c); } else { - snprintf(buf, sizeof(buf), "%03d*%06.3f'%c", int(deg), fabs(min), c); + snprintf(buf, sizeof(buf), "%03d%s%06.3f'%c", int(deg), degSym, fabs(min), c); } break; @@ -1312,9 +1325,9 @@ std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) } } if (c == 'N' || c == 'S') { - snprintf(buf, sizeof(buf), "%02d*%02d'%04.1f\"%c", int(deg), int(min), fabs(sec), c); + snprintf(buf, sizeof(buf), "%02d%s%02d'%04.1f\"%c", int(deg), degSym, int(min), fabs(sec), c); } else { - snprintf(buf, sizeof(buf), "%03d*%02d'%04.1f\"%c", int(deg), int(min), fabs(sec), c); + snprintf(buf, sizeof(buf), "%03d%s%02d'%04.1f\"%c", int(deg), degSym, int(min), fabs(sec), c); } break; @@ -1331,17 +1344,23 @@ std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c) snprintf(buf, sizeof(buf), "%03d* %02d'.%03d%c", int(deg), int(min), int(SGMisc::round((min-int(min))*1000)), c); } break; + + case LatLonFormat::DECIMAL_DEGREES_SYMBOL: + ::snprintf(buf, sizeof(buf), "%3.6f%s%c", deg, degSym, c); + break; } return std::string(buf); } -std::string formatGeodAsString(const SGGeod& geod, LatLonFormat format) +std::string formatGeodAsString(const SGGeod& geod, LatLonFormat format, + DegreeSymbol degreeSymbol) { const char ns = (geod.getLatitudeDeg() > 0.0) ? 'N' : 'S'; const char ew = (geod.getLongitudeDeg() > 0.0) ? 'E' : 'W'; - return formatLatLonValueAsString(geod.getLatitudeDeg(), format, ns) + "," + formatLatLonValueAsString(geod.getLongitudeDeg(), format, ew); + return formatLatLonValueAsString(geod.getLatitudeDeg(), format, ns, degreeSymbol) + "," + + formatLatLonValueAsString(geod.getLongitudeDeg(), format, ew, degreeSymbol); } } // end namespace strutils diff --git a/simgear/misc/strutils.hxx b/simgear/misc/strutils.hxx index e17244fe..cc735982 100644 --- a/simgear/misc/strutils.hxx +++ b/simgear/misc/strutils.hxx @@ -385,16 +385,28 @@ namespace simgear { ZERO_PAD_DEGREES_MINUTES, ZERO_PAD_DEGREES_MINUTES_SECONDS, TRINITY_HOUSE, ///< dd* mm'.mmm X, ddd* mm'.mmm X (Trinity House Navigation standard). + DECIMAL_DEGREES_SYMBOL ///< 88.4*N,4.54*W }; - std::string formatLatLonValueAsString(double deg, LatLonFormat format, char c); + enum class DegreeSymbol + { + ASTERISK = 0, + SPACE, + LATIN1_DEGREE, + UTF8_DEGREE + }; + + std::string formatLatLonValueAsString(double deg, + LatLonFormat format, char c, + DegreeSymbol degreeSymbol = DegreeSymbol::ASTERISK); /** * Format an SGGeod as a string according to the provided rule. * if the SGGeod is invalid (default constructed), will return an empty string */ std::string formatGeodAsString(const SGGeod& geod, - LatLonFormat format = LatLonFormat::DECIMAL_DEGREES); + LatLonFormat format = LatLonFormat::DECIMAL_DEGREES, + DegreeSymbol degreeSymbol = DegreeSymbol::ASTERISK); } // end namespace strutils } // end namespace simgear diff --git a/simgear/misc/strutils_test.cxx b/simgear/misc/strutils_test.cxx index c7c13dd2..8d17e0ed 100644 --- a/simgear/misc/strutils_test.cxx +++ b/simgear/misc/strutils_test.cxx @@ -647,7 +647,18 @@ void test_formatGeod() SG_CHECK_EQUAL(strutils::formatGeodAsString(a, strutils::LatLonFormat::SIGNED_DECIMAL_DEGREES), "55.450000,-3.460000"); SG_CHECK_EQUAL(strutils::formatGeodAsString(a, strutils::LatLonFormat::DEGREES_MINUTES_SECONDS), "55*27'00.0\"N,3*27'36.0\"W"); - + + const auto s = strutils::formatGeodAsString(a, + strutils::LatLonFormat::ZERO_PAD_DEGREES_MINUTES, + strutils::DegreeSymbol::LATIN1_DEGREE); + SG_CHECK_EQUAL(s, "55\xB0" "27.000'N,003\xB0" "27.600'W"); + + // Jakarta, if you care + SGGeod b = SGGeod::fromDeg(106.8278, -6.1568); + const auto s2 = strutils::formatGeodAsString(b, + strutils::LatLonFormat::DECIMAL_DEGREES_SYMBOL, + strutils::DegreeSymbol::UTF8_DEGREE); + SG_CHECK_EQUAL(s2, "6.156800\xC2\xB0S,106.827800\xC2\xB0" "E"); } int main(int argc, char* argv[]) From b1d6a41c65967b54912fd9621d581f051b1524a2 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 27 Jun 2018 14:29:26 +0100 Subject: [PATCH 15/25] =?UTF-8?q?Fix=20a=20bug=20in=20lat-lon=20parsing=20?= =?UTF-8?q?-=20accept=20=E2=80=98*=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, trailing * symbols confused the parser. --- simgear/misc/strutils.cxx | 14 ++++++++++---- simgear/misc/strutils_test.cxx | 10 ++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/simgear/misc/strutils.cxx b/simgear/misc/strutils.cxx index 54b129d3..cb51fff7 100644 --- a/simgear/misc/strutils.cxx +++ b/simgear/misc/strutils.cxx @@ -1137,7 +1137,7 @@ bool matchPropPathToTemplate(const std::string& path, const std::string& templat bool parseStringAsLatLonValue(const std::string& s, double& degrees) { string ss = simplify(s); - auto spacePos = ss.find(' '); + auto spacePos = ss.find_first_of(" *"); if (spacePos == std::string::npos) { degrees = std::stof(ss); @@ -1149,10 +1149,16 @@ bool parseStringAsLatLonValue(const std::string& s, double& degrees) // check for minutes marker auto quotePos = ss.find('\''); if (quotePos == std::string::npos) { - minutes = std::stof(ss.substr(spacePos)); + const auto minutesStr = ss.substr(spacePos+1); + if (!minutesStr.empty()) { + minutes = std::stof(minutesStr); + } } else { - minutes = std::stof(ss.substr(spacePos, quotePos - spacePos)); - seconds = std::stof(ss.substr(quotePos+1)); + minutes = std::stof(ss.substr(spacePos+1, quotePos - spacePos)); + const auto secondsStr = ss.substr(quotePos+1); + if (!secondsStr.empty()) { + seconds = std::stof(secondsStr); + } } if ((seconds < 0.0) || (minutes < 0.0)) { diff --git a/simgear/misc/strutils_test.cxx b/simgear/misc/strutils_test.cxx index 8d17e0ed..00fef42c 100644 --- a/simgear/misc/strutils_test.cxx +++ b/simgear/misc/strutils_test.cxx @@ -627,10 +627,20 @@ void test_parseGeod() SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.0); SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), 56.12, 1e-4); + // trailing degrees + SG_VERIFY(strutils::parseStringAsGeod("56.12*,-3.0*", &a)); + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.0); + SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), 56.12, 1e-4); + // embedded whitepace, DMS notation, NSEW notation SG_VERIFY(strutils::parseStringAsGeod("\t40 30'50\"S, 12 34'56\"W ", &a)); SG_CHECK_EQUAL_EP2(a.getLongitudeDeg(), -12.58222222, 1e-4); SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), -40.5138888, 1e-4); + + // embedded whitepace, DMS notation, NSEW notation, degrees symbol + SG_VERIFY(strutils::parseStringAsGeod("\t40*30'50\"S, 12*34'56\"W ", &a)); + SG_CHECK_EQUAL_EP2(a.getLongitudeDeg(), -12.58222222, 1e-4); + SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), -40.5138888, 1e-4); // signed degrees-minutes SG_VERIFY(strutils::parseStringAsGeod("-45 27.89,-12 34.56", &a)); From 609ac93c10316f1565933c874cc9191f55e19177 Mon Sep 17 00:00:00 2001 From: James Turner Date: Thu, 28 Jun 2018 22:43:30 +0100 Subject: [PATCH 16/25] Lat-lon parsing: accept lower case NSEW, fix precision Thanks to Wkitty for catching both problems. --- simgear/misc/strutils.cxx | 12 ++++++------ simgear/misc/strutils_test.cxx | 23 ++++++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/simgear/misc/strutils.cxx b/simgear/misc/strutils.cxx index cb51fff7..8c0ff464 100644 --- a/simgear/misc/strutils.cxx +++ b/simgear/misc/strutils.cxx @@ -1140,9 +1140,9 @@ bool parseStringAsLatLonValue(const std::string& s, double& degrees) auto spacePos = ss.find_first_of(" *"); if (spacePos == std::string::npos) { - degrees = std::stof(ss); + degrees = std::stod(ss); } else { - degrees = std::stof(ss.substr(0, spacePos)); + degrees = std::stod(ss.substr(0, spacePos)); double minutes = 0.0, seconds = 0.0; @@ -1151,13 +1151,13 @@ bool parseStringAsLatLonValue(const std::string& s, double& degrees) if (quotePos == std::string::npos) { const auto minutesStr = ss.substr(spacePos+1); if (!minutesStr.empty()) { - minutes = std::stof(minutesStr); + minutes = std::stod(minutesStr); } } else { - minutes = std::stof(ss.substr(spacePos+1, quotePos - spacePos)); + minutes = std::stod(ss.substr(spacePos+1, quotePos - spacePos)); const auto secondsStr = ss.substr(quotePos+1); if (!secondsStr.empty()) { - seconds = std::stof(secondsStr); + seconds = std::stod(secondsStr); } } @@ -1171,7 +1171,7 @@ bool parseStringAsLatLonValue(const std::string& s, double& degrees) } // since we simplified, any trailing N/S/E/W must be the last char - const char lastChar = ss.back(); + const char lastChar = ::toupper(ss.back()); if ((lastChar == 'W') || (lastChar == 'S')) { degrees = -degrees; } diff --git a/simgear/misc/strutils_test.cxx b/simgear/misc/strutils_test.cxx index 00fef42c..e321f13b 100644 --- a/simgear/misc/strutils_test.cxx +++ b/simgear/misc/strutils_test.cxx @@ -625,27 +625,32 @@ void test_parseGeod() SGGeod a; SG_VERIFY(strutils::parseStringAsGeod("56.12,-3.0", &a)); SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.0); - SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), 56.12, 1e-4); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), 56.12); + + SG_VERIFY(strutils::parseStringAsGeod("56.12345678s,3.12345678w", &a)); + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.12345678); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -56.12345678); + // trailing degrees SG_VERIFY(strutils::parseStringAsGeod("56.12*,-3.0*", &a)); SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.0); - SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), 56.12, 1e-4); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), 56.12); // embedded whitepace, DMS notation, NSEW notation SG_VERIFY(strutils::parseStringAsGeod("\t40 30'50\"S, 12 34'56\"W ", &a)); - SG_CHECK_EQUAL_EP2(a.getLongitudeDeg(), -12.58222222, 1e-4); - SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), -40.5138888, 1e-4); - + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.58222222); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -40.5138888); + // embedded whitepace, DMS notation, NSEW notation, degrees symbol SG_VERIFY(strutils::parseStringAsGeod("\t40*30'50\"S, 12*34'56\"W ", &a)); - SG_CHECK_EQUAL_EP2(a.getLongitudeDeg(), -12.58222222, 1e-4); - SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), -40.5138888, 1e-4); + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.58222222); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -40.5138888); // signed degrees-minutes SG_VERIFY(strutils::parseStringAsGeod("-45 27.89,-12 34.56", &a)); - SG_CHECK_EQUAL_EP2(a.getLongitudeDeg(), -12.576, 1e-4); - SG_CHECK_EQUAL_EP2(a.getLatitudeDeg(), -45.464833, 1e-4); + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.576); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -45.464833333); SG_VERIFY(strutils::parseStringAsGeod("") == false); SG_VERIFY(strutils::parseStringAsGeod("aaaaaaaa") == false); From d7a413d5e74703640aa6a6f74f220d8dd50f239b Mon Sep 17 00:00:00 2001 From: James Turner Date: Fri, 29 Jun 2018 13:28:02 +0100 Subject: [PATCH 17/25] Lat-lon parsing: allow lon,lat order, and detect order Flightplan/route-manager defaults to lon,lat order. Allow the parser to detect the correct order when NSEW suffixes are provided and consistent. --- simgear/misc/strutils.cxx | 37 ++++++++++++++++++++++++++++------ simgear/misc/strutils.hxx | 13 +++++++++++- simgear/misc/strutils_test.cxx | 22 ++++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/simgear/misc/strutils.cxx b/simgear/misc/strutils.cxx index 8c0ff464..1de78be5 100644 --- a/simgear/misc/strutils.cxx +++ b/simgear/misc/strutils.cxx @@ -1179,7 +1179,21 @@ bool parseStringAsLatLonValue(const std::string& s, double& degrees) return true; } -bool parseStringAsGeod(const std::string& s, SGGeod* result) +namespace { + bool isLatString(const std::string &s) + { + const char lastChar = ::toupper(s.back()); + return (lastChar == 'N') || (lastChar == 'S'); + } + + bool isLonString(const std::string &s) + { + const char lastChar = ::toupper(s.back()); + return (lastChar == 'E') || (lastChar == 'W'); + } +} // of anonymous namespace + +bool parseStringAsGeod(const std::string& s, SGGeod* result, bool assumeLonLatOrder) { if (s.empty()) return false; @@ -1189,16 +1203,27 @@ bool parseStringAsGeod(const std::string& s, SGGeod* result) return false; } - double lat, lon; - if (!parseStringAsLatLonValue(s.substr(0, commaPos), lat) || - !parseStringAsLatLonValue(s.substr(commaPos+1), lon)) - { + auto termA = simplify(s.substr(0, commaPos)), + termB = simplify(s.substr(commaPos+1)); + double valueA, valueB; + if (!parseStringAsLatLonValue(termA, valueA) || !parseStringAsLatLonValue(termB, valueB)) { return false; } if (result) { - *result = SGGeod::fromDeg(lon, lat); + // explicit ordering + if (isLatString(termA) && isLonString(termB)) { + *result = SGGeod::fromDeg(valueB, valueA); + } else if (isLonString(termA) && isLatString(termB)) { + *result = SGGeod::fromDeg(valueA, valueB); + } else { + // implicit ordering + // SGGeod wants longitude, latitude + *result = assumeLonLatOrder ? SGGeod::fromDeg(valueA, valueB) + : SGGeod::fromDeg(valueB, valueA); + } } + return true; } diff --git a/simgear/misc/strutils.hxx b/simgear/misc/strutils.hxx index cc735982..f4b14558 100644 --- a/simgear/misc/strutils.hxx +++ b/simgear/misc/strutils.hxx @@ -367,8 +367,19 @@ namespace simgear { * * Supported formats: * , + * [NS],[EW] + * *'[NS],*'[EW] + * + * Latitude and longitude are parsed seperately so the formats for each + * do not need to agree. Latitude is assumed to precede longitude + * unless assumeLonLatOrder = true + * + * When NSEW characters are used, the order can be swapped and will be + * fixed correctly (longitude then latitude). */ - bool parseStringAsGeod(const std::string& string, SGGeod* result = nullptr); + bool parseStringAsGeod(const std::string& string, + SGGeod* result = nullptr, + bool assumeLonLatOrder = false); // enum values here correspond to existing lon-lat format codes inside // FlightGear (property: /sim/lon-lat-format ) diff --git a/simgear/misc/strutils_test.cxx b/simgear/misc/strutils_test.cxx index e321f13b..07ff8c1c 100644 --- a/simgear/misc/strutils_test.cxx +++ b/simgear/misc/strutils_test.cxx @@ -654,6 +654,28 @@ void test_parseGeod() SG_VERIFY(strutils::parseStringAsGeod("") == false); SG_VERIFY(strutils::parseStringAsGeod("aaaaaaaa") == false); + + // ordering tests + + // normal default order, but explicitly pass as lon,lat + // (should work) + SG_VERIFY(strutils::parseStringAsGeod("3.12345678w, 56.12345678s", &a)); + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.12345678); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -56.12345678); + + + // different default order + // also some embedded whitespace for fun + SG_VERIFY(strutils::parseStringAsGeod(" -12 34.56,\n-45 27.89 ", &a, true)); + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.576); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -45.464833333); + + + // differnet default order, but still set explicitly so should + // use the lat,lon order + SG_VERIFY(strutils::parseStringAsGeod("\t40 30'50\"S, 12 34'56\"W ", &a, true)); + SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.58222222); + SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -40.5138888); } void test_formatGeod() From 4e8df9dcc82ab4785f918d5191c27eaf6b68be36 Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Wed, 4 Jul 2018 09:33:32 +0200 Subject: [PATCH 18/25] Remove a bunch og gcc-8 warnings --- simgear/math/SGMatrix.hxx | 4 ++-- simgear/math/SGVec2.hxx | 4 ++-- simgear/math/SGVec3.hxx | 4 ++-- simgear/math/SGVec4.hxx | 4 ++-- simgear/math/simd.hxx | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/simgear/math/SGMatrix.hxx b/simgear/math/SGMatrix.hxx index 3792185d..a1d5690b 100644 --- a/simgear/math/SGMatrix.hxx +++ b/simgear/math/SGMatrix.hxx @@ -142,10 +142,10 @@ public: T (&sg(void))[4][4] { return _data.ptr(); } /// Readonly raw storage interface - const simd4x4_t (&simd4x4(void) const) + const simd4x4_t &simd4x4(void) const { return _data; } /// Readonly raw storage interface - simd4x4_t (&simd4x4(void)) + simd4x4_t &simd4x4(void) { return _data; } diff --git a/simgear/math/SGVec2.hxx b/simgear/math/SGVec2.hxx index 21746a39..93655cb1 100644 --- a/simgear/math/SGVec2.hxx +++ b/simgear/math/SGVec2.hxx @@ -87,10 +87,10 @@ public: /// Access raw data T (&data(void))[2] { return _data.ptr(); } - const simd4_t (&simd2(void) const) + const simd4_t &simd2(void) const { return _data; } /// Readonly raw storage interface - simd4_t (&simd2(void)) + simd4_t &simd2(void) { return _data; } /// Inplace addition diff --git a/simgear/math/SGVec3.hxx b/simgear/math/SGVec3.hxx index 720a13a0..514f9575 100644 --- a/simgear/math/SGVec3.hxx +++ b/simgear/math/SGVec3.hxx @@ -106,10 +106,10 @@ public: T (&data(void))[3] { return _data.ptr(); } /// Readonly raw storage interface - const simd4_t (&simd3(void) const) + const simd4_t &simd3(void) const { return _data; } /// Readonly raw storage interface - simd4_t (&simd3(void)) + simd4_t &simd3(void) { return _data; } /// Inplace addition diff --git a/simgear/math/SGVec4.hxx b/simgear/math/SGVec4.hxx index 24cef2bb..b62561ab 100644 --- a/simgear/math/SGVec4.hxx +++ b/simgear/math/SGVec4.hxx @@ -99,10 +99,10 @@ public: T (&data(void))[4] { return _data.ptr(); } /// Readonly raw storage interface - const simd4_t (&simd4(void) const) + const simd4_t &simd4(void) const { return _data; } /// Readonly raw storage interface - simd4_t (&simd4(void)) + simd4_t &simd4(void) { return _data; } /// Inplace addition diff --git a/simgear/math/simd.hxx b/simgear/math/simd.hxx index dd584732..a2847775 100644 --- a/simgear/math/simd.hxx +++ b/simgear/math/simd.hxx @@ -352,10 +352,10 @@ public: simd4_t(const simd4_t& v) { simd4 = v.v4(); } simd4_t(const __m128& v) { simd4 = v; } - inline const __m128 (&v4(void) const) { + inline const __m128 &v4(void) const { return simd4; } - inline __m128 (&v4(void)) { + inline __m128 &v4(void) { return simd4; } @@ -1120,11 +1120,11 @@ public: simd4_t(const simd4_t& v) { simd4 = v.v4(); } simd4_t(const __m128i& v) { simd4 = v; } - inline __m128i (&v4(void)) { + inline __m128i &v4(void) { return simd4; } - inline const __m128i (&v4(void) const) { + inline const __m128i &v4(void) const { return simd4; } From 3e081ae8699306a27107e560fdee657262cfca87 Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Wed, 4 Jul 2018 09:35:45 +0200 Subject: [PATCH 19/25] Use CPU optimized versions of endian swap functions when available --- simgear/misc/stdint.hxx | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/simgear/misc/stdint.hxx b/simgear/misc/stdint.hxx index 94fbd60a..49e5817d 100644 --- a/simgear/misc/stdint.hxx +++ b/simgear/misc/stdint.hxx @@ -56,21 +56,42 @@ typedef int ssize_t; inline uint16_t sg_bswap_16(uint16_t x) { +#if defined(__llvm__) || \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && !defined(__ICC) + return __builtin_bswap16(x); +#elif defined(_MSC_VER) && !defined(_DEBUG) + return _byteswap_ushort(x); +#else x = (x >> 8) | (x << 8); return x; +#endif } inline uint32_t sg_bswap_32(uint32_t x) { +#if defined(__llvm__) || \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && !defined(__ICC) + return __builtin_bswap32(x); +#elif defined(_MSC_VER) && !defined(_DEBUG) + return _byteswap_ulong(x); +#else x = ((x >> 8) & 0x00FF00FFL) | ((x << 8) & 0xFF00FF00L); x = (x >> 16) | (x << 16); return x; +#endif } inline uint64_t sg_bswap_64(uint64_t x) { +#if defined(__llvm__) || \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && !defined(__ICC) + return __builtin_bswap64(x); +#elif defined(_MSC_VER) && !defined(_DEBUG) + return _byteswap_uint64(x); +#else x = ((x >> 8) & 0x00FF00FF00FF00FFLL) | ((x << 8) & 0xFF00FF00FF00FF00LL); x = ((x >> 16) & 0x0000FFFF0000FFFFLL) | ((x << 16) & 0xFFFF0000FFFF0000LL); x = (x >> 32) | (x << 32); return x; +#endif } From 126f69434bca204e89721684867109d6a11e217e Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 4 Jul 2018 10:12:37 +0100 Subject: [PATCH 20/25] Refactor untar/unzip code This is to enable reuse in more places, especially scenery extraction and TerraSync --- simgear/io/test_untar.cxx | 80 +++++- simgear/io/untar.cxx | 532 ++++++++++++++++++++++++++---------- simgear/io/untar.hxx | 58 +++- simgear/io/zippy.zip | Bin 0 -> 811 bytes simgear/package/Install.cxx | 176 +----------- 5 files changed, 519 insertions(+), 327 deletions(-) create mode 100644 simgear/io/zippy.zip diff --git a/simgear/io/test_untar.cxx b/simgear/io/test_untar.cxx index 92c4ce29..b98b26e7 100644 --- a/simgear/io/test_untar.cxx +++ b/simgear/io/test_untar.cxx @@ -11,6 +11,7 @@ #include #include +#include #include @@ -31,7 +32,7 @@ void testTarGz() uint8_t* buf = (uint8_t*) alloca(8192); size_t bufSize = f.read((char*) buf, 8192); - SG_VERIFY(TarExtractor::isTarData(buf, bufSize)); + SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData); f.close(); } @@ -44,18 +45,89 @@ void testPlainTar() SGBinaryFile f(p); f.open(SG_IO_IN); - uint8_t* buf = (uint8_t*) alloca(8192); - size_t bufSize = f.read((char*) buf, 8192); + uint8_t* buf = (uint8_t*)alloca(8192); + size_t bufSize = f.read((char*) buf, 8192); - SG_VERIFY(TarExtractor::isTarData(buf, bufSize)); + SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData); f.close(); } +void testExtractStreamed() +{ + SGPath p = SGPath(SRC_DIR); + p.append("test.tar.gz"); + + SGBinaryFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_extract_streamed"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*) alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*) buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + + SG_VERIFY((extractDir / "testDir/hello.c").exists()); + SG_VERIFY((extractDir / "testDir/foo.txt").exists()); +} + +void testExtractLocalFile() +{ + +} + +void testExtractZip() +{ + SGPath p = SGPath(SRC_DIR); + p.append("zippy.zip"); + + SGBinaryFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_extract_zip"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*)alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*)buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + + SG_VERIFY((extractDir / "zippy/dirA/hello.c").exists()); + SG_VERIFY((extractDir / "zippy/bar.xml").exists()); + SG_VERIFY((extractDir / "zippy/long-named.json").exists()); +} + +void testPAXAttributes() +{ + +} + int main(int ac, char ** av) { testTarGz(); testPlainTar(); + testExtractStreamed(); + testExtractZip(); + + std::cout << "all tests passed" << std::endl; return 0; } diff --git a/simgear/io/untar.cxx b/simgear/io/untar.cxx index e6c8a2e8..be07863e 100644 --- a/simgear/io/untar.cxx +++ b/simgear/io/untar.cxx @@ -31,12 +31,78 @@ #include #include #include - +#include #include +#include +#include namespace simgear { + class ArchiveExtractorPrivate + { + public: + ArchiveExtractorPrivate(ArchiveExtractor* o) : + outer(o) + { + assert(outer); + } + + typedef enum { + INVALID = 0, + READING_HEADER, + READING_FILE, + READING_PADDING, + PRE_END_OF_ARCHVE, + END_OF_ARCHIVE, + ERROR_STATE, ///< states above this are error conditions + BAD_ARCHIVE, + BAD_DATA, + FILTER_STOPPED + } State; + + State state = INVALID; + ArchiveExtractor* outer = nullptr; + + virtual void extractBytes(const uint8_t* bytes, size_t count) = 0; + + virtual void flush() = 0; + + SGPath extractRootPath() + { + return outer->_rootPath; + } + + ArchiveExtractor::PathResult filterPath(std::string& pathToExtract) + { + return outer->filterPath(pathToExtract); + } + + + bool isSafePath(const std::string& p) const + { + if (p.empty()) { + return false; + } + + // reject absolute paths + if (p.at(0) == '/') { + return false; + } + + // reject paths containing '..' + size_t doubleDot = p.find(".."); + if (doubleDot != std::string::npos) { + return false; + } + + // on POSIX could use realpath to sanity check + return true; + } + }; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024; const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS; const int ZLIB_DECODE_GZIP_HEADER = 16; @@ -81,24 +147,14 @@ typedef struct #define FIFOTYPE '6' /* FIFO special */ #define CONTTYPE '7' /* reserved */ -class TarExtractorPrivate + const char PAX_GLOBAL_ATTRIBUTES = 'g'; + const char PAX_FILE_ATTRIBUTES = 'x'; + + +class TarExtractorPrivate : public ArchiveExtractorPrivate { public: - typedef enum { - INVALID = 0, - READING_HEADER, - READING_FILE, - READING_PADDING, - PRE_END_OF_ARCHVE, - END_OF_ARCHIVE, - ERROR_STATE, ///< states above this are error conditions - BAD_ARCHIVE, - BAD_DATA, - FILTER_STOPPED - } State; - - SGPath path; - State state; + union { UstarHeaderBlock header; uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE]; @@ -109,17 +165,20 @@ public: size_t currentFileSize; z_stream zlibStream; uint8_t* zlibOutput; - bool haveInitedZLib; - bool uncompressedData; // set if reading a plain .tar (not tar.gz) + bool haveInitedZLib = false; + bool uncompressedData = false; // set if reading a plain .tar (not tar.gz) uint8_t* headerPtr; - TarExtractor* outer; bool skipCurrentEntry = false; - TarExtractorPrivate(TarExtractor* o) : - haveInitedZLib(false), - uncompressedData(false), - outer(o) + TarExtractorPrivate(ArchiveExtractor* o) : + ArchiveExtractorPrivate(o) { + memset(&zlibStream, 0, sizeof(z_stream)); + zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE); + zlibStream.zalloc = Z_NULL; + zlibStream.zfree = Z_NULL; + zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; + zlibStream.next_out = zlibOutput; } ~TarExtractorPrivate() @@ -168,6 +227,77 @@ public: state = newState; } + void extractBytes(const uint8_t* bytes, size_t count) override + { + zlibStream.next_in = (uint8_t*) bytes; + zlibStream.avail_in = count; + + if (!haveInitedZLib) { + // now we have data, see if we're dealing with GZ-compressed data or not + if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) { + // GZIP identification bytes + if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) { + SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed"); + state = TarExtractorPrivate::BAD_DATA; + return; + } + } else { + UstarHeaderBlock* header = (UstarHeaderBlock*)bytes; + if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) { + SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header"); + state = TarExtractorPrivate::BAD_DATA; + return; + } + + uncompressedData = true; + } + + haveInitedZLib = true; + setState(TarExtractorPrivate::READING_HEADER); + } // of init on first-bytes case + + if (uncompressedData) { + processBytes((const char*) bytes, count); + } else { + size_t writtenSize; + // loop, running zlib() inflate and sending output bytes to + // our request body handler. Keep calling inflate until no bytes are + // written, and ZLIB has consumed all available input + do { + zlibStream.next_out = zlibOutput; + zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; + int result = inflate(&zlibStream, Z_NO_FLUSH); + if (result == Z_OK || result == Z_STREAM_END) { + // nothing to do + + } + else if (result == Z_BUF_ERROR) { + // transient error, fall through + } + else { + // _error = result; + SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg); + state = TarExtractorPrivate::BAD_DATA; + return; + } + + writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out; + if (writtenSize > 0) { + processBytes((const char*) zlibOutput, writtenSize); + } + + if (result == Z_STREAM_END) { + break; + } + } while ((zlibStream.avail_in > 0) || (writtenSize > 0)); + } // of Zlib-compressed data + } + + void flush() override + { + // no-op for tar files, we process everything greedily + } + void processHeader() { if (headerIsAllZeros()) { @@ -180,7 +310,7 @@ public: } if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) { - SG_LOG(SG_IO, SG_WARN, "magic is wrong"); + SG_LOG(SG_IO, SG_WARN, "Untar: magic is wrong"); state = BAD_ARCHIVE; return; } @@ -193,15 +323,15 @@ public: skipCurrentEntry = true; } - auto result = outer->filterPath(tarPath); - if (result == TarExtractor::Stop) { + auto result = filterPath(tarPath); + if (result == ArchiveExtractor::Stop) { setState(FILTER_STOPPED); return; - } else if (result == TarExtractor::Skipped) { + } else if (result == ArchiveExtractor::Skipped) { skipCurrentEntry = true; } - SGPath p = path / tarPath; + SGPath p = extractRootPath() / tarPath; if (header.typeflag == DIRTYPE) { if (!skipCurrentEntry) { Dir dir(p); @@ -260,133 +390,247 @@ public: return true; } - - bool isSafePath(const std::string& p) const - { - if (p.empty()) { - return false; - } - - // reject absolute paths - if (p.at(0) == '/') { - return false; - } - - // reject paths containing '..' - size_t doubleDot = p.find(".."); - if (doubleDot != std::string::npos) { - return false; - } - - // on POSIX could use realpath to sanity check - return true; - } }; -TarExtractor::TarExtractor(const SGPath& rootPath) : - d(new TarExtractorPrivate(this)) -{ +/////////////////////////////////////////////////////////////////////////////// - d->path = rootPath; - d->state = TarExtractorPrivate::INVALID; - - memset(&d->zlibStream, 0, sizeof(z_stream)); - d->zlibOutput = (unsigned char*) malloc(ZLIB_DECOMPRESS_BUFFER_SIZE); - d->zlibStream.zalloc = Z_NULL; - d->zlibStream.zfree = Z_NULL; - - d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; - d->zlibStream.next_out = d->zlibOutput; +extern "C" { + void fill_memory_filefunc(zlib_filefunc_def*); } -TarExtractor::~TarExtractor() +class ZipExtractorPrivate : public ArchiveExtractorPrivate +{ +public: + std::string m_buffer; + + ZipExtractorPrivate(ArchiveExtractor* outer) : + ArchiveExtractorPrivate(outer) + { + + } + + ~ZipExtractorPrivate() + { + + } + + void extractBytes(const uint8_t* bytes, size_t count) override + { + // becuase the .zip central directory is at the end of the file, + // we have no choice but to simply buffer bytes here until flush() + // is called + m_buffer.append((const char*) bytes, count); + } + + void flush() override + { + zlib_filefunc_def memoryAccessFuncs; + fill_memory_filefunc(&memoryAccessFuncs); + + char bufferName[128]; + ::snprintf(bufferName, 128, "%p+%llx", m_buffer.data(), m_buffer.size()); + unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs); + + const size_t BUFFER_SIZE = 32 * 1024; + void* buf = malloc(BUFFER_SIZE); + + try { + int result = unzGoToFirstFile(zip); + if (result != UNZ_OK) { + throw sg_exception("failed to go to first file in archive"); + } + + while (true) { + extractCurrentFile(zip, (char*)buf, BUFFER_SIZE); + if (state == FILTER_STOPPED) { + break; + } + + result = unzGoToNextFile(zip); + if (result == UNZ_END_OF_LIST_OF_FILE) { + break; + } + else if (result != UNZ_OK) { + throw sg_io_exception("failed to go to next file in the archive"); + } + } + state = END_OF_ARCHIVE; + } + catch (sg_exception&) { + state = BAD_ARCHIVE; + } + + free(buf); + unzClose(zip); + } + + void extractCurrentFile(unzFile zip, char* buffer, size_t bufferSize) + { + unz_file_info fileInfo; + unzGetCurrentFileInfo(zip, &fileInfo, + buffer, bufferSize, + NULL, 0, /* extra field */ + NULL, 0 /* comment field */); + + std::string name(buffer); + if (!isSafePath(name)) { + throw sg_format_exception("Bad zip path", name); + } + + auto filterResult = filterPath(name); + if (filterResult == ArchiveExtractor::Stop) { + state = FILTER_STOPPED; + return; + } + else if (filterResult == ArchiveExtractor::Skipped) { + return; + } + + if (fileInfo.uncompressed_size == 0) { + // assume it's a directory for now + // since we create parent directories when extracting + // a path, we're done here + return; + } + + int result = unzOpenCurrentFile(zip); + if (result != UNZ_OK) { + throw sg_io_exception("opening current zip file failed", sg_location(name)); + } + + sg_ofstream outFile; + bool eof = false; + SGPath path = extractRootPath() / name; + + // create enclosing directory heirarchy as required + Dir parentDir(path.dir()); + if (!parentDir.exists()) { + bool ok = parentDir.create(0755); + if (!ok) { + throw sg_io_exception("failed to create directory heirarchy for extraction", path); + } + } + + outFile.open(path, std::ios::binary | std::ios::trunc | std::ios::out); + if (outFile.fail()) { + throw sg_io_exception("failed to open output file for writing", path); + } + + while (!eof) { + int bytes = unzReadCurrentFile(zip, buffer, bufferSize); + if (bytes < 0) { + throw sg_io_exception("unzip failure reading curent archive", sg_location(name)); + } + else if (bytes == 0) { + eof = true; + } + else { + outFile.write(buffer, bytes); + } + } + + outFile.close(); + unzCloseCurrentFile(zip); + } +}; + +////////////////////////////////////////////////////////////////////////////// + +ArchiveExtractor::ArchiveExtractor(const SGPath& rootPath) : + _rootPath(rootPath) +{ +} + +ArchiveExtractor::~ArchiveExtractor() { } -void TarExtractor::extractBytes(const char* bytes, size_t count) +void ArchiveExtractor::extractBytes(const uint8_t* bytes, size_t count) { - if (d->state >= TarExtractorPrivate::ERROR_STATE) { + if (!d) { + _prebuffer.append((char*) bytes, count); + auto r = determineType((uint8_t*) _prebuffer.data(), _prebuffer.size()); + if (r == InsufficientData) { + return; + } + + if (r == TarData) { + d.reset(new TarExtractorPrivate(this)); + } + else if (r == ZipData) { + d.reset(new ZipExtractorPrivate(this)); + } + else { + SG_LOG(SG_IO, SG_ALERT, "Invcalid archive type"); + _invalidDataType = true; + return; + } + + // if hit here, we created the extractor. Feed the prefbuffer + // bytes through it + d->extractBytes((uint8_t*) _prebuffer.data(), _prebuffer.size()); + _prebuffer.clear(); + return; + } + + if (d->state >= ArchiveExtractorPrivate::ERROR_STATE) { return; } - d->zlibStream.next_in = (uint8_t*) bytes; - d->zlibStream.avail_in = count; - - if (!d->haveInitedZLib) { - // now we have data, see if we're dealing with GZ-compressed data or not - uint8_t* ubytes = (uint8_t*) bytes; - if ((ubytes[0] == 0x1f) && (ubytes[1] == 0x8b)) { - // GZIP identification bytes - if (inflateInit2(&d->zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) { - SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed"); - d->state = TarExtractorPrivate::BAD_DATA; - return; - } - } else { - UstarHeaderBlock* header = (UstarHeaderBlock*) bytes; - if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) { - SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header"); - d->state = TarExtractorPrivate::BAD_DATA; - return; - } - - d->uncompressedData = true; - } - - d->haveInitedZLib = true; - d->setState(TarExtractorPrivate::READING_HEADER); - } // of init on first-bytes case - - if (d->uncompressedData) { - d->processBytes(bytes, count); - } else { - size_t writtenSize; - // loop, running zlib() inflate and sending output bytes to - // our request body handler. Keep calling inflate until no bytes are - // written, and ZLIB has consumed all available input - do { - d->zlibStream.next_out = d->zlibOutput; - d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; - int result = inflate(&d->zlibStream, Z_NO_FLUSH); - if (result == Z_OK || result == Z_STREAM_END) { - // nothing to do - - } else if (result == Z_BUF_ERROR) { - // transient error, fall through - } else { - // _error = result; - SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << d->zlibStream.msg); - d->state = TarExtractorPrivate::BAD_DATA; - return; - } - - writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - d->zlibStream.avail_out; - if (writtenSize > 0) { - d->processBytes((const char*) d->zlibOutput, writtenSize); - } - - if (result == Z_STREAM_END) { - break; - } - } while ((d->zlibStream.avail_in > 0) || (writtenSize > 0)); - } // of Zlib-compressed data + d->extractBytes(bytes, count); } -bool TarExtractor::isAtEndOfArchive() const +void ArchiveExtractor::flush() { - return (d->state == TarExtractorPrivate::END_OF_ARCHIVE); + if (!d) + return; + + d->flush(); } -bool TarExtractor::hasError() const +bool ArchiveExtractor::isAtEndOfArchive() const { - return (d->state >= TarExtractorPrivate::ERROR_STATE); + if (!d) + return false; + + return (d->state == ArchiveExtractorPrivate::END_OF_ARCHIVE); } -bool TarExtractor::isTarData(const uint8_t* bytes, size_t count) +bool ArchiveExtractor::hasError() const +{ + if (_invalidDataType) + return true; + + if (!d) + return false; + + return (d->state >= ArchiveExtractorPrivate::ERROR_STATE); +} + +ArchiveExtractor::DetermineResult ArchiveExtractor::determineType(const uint8_t* bytes, size_t count) +{ + // check for ZIP + if (count < 4) { + return InsufficientData; + } + + if (memcmp(bytes, "PK\x03\x04", 4) == 0) { + return ZipData; + } + + auto r = isTarData(bytes, count); + if ((r == TarData) || (r == InsufficientData)) + return r; + + return Invalid; +} + + +ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* bytes, size_t count) { if (count < 2) { - return false; + return InsufficientData; } UstarHeaderBlock* header = 0; @@ -404,21 +648,20 @@ bool TarExtractor::isTarData(const uint8_t* bytes, size_t count) if (inflateInit2(&z, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) { inflateEnd(&z); - return false; + return Invalid; } int result = inflate(&z, Z_SYNC_FLUSH); if (result != Z_OK) { SG_LOG(SG_IO, SG_WARN, "inflate failed:" << result); inflateEnd(&z); - return false; // not tar data + return Invalid; // not tar data } size_t written = 4096 - z.avail_out; if (written < TAR_HEADER_BLOCK_SIZE) { - SG_LOG(SG_IO, SG_WARN, "insufficient data for header"); inflateEnd(&z); - return false; + return InsufficientData; } header = reinterpret_cast(zlibOutput); @@ -426,22 +669,25 @@ bool TarExtractor::isTarData(const uint8_t* bytes, size_t count) } else { // uncompressed tar if (count < TAR_HEADER_BLOCK_SIZE) { - SG_LOG(SG_IO, SG_WARN, "insufficient data for header"); - return false; + return InsufficientData; } header = (UstarHeaderBlock*) bytes; } if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) { - SG_LOG(SG_IO, SG_WARN, "not a tar file"); - return false; + return Invalid; } - return true; + return TarData; } -auto TarExtractor::filterPath(std::string& pathToExtract) +void ArchiveExtractor::extractLocalFile(const SGPath& archiveFile) +{ + +} + +auto ArchiveExtractor::filterPath(std::string& pathToExtract) -> PathResult { SG_UNUSED(pathToExtract); diff --git a/simgear/io/untar.hxx b/simgear/io/untar.hxx index 0d963df2..1b3d9ccc 100644 --- a/simgear/io/untar.hxx +++ b/simgear/io/untar.hxx @@ -21,40 +21,68 @@ #include #include -#include // for uint8_t +#include + #include namespace simgear { -class TarExtractorPrivate; +class ArchiveExtractorPrivate; -class TarExtractor +class ArchiveExtractor { public: - TarExtractor(const SGPath& rootPath); - ~TarExtractor(); + ArchiveExtractor(const SGPath& rootPath); + ~ArchiveExtractor(); - static bool isTarData(const uint8_t* bytes, size_t count); + enum DetermineResult + { + Invalid, + InsufficientData, + TarData, + ZipData + }; - void extractBytes(const char* bytes, size_t count); + static DetermineResult determineType(const uint8_t* bytes, size_t count); + + /** + * @brief API to extract a local zip or tar.gz + */ + void extractLocalFile(const SGPath& archiveFile); + + /** + * @brief API to extract from memory - this can be called multiple + * times for streamking from a network socket etc + */ + void extractBytes(const uint8_t* bytes, size_t count); + + void flush(); bool isAtEndOfArchive() const; bool hasError() const; + enum PathResult { + Accepted, + Skipped, + Modified, + Stop + }; + protected: - enum PathResult { - Accepted, - Skipped, - Modified, - Stop - }; + virtual PathResult filterPath(std::string& pathToExtract); private: - friend class TarExtractorPrivate; - std::unique_ptr d; + static DetermineResult isTarData(const uint8_t* bytes, size_t count); + + friend class ArchiveExtractorPrivate; + std::unique_ptr d; + + SGPath _rootPath; + std::string _prebuffer; // store bytes before type is determined + bool _invalidDataType = false; }; } // of namespace simgear diff --git a/simgear/io/zippy.zip b/simgear/io/zippy.zip new file mode 100644 index 0000000000000000000000000000000000000000..5c0722d5d1b571e581c705d726d9e24aa7eea141 GIT binary patch literal 811 zcmWIWW@Zs#0D-659{Ye9P=XCeS7jCyRO$x+)o_7D9|6_Kw@WAR0(meDQInKdq*sxf zW0O&mn`39A52ir+AZEWrGn*T3c1mWEBa#L0feQO%{9ft-6@ai1s=|!aoSb~UWL~b! zyb^`n#LPSm5F@cDJy}N~IU})13&bvqj@9Jls^;ZV00LgFT7>g&08L+Cq9?BaG!=wJ zff(Zaocz3W-Mqxy)D*p};`}_QW~D?WD-$Cfh)GIGN>;{th9-t!BN>_O8E}Q03eZ>( z5P(onE4aW`GB7YQNH9G6DAgn(e1#E4Z%TR@4B`iPgUrPh)-ZDwfaZeOC?St*{FQi> zdXRDJi%umV(-7lvMG4G!kh5dp#)BgW+3@BgciKRP!f2Qukll_MjWEL%fCjgs8V-$A zWIN #include -extern "C" { - void fill_memory_filefunc (zlib_filefunc_def*); -} - namespace simgear { namespace pkg { @@ -57,9 +53,9 @@ public: throw sg_exception("no package download URLs"); } - if (m_owner->package()->properties()->hasChild("archive-type")) { - setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type")); - } + // if (m_owner->package()->properties()->hasChild("archive-type")) { + // setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type")); + //} // TODO randomise order of m_urls @@ -110,17 +106,17 @@ protected: Dir d(m_extractPath); d.create(0755); + m_extractor.reset(new ArchiveExtractor(m_extractPath)); memset(&m_md5, 0, sizeof(SG_MD5_CTX)); SG_MD5Init(&m_md5); } virtual void gotBodyData(const char* s, int n) { - m_buffer += std::string(s, n); - SG_MD5Update(&m_md5, (unsigned char*) s, n); - - m_downloaded = m_buffer.size(); - m_owner->installProgress(m_buffer.size(), responseLength()); + const uint8_t* ubytes = (uint8_t*) s; + SG_MD5Update(&m_md5, ubytes, n); + m_owner->installProgress(m_downloaded, responseLength()); + m_extractor->extractBytes(ubytes, n); } virtual void onDone() @@ -151,7 +147,8 @@ protected: return; } - if (!extract()) { + m_extractor->flush(); + if (m_extractor->hasError() || !m_extractor->isAtEndOfArchive()) { SG_LOG(SG_GENERAL, SG_WARN, "archive extraction failed"); doFailure(Delegate::FAIL_EXTRACT); return; @@ -207,151 +204,6 @@ protected: } private: - void setArchiveTypeFromExtension(const std::string& ext) - { - if (ext.empty()) - return; - - if (ext == "zip") { - m_archiveType = ZIP; - return; - } - - if ((ext == "tar.gz") || (ext == "tgz")) { - m_archiveType = TAR_GZ; - return; - } - } - - void extractCurrentFile(unzFile zip, char* buffer, size_t bufferSize) - { - unz_file_info fileInfo; - unzGetCurrentFileInfo(zip, &fileInfo, - buffer, bufferSize, - NULL, 0, /* extra field */ - NULL, 0 /* comment field */); - - std::string name(buffer); - // no absolute paths, no 'up' traversals - // we could also look for suspicious file extensions here (forbid .dll, .exe, .so) - if ((name[0] == '/') || (name.find("../") != std::string::npos) || (name.find("..\\") != std::string::npos)) { - throw sg_format_exception("Bad zip path", name); - } - - if (fileInfo.uncompressed_size == 0) { - // assume it's a directory for now - // since we create parent directories when extracting - // a path, we're done here - return; - } - - int result = unzOpenCurrentFile(zip); - if (result != UNZ_OK) { - throw sg_io_exception("opening current zip file failed", sg_location(name)); - } - - sg_ofstream outFile; - bool eof = false; - SGPath path(m_extractPath); - path.append(name); - - // create enclosing directory heirarchy as required - Dir parentDir(path.dir()); - if (!parentDir.exists()) { - bool ok = parentDir.create(0755); - if (!ok) { - throw sg_io_exception("failed to create directory heirarchy for extraction", path); - } - } - - outFile.open(path, std::ios::binary | std::ios::trunc | std::ios::out); - if (outFile.fail()) { - throw sg_io_exception("failed to open output file for writing", path); - } - - while (!eof) { - int bytes = unzReadCurrentFile(zip, buffer, bufferSize); - if (bytes < 0) { - throw sg_io_exception("unzip failure reading curent archive", sg_location(name)); - } else if (bytes == 0) { - eof = true; - } else { - outFile.write(buffer, bytes); - } - } - - outFile.close(); - unzCloseCurrentFile(zip); - } - - bool extract() - { - const std::string u(url()); - const size_t ul(u.length()); - - if (m_archiveType == AUTO_DETECT) { - if (u.rfind(".zip") == (ul - 4)) { - m_archiveType = ZIP; - } else if (u.rfind(".tar.gz") == (ul - 7)) { - m_archiveType = TAR_GZ; - } - // we will fall through to the error case now - } - - if (m_archiveType == ZIP) { - return extractUnzip(); - } else if (m_archiveType == TAR_GZ) { - return extractTar(); - } - - SG_LOG(SG_IO, SG_WARN, "unsupported archive format:" << u); - return false; - } - - bool extractUnzip() - { - bool result = true; - zlib_filefunc_def memoryAccessFuncs; - fill_memory_filefunc(&memoryAccessFuncs); - - char bufferName[128]; - snprintf(bufferName, 128, "%p+%lx", m_buffer.data(), m_buffer.size()); - unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs); - - const size_t BUFFER_SIZE = 32 * 1024; - void* buf = malloc(BUFFER_SIZE); - - try { - int result = unzGoToFirstFile(zip); - if (result != UNZ_OK) { - throw sg_exception("failed to go to first file in archive"); - } - - while (true) { - extractCurrentFile(zip, (char*) buf, BUFFER_SIZE); - result = unzGoToNextFile(zip); - if (result == UNZ_END_OF_LIST_OF_FILE) { - break; - } else if (result != UNZ_OK) { - throw sg_io_exception("failed to go to next file in the archive"); - } - } - } catch (sg_exception& ) { - result = false; - } - - free(buf); - unzClose(zip); - return result; - } - - bool extractTar() - { - TarExtractor tx(m_extractPath); - tx.extractBytes(m_buffer.data(), m_buffer.size()); - return !tx.hasError() && tx.isAtEndOfArchive(); - } - void doFailure(Delegate::StatusCode aReason) { Dir dir(m_extractPath); @@ -364,19 +216,13 @@ private: m_owner->installResult(aReason); } - enum ArchiveType { - AUTO_DETECT = 0, - ZIP, - TAR_GZ - }; InstallRef m_owner; - ArchiveType m_archiveType = AUTO_DETECT; string_list m_urls; SG_MD5_CTX m_md5; - std::string m_buffer; SGPath m_extractPath; size_t m_downloaded; + std::unique_ptr m_extractor; }; //////////////////////////////////////////////////////////////////// From 05d8ab3000745332ec0a5c5cc5fdb91caae62b2c Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 4 Jul 2018 14:08:57 +0100 Subject: [PATCH 21/25] Fix Windows build Clean out legacy crap from stdint.h while we're here --- simgear/misc/stdint.hxx | 44 ++++------------------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/simgear/misc/stdint.hxx b/simgear/misc/stdint.hxx index 49e5817d..f52b3d0e 100644 --- a/simgear/misc/stdint.hxx +++ b/simgear/misc/stdint.hxx @@ -12,49 +12,13 @@ // $Id$ -////////////////////////////////////////////////////////////////////// -// -// There are many sick systems out there: -// -// check for sizeof(float) and sizeof(double) -// if sizeof(float) != 4 this code must be patched -// if sizeof(double) != 8 this code must be patched -// -// Those are comments I fetched out of glibc source: -// - s390 is big-endian -// - Sparc is big-endian, but v9 supports endian conversion -// on loads/stores and GCC supports such a mode. Be prepared. -// - The MIPS architecture has selectable endianness. -// - x86_64 is little-endian. -// - CRIS is little-endian. -// - m68k is big-endian. -// - Alpha is little-endian. -// - PowerPC can be little or big endian. -// - SH is bi-endian but with a big-endian FPU. -// - hppa1.1 big-endian. -// - ARM is (usually) little-endian but with a big-endian FPU. -// -////////////////////////////////////////////////////////////////////// +#include +#include // for _byteswap_foo on Win32 - -#ifdef _MSC_VER -typedef signed char int8_t; -typedef signed short int16_t; -typedef signed int int32_t; -typedef signed __int64 int64_t; -typedef unsigned char uint8_t; -typedef unsigned short uint16_t; -typedef unsigned int uint32_t; -typedef unsigned __int64 uint64_t; - -typedef int ssize_t; -#elif defined(sgi) || defined(__sun) -# include -#else -# include +#if defined(_MSC_VER) +using ssize_t = int64_t; // this is a POSIX type, not a C one #endif - inline uint16_t sg_bswap_16(uint16_t x) { #if defined(__llvm__) || \ (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && !defined(__ICC) From f2f465960bbf0608707dbc3d028c1ce3167c3f6c Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 4 Jul 2018 23:47:03 +0100 Subject: [PATCH 22/25] Tar code: handle symlink and PAX extensions This now accepts the GitLab .tgz archives without problems, eg the Tu-144. --- simgear/io/badTar.tgz | Bin 0 -> 637 bytes simgear/io/test_untar.cxx | 61 ++++++++++++++++++++++++- simgear/io/untar.cxx | 93 ++++++++++++++++++++++++++++++++++---- 3 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 simgear/io/badTar.tgz diff --git a/simgear/io/badTar.tgz b/simgear/io/badTar.tgz new file mode 100644 index 0000000000000000000000000000000000000000..ecdd73a31ba7a3442e035f88c04b6dc357ba332e GIT binary patch literal 637 zcmV-@0)qV?iwFRbO+8xx1MS+~ZksR|24Hv6v~?G-V;}5jQmHW*q@87pv`y;Iv}M}P zDkpJLRtF+*r1oxl*z4I9%}Ll6h5VsfL{;8LL8J)*!^aB-YGnQ+G1u>9|3j7XTwniy^S|{Mekg4H<2Vq6G8Zej&4JgS zJb(RyJR{^>_UPp|`jeKXpOIrba68=FVfy#)-DcV9biS5vr@wfx!`r1vj{blCrk9O8 zIUd`6ZKRCsOC#T1-JG{S8#Ve~sJtVG1tHzBP71@={mXI&M+EE7z z#>8^|g&!gRP57j=&i361*qc@7|E|ovzSho`QS1C4#Ww#Divr}o5jg)Jv}*ocr+pO4Xt%9ah^IzPJx|vRTCpt-QRNot?!$&To z#{C}=`T=D;V8SPq7rk7mw7AZUgI26@;Q3$G|8u?B>-7J){lZ@Vg)xr*%|QR}pshIm zJ3r`d@Bgvo{Bwc(zm1r8{k~j$EW3XDZVxo8n*TwTdFoat9T|?Idrx*bnN| zKd^m4+yC 0) { @@ -197,13 +212,7 @@ public: currentFile->close(); currentFile.reset(); } - size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE; - if (pad) { - bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad; - setState(READING_PADDING); - } else { - setState(READING_HEADER); - } + readPaddingIfRequired(); } else if (state == READING_HEADER) { processHeader(); } else if (state == PRE_END_OF_ARCHVE) { @@ -212,6 +221,12 @@ public: } else { // what does the spec say here? } + } else if (state == READING_PAX_GLOBAL_ATTRIBUTES) { + parsePAXAttributes(true); + readPaddingIfRequired(); + } else if (state == READING_PAX_FILE_ATTRIBUTES) { + parsePAXAttributes(false); + readPaddingIfRequired(); } else if (state == READING_PADDING) { setState(READING_HEADER); } @@ -317,7 +332,11 @@ public: skipCurrentEntry = false; std::string tarPath = std::string(header.prefix) + std::string(header.fileName); - + if (!paxPathName.empty()) { + tarPath = paxPathName; + paxPathName.clear(); // clear for next file + } + if (!isSafePath(tarPath)) { SG_LOG(SG_IO, SG_WARN, "bad tar path:" << tarPath); skipCurrentEntry = true; @@ -346,6 +365,20 @@ public: currentFile->open(SG_IO_OUT); } setState(READING_FILE); + } else if (header.typeflag == PAX_GLOBAL_HEADER) { + setState(READING_PAX_GLOBAL_ATTRIBUTES); + currentFileSize = ::strtol(header.size, NULL, 8); + bytesRemaining = currentFileSize; + paxAttributes.clear(); + } else if (header.typeflag == PAX_FILE_ATTRIBUTES) { + setState(READING_PAX_FILE_ATTRIBUTES); + currentFileSize = ::strtol(header.size, NULL, 8); + bytesRemaining = currentFileSize; + paxAttributes.clear(); + } else if ((header.typeflag == SYMTYPE) || (header.typeflag == LNKTYPE)) { + SG_LOG(SG_IO, SG_WARN, "Tarball contains a link or symlink, will be skipped:" << tarPath); + skipCurrentEntry = true; + setState(READING_HEADER); } else { SG_LOG(SG_IO, SG_WARN, "Unsupported tar file type:" << header.typeflag); state = BAD_ARCHIVE; @@ -370,6 +403,9 @@ public: headerPtr += curBytes; } else if (state == READING_PADDING) { bytesRemaining -= curBytes; + } else if ((state == READING_PAX_FILE_ATTRIBUTES) || (state == READING_PAX_GLOBAL_ATTRIBUTES)) { + bytesRemaining -= curBytes; + paxAttributes.append(bytes, curBytes); } checkEndOfState(); @@ -390,6 +426,39 @@ public: return true; } + + // https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.htm#paxex + void parsePAXAttributes(bool areGlobal) + { + auto lineStart = 0; + for (;;) { + auto firstSpace = paxAttributes.find(' ', lineStart); + auto firstEq = paxAttributes.find('=', lineStart); + if ((firstEq == std::string::npos) || (firstSpace == std::string::npos)) { + SG_LOG(SG_IO, SG_WARN, "Malfroemd PAX attributes in tarfile"); + break; + } + + uint32_t lengthBytes = std::stoul(paxAttributes.substr(lineStart, firstSpace)); + uint32_t dataBytes = lengthBytes - (firstEq + 1); + std::string name = paxAttributes.substr(firstSpace+1, firstEq - (firstSpace + 1)); + + // dataBytes - 1 here to trim off the trailing newline + std::string data = paxAttributes.substr(firstEq+1, dataBytes - 1); + + processPAXAttribute(areGlobal, name, data); + + lineStart += lengthBytes; + } + } + + void processPAXAttribute(bool isGlobalAttr, const std::string& attrName, const std::string& data) + { + if (!isGlobalAttr && (attrName == "path")) { + // data is UTF-8 encoded path name + paxPathName = data; + } + } }; /////////////////////////////////////////////////////////////////////////////// @@ -428,7 +497,11 @@ public: fill_memory_filefunc(&memoryAccessFuncs); char bufferName[128]; - ::snprintf(bufferName, 128, "%p+%llx", m_buffer.data(), m_buffer.size()); +#if defined(SG_WINDOWS) + ::snprintf(bufferName, 128, "%p+%llx", m_buffer.data(), m_buffer.size()); +#else + ::snprintf(bufferName, 128, "%p+%lx", m_buffer.data(), m_buffer.size()); +#endif unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs); const size_t BUFFER_SIZE = 32 * 1024; From 0e9e5f77cceddf4a1c9a67c1283c3d29a9f2436a Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Thu, 5 Jul 2018 10:29:03 +0200 Subject: [PATCH 23/25] It turns out the 16-bit version was not introduced until gcc-4.8 --- simgear/misc/stdint.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simgear/misc/stdint.hxx b/simgear/misc/stdint.hxx index f52b3d0e..f7031edc 100644 --- a/simgear/misc/stdint.hxx +++ b/simgear/misc/stdint.hxx @@ -21,7 +21,7 @@ using ssize_t = int64_t; // this is a POSIX type, not a C one inline uint16_t sg_bswap_16(uint16_t x) { #if defined(__llvm__) || \ - (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && !defined(__ICC) + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && !defined(__ICC) return __builtin_bswap16(x); #elif defined(_MSC_VER) && !defined(_DEBUG) return _byteswap_ushort(x); From 3dfce43de2e02efcd8f13f00dc8f70966cffaab4 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 18 Jul 2018 07:20:18 +0100 Subject: [PATCH 24/25] Fixes to HTTP repo tests on Windows. Large alloca fails, and SGFIle dtor doesn't close the handle, which breaks _wunlink on Windows :( --- simgear/io/HTTPRepository.cxx | 4 ++++ simgear/io/sg_file.hxx | 2 +- simgear/io/test_repository.cxx | 8 ++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/simgear/io/HTTPRepository.cxx b/simgear/io/HTTPRepository.cxx index 2c1c61c6..acac1cf8 100644 --- a/simgear/io/HTTPRepository.cxx +++ b/simgear/io/HTTPRepository.cxx @@ -749,6 +749,10 @@ HTTPRepository::failure() const code = HTTPRepository::REPO_ERROR_CANCELLED; } + if (file) { + file->close(); + } + file.reset(); if (pathInRepo.exists()) { pathInRepo.remove(); diff --git a/simgear/io/sg_file.hxx b/simgear/io/sg_file.hxx index 8385c8b3..613f097c 100644 --- a/simgear/io/sg_file.hxx +++ b/simgear/io/sg_file.hxx @@ -62,7 +62,7 @@ public: SGFile( int existingFd ); /** Destructor */ - ~SGFile(); + virtual ~SGFile(); // open the file based on specified direction bool open( const SGProtocolDir dir ); diff --git a/simgear/io/test_repository.cxx b/simgear/io/test_repository.cxx index 07ba9728..666ff5c9 100644 --- a/simgear/io/test_repository.cxx +++ b/simgear/io/test_repository.cxx @@ -309,7 +309,8 @@ std::string test_computeHashForPath(const SGPath& p) return std::string(); sha1nfo info; sha1_init(&info); - char* buf = static_cast(alloca(1024 * 1024)); + char* buf = static_cast(malloc(1024 * 1024)); + assert(buf); size_t readLen; SGBinaryFile f(p); @@ -319,6 +320,9 @@ std::string test_computeHashForPath(const SGPath& p) sha1_write(&info, buf, readLen); } + f.close(); + free(buf); + std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH); return strutils::encodeHex(hashBytes); } @@ -725,7 +729,7 @@ void testCopyInstalledChildren(HTTP::Client* cl) verifyRequestCount("dirJ/fileJC", 1); verifyRequestCount("dirJ/fileJD", 1); - std::cout << "Copy installed children" << std::endl; + std::cout << "passed Copy installed children" << std::endl; } int main(int argc, char* argv[]) From a0c4913f84e8b2581f9a6298344f3e888e9e6c5e Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 18 Jul 2018 16:30:44 +0100 Subject: [PATCH 25/25] Fix .dirindex preservation on Windows Ensure we don't treat the special files as regular children when updating. --- simgear/io/HTTPRepository.cxx | 27 +++++++++++++++------------ simgear/io/test_repository.cxx | 33 +++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/simgear/io/HTTPRepository.cxx b/simgear/io/HTTPRepository.cxx index acac1cf8..fb2cd9fe 100644 --- a/simgear/io/HTTPRepository.cxx +++ b/simgear/io/HTTPRepository.cxx @@ -323,32 +323,35 @@ public: toBeUpdated, orphans; simgear::Dir d(absolutePath()); PathList fsChildren = d.children(0); - PathList::const_iterator it = fsChildren.begin(); - for (; it != fsChildren.end(); ++it) { - ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType, - it->file(), ""); + for (const auto& child : fsChildren) { + const auto& fileName = child.file(); + if ((fileName == ".dirindex") || (fileName == ".hashes")) { + continue; + } + + ChildInfo info(child.isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType, fileName, ""); std::string hash = hashForChild(info); - ChildInfoList::iterator c = findIndexChild(it->file()); + ChildInfoList::iterator c = findIndexChild(fileName); if (c == children.end()) { - orphans.push_back(it->file()); + orphans.push_back(fileName); } else if (c->hash != hash) { #if 0 - SG_LOG(SG_TERRASYNC, SG_DEBUG, "hash mismatch'" << it->file() ); + SG_LOG(SG_TERRASYNC, SG_DEBUG, "hash mismatch'" << fileName); // file exists, but hash mismatch, schedule update if (!hash.empty()) { - SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists but hash is wrong for:" << it->file() ); + SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists but hash is wrong for:" << fileName); SG_LOG(SG_TERRASYNC, SG_DEBUG, "on disk:" << hash << " vs in info:" << c->hash); } #endif - toBeUpdated.push_back(it->file() ); + toBeUpdated.push_back(fileName); } else { // file exists and hash is valid. If it's a directory, // perform a recursive check. if (c->type == ChildInfo::DirectoryType) { - HTTPDirectory* childDir = childDirectory(it->file()); + HTTPDirectory* childDir = childDirectory(fileName); childDir->updateChildrenBasedOnHash(); } } @@ -356,7 +359,7 @@ public: // remove existing file system children from the index list, // so we can detect new children // https://en.wikibooks.org/wiki/More_C%2B%2B_Idioms/Erase-Remove - indexNames.erase(std::remove(indexNames.begin(), indexNames.end(), it->file()), indexNames.end()); + indexNames.erase(std::remove(indexNames.begin(), indexNames.end(), fileName), indexNames.end()); } // of real children iteration // all remaining names in indexChilden are new children @@ -824,7 +827,7 @@ HTTPRepository::failure() const // dir index data has changed, so write to disk and update // the hash accordingly - sg_ofstream of(pathInRepo(), std::ios::trunc | std::ios::out); + sg_ofstream of(pathInRepo(), std::ios::trunc | std::ios::out | std::ios::binary); if (!of.is_open()) { throw sg_io_exception("Failed to open directory index file for writing", pathInRepo()); } diff --git a/simgear/io/test_repository.cxx b/simgear/io/test_repository.cxx index 666ff5c9..0148fc5c 100644 --- a/simgear/io/test_repository.cxx +++ b/simgear/io/test_repository.cxx @@ -437,6 +437,34 @@ void testBasicClone(HTTP::Client* cl) std::cout << "Passed test: basic clone and update" << std::endl; } +void testUpdateNoChanges(HTTP::Client* cl) +{ + std::unique_ptr repo; + SGPath p(simgear::Dir::current().path()); + p.append("http_repo_basic"); // same as before + + global_repo->clearRequestCounts(); + + repo.reset(new HTTPRepository(p, cl)); + repo->setBaseUrl("http://localhost:2000/repo"); + repo->update(); + + waitForUpdateComplete(cl, repo.get()); + + verifyFileState(p, "fileA"); + verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA"); + + verifyRequestCount("dirA", 0); + verifyRequestCount("dirB", 0); + verifyRequestCount("dirB/subdirA", 0); + verifyRequestCount("dirB/subdirA/fileBAA", 0); + verifyRequestCount("dirC", 0); + verifyRequestCount("dirC/fileCA", 0); + + std::cout << "Passed test:no changes update" << std::endl; + +} + void testModifyLocalFiles(HTTP::Client* cl) { std::unique_ptr repo; @@ -473,10 +501,6 @@ void testModifyLocalFiles(HTTP::Client* cl) std::cout << "Passed test: identify and fix locally modified files" << std::endl; } -void testNoChangesUpdate() -{ - -} void testMergeExistingFileWithoutDownload(HTTP::Client* cl) { @@ -755,6 +779,7 @@ int main(int argc, char* argv[]) global_repo->defineFile("dirC/subdirA/subsubA/fileCAAA"); testBasicClone(&cl); + testUpdateNoChanges(&cl); testModifyLocalFiles(&cl);