diff --git a/simgear/io/CMakeLists.txt b/simgear/io/CMakeLists.txt index 7a32073c..a3ce2094 100644 --- a/simgear/io/CMakeLists.txt +++ b/simgear/io/CMakeLists.txt @@ -75,4 +75,8 @@ add_simgear_autotest(test_untar test_untar.cxx) set_target_properties(test_untar PROPERTIES COMPILE_DEFINITIONS "SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"" ) +add_simgear_autotest(test_mmap test_mmap.cxx) +set_target_properties(test_mmap PROPERTIES + COMPILE_DEFINITIONS "SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"" ) + endif(ENABLE_TESTS) diff --git a/simgear/io/sg_mmap.cxx b/simgear/io/sg_mmap.cxx index 7e346b22..5d2879e8 100644 --- a/simgear/io/sg_mmap.cxx +++ b/simgear/io/sg_mmap.cxx @@ -126,40 +126,58 @@ bool SGMMapFile::open( const SGProtocolDir d ) { return true; } +off_t SGMMapFile::forward(off_t amount) { + if ((size_t)amount > size - offset) { + amount = size - offset; + } + offset += amount; + return amount; +} + +const char* SGMMapFile::advance(off_t amount) { + const char *ptr = buffer + offset; + + off_t advanced = forward(amount); + if (advanced != amount) + return nullptr; + + return ptr; +} + +ssize_t SGMMapFile::_read(void *buf, size_t count) { + const char *ptr = buffer + offset; + size_t result = forward(count); + + memcpy(buf, ptr, result); + + return result; +} + +ssize_t SGMMapFile::_write(const void *buf, size_t count) { + char *ptr = buffer + offset; + size_t result = forward(count); + + memcpy(ptr, buf, result); + + return result; +} + // read a block of data of specified size int SGMMapFile::read( char *buf, int length ) { - size_t read_size = length; - size_t result = length; - size_t pos = offset; - - if (read_size > size - offset) { - read_size = size - offset; - result = 0; // eof - } - // read a chunk - memcpy(buf, buffer+offset, read_size); - offset += read_size; - + ssize_t result = _read(buf, length); if ( length > 0 && result == 0 ) { if (repeat < 0 || iteration < repeat - 1) { iteration++; // loop reading the file, unless it is empty - - off_t fileLen = pos; + off_t fileLen = offset; // lseek(0, SEEK_CUR) if (fileLen == 0) { eof_flag = true; return 0; } else { - offset = 0; - if (read_size > size) { - read_size = size; - result = 0; // eof - } - memcpy(buf, buffer, read_size); - offset += read_size; - return result; + offset = 0; // lseek(0, SEEK_SET) + return _read(buf, length); } } else { eof_flag = true; @@ -168,67 +186,36 @@ int SGMMapFile::read( char *buf, int length ) { return result; } -const char* SGMMapFile::advance(size_t len) { - if (len >= size - offset) - return nullptr; - - const char *ptr = buffer + offset; - offset += len; - - return ptr; -} - int SGMMapFile::read( char *buf, int length, int num ) { - if (length == 0) - return 0; - - size_t size = num*length; - return read(buf, size)/length; + return _read(buf, num*length)/length; } // read a line of data, length is max size of input buffer int SGMMapFile::readline( char *buf, int length ) { - size_t read_size = length; - size_t result = length; - size_t pos = offset; - - if (read_size > size - offset) { - read_size = size - offset; - result = 0; // eof - } + int pos = offset; // pos = lseek(0, SEEK_CUR) // read a chunk - memcpy(buf, buffer+offset, read_size); - offset += read_size; - + ssize_t result = _read(buf, length); if ( length > 0 && result == 0 ) { if ((repeat < 0 || iteration < repeat - 1) && pos != 0) { iteration++; - pos = 0; - result = length; - read_size = length; - if (read_size > size) { - read_size = size; - result = 0; // eof - } - - memcpy(buf, buffer, read_size); - offset += read_size; + pos = offset = 0; // pos = lseek(0, SEEK_SET) + result = _read(buf, length); } else { eof_flag = true; } } // find the end of line and reset position - size_t i; + int i; for ( i = 0; i < result && buf[i] != '\n'; ++i ); if ( buf[i] == '\n' ) { result = i + 1; } else { result = i; } - offset = pos + result; + offset = pos + result; // lseek(pos+result, SEEK_SET) // just in case ... buf[ result ] = '\0'; @@ -236,39 +223,26 @@ int SGMMapFile::readline( char *buf, int length ) { return result; } -std::string SGMMapFile::read_all() -{ +std::string SGMMapFile::read_all() { return std::string(buffer, size); } // write data to a file int SGMMapFile::write( const char *buf, const int length ) { - size_t write_size = length; + int result = _write(buf, length); - if (write_size > size - offset) { - SG_LOG( SG_IO, SG_ALERT, "Attempting to write beyond the mmap buffer size: " << file_name ); - write_size = size - offset; + if ( result != length ) { + SG_LOG( SG_IO, SG_ALERT, "Error writing data: " << file_name ); } - memcpy(buffer+offset, buf, write_size); - offset += write_size; - - return write_size; + return result; } // write null terminated string to a file int SGMMapFile::writestring( const char *str ) { - size_t write_size = std::strlen( str ); - if (write_size > size - offset) { - SG_LOG( SG_IO, SG_ALERT, "Attempting to write beyond the mmap buffer size: " << file_name ); - write_size = size - offset; - } - - memcpy(buffer+offset, str, write_size); - offset += write_size; - - return write_size; + int length = std::strlen( str ); + return write( str, length ); } diff --git a/simgear/io/sg_mmap.hxx b/simgear/io/sg_mmap.hxx index 74f5c081..a330f1e7 100644 --- a/simgear/io/sg_mmap.hxx +++ b/simgear/io/sg_mmap.hxx @@ -50,7 +50,7 @@ class SGMMapFile : public SGIOChannel { int extraoflags = 0; char *buffer = nullptr; - size_t offset = 0; + off_t offset = 0; size_t size = 0; #ifdef WIN32 typedef struct @@ -59,21 +59,20 @@ class SGMMapFile : public SGIOChannel { void *p; } SIMPLE_UNMMAP; SIMPLE_UNMMAP un; -#else - int un; // referenced but not used -#endif -#ifdef WIN32 -/* - * map 'filename' and return a pointer to it. - */ + // map the file descriptor and return a pointer to the buffer. static void *simple_mmap(int, size_t, SIMPLE_UNMMAP *); static void simple_unmmap(void*, size_t, SIMPLE_UNMMAP *); #else + int un; // referenced but not used + # define simple_mmap(a, b, c) mmap(0, (b), PROT_READ, MAP_PRIVATE, (a), 0L) # define simple_unmmap(a, b, c) munmap((a), (b)) #endif + ssize_t _read(void *buf, size_t count); + ssize_t _write(const void *buf, size_t count); + public: SGMMapFile(); @@ -116,11 +115,22 @@ public: // get the pointer to the start of the buffer inline const char *get() { return buffer; } - // get the pointer at the current offset and increase the offset by len - const char* advance(size_t len); + // get the pointer at the current offset and increase the offset by amount + // returns nullptr if the offset pointer would end up beyond the mmap + // buffer size + const char* advance(off_t amount); + // return the size of the mmaped area inline size_t get_size() { return size; } + // forward by amount, returns the amount which could be forwarded + // without the offset getting beyond the mmap buffer size. + // returns 0 on end of file. + off_t forward(off_t amount); + + // rewind the offset pointer + inline void rewind() { offset = 0; } + // write data to a file int write( const char *buf, const int length ); diff --git a/simgear/io/test_mmap.cxx b/simgear/io/test_mmap.cxx new file mode 100644 index 00000000..b9999b2d --- /dev/null +++ b/simgear/io/test_mmap.cxx @@ -0,0 +1,219 @@ +//////////////////////////////////////////////////////////////////////// +// Test harness. +//////////////////////////////////////////////////////////////////////// + +#include +#include + +#include + +#include "untar.hxx" + +#include +#include +#include +#include + + +using std::cout; +using std::cerr; +using std::endl; + +using namespace simgear; + +void testTarGz() +{ + SGPath p = SGPath(SRC_DIR); + p.append("test.tar.gz"); + + SGMMapFile f(p); + f.open(SG_IO_IN); + + uint8_t* buf = (uint8_t*) alloca(8192); + size_t bufSize = f.read((char*) buf, 8192); + + SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::GZData); + + f.close(); +} + +void testPlainTar() +{ + SGPath p = SGPath(SRC_DIR); + p.append("test2.tar"); + + SGMMapFile f(p); + f.open(SG_IO_IN); + + uint8_t* buf = (uint8_t*)alloca(8192); + size_t bufSize = f.read((char*) buf, 8192); + + SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData); + + f.close(); +} + +void testExtractStreamed() +{ + SGPath p = SGPath(SRC_DIR); + p.append("test.tar.gz"); + + SGMMapFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_extract_streamed"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*) alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*) buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + + SG_VERIFY((extractDir / "testDir/hello.c").exists()); + SG_VERIFY((extractDir / "testDir/foo.txt").exists()); +} + +void testExtractLocalFile() +{ + +} + +void testFilterTar() +{ + SGPath p = SGPath(SRC_DIR); + p.append("badTar.tgz"); + + SGMMapFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_filter_tar"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*) alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*) buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + + SG_VERIFY((extractDir / "tarWithBadContent/regular-file.txt").exists()); + SG_VERIFY(!(extractDir / "tarWithBadContent/symbolic-linked.png").exists()); + SG_VERIFY((extractDir / "tarWithBadContent/screenshot.png").exists()); + SG_VERIFY((extractDir / "tarWithBadContent/dirOne/subDirA").exists()); + SG_VERIFY(!(extractDir / "tarWithBadContent/dirOne/subDirA/linked.txt").exists()); + + +} + +void testExtractZip() +{ + SGPath p = SGPath(SRC_DIR); + p.append("zippy.zip"); + + SGMMapFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_extract_zip"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*)alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*)buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + + SG_VERIFY((extractDir / "zippy/dirA/hello.c").exists()); + SG_VERIFY((extractDir / "zippy/bar.xml").exists()); + SG_VERIFY((extractDir / "zippy/long-named.json").exists()); +} + +void testPAXAttributes() +{ + SGPath p = SGPath(SRC_DIR); + p.append("pax-extended.tar"); + + SGMMapFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_pax_extended"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*) alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*) buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + +} + +void testExtractXZ() +{ + SGPath p = SGPath(SRC_DIR); + p.append("test.tar.xz"); + + SGMMapFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_extract_xz"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*)alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*)buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + + SG_VERIFY((extractDir / "testDir/hello.c").exists()); + SG_VERIFY((extractDir / "testDir/foo.txt").exists()); +} + +int main(int ac, char ** av) +{ + testTarGz(); + testPlainTar(); + testFilterTar(); + testExtractStreamed(); + testExtractZip(); + testExtractXZ(); + + // disabled to avoiding checking in large PAX archive + // testPAXAttributes(); + + std::cout << "all tests passed" << std::endl; + return 0; +}