Make life easier by implementing the system read and write function and call that from the classes read and write functions. Add a forward and rewind function. Add a test_untar derived mmap test utility.

simgear/io/CMakeLists.txt

@@ -75,4 +75,8 @@ add_simgear_autotest(test_untar test_untar.cxx)
 set_target_properties(test_untar PROPERTIES
   COMPILE_DEFINITIONS "SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"" )
 
+add_simgear_autotest(test_mmap test_mmap.cxx)
+set_target_properties(test_mmap PROPERTIES
+  COMPILE_DEFINITIONS "SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"" )
+
 endif(ENABLE_TESTS)

simgear/io/sg_mmap.cxx

@@ -126,109 +126,96 @@ bool SGMMapFile::open( const SGProtocolDir d ) {
     return true;
 }
 
-
+off_t SGMMapFile::forward(off_t amount) {
-// read a block of data of specified size
+    if ((size_t)amount > size - offset) {
-int SGMMapFile::read( char *buf, int length ) {
+        amount = size - offset;
-    size_t read_size = length;
-    size_t result = length;
-    size_t pos = offset;
-
-    if (read_size > size - offset) {
-        read_size = size - offset;
-        result = 0; // eof
     }
-
+    offset += amount;
-    // read a chunk
+    return amount;
-    memcpy(buf, buffer+offset, read_size);
-    offset += read_size;
-
-    if ( length > 0 && result == 0 ) {
-        if (repeat < 0 || iteration < repeat - 1) {
-            iteration++;
-            // loop reading the file, unless it is empty
-            
-            off_t fileLen = pos;
-            if (fileLen == 0) {
-                eof_flag = true;
-                return 0;
-            } else {
-                offset = 0;
-                if (read_size > size) {
-                    read_size = size;
-                    result = 0; // eof
-                }
-                memcpy(buf, buffer, read_size);
-                offset += read_size;
-                return result;
-            }
-        } else {
-            eof_flag = true;
-        }
-    }
-    return result;
 }
 
-const char* SGMMapFile::advance(size_t len) {
+const char* SGMMapFile::advance(off_t amount) {
-   if (len >= size - offset)
-      return nullptr;
-
    const char *ptr = buffer + offset;
-   offset += len;
+
+   off_t advanced = forward(amount);
+   if (advanced != amount)
+      return nullptr;
 
    return ptr;
 }
 
-int SGMMapFile::read( char *buf, int length, int num ) {
+ssize_t SGMMapFile::_read(void *buf, size_t count) {
-    if (length == 0)
+    const char *ptr = buffer + offset;
-        return 0;
+    size_t result = forward(count);
 
-    size_t size = num*length;
+    memcpy(buf, ptr, result);
-    return read(buf, size)/length;
+
+    return result;
+}
+
+ssize_t SGMMapFile::_write(const void *buf, size_t count) {
+    char *ptr = buffer + offset;
+    size_t result = forward(count);
+
+    memcpy(ptr, buf, result);
+
+    return result;
+}
+
+
+// read a block of data of specified size
+int SGMMapFile::read( char *buf, int length ) {
+    // read a chunk
+    ssize_t result = _read(buf, length);
+    if ( length > 0 && result == 0 ) {
+        if (repeat < 0 || iteration < repeat - 1) {
+            iteration++;
+            // loop reading the file, unless it is empty
+            off_t fileLen = offset;		// lseek(0, SEEK_CUR)
+            if (fileLen == 0) {
+                eof_flag = true;
+                return 0;
+            } else {
+                offset = 0;			// lseek(0, SEEK_SET)
+                return _read(buf, length);
+            }
+        } else {
+            eof_flag = true;
+        }
+    }
+    return result;
+}
+
+int SGMMapFile::read( char *buf, int length, int num ) {
+    return _read(buf, num*length)/length;
 }
 
 // read a line of data, length is max size of input buffer
 int SGMMapFile::readline( char *buf, int length ) {
-    size_t read_size = length;
+    int pos = offset;				// pos = lseek(0, SEEK_CUR)
-    size_t result = length;
-    size_t pos = offset;
-
-    if (read_size > size - offset) {
-        read_size = size - offset;
-        result = 0; // eof
-    }
 
     // read a chunk
-    memcpy(buf, buffer+offset, read_size);
+    ssize_t result = _read(buf, length);
-    offset += read_size;
-
     if ( length > 0 && result == 0 ) {
         if ((repeat < 0 || iteration < repeat - 1) && pos != 0) {
             iteration++;
 
-            pos = 0;
+            pos = offset = 0;			// pos = lseek(0, SEEK_SET)
-            result = length;
+            result = _read(buf, length);
-            read_size = length;
-            if (read_size > size) {
-                read_size = size;
-                result = 0; // eof
-            }
-            
-            memcpy(buf, buffer, read_size);
-            offset += read_size;
         } else {
             eof_flag = true;
         }
     }
 
     // find the end of line and reset position
-    size_t i;
+    int i;
     for ( i = 0; i < result && buf[i] != '\n'; ++i );
     if ( buf[i] == '\n' ) {
 	result = i + 1;
     } else {
 	result = i;
     }
-    offset = pos + result;
+    offset = pos + result;			// lseek(pos+result, SEEK_SET)
 
     // just in case ...
     buf[ result ] = '\0';
@@ -236,39 +223,26 @@ int SGMMapFile::readline( char *buf, int length ) {
     return result;
 }
 
-std::string SGMMapFile::read_all()
+std::string SGMMapFile::read_all() {
-{
     return std::string(buffer, size);
 }
 
 // write data to a file
 int SGMMapFile::write( const char *buf, const int length ) {
-     size_t write_size = length;
+    int result = _write(buf, length);
 
-     if (write_size > size - offset) {
+    if ( result != length ) {
-        SG_LOG( SG_IO, SG_ALERT, "Attempting to write beyond the mmap buffer size: " << file_name );
+        SG_LOG( SG_IO, SG_ALERT, "Error writing data: " << file_name );
-        write_size = size - offset;
     }
 
-    memcpy(buffer+offset, buf, write_size);
+    return result;
-    offset += write_size;
-
-    return write_size;
 }
 
 
 // write null terminated string to a file
 int SGMMapFile::writestring( const char *str ) {
-    size_t write_size = std::strlen( str );
+    int length = std::strlen( str );
-    if (write_size > size - offset) {
+    return write( str, length );
-        SG_LOG( SG_IO, SG_ALERT, "Attempting to write beyond the mmap buffer size: " << file_name );
-        write_size = size - offset;
-    }
-
-    memcpy(buffer+offset, str, write_size);
-    offset += write_size;
-
-    return write_size;
 }
 
 

simgear/io/sg_mmap.hxx

@@ -50,7 +50,7 @@ class SGMMapFile : public SGIOChannel {
     int extraoflags = 0;
 
     char *buffer = nullptr;
-    size_t offset = 0;
+    off_t offset = 0;
     size_t size = 0;
 #ifdef WIN32
     typedef struct
@@ -59,21 +59,20 @@ class SGMMapFile : public SGIOChannel {
         void *p;
     } SIMPLE_UNMMAP;
     SIMPLE_UNMMAP un;
-#else
-    int un;			// referenced but not used
-#endif
 
-#ifdef WIN32
+    // map the file descriptor and return a pointer to the buffer.
-/*
- * map 'filename' and return a pointer to it.
- */
     static void *simple_mmap(int, size_t, SIMPLE_UNMMAP *);
     static void simple_unmmap(void*, size_t, SIMPLE_UNMMAP *);
 #else
+    int un;			// referenced but not used
+
 # define simple_mmap(a, b, c)   mmap(0, (b), PROT_READ, MAP_PRIVATE, (a), 0L)
 # define simple_unmmap(a, b, c) munmap((a), (b))
 #endif
 
+   ssize_t _read(void *buf, size_t count);
+   ssize_t _write(const void *buf, size_t count);
+
 public:
 
     SGMMapFile();
@@ -116,11 +115,22 @@ public:
     // get the pointer to the start of the buffer
     inline const char *get() { return buffer; }
 
-    // get the pointer at the current offset and increase the offset by len
+    // get the pointer at the current offset and increase the offset by amount
-    const char* advance(size_t len);
+    // returns nullptr if the offset pointer would end up beyond the mmap
+    // buffer size
+    const char* advance(off_t amount);
 
+    // return the size of the mmaped area
     inline size_t get_size() { return size; }
 
+    // forward by amount, returns the amount which could be forwarded
+    // without the offset getting beyond the mmap buffer size.
+     // returns 0 on end of file.
+    off_t forward(off_t amount);
+
+    // rewind the offset pointer
+    inline void rewind() { offset = 0; }
+
     // write data to a file
     int write( const char *buf, const int length );
 

simgear/io/test_mmap.cxx

@@ -0,0 +1,219 @@
+////////////////////////////////////////////////////////////////////////
+// Test harness.
+////////////////////////////////////////////////////////////////////////
+
+#include <simgear_config.h>
+#include <simgear/compiler.h>
+
+#include <iostream>
+
+#include "untar.hxx"
+
+#include <simgear/misc/test_macros.hxx>
+#include <simgear/misc/sg_path.hxx>
+#include <simgear/misc/sg_dir.hxx>
+#include <simgear/io/sg_mmap.hxx>
+
+
+using std::cout;
+using std::cerr;
+using std::endl;
+
+using namespace simgear;
+
+void testTarGz()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("test.tar.gz");
+
+    SGMMapFile f(p);
+    f.open(SG_IO_IN);
+
+    uint8_t* buf = (uint8_t*) alloca(8192);
+    size_t bufSize = f.read((char*) buf, 8192);
+
+    SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::GZData);
+
+    f.close();
+}
+
+void testPlainTar()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("test2.tar");
+
+    SGMMapFile f(p);
+    f.open(SG_IO_IN);
+
+	uint8_t* buf = (uint8_t*)alloca(8192);
+	size_t bufSize = f.read((char*) buf, 8192);
+
+	SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData);
+
+    f.close();
+}
+
+void testExtractStreamed()
+{
+	SGPath p = SGPath(SRC_DIR);
+	p.append("test.tar.gz");
+
+	SGMMapFile f(p);
+	f.open(SG_IO_IN);
+
+	SGPath extractDir = simgear::Dir::current().path() / "test_extract_streamed";
+	simgear::Dir pd(extractDir);
+	pd.removeChildren();
+
+	ArchiveExtractor ex(extractDir);
+
+	uint8_t* buf = (uint8_t*) alloca(128);
+	while (!f.eof()) {
+		size_t bufSize = f.read((char*) buf, 128);
+		ex.extractBytes(buf, bufSize);
+	}
+
+	ex.flush();
+	SG_VERIFY(ex.isAtEndOfArchive());
+	SG_VERIFY(ex.hasError() == false);
+
+	SG_VERIFY((extractDir / "testDir/hello.c").exists());
+	SG_VERIFY((extractDir / "testDir/foo.txt").exists());
+}
+
+void testExtractLocalFile()
+{
+
+}
+
+void testFilterTar()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("badTar.tgz");
+    
+    SGMMapFile f(p);
+    f.open(SG_IO_IN);
+    
+    SGPath extractDir = simgear::Dir::current().path() / "test_filter_tar";
+    simgear::Dir pd(extractDir);
+    pd.removeChildren();
+    
+    ArchiveExtractor ex(extractDir);
+    
+    uint8_t* buf = (uint8_t*) alloca(128);
+    while (!f.eof()) {
+        size_t bufSize = f.read((char*) buf, 128);
+        ex.extractBytes(buf, bufSize);
+    }
+    
+    ex.flush();
+    SG_VERIFY(ex.isAtEndOfArchive());
+    SG_VERIFY(ex.hasError() == false);
+    
+    SG_VERIFY((extractDir / "tarWithBadContent/regular-file.txt").exists());
+    SG_VERIFY(!(extractDir / "tarWithBadContent/symbolic-linked.png").exists());
+    SG_VERIFY((extractDir / "tarWithBadContent/screenshot.png").exists());
+    SG_VERIFY((extractDir / "tarWithBadContent/dirOne/subDirA").exists());
+    SG_VERIFY(!(extractDir / "tarWithBadContent/dirOne/subDirA/linked.txt").exists());
+
+
+}
+
+void testExtractZip()
+{
+	SGPath p = SGPath(SRC_DIR);
+	p.append("zippy.zip");
+
+	SGMMapFile f(p);
+	f.open(SG_IO_IN);
+
+	SGPath extractDir = simgear::Dir::current().path() / "test_extract_zip";
+	simgear::Dir pd(extractDir);
+	pd.removeChildren();
+
+	ArchiveExtractor ex(extractDir);
+
+	uint8_t* buf = (uint8_t*)alloca(128);
+	while (!f.eof()) {
+		size_t bufSize = f.read((char*)buf, 128);
+		ex.extractBytes(buf, bufSize);
+	}
+
+	ex.flush();
+	SG_VERIFY(ex.isAtEndOfArchive());
+	SG_VERIFY(ex.hasError() == false);
+
+	SG_VERIFY((extractDir / "zippy/dirA/hello.c").exists());
+	SG_VERIFY((extractDir / "zippy/bar.xml").exists());
+	SG_VERIFY((extractDir / "zippy/long-named.json").exists());
+}
+
+void testPAXAttributes()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("pax-extended.tar");
+    
+    SGMMapFile f(p);
+    f.open(SG_IO_IN);
+    
+    SGPath extractDir = simgear::Dir::current().path() / "test_pax_extended";
+    simgear::Dir pd(extractDir);
+    pd.removeChildren();
+    
+    ArchiveExtractor ex(extractDir);
+    
+    uint8_t* buf = (uint8_t*) alloca(128);
+    while (!f.eof()) {
+        size_t bufSize = f.read((char*) buf, 128);
+        ex.extractBytes(buf, bufSize);
+    }
+    
+    ex.flush();
+    SG_VERIFY(ex.isAtEndOfArchive());
+    SG_VERIFY(ex.hasError() == false);
+
+}
+
+void testExtractXZ()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("test.tar.xz");
+
+    SGMMapFile f(p);
+    f.open(SG_IO_IN);
+
+    SGPath extractDir = simgear::Dir::current().path() / "test_extract_xz";
+    simgear::Dir pd(extractDir);
+    pd.removeChildren();
+
+    ArchiveExtractor ex(extractDir);
+
+    uint8_t* buf = (uint8_t*)alloca(128);
+    while (!f.eof()) {
+        size_t bufSize = f.read((char*)buf, 128);
+        ex.extractBytes(buf, bufSize);
+    }
+
+    ex.flush();
+    SG_VERIFY(ex.isAtEndOfArchive());
+    SG_VERIFY(ex.hasError() == false);
+
+    SG_VERIFY((extractDir / "testDir/hello.c").exists());
+    SG_VERIFY((extractDir / "testDir/foo.txt").exists());
+}
+
+int main(int ac, char ** av)
+{
+    testTarGz();
+    testPlainTar();
+    testFilterTar();
+	testExtractStreamed();
+	testExtractZip();
+    testExtractXZ();
+
+    // disabled to avoiding checking in large PAX archive
+    // testPAXAttributes();
+    
+	std::cout << "all tests passed" << std::endl;
+    return 0;
+}