FGFS-4.1/simgear
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make life easier by implementing the system read and write function and call that from the classes read and write functions. Add a forward and rewind function. Add a test_untar derived mmap test utility.

Browse Source
This commit is contained in:
Erik Hofman 2021-03-16 09:51:37 +01:00
parent cf2fe76bb8
commit e8cbcebad8
4 changed files with 297 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
simgear/io/CMakeLists.txt
View File

@ -75,4 +75,8 @@ add_simgear_autotest(test_untar test_untar.cxx)
set_target_properties(test_untar PROPERTIES
COMPILE_DEFINITIONS "SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"" )
add_simgear_autotest(test_mmap test_mmap.cxx)
set_target_properties(test_mmap PROPERTIES
COMPILE_DEFINITIONS "SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"" )
endif(ENABLE_TESTS)

158
simgear/io/sg_mmap.cxx
View File

@ -126,109 +126,96 @@ bool SGMMapFile::open( const SGProtocolDir d ) {
return true;
}
// read a block of data of specified size
int SGMMapFile::read( char *buf, int length ) {
size_t read_size = length;
size_t result = length;
size_t pos = offset;
if (read_size > size - offset) {
read_size = size - offset;
result = 0; // eof
off_t SGMMapFile::forward(off_t amount) {
if ((size_t)amount > size - offset) {
amount = size - offset;
}
// read a chunk
memcpy(buf, buffer+offset, read_size);
offset += read_size;
if ( length > 0 && result == 0 ) {
if (repeat < 0 || iteration < repeat - 1) {
iteration++;
// loop reading the file, unless it is empty
off_t fileLen = pos;
if (fileLen == 0) {
eof_flag = true;
return 0;
} else {
offset = 0;
if (read_size > size) {
read_size = size;
result = 0; // eof
}
memcpy(buf, buffer, read_size);
offset += read_size;
return result;
}
} else {
eof_flag = true;
}
}
return result;
offset += amount;
return amount;
}
const char* SGMMapFile::advance(size_t len) {
if (len >= size - offset)
return nullptr;
const char* SGMMapFile::advance(off_t amount) {
const char *ptr = buffer + offset;
offset += len;
off_t advanced = forward(amount);
if (advanced != amount)
return nullptr;
return ptr;
}
int SGMMapFile::read( char *buf, int length, int num ) {
if (length == 0)
return 0;
ssize_t SGMMapFile::_read(void *buf, size_t count) {
const char *ptr = buffer + offset;
size_t result = forward(count);
size_t size = num*length;
return read(buf, size)/length;
memcpy(buf, ptr, result);
return result;
}
ssize_t SGMMapFile::_write(const void *buf, size_t count) {
char *ptr = buffer + offset;
size_t result = forward(count);
memcpy(ptr, buf, result);
return result;
}
// read a block of data of specified size
int SGMMapFile::read( char *buf, int length ) {
// read a chunk
ssize_t result = _read(buf, length);
if ( length > 0 && result == 0 ) {
if (repeat < 0 || iteration < repeat - 1) {
iteration++;
// loop reading the file, unless it is empty
off_t fileLen = offset; // lseek(0, SEEK_CUR)
if (fileLen == 0) {
eof_flag = true;
return 0;
} else {
offset = 0; // lseek(0, SEEK_SET)
return _read(buf, length);
}
} else {
eof_flag = true;
}
}
return result;
}
int SGMMapFile::read( char *buf, int length, int num ) {
return _read(buf, num*length)/length;
}
// read a line of data, length is max size of input buffer
int SGMMapFile::readline( char *buf, int length ) {
size_t read_size = length;
size_t result = length;
size_t pos = offset;
if (read_size > size - offset) {
read_size = size - offset;
result = 0; // eof
}
int pos = offset; // pos = lseek(0, SEEK_CUR)
// read a chunk
memcpy(buf, buffer+offset, read_size);
offset += read_size;
ssize_t result = _read(buf, length);
if ( length > 0 && result == 0 ) {
if ((repeat < 0 || iteration < repeat - 1) && pos != 0) {
iteration++;
pos = 0;
result = length;
read_size = length;
if (read_size > size) {
read_size = size;
result = 0; // eof
}
memcpy(buf, buffer, read_size);
offset += read_size;
pos = offset = 0; // pos = lseek(0, SEEK_SET)
result = _read(buf, length);
} else {
eof_flag = true;
}
}
// find the end of line and reset position
size_t i;
int i;
for ( i = 0; i < result && buf[i] != '\n'; ++i );
if ( buf[i] == '\n' ) {
result = i + 1;
} else {
result = i;
}
offset = pos + result;
offset = pos + result; // lseek(pos+result, SEEK_SET)
// just in case ...
buf[ result ] = '\0';
@ -236,39 +223,26 @@ int SGMMapFile::readline( char *buf, int length ) {
return result;
}
std::string SGMMapFile::read_all()
{
std::string SGMMapFile::read_all() {
return std::string(buffer, size);
}
// write data to a file
int SGMMapFile::write( const char *buf, const int length ) {
size_t write_size = length;
int result = _write(buf, length);
if (write_size > size - offset) {
SG_LOG( SG_IO, SG_ALERT, "Attempting to write beyond the mmap buffer size: " << file_name );
write_size = size - offset;
if ( result != length ) {
SG_LOG( SG_IO, SG_ALERT, "Error writing data: " << file_name );
}
memcpy(buffer+offset, buf, write_size);
offset += write_size;
return write_size;
return result;
}
// write null terminated string to a file
int SGMMapFile::writestring( const char *str ) {
size_t write_size = std::strlen( str );
if (write_size > size - offset) {
SG_LOG( SG_IO, SG_ALERT, "Attempting to write beyond the mmap buffer size: " << file_name );
write_size = size - offset;
}
memcpy(buffer+offset, str, write_size);
offset += write_size;
return write_size;
int length = std::strlen( str );
return write( str, length );
}

30
simgear/io/sg_mmap.hxx
View File

@ -50,7 +50,7 @@ class SGMMapFile : public SGIOChannel {
int extraoflags = 0;
char *buffer = nullptr;
size_t offset = 0;
off_t offset = 0;
size_t size = 0;
#ifdef WIN32
typedef struct
@ -59,21 +59,20 @@ class SGMMapFile : public SGIOChannel {
void *p;
} SIMPLE_UNMMAP;
SIMPLE_UNMMAP un;
#else
int un; // referenced but not used
#endif
#ifdef WIN32
/*
* map 'filename' and return a pointer to it.
*/
// map the file descriptor and return a pointer to the buffer.
static void *simple_mmap(int, size_t, SIMPLE_UNMMAP *);
static void simple_unmmap(void*, size_t, SIMPLE_UNMMAP *);
#else
int un; // referenced but not used
# define simple_mmap(a, b, c) mmap(0, (b), PROT_READ, MAP_PRIVATE, (a), 0L)
# define simple_unmmap(a, b, c) munmap((a), (b))
#endif
ssize_t _read(void *buf, size_t count);
ssize_t _write(const void *buf, size_t count);
public:
SGMMapFile();
@ -116,11 +115,22 @@ public:
// get the pointer to the start of the buffer
inline const char *get() { return buffer; }
// get the pointer at the current offset and increase the offset by len
const char* advance(size_t len);
// get the pointer at the current offset and increase the offset by amount
// returns nullptr if the offset pointer would end up beyond the mmap
// buffer size
const char* advance(off_t amount);
// return the size of the mmaped area
inline size_t get_size() { return size; }
// forward by amount, returns the amount which could be forwarded
// without the offset getting beyond the mmap buffer size.
// returns 0 on end of file.
off_t forward(off_t amount);
// rewind the offset pointer
inline void rewind() { offset = 0; }
// write data to a file
int write( const char *buf, const int length );

219
simgear/io/test_mmap.cxx Normal file
View File

@ -0,0 +1,219 @@
////////////////////////////////////////////////////////////////////////
// Test harness.
////////////////////////////////////////////////////////////////////////
#include <simgear_config.h>
#include <simgear/compiler.h>
#include <iostream>
#include "untar.hxx"
#include <simgear/misc/test_macros.hxx>
#include <simgear/misc/sg_path.hxx>
#include <simgear/misc/sg_dir.hxx>
#include <simgear/io/sg_mmap.hxx>
using std::cout;
using std::cerr;
using std::endl;
using namespace simgear;
void testTarGz()
{
SGPath p = SGPath(SRC_DIR);
p.append("test.tar.gz");
SGMMapFile f(p);
f.open(SG_IO_IN);
uint8_t* buf = (uint8_t*) alloca(8192);
size_t bufSize = f.read((char*) buf, 8192);
SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::GZData);
f.close();
}
void testPlainTar()
{
SGPath p = SGPath(SRC_DIR);
p.append("test2.tar");
SGMMapFile f(p);
f.open(SG_IO_IN);
uint8_t* buf = (uint8_t*)alloca(8192);
size_t bufSize = f.read((char*) buf, 8192);
SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData);
f.close();
}
void testExtractStreamed()
{
SGPath p = SGPath(SRC_DIR);
p.append("test.tar.gz");
SGMMapFile f(p);
f.open(SG_IO_IN);
SGPath extractDir = simgear::Dir::current().path() / "test_extract_streamed";
simgear::Dir pd(extractDir);
pd.removeChildren();
ArchiveExtractor ex(extractDir);
uint8_t* buf = (uint8_t*) alloca(128);
while (!f.eof()) {
size_t bufSize = f.read((char*) buf, 128);
ex.extractBytes(buf, bufSize);
}
ex.flush();
SG_VERIFY(ex.isAtEndOfArchive());
SG_VERIFY(ex.hasError() == false);
SG_VERIFY((extractDir / "testDir/hello.c").exists());
SG_VERIFY((extractDir / "testDir/foo.txt").exists());
}
void testExtractLocalFile()
{
}
void testFilterTar()
{
SGPath p = SGPath(SRC_DIR);
p.append("badTar.tgz");
SGMMapFile f(p);
f.open(SG_IO_IN);
SGPath extractDir = simgear::Dir::current().path() / "test_filter_tar";
simgear::Dir pd(extractDir);
pd.removeChildren();
ArchiveExtractor ex(extractDir);
uint8_t* buf = (uint8_t*) alloca(128);
while (!f.eof()) {
size_t bufSize = f.read((char*) buf, 128);
ex.extractBytes(buf, bufSize);
}
ex.flush();
SG_VERIFY(ex.isAtEndOfArchive());
SG_VERIFY(ex.hasError() == false);
SG_VERIFY((extractDir / "tarWithBadContent/regular-file.txt").exists());
SG_VERIFY(!(extractDir / "tarWithBadContent/symbolic-linked.png").exists());
SG_VERIFY((extractDir / "tarWithBadContent/screenshot.png").exists());
SG_VERIFY((extractDir / "tarWithBadContent/dirOne/subDirA").exists());
SG_VERIFY(!(extractDir / "tarWithBadContent/dirOne/subDirA/linked.txt").exists());
}
void testExtractZip()
{
SGPath p = SGPath(SRC_DIR);
p.append("zippy.zip");
SGMMapFile f(p);
f.open(SG_IO_IN);
SGPath extractDir = simgear::Dir::current().path() / "test_extract_zip";
simgear::Dir pd(extractDir);
pd.removeChildren();
ArchiveExtractor ex(extractDir);
uint8_t* buf = (uint8_t*)alloca(128);
while (!f.eof()) {
size_t bufSize = f.read((char*)buf, 128);
ex.extractBytes(buf, bufSize);
}
ex.flush();
SG_VERIFY(ex.isAtEndOfArchive());
SG_VERIFY(ex.hasError() == false);
SG_VERIFY((extractDir / "zippy/dirA/hello.c").exists());
SG_VERIFY((extractDir / "zippy/bar.xml").exists());
SG_VERIFY((extractDir / "zippy/long-named.json").exists());
}
void testPAXAttributes()
{
SGPath p = SGPath(SRC_DIR);
p.append("pax-extended.tar");
SGMMapFile f(p);
f.open(SG_IO_IN);
SGPath extractDir = simgear::Dir::current().path() / "test_pax_extended";
simgear::Dir pd(extractDir);
pd.removeChildren();
ArchiveExtractor ex(extractDir);
uint8_t* buf = (uint8_t*) alloca(128);
while (!f.eof()) {
size_t bufSize = f.read((char*) buf, 128);
ex.extractBytes(buf, bufSize);
}
ex.flush();
SG_VERIFY(ex.isAtEndOfArchive());
SG_VERIFY(ex.hasError() == false);
}
void testExtractXZ()
{
SGPath p = SGPath(SRC_DIR);
p.append("test.tar.xz");
SGMMapFile f(p);
f.open(SG_IO_IN);
SGPath extractDir = simgear::Dir::current().path() / "test_extract_xz";
simgear::Dir pd(extractDir);
pd.removeChildren();
ArchiveExtractor ex(extractDir);
uint8_t* buf = (uint8_t*)alloca(128);
while (!f.eof()) {
size_t bufSize = f.read((char*)buf, 128);
ex.extractBytes(buf, bufSize);
}
ex.flush();
SG_VERIFY(ex.isAtEndOfArchive());
SG_VERIFY(ex.hasError() == false);
SG_VERIFY((extractDir / "testDir/hello.c").exists());
SG_VERIFY((extractDir / "testDir/foo.txt").exists());
}
int main(int ac, char ** av)
{
testTarGz();
testPlainTar();
testFilterTar();
testExtractStreamed();
testExtractZip();
testExtractXZ();
// disabled to avoiding checking in large PAX archive
// testPAXAttributes();
std::cout << "all tests passed" << std::endl;
return 0;
}