FGFS-4.1/simgear
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial Tar package support.

Browse Source 
Needs proper testing, but basic unit-test passes.
This commit is contained in:
James Turner 2016-05-04 20:35:48 +01:00
parent 5b71ede2ea
commit 871b418242
12 changed files with 541 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
simgear/package/CMakeLists.txt
View File

@ -1,7 +1,7 @@
include (SimGearComponent)
set(HEADERS
set(HEADERS
Catalog.hxx
Package.hxx
Install.hxx
@ -9,7 +9,7 @@ set(HEADERS
Delegate.hxx
)
set(SOURCES
set(SOURCES
Catalog.cxx
Package.cxx
Install.cxx
@ -18,6 +18,7 @@ set(SOURCES
md5.h md5.c
ioapi.c ioapi_mem.c ioapi.h
unzip.h unzip.c
untar.hxx untar.cxx
)
simgear_component(package package "${SOURCES}" "${HEADERS}")
@ -34,7 +35,7 @@ target_link_libraries(catalog_test ${TEST_LIBS})
set_target_properties(catalog_test PROPERTIES
COMPILE_DEFINITIONS "SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\"" )
add_test(catalog_test ${EXECUTABLE_OUTPUT_PATH}/catalog_test)
endif(ENABLE_TESTS)
endif(ENABLE_TESTS)

42
simgear/package/CatalogTest.cxx
View File

@ -133,7 +133,7 @@ int parseTest()
COMPARE(cat->description(), "First test catalog");
// check the packages too
COMPARE(cat->packages().size(), 3);
COMPARE(cat->packages().size(), 4);
pkg::PackageRef p1 = cat->packages().front();
COMPARE(p1->catalog(), cat.ptr());
@ -208,7 +208,7 @@ void testAddCatalog(HTTP::Client* cl)
p.append("org.flightgear.test.catalog1");
p.append("catalog.xml");
VERIFY(p.exists());
COMPARE(root->allPackages().size(), 3);
COMPARE(root->allPackages().size(), 4);
COMPARE(root->catalogs().size(), 1);
pkg::PackageRef p1 = root->getPackageById("alpha");
@ -403,6 +403,42 @@ void testRefreshCatalog(HTTP::Client* cl)
COMPARE(root->getPackageById("common-sounds")->revision(), 11);
}
void testInstallTarPackage(HTTP::Client* cl)
{
SGPath rootPath(simgear::Dir::current().path());
rootPath.append("pkg_install_tar");
simgear::Dir pd(rootPath);
pd.removeChildren();
pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
// specify a test dir
root->setHTTPClient(cl);
pkg::CatalogRef c = pkg::Catalog::createFromUrl(root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
waitForUpdateComplete(cl, root);
pkg::PackageRef p1 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
COMPARE(p1->id(), "b737-NG");
pkg::InstallRef ins = p1->install();
VERIFY(ins->isQueued());
waitForUpdateComplete(cl, root);
VERIFY(p1->isInstalled());
VERIFY(p1->existingInstall() == ins);
// verify on disk state
SGPath p(rootPath);
p.append("org.flightgear.test.catalog1");
p.append("Aircraft");
p.append("b737NG");
COMPARE(p, ins->path());
p.append("b737-900-set.xml");
VERIFY(p.exists());
}
int main(int argc, char* argv[])
{
@ -425,6 +461,8 @@ int main(int argc, char* argv[])
testRefreshCatalog(&cl);
testInstallTarPackage(&cl);
std::cout << "Successfully passed all tests!" << std::endl;
return EXIT_SUCCESS;
}

28
simgear/package/Install.cxx
View File

@ -22,6 +22,7 @@
#include <simgear/package/unzip.h>
#include <simgear/package/md5.h>
#include <simgear/package/untar.hxx>
#include <simgear/structure/exception.hxx>
#include <simgear/props/props_io.hxx>
@ -144,8 +145,8 @@ protected:
return;
}
if (!extractUnzip()) {
SG_LOG(SG_GENERAL, SG_WARN, "zip extraction failed");
if (!extract()) {
SG_LOG(SG_GENERAL, SG_WARN, "archive extraction failed");
doFailure(Delegate::FAIL_EXTRACT);
return;
}
@ -250,6 +251,22 @@ private:
unzCloseCurrentFile(zip);
}
bool extract()
{
const std::string u(url());
const size_t ul(u.length());
if (u.rfind(".zip") == (ul - 4)) {
return extractUnzip();
}
if (u.rfind(".tar.gz") == (ul - 7)) {
return extractTar();
}
SG_LOG(SG_IO, SG_WARN, "unsupported archive format:" << u);
return false;
}
bool extractUnzip()
{
bool result = true;
@ -287,6 +304,13 @@ private:
return result;
}
bool extractTar()
{
TarExtractor tx(m_extractPath);
tx.extractBytes(m_buffer.data(), m_buffer.size());
return !tx.hasError() && tx.isAtEndOfArchive();
}
void doFailure(Delegate::StatusCode aReason)
{
Dir dir(m_extractPath);

BIN
simgear/package/catalogTest1/b737.tar.gz Normal file
View File

Binary file not shown.

7
simgear/package/catalogTest1/b737NG/b737-700-set.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0"?>
<PropertyList>
<sim>
<description>Boeing 737-700</description>
</sim>
</PropertyList>

7
simgear/package/catalogTest1/b737NG/b737-800-set.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0"?>
<PropertyList>
<sim>
<description>Boeing 737-800</description>
</sim>
</PropertyList>

7
simgear/package/catalogTest1/b737NG/b737-900-set.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0"?>
<PropertyList>
<sim>
<description>Boeing 737-900</description>
</sim>
</PropertyList>

9
simgear/package/catalogTest1/b737NG/b737-common-set.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0"?>
<PropertyList>
<sim>
<tags>
<tag>boeing</tag>
</tags>
</sim>
</PropertyList>

24
simgear/package/catalogTest1/catalog-v2.xml
View File

@ -78,6 +78,30 @@
<md5>acf9eb89cf396eb42f8823d9cdf17584</md5>
</package>
<package>
<id>b737-NG</id>
<name>Boeing 737 NG</name>
<dir>b737NG</dir>
<description>A popular twin-engined narrow body jet</description>
<revision type="int">112</revision>
<file-size-bytes type="int">860</file-size-bytes>
<tag>boeing</tag>
<tag>jet</tag>
<tag>ifr</tag>
<rating>
<FDM type="int">5</FDM>
<systems type="int">5</systems>
<model type="int">4</model>
<cockpit type="int">4</cockpit>
</rating>
<md5>a94ca5704f305b90767f40617d194ed6</md5>
<url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
</package>
<package>
<id>dc3</id>
<name>DC-3</name>

23
simgear/package/catalogTest1/catalog.xml
View File

@ -68,6 +68,29 @@
</package>
<package>
<id>b737-NG</id>
<name>Boeing 737 NG</name>
<dir>b737NG</dir>
<description>A popular twin-engined narrow body jet</description>
<revision type="int">111</revision>
<file-size-bytes type="int">860</file-size-bytes>
<tag>boeing</tag>
<tag>jet</tag>
<tag>ifr</tag>
<rating>
<FDM type="int">5</FDM>
<systems type="int">5</systems>
<model type="int">4</model>
<cockpit type="int">4</cockpit>
</rating>
<md5>a94ca5704f305b90767f40617d194ed6</md5>
<url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
</package>
<package>
<id>common-sounds</id>
<name>Common sound files for test catalog aircraft</name>

341
simgear/package/untar.cxx Normal file
View File

@ -0,0 +1,341 @@
// Copyright (C) 2016 James Turner - <zakalawe@mac.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//
#include "untar.hxx"
#include <cstdlib>
#include <cassert>
#include <zlib.h>
#include <simgear/io/sg_file.hxx>
#include <simgear/misc/sg_dir.hxx>
#include <simgear/debug/logstream.hxx>
namespace simgear
{
namespace pkg
{
const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024;
const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS;
const int ZLIB_DECODE_GZIP_HEADER = 16;
/* tar Header Block, from POSIX 1003.1-1990. */
typedef struct
{
char fileName[100];
char mode[8]; /* 100 */
char uid[8]; /* 108 */
char gid[8]; /* 116 */
char size[12]; /* 124 */
char mtime[12]; /* 136 */
char chksum[8]; /* 148 */
char typeflag; /* 156 */
char linkname[100]; /* 157 */
char magic[6]; /* 257 */
char version[2]; /* 263 */
char uname[32]; /* 265 */
char gname[32]; /* 297 */
char devmajor[8]; /* 329 */
char devminor[8]; /* 337 */
char prefix[155]; /* 345 */
} UstarHeaderBlock;
const size_t TAR_HEADER_BLOCK_SIZE = 512;
#define TMAGIC "ustar" /* ustar and a null */
#define TMAGLEN 6
#define TVERSION "00" /* 00 and no null */
#define TVERSLEN 2
/* Values used in typeflag field. */
#define REGTYPE '0' /* regular file */
#define AREGTYPE '\0' /* regular file */
#define LNKTYPE '1' /* link */
#define SYMTYPE '2' /* reserved */
#define CHRTYPE '3' /* character special */
#define BLKTYPE '4' /* block special */
#define DIRTYPE '5' /* directory */
#define FIFOTYPE '6' /* FIFO special */
#define CONTTYPE '7' /* reserved */
class TarExtractorPrivate
{
public:
typedef enum {
INVALID = 0,
READING_HEADER,
READING_FILE,
READING_PADDING,
PRE_END_OF_ARCHVE,
END_OF_ARCHIVE,
ERROR_STATE, ///< states above this are error conditions
BAD_ARCHIVE,
BAD_DATA
} State;
SGPath path;
State state;
union {
UstarHeaderBlock header;
uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE];
};
size_t bytesRemaining;
std::auto_ptr<SGFile> currentFile;
size_t currentFileSize;
z_stream zlibStream;
uint8_t* zlibOutput;
bool haveInitedZLib;
uint8_t* headerPtr;
TarExtractorPrivate() :
haveInitedZLib(false)
{
}
~TarExtractorPrivate()
{
free(zlibOutput);
}
void checkEndOfState()
{
if (bytesRemaining > 0) {
return;
}
if (state == READING_FILE) {
currentFile->close();
size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE;
if (pad) {
bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad;
setState(READING_PADDING);
} else {
setState(READING_HEADER);
}
} else if (state == READING_HEADER) {
processHeader();
} else if (state == PRE_END_OF_ARCHVE) {
if (headerIsAllZeros()) {
setState(END_OF_ARCHIVE);
} else {
// what does the spec say here?
}
} else if (state == READING_PADDING) {
setState(READING_HEADER);
}
}
void setState(State newState)
{
if ((newState == READING_HEADER) || (newState == PRE_END_OF_ARCHVE)) {
bytesRemaining = TAR_HEADER_BLOCK_SIZE;
headerPtr = headerBytes;
}
state = newState;
}
void processHeader()
{
if (headerIsAllZeros()) {
if (state == PRE_END_OF_ARCHVE) {
setState(END_OF_ARCHIVE);
} else {
setState(PRE_END_OF_ARCHVE);
}
return;
}
if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) {
SG_LOG(SG_IO, SG_WARN, "magic is wrong");
state = BAD_ARCHIVE;
return;
}
std::string tarPath = std::string(header.prefix) + std::string(header.fileName);
if (!isSafePath(tarPath)) {
//state = BAD_ARCHIVE;
SG_LOG(SG_IO, SG_WARN, "bad tar path:" << tarPath);
//return;
}
SGPath p = path;
p.append(tarPath);
if (header.typeflag == DIRTYPE) {
Dir dir(p);
dir.create(0755);
setState(READING_HEADER);
} else if ((header.typeflag == REGTYPE) || (header.typeflag == AREGTYPE)) {
currentFileSize = ::strtol(header.size, NULL, 8);
bytesRemaining = currentFileSize;
currentFile.reset(new SGBinaryFile(p.str()));
currentFile->open(SG_IO_OUT);
setState(READING_FILE);
} else {
SG_LOG(SG_IO, SG_WARN, "Unsupported tar file type:" << header.typeflag);
state = BAD_ARCHIVE;
}
}
void processBytes(const char* bytes, size_t count)
{
if ((state >= ERROR_STATE) || (state == END_OF_ARCHIVE)) {
return;
}
size_t curBytes = std::min(bytesRemaining, count);
if (state == READING_FILE) {
currentFile->write(bytes, curBytes);
bytesRemaining -= curBytes;
} else if ((state == READING_HEADER) || (state == PRE_END_OF_ARCHVE) || (state == END_OF_ARCHIVE)) {
memcpy(headerPtr, bytes, curBytes);
bytesRemaining -= curBytes;
headerPtr += curBytes;
} else if (state == READING_PADDING) {
bytesRemaining -= curBytes;
}
checkEndOfState();
if (count > curBytes) {
// recurse with unprocessed bytes
processBytes(bytes + curBytes, count - curBytes);
}
}
bool headerIsAllZeros() const
{
char* headerAsChar = (char*) &header;
for (size_t i=0; i < offsetof(UstarHeaderBlock, magic); ++i) {
if (*headerAsChar++ != 0) {
return false;
}
}
return true;
}
bool isSafePath(const std::string& p) const
{
if (p.empty()) {
return false;
}
// reject absolute paths
if (p.front() == '/') {
return false;
}
// reject paths containing '..'
size_t doubleDot = p.find("..");
if (doubleDot != std::string::npos) {
return false;
}
// on POSIX could use realpath to sanity check
return true;
}
};
TarExtractor::TarExtractor(const SGPath& rootPath) :
d(new TarExtractorPrivate)
{
d->path = rootPath;
d->state = TarExtractorPrivate::INVALID;
memset(&d->zlibStream, 0, sizeof(z_stream));
d->zlibOutput = (unsigned char*) malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
d->zlibStream.zalloc = Z_NULL;
d->zlibStream.zfree = Z_NULL;
d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
d->zlibStream.next_out = d->zlibOutput;
}
void TarExtractor::extractBytes(const char* bytes, size_t count)
{
if (d->state >= TarExtractorPrivate::ERROR_STATE) {
return;
}
d->zlibStream.next_in = (uint8_t*) bytes;
d->zlibStream.avail_in = count;
if (!d->haveInitedZLib) {
if (inflateInit2(&d->zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
d->state = TarExtractorPrivate::BAD_DATA;
return;
} else {
d->haveInitedZLib = true;
d->setState(TarExtractorPrivate::READING_HEADER);
}
}
size_t writtenSize;
// loop, running zlib() inflate and sending output bytes to
// our request body handler. Keep calling inflate until no bytes are
// written, and ZLIB has consumed all available input
do {
d->zlibStream.next_out = d->zlibOutput;
d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
int result = inflate(&d->zlibStream, Z_NO_FLUSH);
if (result == Z_OK || result == Z_STREAM_END) {
// nothing to do
} else if (result == Z_BUF_ERROR) {
// transient error, fall through
} else {
// _error = result;
SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << d->zlibStream.msg);
d->state = TarExtractorPrivate::BAD_DATA;
return;
}
writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - d->zlibStream.avail_out;
if (writtenSize > 0) {
d->processBytes((const char*) d->zlibOutput, writtenSize);
}
if (result == Z_STREAM_END) {
break;
}
} while ((d->zlibStream.avail_in > 0) || (writtenSize > 0));
}
bool TarExtractor::isAtEndOfArchive() const
{
return (d->state == TarExtractorPrivate::END_OF_ARCHIVE);
}
bool TarExtractor::hasError() const
{
return (d->state >= TarExtractorPrivate::ERROR_STATE);
}
} // of pkg
} // of simgear

52
simgear/package/untar.hxx Normal file
View File

@ -0,0 +1,52 @@
// Copyright (C) 2016 James Turner - <zakalawe@mac.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//
#ifndef SG_PKG_UNTAR_HXX
#define SG_PKG_UNTAR_HXX
#include <memory>
#include <simgear/misc/sg_path.hxx>
namespace simgear
{
namespace pkg
{
class TarExtractorPrivate;
class TarExtractor
{
public:
TarExtractor(const SGPath& rootPath);
void extractBytes(const char* bytes, size_t count);
bool isAtEndOfArchive() const;
bool hasError() const;
private:
std::auto_ptr<TarExtractorPrivate> d;
};
} // of namespace pkg
} // of namespace simgear
#endif