FGFS-4.1/simgear
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TerraSync: stronger fix for handling 0-byte files

Browse Source 
Change logic so we create an empty file for such cases, i.e exactly
matching the repository. This simplifies logic in downstream code,
compared with not creating a local file.

Add a test-case to cover this

Modify TerraSync to detect a failure of Airports_archive downloading,
and fall back to file-by-file updating.
This commit is contained in:
James Turner 2020-12-03 16:40:48 +00:00
parent 57a4dc53f2
commit 191d546b54
3 changed files with 76 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
simgear/io/HTTPRepository.cxx
View File

@ -979,55 +979,74 @@ HTTPRepository::failure() const
{
pathInRepo = _directory->absolutePath();
pathInRepo.append(fileName);
sha1_init(&hashContext);
}
protected:
void gotBodyData(const char *s, int n) override {
if (!file.get()) {
file.reset(new SGBinaryFile(pathInRepo));
if (!file->open(SG_IO_OUT)) {
SG_LOG(SG_TERRASYNC, SG_WARN,
"unable to create file " << pathInRepo);
_directory->repository()->http->cancelRequest(
this, "Unable to create output file:" + pathInRepo.utf8Str());
}
void gotBodyData(const char* s, int n) override
{
if (!file.get()) {
const bool ok = createOutputFile();
if (!ok) {
_directory->repository()->http->cancelRequest(
this, "Unable to create output file:" + pathInRepo.utf8Str());
}
}
sha1_write(&hashContext, s, n);
file->write(s, n);
}
sha1_write(&hashContext, s, n);
file->write(s, n);
}
bool createOutputFile()
{
file.reset(new SGBinaryFile(pathInRepo));
if (!file->open(SG_IO_OUT)) {
SG_LOG(SG_TERRASYNC, SG_WARN,
"unable to create file " << pathInRepo);
return false;
}
void onDone() override {
if (file) {
file->close();
}
if (responseCode() == 200) {
std::string hash =
strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
_directory->didUpdateFile(fileName, hash, contentSize());
} else if (responseCode() == 404) {
SG_LOG(SG_TERRASYNC, SG_WARN,
"terrasync file not found on server: "
<< fileName << " for " << _directory->absolutePath());
_directory->didFailToUpdateFile(
fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
} else {
SG_LOG(SG_TERRASYNC, SG_WARN,
"terrasync file download error on server: "
<< fileName << " for " << _directory->absolutePath()
<< "\n\tserver responded: " << responseCode() << "/"
<< responseReason());
_directory->didFailToUpdateFile(fileName,
HTTPRepository::REPO_ERROR_HTTP);
// should we every retry here?
sha1_init(&hashContext);
return true;
}
_directory->repository()->finishedRequest(
this, HTTPRepoPrivate::RequestFinish::Done);
}
void onDone() override
{
const bool is200Response = (responseCode() == 200);
if (!file && is200Response) {
// if the server defines a zero-byte file, we will never call
// gotBodyData, so create the file here
// this ensures all the logic below works as expected
createOutputFile();
}
if (file) {
file->close();
}
if (is200Response) {
std::string hash =
strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
_directory->didUpdateFile(fileName, hash, contentSize());
} else if (responseCode() == 404) {
SG_LOG(SG_TERRASYNC, SG_WARN,
"terrasync file not found on server: "
<< fileName << " for " << _directory->absolutePath());
_directory->didFailToUpdateFile(
fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
} else {
SG_LOG(SG_TERRASYNC, SG_WARN,
"terrasync file download error on server: "
<< fileName << " for " << _directory->absolutePath()
<< "\n\tserver responded: " << responseCode() << "/"
<< responseReason());
_directory->didFailToUpdateFile(fileName,
HTTPRepository::REPO_ERROR_HTTP);
// should we every retry here?
}
_directory->repository()->finishedRequest(
this, HTTPRepoPrivate::RequestFinish::Done);
}
void onFail() override {
HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;

7
simgear/io/test_repository.cxx
View File

@ -31,6 +31,10 @@ using TestApi = simgear::HTTP::TestApi;
std::string dataForFile(const std::string& parentName, const std::string& name, int revision)
{
if (name == "zeroByteFile") {
return {};
}
std::ostringstream os;
// random content but which definitely depends on our tree location
// and revision.
@ -446,6 +450,7 @@ void testBasicClone(HTTP::Client* cl)
verifyFileState(p, "fileA");
verifyFileState(p, "dirA/subdirA/fileAAA");
verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
verifyFileState(p, "dirA/subdirA/zeroByteFile");
global_repo->findEntry("fileA")->revision++;
global_repo->findEntry("dirB/subdirA/fileBAA")->revision++;
@ -911,6 +916,8 @@ int main(int argc, char* argv[])
global_repo->defineFile("dirA/fileAC");
global_repo->defineFile("dirA/subdirA/fileAAA");
global_repo->defineFile("dirA/subdirA/fileAAB");
global_repo->defineFile("dirA/subdirA/zeroByteFile");
global_repo->defineFile("dirB/subdirA/fileBAA");
global_repo->defineFile("dirB/subdirA/fileBAB");
global_repo->defineFile("dirB/subdirA/fileBAC");

9
simgear/scene/tsync/terrasync.cxx
View File

@ -571,6 +571,15 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
notFound(slot.currentItem);
} else if (res != HTTPRepository::REPO_NO_ERROR) {
fail(slot.currentItem);
// in case the Airports_archive download fails, create the
// directory, so that next sync, we do a manual sync
if ((slot.currentItem._type == SyncItem::AirportData) && slot.isNewDirectory) {
SG_LOG(SG_TERRASYNC, SG_ALERT, "Failed to download Airports_archive, will download discrete files next time");
simgear::Dir d(_local_dir + "/Airports");
d.create(0755);
_completedTiles.erase(slot.currentItem._dir);
}
} else {
updated(slot.currentItem, slot.isNewDirectory);
SG_LOG(SG_TERRASYNC, SG_DEBUG, "sync of " << slot.repository->baseUrl() << " finished ("