FGFS-4.1/simgear
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TerraSync: avoid 404s to probe missing tiles

Browse Source 
Use the top-level dirIndex files to determine if a 1x1 tile dir
exists, instead of proving the server via a 404. This reduces the
number of requests we make considerably, which is … important.
This commit is contained in:
James Turner 2020-10-30 16:45:45 +00:00
parent 1ab1cae199
commit 0dd046065d
1 changed files with 89 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
simgear/scene/tsync/terrasync.cxx
View File

@ -333,6 +333,10 @@ public:
void runInternal();
void updateSyncSlot(SyncSlot& slot);
void beginSyncAirports(SyncSlot& slot);
void beginSyncTile(SyncSlot& slot);
void beginNormalSync(SyncSlot& slot);
void drainWaitingTiles();
// commond helpers between both internal and external models
@ -588,41 +592,14 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
SGPath path(_local_dir);
path.append(slot.currentItem._dir);
slot.isNewDirectory = !path.exists();
if (slot.isNewDirectory) {
int rc = path.create_dir( 0755 );
if (rc) {
SG_LOG(SG_TERRASYNC,SG_ALERT,
"Cannot create directory '" << path << "', return code = " << rc );
fail(slot.currentItem);
return;
}
} // of creating directory step
// optimise initial Airport download
if (slot.isNewDirectory &&
(slot.currentItem._type == SyncItem::AirportData)) {
SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
// we want to sync the 'root' TerraSync dir, but not all of it, just
// the Airports_archive.tar.gz file so we use our TerraSync local root
// as the path (since the archive will add Airports/)
slot.repository.reset(new HTTPRepository(_local_dir, &_http));
slot.repository->setBaseUrl(_httpServer + "/");
// filter callback to *only* sync the Airport_archive tarball,
// and ensure no other contents are touched
auto f = [](const HTTPRepository::SyncItem &item) {
if (!item.directory.empty())
return false;
return (item.filename.find("Airports_archive.") == 0);
};
slot.repository->setFilter(f);
const auto type = slot.currentItem._type;
if (type == SyncItem::AirportData) {
beginSyncAirports(slot);
} else if (type == SyncItem::Tile) {
beginSyncTile(slot);
} else {
slot.repository.reset(new HTTPRepository(path, &_http));
slot.repository->setBaseUrl(_httpServer + "/" +
slot.currentItem._dir);
beginNormalSync(slot);
}
if (_installRoot.exists()) {
@ -651,6 +628,85 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
}
}
void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
{
if (!slot.isNewDirectory) {
beginNormalSync(slot);
return;
}
SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
// we want to sync the 'root' TerraSync dir, but not all of it, just
// the Airports_archive.tar.gz file so we use our TerraSync local root
// as the path (since the archive will add Airports/)
slot.repository.reset(new HTTPRepository(_local_dir, &_http));
slot.repository->setBaseUrl(_httpServer + "/");
// filter callback to *only* sync the Airport_archive tarball,
// and ensure no other contents are touched
auto f = [](const HTTPRepository::SyncItem& item) {
if (!item.directory.empty())
return false;
return (item.filename.find("Airports_archive.") == 0);
};
slot.repository->setFilter(f);
}
void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
{
// avoid 404 requests by doing a sync which excludes all paths
// except our tile path. In the case of a missing 1x1 tile, we will
// stop becuase all directories are filtered out, which is what we want
auto comps = strutils::split(slot.currentItem._dir, "/");
if (comps.size() != 3) {
SG_LOG(SG_TERRASYNC, SG_ALERT, "Bad tile path:" << slot.currentItem._dir);
beginNormalSync(slot);
return;
}
const auto tileCategory = comps.front();
const auto tenByTenDir = comps.at(1);
const auto oneByOneDir = comps.at(2);
const auto path = SGPath::fromUtf8(_local_dir) / tileCategory;
slot.repository.reset(new HTTPRepository(path, &_http));
slot.repository->setBaseUrl(_httpServer + "/" + tileCategory);
const auto dirPrefix = tenByTenDir + "/" + oneByOneDir;
// filter callback to *only* sync the 1x1 dir we want, if it exists
// if doesn't, we'll simply stop, which is what we want
auto f = [tenByTenDir, oneByOneDir, dirPrefix](const HTTPRepository::SyncItem& item) {
// only allow the specific 10x10 and 1x1 dirs we want
if (item.directory.empty()) {
return item.filename == tenByTenDir;
} else if (item.directory == tenByTenDir) {
return item.filename == oneByOneDir;
}
// allow arbitrary children below dirPrefix, including sub-dirs
if (item.directory.find(dirPrefix) == 0) {
return true;
}
SG_LOG(SG_TERRASYNC, SG_ALERT, "Tile sync: saw weird path:" << item.directory << " file " << item.filename);
return false;
};
slot.repository->setFilter(f);
}
void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
{
SGPath path(_local_dir);
path.append(slot.currentItem._dir);
slot.repository.reset(new HTTPRepository(path, &_http));
slot.repository->setBaseUrl(_httpServer + "/" + slot.currentItem._dir);
}
void SGTerraSync::WorkerThread::runInternal()
{
while (!_stop) {