FGFS-4.1/simgear
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Packages: randomise mirror URL selection

Browse Source 
Also add retry on mirrors when a download fails.
This commit is contained in:
James Turner 2020-03-21 18:31:08 +00:00
parent 75f40fad95
commit 40725a76ab
8 changed files with 107 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
simgear/package/CatalogTest.cxx
View File

@ -62,6 +62,8 @@ unsigned int global_catalogVersion = 0;
bool global_failRequests = false;
bool global_fail747Request = true;
int global_737RequestsToFailCount = 0;
class TestPackageChannel : public TestServerChannel
{
public:
@ -106,6 +108,15 @@ public:
}
}
if ((path.find("/mirror") == 0) && (path.find("/b737.tar.gz") == 8)) {
if (global_737RequestsToFailCount > 0) {
sendErrorResponse(404, false, "Mirror failure");
--global_737RequestsToFailCount;
return;
}
path = "/catalogTest1/b737.tar.gz";
}
localPath.append(path);
// SG_LOG(SG_IO, SG_INFO, "local path is:" << localPath.str());
@ -329,7 +340,9 @@ int parseTest()
SG_VERIFY(p3->matches(queryText.ptr()));
}
string_list urls = p3->downloadUrls();
SG_CHECK_EQUAL(urls.size(), 3);
SG_CHECK_EQUAL(urls.at(1), "http://localhost:2000/mirrorB/b737.tar.gz");
delete root;
return EXIT_SUCCESS;
@ -1136,9 +1149,44 @@ void testInstallBadPackage(HTTP::Client* cl)
SG_CHECK_EQUAL(ins->path(), rootPath / "org.flightgear.test.catalog1" / "Aircraft" / "b744");
}
void testMirrorsFailure(HTTP::Client* cl)
{
global_catalogVersion = 0;
// there's three mirrors defined, so se tthe first twom attempts to fail
global_737RequestsToFailCount = 2;
SGPath rootPath(simgear::Dir::current().path());
rootPath.append("pkg_install_mirror_fail");
simgear::Dir pd(rootPath);
pd.removeChildren();
pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
// specify a test dir
root->setHTTPClient(cl);
pkg::CatalogRef c = pkg::Catalog::createFromUrl(root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
waitForUpdateComplete(cl, root);
pkg::PackageRef p1 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
pkg::InstallRef ins = p1->install();
bool didFail = false;
ins->fail([&didFail, &ins](pkg::Install* ourInstall) {
SG_CHECK_EQUAL(ins, ourInstall);
didFail = true;
});
waitForUpdateComplete(cl, root);
SG_VERIFY(p1->isInstalled());
SG_VERIFY(!didFail);
SG_VERIFY(p1->existingInstall() == ins);
SG_CHECK_EQUAL(ins->status(), pkg::Delegate::STATUS_SUCCESS);
}
int main(int argc, char* argv[])
{
// sglog().setLogLevels( SG_ALL, SG_DEBUG );
sglog().setLogLevels( SG_ALL, SG_WARN );
HTTP::Client cl;
cl.setMaxConnections(1);
@ -1150,7 +1198,7 @@ int main(int argc, char* argv[])
parseTest();
parseInvalidTest();
testInstallPackage(&cl);
testUninstall(&cl);
@ -1179,6 +1227,8 @@ int main(int argc, char* argv[])
testInstallBadPackage(&cl);
SG_LOG(SG_GENERAL, SG_INFO, "Successfully passed all tests!");
testMirrorsFailure(&cl);
cerr << "Successfully passed all tests!" << endl;
return EXIT_SUCCESS;
}

55
simgear/package/Install.cxx
View File

@ -18,8 +18,8 @@
#include <simgear_config.h>
#include <simgear/package/Install.hxx>
#include <boost/foreach.hpp>
#include <fstream>
#include <cstdlib>
#include <simgear/package/unzip.h>
#include <simgear/package/md5.h>
@ -43,21 +43,17 @@ namespace pkg {
class Install::PackageArchiveDownloader : public HTTP::Request
{
public:
PackageArchiveDownloader(InstallRef aOwner) :
PackageArchiveDownloader(InstallRef aOwner, const string_list& urls) :
HTTP::Request("" /* dummy URL */),
m_owner(aOwner)
{
m_urls = m_owner->package()->downloadUrls();
if (m_urls.empty()) {
throw sg_exception("no package download URLs");
m_urls = urls;
if (urls.empty()) {
m_urls = m_owner->package()->downloadUrls();
}
// if (m_owner->package()->properties()->hasChild("archive-type")) {
// setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type"));
//}
selectMirrorUrl();
// TODO randomise order of m_urls
m_extractPath = aOwner->path().dir();
m_extractPath.append("_extract_" + aOwner->package()->md5());
@ -93,12 +89,19 @@ public:
return (m_downloaded * 100) / responseLength();
}
protected:
virtual std::string url() const
void selectMirrorUrl()
{
return m_urls.front();
const int randomizedIndex = rand() % m_urls.size();
m_activeURL = m_urls.at(randomizedIndex);
m_urls.erase(m_urls.begin() + randomizedIndex);
}
virtual std::string url() const override
{
return m_activeURL;
}
virtual void responseHeadersComplete()
void responseHeadersComplete() override
{
Request::responseHeadersComplete();
@ -112,7 +115,7 @@ protected:
m_owner->startDownload();
}
virtual void gotBodyData(const char* s, int n)
void gotBodyData(const char* s, int n) override
{
const uint8_t* ubytes = (uint8_t*) s;
SG_MD5Update(&m_md5, ubytes, n);
@ -121,10 +124,10 @@ protected:
m_extractor->extractBytes(ubytes, n);
}
virtual void onDone()
void onDone() override
{
if (responseCode() != 200) {
SG_LOG(SG_GENERAL, SG_ALERT, "download failure:" << responseCode() <<
SG_LOG(SG_GENERAL, SG_WARN, "download failure:" << responseCode() <<
"\n\t" << url());
Delegate::StatusCode code = Delegate::FAIL_DOWNLOAD;
if (responseCode() == 404) {
@ -141,7 +144,7 @@ protected:
strutils::encodeHex(digest, MD5_DIGEST_LENGTH);
if (hex_md5 != m_owner->package()->md5()) {
SG_LOG(SG_GENERAL, SG_ALERT, "md5 verification failed:\n"
SG_LOG(SG_GENERAL, SG_WARN, "md5 verification failed:\n"
<< "\t" << hex_md5 << "\n\t"
<< m_owner->package()->md5() << "\n\t"
<< "downloading from:" << url());
@ -196,7 +199,7 @@ protected:
m_owner->installResult(Delegate::STATUS_SUCCESS);
}
virtual void onFail()
void onFail() override
{
if (responseCode() == -1) {
doFailure(Delegate::USER_CANCELLED);
@ -212,6 +215,18 @@ private:
if (dir.exists()) {
dir.remove(true /* recursive */);
}
const auto canRetry = (aReason == Delegate::FAIL_NOT_FOUND) || (aReason == Delegate::FAIL_DOWNLOAD);
if (canRetry && !m_urls.empty()) {
SG_LOG(SG_GENERAL, SG_WARN, "archive download failed from:" << m_activeURL
<< "\n\twill retry with next mirror");
// becuase selectMirrorUrl erased the active URL from m_urls,
// this new request will select one of the other mirrors
auto retryDownload = new PackageArchiveDownloader(m_owner, m_urls);
m_owner->m_download.reset(retryDownload);
m_owner->package()->catalog()->root()->makeHTTPRequest(retryDownload);
return;
}
// TODO - try other mirrors
m_owner->m_download.reset(); // ensure we get cleaned up
@ -220,6 +235,7 @@ private:
InstallRef m_owner;
string m_activeURL;
string_list m_urls;
SG_MD5_CTX m_md5;
SGPath m_extractPath;
@ -283,7 +299,7 @@ void Install::startUpdate()
return; // already active
}
m_download = new PackageArchiveDownloader(this);
m_download = new PackageArchiveDownloader(this, {});
m_package->catalog()->root()->makeHTTPRequest(m_download);
m_package->catalog()->root()->startInstall(this);
}
@ -343,6 +359,7 @@ void Install::cancelDownload()
}
m_package->catalog()->root()->cancelDownload(this);
m_download.clear();
}
SGPath Install::primarySetPath() const

2
simgear/package/Root.cxx
View File

@ -660,7 +660,7 @@ void Root::startNext(InstallRef aCurrent)
void Root::finishInstall(InstallRef aInstall, Delegate::StatusCode aReason)
{
if (aReason != Delegate::STATUS_SUCCESS) {
SG_LOG(SG_GENERAL, SG_ALERT, "failed to install package:"
SG_LOG(SG_GENERAL, SG_WARN, "failed to install package:"
<< aInstall->package()->id() << ":" << aReason);
}

4
simgear/package/catalogTest1/catalog-alt.xml
View File

@ -123,7 +123,9 @@
</rating>
<md5>a94ca5704f305b90767f40617d194ed6</md5>
<url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
<url>http://localhost:2000/mirrorA/b737.tar.gz</url>
<url>http://localhost:2000/mirrorB/b737.tar.gz</url>
<url>http://localhost:2000/mirrorC/b737.tar.gz</url>
</package>

4
simgear/package/catalogTest1/catalog-v10.xml
View File

@ -163,7 +163,9 @@
</rating>
<md5>a94ca5704f305b90767f40617d194ed6</md5>
<url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
<url>http://localhost:2000/mirrorA/b737.tar.gz</url>
<url>http://localhost:2000/mirrorB/b737.tar.gz</url>
<url>http://localhost:2000/mirrorC/b737.tar.gz</url>
</package>
<package>

5
simgear/package/catalogTest1/catalog-v2.xml
View File

@ -150,7 +150,10 @@
</rating>
<md5>a94ca5704f305b90767f40617d194ed6</md5>
<url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
<url>http://localhost:2000/mirrorA/b737.tar.gz</url>
<url>http://localhost:2000/mirrorB/b737.tar.gz</url>
<url>http://localhost:2000/mirrorC/b737.tar.gz</url>
<preview>
<type>exterior</type>

4
simgear/package/catalogTest1/catalog-v9.xml
View File

@ -163,7 +163,9 @@
</rating>
<md5>a94ca5704f305b90767f40617d194ed6</md5>
<url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
<url>http://localhost:2000/mirrorA/b737.tar.gz</url>
<url>http://localhost:2000/mirrorB/b737.tar.gz</url>
<url>http://localhost:2000/mirrorC/b737.tar.gz</url>
</package>
<package>

4
simgear/package/catalogTest1/catalog.xml
View File

@ -191,7 +191,9 @@
</rating>
<md5>a94ca5704f305b90767f40617d194ed6</md5>
<url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
<url>http://localhost:2000/mirrorA/b737.tar.gz</url>
<url>http://localhost:2000/mirrorB/b737.tar.gz</url>
<url>http://localhost:2000/mirrorC/b737.tar.gz</url>
</package>
<package>