FGFS-4.1/simgear
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Catalog migration: migrate packages too

Browse Source 
When doing a catalog migration to a new ID (eg, 2018 -> 2020), also
mark the installed packages for installation, on the new catalog.

Related to this, when manually removing a catalog, record this fact,
so we don’t re-add it automatically due to migration.

Add unit-tests covering both of these cases.
This commit is contained in:
Automatic Release Builder 2020-10-22 21:27:47 +01:00
parent 444e2ffb2d
commit 1568ed8b97
8 changed files with 412 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
simgear/package/Catalog.cxx
View File

@ -643,16 +643,53 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
return;
}
// we have an alternate ID, and it's differnt from our ID, so let's
// we have an alternate ID, and it's different from our ID, so let's
// define a new catalog
if (!altId.empty()) {
SG_LOG(SG_GENERAL, SG_INFO, "Adding new catalog:" << altId << " as version alternate for " << id());
// new catalog being added
createFromUrl(root(), altUrl);
// and we can go idle now
// don't auto-re-add Catalogs the user has explicilty rmeoved, that would
// suck
const auto removedByUser = root()->explicitlyRemovedCatalogs();
auto it = std::find(removedByUser.begin(), removedByUser.end(), altId);
if (it != removedByUser.end()) {
changeStatus(Delegate::FAIL_VERSION);
return;
}
SG_LOG(SG_GENERAL, SG_WARN,
"Adding new catalog:" << altId << " as version alternate for "
<< id());
// new catalog being added
auto newCat = createFromUrl(root(), altUrl);
bool didRun = false;
newCat->m_migratedFrom = this;
auto migratePackagesCb = [didRun](Catalog *c) mutable {
// removing callbacks is awkward, so use this
// flag to only run once. (and hence, we need to be mutable)
if (didRun)
return;
if (c->status() == Delegate::STATUS_REFRESHED) {
didRun = true;
string_list existing;
for (const auto &pack : c->migratedFrom()->installedPackages()) {
existing.push_back(pack->id());
}
const int count = c->markPackagesForInstallation(existing);
SG_LOG(
SG_GENERAL, SG_INFO,
"Marked " << count
<< " packages from previous catalog for installation");
}
};
newCat->addStatusCallback(migratePackagesCb);
// and we can go idle now
changeStatus(Delegate::FAIL_VERSION);
return;
}
SG_LOG(SG_GENERAL, SG_INFO, "Migrating catalog " << id() << " to new URL:" << altUrl);
@ -661,6 +698,26 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
root()->makeHTTPRequest(dl);
}
int Catalog::markPackagesForInstallation(const string_list &packageIds) {
int result = 0;
for (const auto &id : packageIds) {
auto ourPkg = getPackageById(id);
if (!ourPkg)
continue;
auto existing = ourPkg->existingInstall();
if (!existing) {
ourPkg->markForInstall();
++result;
}
} // of outer package ID candidates iteration
return result;
}
CatalogRef Catalog::migratedFrom() const { return m_migratedFrom; }
} // of namespace pkg
} // of namespace simgear

32
simgear/package/Catalog.hxx
View File

@ -23,6 +23,7 @@
#include <map>
#include <simgear/misc/sg_path.hxx>
#include <simgear/misc/strutils.hxx>
#include <simgear/props/props.hxx>
#include <simgear/structure/SGReferenced.hxx>
@ -93,7 +94,7 @@ public:
/**
* retrieve all the packages in the catalog which are installed
* and have a pendig update
* and have a pending update
*/
PackageList packagesNeedingUpdate() const;
@ -151,7 +152,32 @@ public:
bool isUserEnabled() const;
void setUserEnabled(bool b);
private:
/**
* Given a list of package IDs, mark all which exist in this package,
* for installation. ANy packahe IDs not present in this catalog,
* will be ignored.
*
* @result The number for packages newly marked for installation.
*/
int markPackagesForInstallation(const string_list &packageIds);
/**
* When a catalog is added due to migration, this will contain the
* Catalog which triggered the add. Usually this will be a catalog
* corresponding to an earlier version.
*
* Note it's only valid at the time, the migration actually took place;
* when the new catalog is loaded from disk, this value will return
* null.
*
* This is intended to allow Uis to show a 'catalog was migrated'
* feedback, when they see a catalog refresh, which has a non-null
* value of this method.
*/
CatalogRef migratedFrom() const;
private:
Catalog(Root* aRoot);
class Downloader;
@ -197,6 +223,8 @@ private:
PackageWeakMap m_variantDict;
function_list<Callback> m_statusCallbacks;
CatalogRef m_migratedFrom;
};
} // of namespace pkg

137
simgear/package/CatalogTest.cxx
View File

@ -827,7 +827,10 @@ void testVersionMigrateToId(HTTP::Client* cl)
it = std::find(enabledCats.begin(), enabledCats.end(), altCat);
SG_VERIFY(it != enabledCats.end());
SG_CHECK_EQUAL(altCat->packagesNeedingUpdate().size(),
1); // should be the 737
// install a parallel package from the new catalog
pkg::PackageRef p2 = root->getPackageById("org.flightgear.test.catalog-alt.b737-NG");
SG_CHECK_EQUAL(p2->id(), "b737-NG");
@ -840,8 +843,8 @@ void testVersionMigrateToId(HTTP::Client* cl)
pkg::PackageRef p3 = root->getPackageById("b737-NG");
SG_CHECK_EQUAL(p2, p3);
}
// test that re-init-ing doesn't mirgate again
// test that re-init-ing doesn't migrate again
{
pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
root->setHTTPClient(cl);
@ -1184,6 +1187,128 @@ void testMirrorsFailure(HTTP::Client* cl)
}
void testMigrateInstalled(HTTP::Client *cl) {
SGPath rootPath(simgear::Dir::current().path());
rootPath.append("pkg_migrate_installed");
simgear::Dir pd(rootPath);
pd.removeChildren();
pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
root->setHTTPClient(cl);
pkg::CatalogRef oldCatalog, newCatalog;
{
oldCatalog = pkg::Catalog::createFromUrl(
root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
waitForUpdateComplete(cl, root);
pkg::PackageRef p1 =
root->getPackageById("org.flightgear.test.catalog1.b747-400");
p1->install();
auto p2 = root->getPackageById("org.flightgear.test.catalog1.c172p");
p2->install();
auto p3 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
p3->install();
waitForUpdateComplete(cl, root);
}
{
newCatalog = pkg::Catalog::createFromUrl(
root.ptr(), "http://localhost:2000/catalogTest2/catalog.xml");
waitForUpdateComplete(cl, root);
string_list existing;
for (const auto &pack : oldCatalog->installedPackages()) {
existing.push_back(pack->id());
}
SG_CHECK_EQUAL(4, existing.size());
int result = newCatalog->markPackagesForInstallation(existing);
SG_CHECK_EQUAL(2, result);
SG_CHECK_EQUAL(2, newCatalog->packagesNeedingUpdate().size());
auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG");
auto ins = p1->existingInstall();
SG_CHECK_EQUAL(0, ins->revsion());
}
{
root->scheduleAllUpdates();
waitForUpdateComplete(cl, root);
SG_CHECK_EQUAL(0, newCatalog->packagesNeedingUpdate().size());
auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG");
auto ins = p1->existingInstall();
SG_CHECK_EQUAL(ins->revsion(), p1->revision());
}
}
void testDontMigrateRemoved(HTTP::Client *cl) {
global_catalogVersion = 2; // version which has migration info
SGPath rootPath(simgear::Dir::current().path());
rootPath.append("cat_dont_migrate_id");
simgear::Dir pd(rootPath);
pd.removeChildren();
// install and mnaully remove the alt catalog
{
pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
root->setHTTPClient(cl);
pkg::CatalogRef c = pkg::Catalog::createFromUrl(
root.ptr(), "http://localhost:2000/catalogTest1/catalog-alt.xml");
waitForUpdateComplete(cl, root);
root->removeCatalogById("org.flightgear.test.catalog-alt");
}
// install the migration catalog
{
pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
root->setHTTPClient(cl);
pkg::CatalogRef c = pkg::Catalog::createFromUrl(
root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
waitForUpdateComplete(cl, root);
SG_VERIFY(c->isEnabled());
}
// change version to an alternate one
{
pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
auto removed = root->explicitlyRemovedCatalogs();
auto j = std::find(removed.begin(), removed.end(),
"org.flightgear.test.catalog-alt");
SG_VERIFY(j != removed.end());
root->setHTTPClient(cl);
// this would tirgger migration, but we blocked it
root->refresh(true);
waitForUpdateComplete(cl, root);
pkg::CatalogRef cat = root->getCatalogById("org.flightgear.test.catalog1");
SG_VERIFY(!cat->isEnabled());
SG_CHECK_EQUAL(cat->status(), pkg::Delegate::FAIL_VERSION);
SG_CHECK_EQUAL(cat->id(), "org.flightgear.test.catalog1");
SG_CHECK_EQUAL(cat->url(),
"http://localhost:2000/catalogTest1/catalog.xml");
auto enabledCats = root->catalogs();
auto it = std::find(enabledCats.begin(), enabledCats.end(), cat);
SG_VERIFY(it == enabledCats.end());
// check the new catalog
auto altCat = root->getCatalogById("org.flightgear.test.catalog-alt");
SG_VERIFY(altCat.get() == nullptr);
}
}
int main(int argc, char* argv[])
{
sglog().setLogLevels( SG_ALL, SG_WARN );
@ -1228,7 +1353,11 @@ int main(int argc, char* argv[])
testInstallBadPackage(&cl);
testMirrorsFailure(&cl);
testMigrateInstalled(&cl);
testDontMigrateRemoved(&cl);
cerr << "Successfully passed all tests!" << endl;
return EXIT_SUCCESS;
}

33
simgear/package/Package.cxx
View File

@ -198,6 +198,11 @@ InstallRef Package::install()
{
InstallRef ins = existingInstall();
if (ins) {
// if there's updates, treat this as a 'start update' request
if (ins->hasUpdate()) {
m_catalog->root()->scheduleToUpdate(ins);
}
return ins;
}
@ -210,13 +215,39 @@ InstallRef Package::install()
return ins;
}
InstallRef Package::markForInstall() {
InstallRef ins = existingInstall();
if (ins) {
return ins;
}
const auto pd = pathOnDisk();
Dir dir(pd);
if (!dir.create(0700)) {
SG_LOG(SG_IO, SG_ALERT,
"Package::markForInstall: couldn't create directory at:" << pd);
return {};
}
ins = new Install{this, pd};
_install_cb(this, ins); // not sure if we should trigger the callback for this
// repeat for dependencies to be kind
for (auto dep : dependencies()) {
dep->markForInstall();
}
return ins;
}
InstallRef Package::existingInstall(const InstallCallback& cb) const
{
InstallRef install;
try {
install = m_catalog->root()->existingInstallForPackage(const_cast<Package*>(this));
} catch (std::exception& ) {
return InstallRef();
return {};
}
if( cb )

7
simgear/package/Package.hxx
View File

@ -61,6 +61,13 @@ public:
InstallRef
existingInstall(const InstallCallback& cb = InstallCallback()) const;
/**
* Mark this package for installation, but don't actually start the
* download process. This creates the on-disk placeholder, so
* the package will appear an eededing to be updated.
*/
InstallRef markForInstall();
bool isInstalled() const;
/**

45
simgear/package/Root.cxx
View File

@ -283,6 +283,32 @@ public:
fireDataForThumbnail(url, reinterpret_cast<const uint8_t*>(bytes.data()), bytes.size());
}
void writeRemovedCatalogsFile() const {
SGPath p = path / "RemovedCatalogs";
sg_ofstream stream(p, std::ios::out | std::ios::trunc | std::ios::binary);
for (const auto &cid : manuallyRemovedCatalogs) {
stream << cid << "\n";
}
stream.close();
}
void loadRemovedCatalogsFile() {
manuallyRemovedCatalogs.clear();
SGPath p = path / "RemovedCatalogs";
if (!p.exists())
return;
sg_ifstream stream(p, std::ios::in);
while (!stream.eof()) {
std::string line;
std::getline(stream, line);
const auto trimmed = strutils::strip(line);
if (!trimmed.empty()) {
manuallyRemovedCatalogs.push_back(trimmed);
}
} // of lines iteration
}
DelegateVec delegates;
SGPath path;
@ -312,6 +338,9 @@ public:
typedef std::map<PackageRef, InstallRef> InstallCache;
InstallCache m_installs;
/// persistent list of catalogs the user has manually removed
string_list manuallyRemovedCatalogs;
};
@ -400,6 +429,8 @@ Root::Root(const SGPath& aPath, const std::string& aVersion) :
thumbsCacheDir.create(0755);
}
d->loadRemovedCatalogsFile();
for (SGPath c : dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) {
// note this will set the catalog status, which will insert into
// disabled catalogs automatically if necesary
@ -621,6 +652,13 @@ void Root::scheduleToUpdate(InstallRef aInstall)
}
}
void Root::scheduleAllUpdates() {
auto toBeUpdated = packagesNeedingUpdate(); // make a copy
for (const auto &u : toBeUpdated) {
scheduleToUpdate(u->existingInstall());
}
}
bool Root::isInstallQueued(InstallRef aInstall) const
{
auto it = std::find(d->updateDeque.begin(), d->updateDeque.end(), aInstall);
@ -783,6 +821,9 @@ bool Root::removeCatalogById(const std::string& aId)
<< "failed to remove directory");
}
d->manuallyRemovedCatalogs.push_back(aId);
d->writeRemovedCatalogsFile();
// notify that a catalog is being removed
d->firePackagesChanged();
@ -854,6 +895,10 @@ void Root::unregisterInstall(InstallRef ins)
d->fireFinishUninstall(ins->package());
}
string_list Root::explicitlyRemovedCatalogs() const {
return d->manuallyRemovedCatalogs;
}
} // of namespace pkg
} // of namespace simgear

17
simgear/package/Root.hxx
View File

@ -155,7 +155,22 @@ public:
void requestThumbnailData(const std::string& aUrl);
bool isInstallQueued(InstallRef aInstall) const;
private:
/**
* Mark all 'to be updated' packages for update now
*/
void scheduleAllUpdates();
/**
* @brief list of catalog IDs, the user has explicitly removed via
* removeCatalogById(). This is important to allow the user to opt-out
* of migrated packages.
*
* This information is stored in a helper file, in the root directory
*/
string_list explicitlyRemovedCatalogs() const;
private:
friend class Install;
friend class Catalog;
friend class Package;

86
simgear/package/catalogTest2/catalog.xml Normal file
View File

@ -0,0 +1,86 @@
<?xml version="1.0"?>
<PropertyList>
<id>org.flightgear.test.catalog2</id>
<description>Second test catalog</description>
<url>http://localhost:2000/catalogTest2/catalog.xml</url>
<catalog-version>4</catalog-version>
<version>8.1.*</version>
<version>8.0.0</version>
<version>8.2.0</version>
<package>
<id>alpha</id>
<name>Alpha package</name>
<revision type="int">8</revision>
<file-size-bytes type="int">593</file-size-bytes>
<md5>a469c4b837f0521db48616cfe65ac1ea</md5>
<url>http://localhost:2000/catalogTest1/alpha.zip</url>
<dir>alpha</dir>
</package>
<package>
<id>b737-NG</id>
<name>Boeing 737 NG</name>
<dir>b737NG</dir>
<description>A popular twin-engined narrow body jet</description>
<revision type="int">111</revision>
<file-size-bytes type="int">860</file-size-bytes>
<tag>boeing</tag>
<tag>jet</tag>
<tag>ifr</tag>
<!-- not within a localized element -->
<de>
<description>German description of B737NG XYZ</description>
</de>
<fr>
<description>French description of B737NG</description>
</fr>
<rating>
<FDM type="int">5</FDM>
<systems type="int">5</systems>
<model type="int">4</model>
<cockpit type="int">4</cockpit>
</rating>
<md5>a94ca5704f305b90767f40617d194ed6</md5>
<url>http://localhost:2000/mirrorA/b737.tar.gz</url>
<url>http://localhost:2000/mirrorB/b737.tar.gz</url>
<url>http://localhost:2000/mirrorC/b737.tar.gz</url>
</package>
<package>
<id>b747-400</id>
<name>Boeing 747-400</name>
<dir>b744</dir>
<description>A popular four-engined wide-body jet</description>
<revision type="int">111</revision>
<file-size-bytes type="int">860</file-size-bytes>
<tag>boeing</tag>
<tag>jet</tag>
<tag>ifr</tag>
<rating>
<FDM type="int">5</FDM>
<systems type="int">5</systems>
<model type="int">4</model>
<cockpit type="int">4</cockpit>
</rating>
<md5>4d3f7417d74f811aa20ccc4f35673d20</md5>
<!-- this URL will sometimes fail, on purpose -->
<url>http://localhost:2000/catalogTest1/b747.tar.gz</url>
</package>
</PropertyList>