diff --git a/simgear/package/Catalog.cxx b/simgear/package/Catalog.cxx index 7cb8bc7d..6422bf76 100644 --- a/simgear/package/Catalog.cxx +++ b/simgear/package/Catalog.cxx @@ -643,16 +643,53 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt) return; } - // we have an alternate ID, and it's differnt from our ID, so let's + // we have an alternate ID, and it's different from our ID, so let's // define a new catalog if (!altId.empty()) { - SG_LOG(SG_GENERAL, SG_INFO, "Adding new catalog:" << altId << " as version alternate for " << id()); - // new catalog being added - createFromUrl(root(), altUrl); - - // and we can go idle now + // don't auto-re-add Catalogs the user has explicilty rmeoved, that would + // suck + const auto removedByUser = root()->explicitlyRemovedCatalogs(); + auto it = std::find(removedByUser.begin(), removedByUser.end(), altId); + if (it != removedByUser.end()) { changeStatus(Delegate::FAIL_VERSION); return; + } + + SG_LOG(SG_GENERAL, SG_WARN, + "Adding new catalog:" << altId << " as version alternate for " + << id()); + // new catalog being added + auto newCat = createFromUrl(root(), altUrl); + + bool didRun = false; + newCat->m_migratedFrom = this; + + auto migratePackagesCb = [didRun](Catalog *c) mutable { + // removing callbacks is awkward, so use this + // flag to only run once. (and hence, we need to be mutable) + if (didRun) + return; + + if (c->status() == Delegate::STATUS_REFRESHED) { + didRun = true; + + string_list existing; + for (const auto &pack : c->migratedFrom()->installedPackages()) { + existing.push_back(pack->id()); + } + + const int count = c->markPackagesForInstallation(existing); + SG_LOG( + SG_GENERAL, SG_INFO, + "Marked " << count + << " packages from previous catalog for installation"); + } + }; + + newCat->addStatusCallback(migratePackagesCb); + // and we can go idle now + changeStatus(Delegate::FAIL_VERSION); + return; } SG_LOG(SG_GENERAL, SG_INFO, "Migrating catalog " << id() << " to new URL:" << altUrl); @@ -661,6 +698,26 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt) root()->makeHTTPRequest(dl); } +int Catalog::markPackagesForInstallation(const string_list &packageIds) { + int result = 0; + + for (const auto &id : packageIds) { + auto ourPkg = getPackageById(id); + if (!ourPkg) + continue; + + auto existing = ourPkg->existingInstall(); + if (!existing) { + ourPkg->markForInstall(); + ++result; + } + } // of outer package ID candidates iteration + + return result; +} + +CatalogRef Catalog::migratedFrom() const { return m_migratedFrom; } + } // of namespace pkg } // of namespace simgear diff --git a/simgear/package/Catalog.hxx b/simgear/package/Catalog.hxx index 7c421b88..c83d5c45 100644 --- a/simgear/package/Catalog.hxx +++ b/simgear/package/Catalog.hxx @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -93,7 +94,7 @@ public: /** * retrieve all the packages in the catalog which are installed - * and have a pendig update + * and have a pending update */ PackageList packagesNeedingUpdate() const; @@ -151,7 +152,32 @@ public: bool isUserEnabled() const; void setUserEnabled(bool b); -private: + + /** + * Given a list of package IDs, mark all which exist in this package, + * for installation. ANy packahe IDs not present in this catalog, + * will be ignored. + * + * @result The number for packages newly marked for installation. + */ + int markPackagesForInstallation(const string_list &packageIds); + + /** + * When a catalog is added due to migration, this will contain the + * Catalog which triggered the add. Usually this will be a catalog + * corresponding to an earlier version. + * + * Note it's only valid at the time, the migration actually took place; + * when the new catalog is loaded from disk, this value will return + * null. + * + * This is intended to allow Uis to show a 'catalog was migrated' + * feedback, when they see a catalog refresh, which has a non-null + * value of this method. + */ + CatalogRef migratedFrom() const; + + private: Catalog(Root* aRoot); class Downloader; @@ -197,6 +223,8 @@ private: PackageWeakMap m_variantDict; function_list m_statusCallbacks; + + CatalogRef m_migratedFrom; }; } // of namespace pkg diff --git a/simgear/package/CatalogTest.cxx b/simgear/package/CatalogTest.cxx index edd33f34..09246d70 100644 --- a/simgear/package/CatalogTest.cxx +++ b/simgear/package/CatalogTest.cxx @@ -827,7 +827,10 @@ void testVersionMigrateToId(HTTP::Client* cl) it = std::find(enabledCats.begin(), enabledCats.end(), altCat); SG_VERIFY(it != enabledCats.end()); - + + SG_CHECK_EQUAL(altCat->packagesNeedingUpdate().size(), + 1); // should be the 737 + // install a parallel package from the new catalog pkg::PackageRef p2 = root->getPackageById("org.flightgear.test.catalog-alt.b737-NG"); SG_CHECK_EQUAL(p2->id(), "b737-NG"); @@ -840,8 +843,8 @@ void testVersionMigrateToId(HTTP::Client* cl) pkg::PackageRef p3 = root->getPackageById("b737-NG"); SG_CHECK_EQUAL(p2, p3); } - - // test that re-init-ing doesn't mirgate again + + // test that re-init-ing doesn't migrate again { pkg::RootRef root(new pkg::Root(rootPath, "7.5")); root->setHTTPClient(cl); @@ -1184,6 +1187,128 @@ void testMirrorsFailure(HTTP::Client* cl) } +void testMigrateInstalled(HTTP::Client *cl) { + SGPath rootPath(simgear::Dir::current().path()); + rootPath.append("pkg_migrate_installed"); + simgear::Dir pd(rootPath); + pd.removeChildren(); + + pkg::RootRef root(new pkg::Root(rootPath, "8.1.2")); + root->setHTTPClient(cl); + + pkg::CatalogRef oldCatalog, newCatalog; + + { + oldCatalog = pkg::Catalog::createFromUrl( + root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml"); + waitForUpdateComplete(cl, root); + + pkg::PackageRef p1 = + root->getPackageById("org.flightgear.test.catalog1.b747-400"); + p1->install(); + auto p2 = root->getPackageById("org.flightgear.test.catalog1.c172p"); + p2->install(); + auto p3 = root->getPackageById("org.flightgear.test.catalog1.b737-NG"); + p3->install(); + waitForUpdateComplete(cl, root); + } + + { + newCatalog = pkg::Catalog::createFromUrl( + root.ptr(), "http://localhost:2000/catalogTest2/catalog.xml"); + waitForUpdateComplete(cl, root); + + string_list existing; + for (const auto &pack : oldCatalog->installedPackages()) { + existing.push_back(pack->id()); + } + + SG_CHECK_EQUAL(4, existing.size()); + + int result = newCatalog->markPackagesForInstallation(existing); + SG_CHECK_EQUAL(2, result); + SG_CHECK_EQUAL(2, newCatalog->packagesNeedingUpdate().size()); + + auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG"); + auto ins = p1->existingInstall(); + SG_CHECK_EQUAL(0, ins->revsion()); + } + + { + root->scheduleAllUpdates(); + waitForUpdateComplete(cl, root); + + SG_CHECK_EQUAL(0, newCatalog->packagesNeedingUpdate().size()); + + auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG"); + auto ins = p1->existingInstall(); + SG_CHECK_EQUAL(ins->revsion(), p1->revision()); + } +} + +void testDontMigrateRemoved(HTTP::Client *cl) { + global_catalogVersion = 2; // version which has migration info + SGPath rootPath(simgear::Dir::current().path()); + rootPath.append("cat_dont_migrate_id"); + simgear::Dir pd(rootPath); + pd.removeChildren(); + + // install and mnaully remove the alt catalog + + { + pkg::RootRef root(new pkg::Root(rootPath, "8.1.2")); + root->setHTTPClient(cl); + + pkg::CatalogRef c = pkg::Catalog::createFromUrl( + root.ptr(), "http://localhost:2000/catalogTest1/catalog-alt.xml"); + waitForUpdateComplete(cl, root); + + root->removeCatalogById("org.flightgear.test.catalog-alt"); + } + + // install the migration catalog + { + pkg::RootRef root(new pkg::Root(rootPath, "8.1.2")); + root->setHTTPClient(cl); + + pkg::CatalogRef c = pkg::Catalog::createFromUrl( + root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml"); + waitForUpdateComplete(cl, root); + SG_VERIFY(c->isEnabled()); + } + + // change version to an alternate one + { + pkg::RootRef root(new pkg::Root(rootPath, "7.5")); + + auto removed = root->explicitlyRemovedCatalogs(); + auto j = std::find(removed.begin(), removed.end(), + "org.flightgear.test.catalog-alt"); + SG_VERIFY(j != removed.end()); + + root->setHTTPClient(cl); + + // this would tirgger migration, but we blocked it + root->refresh(true); + waitForUpdateComplete(cl, root); + + pkg::CatalogRef cat = root->getCatalogById("org.flightgear.test.catalog1"); + SG_VERIFY(!cat->isEnabled()); + SG_CHECK_EQUAL(cat->status(), pkg::Delegate::FAIL_VERSION); + SG_CHECK_EQUAL(cat->id(), "org.flightgear.test.catalog1"); + SG_CHECK_EQUAL(cat->url(), + "http://localhost:2000/catalogTest1/catalog.xml"); + + auto enabledCats = root->catalogs(); + auto it = std::find(enabledCats.begin(), enabledCats.end(), cat); + SG_VERIFY(it == enabledCats.end()); + + // check the new catalog + auto altCat = root->getCatalogById("org.flightgear.test.catalog-alt"); + SG_VERIFY(altCat.get() == nullptr); + } +} + int main(int argc, char* argv[]) { sglog().setLogLevels( SG_ALL, SG_WARN ); @@ -1228,7 +1353,11 @@ int main(int argc, char* argv[]) testInstallBadPackage(&cl); testMirrorsFailure(&cl); - + + testMigrateInstalled(&cl); + + testDontMigrateRemoved(&cl); + cerr << "Successfully passed all tests!" << endl; return EXIT_SUCCESS; } diff --git a/simgear/package/Package.cxx b/simgear/package/Package.cxx index e572c4a0..7feafe53 100644 --- a/simgear/package/Package.cxx +++ b/simgear/package/Package.cxx @@ -198,6 +198,11 @@ InstallRef Package::install() { InstallRef ins = existingInstall(); if (ins) { + // if there's updates, treat this as a 'start update' request + if (ins->hasUpdate()) { + m_catalog->root()->scheduleToUpdate(ins); + } + return ins; } @@ -210,13 +215,39 @@ InstallRef Package::install() return ins; } +InstallRef Package::markForInstall() { + InstallRef ins = existingInstall(); + if (ins) { + return ins; + } + + const auto pd = pathOnDisk(); + + Dir dir(pd); + if (!dir.create(0700)) { + SG_LOG(SG_IO, SG_ALERT, + "Package::markForInstall: couldn't create directory at:" << pd); + return {}; + } + + ins = new Install{this, pd}; + _install_cb(this, ins); // not sure if we should trigger the callback for this + + // repeat for dependencies to be kind + for (auto dep : dependencies()) { + dep->markForInstall(); + } + + return ins; +} + InstallRef Package::existingInstall(const InstallCallback& cb) const { InstallRef install; try { install = m_catalog->root()->existingInstallForPackage(const_cast(this)); } catch (std::exception& ) { - return InstallRef(); + return {}; } if( cb ) diff --git a/simgear/package/Package.hxx b/simgear/package/Package.hxx index dc6958a8..88408c54 100644 --- a/simgear/package/Package.hxx +++ b/simgear/package/Package.hxx @@ -61,6 +61,13 @@ public: InstallRef existingInstall(const InstallCallback& cb = InstallCallback()) const; + /** + * Mark this package for installation, but don't actually start the + * download process. This creates the on-disk placeholder, so + * the package will appear an eededing to be updated. + */ + InstallRef markForInstall(); + bool isInstalled() const; /** diff --git a/simgear/package/Root.cxx b/simgear/package/Root.cxx index 193199c3..e358c7d6 100755 --- a/simgear/package/Root.cxx +++ b/simgear/package/Root.cxx @@ -283,6 +283,32 @@ public: fireDataForThumbnail(url, reinterpret_cast(bytes.data()), bytes.size()); } + void writeRemovedCatalogsFile() const { + SGPath p = path / "RemovedCatalogs"; + sg_ofstream stream(p, std::ios::out | std::ios::trunc | std::ios::binary); + for (const auto &cid : manuallyRemovedCatalogs) { + stream << cid << "\n"; + } + stream.close(); + } + + void loadRemovedCatalogsFile() { + manuallyRemovedCatalogs.clear(); + SGPath p = path / "RemovedCatalogs"; + if (!p.exists()) + return; + + sg_ifstream stream(p, std::ios::in); + while (!stream.eof()) { + std::string line; + std::getline(stream, line); + const auto trimmed = strutils::strip(line); + if (!trimmed.empty()) { + manuallyRemovedCatalogs.push_back(trimmed); + } + } // of lines iteration + } + DelegateVec delegates; SGPath path; @@ -312,6 +338,9 @@ public: typedef std::map InstallCache; InstallCache m_installs; + + /// persistent list of catalogs the user has manually removed + string_list manuallyRemovedCatalogs; }; @@ -400,6 +429,8 @@ Root::Root(const SGPath& aPath, const std::string& aVersion) : thumbsCacheDir.create(0755); } + d->loadRemovedCatalogsFile(); + for (SGPath c : dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) { // note this will set the catalog status, which will insert into // disabled catalogs automatically if necesary @@ -621,6 +652,13 @@ void Root::scheduleToUpdate(InstallRef aInstall) } } +void Root::scheduleAllUpdates() { + auto toBeUpdated = packagesNeedingUpdate(); // make a copy + for (const auto &u : toBeUpdated) { + scheduleToUpdate(u->existingInstall()); + } +} + bool Root::isInstallQueued(InstallRef aInstall) const { auto it = std::find(d->updateDeque.begin(), d->updateDeque.end(), aInstall); @@ -783,6 +821,9 @@ bool Root::removeCatalogById(const std::string& aId) << "failed to remove directory"); } + d->manuallyRemovedCatalogs.push_back(aId); + d->writeRemovedCatalogsFile(); + // notify that a catalog is being removed d->firePackagesChanged(); @@ -854,6 +895,10 @@ void Root::unregisterInstall(InstallRef ins) d->fireFinishUninstall(ins->package()); } +string_list Root::explicitlyRemovedCatalogs() const { + return d->manuallyRemovedCatalogs; +} + } // of namespace pkg } // of namespace simgear diff --git a/simgear/package/Root.hxx b/simgear/package/Root.hxx index 2c598166..a1114461 100644 --- a/simgear/package/Root.hxx +++ b/simgear/package/Root.hxx @@ -155,7 +155,22 @@ public: void requestThumbnailData(const std::string& aUrl); bool isInstallQueued(InstallRef aInstall) const; -private: + + /** + * Mark all 'to be updated' packages for update now + */ + void scheduleAllUpdates(); + + /** + * @brief list of catalog IDs, the user has explicitly removed via + * removeCatalogById(). This is important to allow the user to opt-out + * of migrated packages. + * + * This information is stored in a helper file, in the root directory + */ + string_list explicitlyRemovedCatalogs() const; + + private: friend class Install; friend class Catalog; friend class Package; diff --git a/simgear/package/catalogTest2/catalog.xml b/simgear/package/catalogTest2/catalog.xml new file mode 100644 index 00000000..b72f467a --- /dev/null +++ b/simgear/package/catalogTest2/catalog.xml @@ -0,0 +1,86 @@ + + + + org.flightgear.test.catalog2 + Second test catalog + http://localhost:2000/catalogTest2/catalog.xml + 4 + + 8.1.* + 8.0.0 + 8.2.0 + + + alpha + Alpha package + 8 + 593 + + a469c4b837f0521db48616cfe65ac1ea + http://localhost:2000/catalogTest1/alpha.zip + + alpha + + + + + + b737-NG + Boeing 737 NG + b737NG + A popular twin-engined narrow body jet + 111 + 860 + + boeing + jet + ifr + + + + German description of B737NG XYZ + + + French description of B737NG + + + + 5 + 5 + 4 + 4 + + + a94ca5704f305b90767f40617d194ed6 + http://localhost:2000/mirrorA/b737.tar.gz + http://localhost:2000/mirrorB/b737.tar.gz + http://localhost:2000/mirrorC/b737.tar.gz + + + + b747-400 + Boeing 747-400 + b744 + A popular four-engined wide-body jet + 111 + 860 + + boeing + jet + ifr + + + + + 5 + 5 + 4 + 4 + + + 4d3f7417d74f811aa20ccc4f35673d20 + + http://localhost:2000/catalogTest1/b747.tar.gz + + +