update index to be pimpl

also remove the ability for multiple sources from it

loader/include/Geode/loader/Index.hpp

@@ -8,17 +8,63 @@
 #include <unordered_set>
 
 namespace geode {
+    class Index;
+
+    /**
+     * Status signifying an index-related download has been finished
+     */
     using UpdateFinished = std::monostate;
+    /**
+     * Status signifying an index-related download is in progress. First element 
+     * in pair is percentage downloaded, second is status string
+     */
     using UpdateProgress = std::pair<uint8_t, std::string>;
+    /**
+     * Status signifying an index-related download has failed. Consists of the 
+     * error string
+     */
     using UpdateFailed = std::string;
+    /**
+     * Status code for an index-related download
+     */
     using UpdateStatus = std::variant<UpdateFinished, UpdateProgress, UpdateFailed>;
 
+    /**
+     * Event for when a mod is being installed from the index. Automatically 
+     * broadcast by the mods index; use ModInstallFilter to listen to these 
+     * events
+     */
     struct GEODE_DLL ModInstallEvent : public Event {
+        /**
+         * The ID of the mod being installed
+         */
         const std::string modID;
+        /**
+         * The current status of the installation
+         */
         const UpdateStatus status;
+    
+    private:
         ModInstallEvent(std::string const& id, const UpdateStatus status);
+
+        friend class Index;
     };
 
+    /**
+     * Basic filter for listening to mod installation events. Always propagates 
+     * the event down the chain
+     * @example 
+     * // Install "steve.hotdogs" and listen for its installation progress
+     * 
+     * // Create a listener that listens for when steve.hotdogs is being installed
+     * auto listener = EventListener<ModInstallFilter>(+[](ModInstallEvent* ev) {
+     *     // Check the event status using std::visit or other
+     * }, ModInstallFilter("steve.hotdogs"));
+     * // Get the latest version of steve.hotdogs from the index and install it
+     * if (auto mod = Index::get()->getMajorItem("steve.hotdogs")) {
+     *     Index::get()->install(mod);
+     * }
+     */
 	class GEODE_DLL ModInstallFilter : public EventFilter<ModInstallEvent> {
     protected:
         std::string m_id;
@@ -31,11 +77,18 @@ namespace geode {
         ModInstallFilter(ModInstallFilter const&) = default;
 	};
 
+    /**
+     * Event broadcast when the index is being updated
+     */
     struct GEODE_DLL IndexUpdateEvent : public Event {
         const UpdateStatus status;
         IndexUpdateEvent(const UpdateStatus status);
     };
 
+    /**
+     * Basic filter for listening to index update events. Always propagates 
+     * the event down the chain
+     */
     class GEODE_DLL IndexUpdateFilter : public EventFilter<IndexUpdateEvent> {
     public:
         using Callback = void(IndexUpdateEvent*);
@@ -45,32 +98,24 @@ namespace geode {
         IndexUpdateFilter(IndexUpdateFilter const&) = default;
     };
 
-    struct IndexSourceImpl;
+    class GEODE_DLL IndexItem final {
-    struct GEODE_DLL IndexSourceImplDeleter {
+    public:
-        void operator()(IndexSourceImpl* src);
+        class Impl;
-    };
-    struct SourceUpdateEvent;
-    using IndexSourcePtr = std::unique_ptr<IndexSourceImpl, IndexSourceImplDeleter>;
 
-    struct GEODE_DLL IndexItem {
+    private:
-        std::string sourceRepository;
+        std::unique_ptr<Impl> m_impl;
-        ghc::filesystem::path path;
-        ModInfo info;
-        struct {
-            std::string url;
-            std::string hash;
-            std::unordered_set<PlatformID> platforms;
-        } download;
-        bool isFeatured;
-        std::unordered_set<std::string> tags;
 
-        /**
+    public:
-         * Create IndexItem from a directory
+        ghc::filesystem::path getPath() const;
-         */
+        ModInfo getModInfo() const;
-        static Result<std::shared_ptr<IndexItem>> createFromDir(
+        std::string getDownloadURL() const;
-            std::string const& sourceRepository,
+        std::string getPackageHash() const;
-            ghc::filesystem::path const& dir
+        std::unordered_set<PlatformID> getAvailablePlatforms() const;
-        );
+        bool isFeatured() const;
+        std::unordered_set<std::string> getTags() const;
+
+        IndexItem();
+        ~IndexItem();
     };
     using IndexItemHandle = std::shared_ptr<IndexItem>;
 
@@ -85,38 +130,19 @@ namespace geode {
         std::vector<IndexItemHandle> list;
     };
 
-    class GEODE_DLL Index final {
+    static constexpr size_t MAX_INDEX_API_VERSION = 0;
-    protected:
-        // for once, the fact that std::map is ordered is useful (this makes 
-        // getting the latest version of a mod as easy as items.rbegin())
-        using ItemVersions = std::map<size_t, IndexItemHandle>;
 
-        std::vector<IndexSourcePtr> m_sources;
+    class GEODE_DLL Index final {
-        std::unordered_map<std::string, UpdateStatus> m_sourceStatuses;
+    private:
-        std::unordered_map<
+        class Impl;
-            IndexItemHandle,
+        std::unique_ptr<Impl> m_impl;
-            utils::web::SentAsyncWebRequestHandle
-        > m_runningInstallations;
-        std::atomic<bool> m_triedToUpdate = false;
-        std::unordered_map<std::string, ItemVersions> m_items;
 
         Index();
-
+        ~Index();
-        void onSourceUpdate(SourceUpdateEvent* event);
-        void checkSourceUpdates(IndexSourceImpl* src);
-        void downloadSource(IndexSourceImpl* src);
-        void updateSourceFromLocal(IndexSourceImpl* src);
-        void cleanupItems();
-
-        void installNext(size_t index, IndexInstallList const& list);
 
     public:
         static Index* get();
 
-        void addSource(std::string const& repository);
-        void removeSource(std::string const& repository);
-        std::vector<std::string> getSources() const;
-
         /**
          * Get all tags
          */
@@ -206,13 +232,17 @@ namespace geode {
         Result<IndexInstallList> getInstallList(IndexItemHandle item) const;
         /**
          * Install an index item. Add an event listener for the ModInstallEvent 
-         * class to track the installation progress
+         * class to track the installation progress. Automatically also downloads 
+         * all missing dependencies for the item
          * @param item Item to install
          */
         void install(IndexItemHandle item);
         /**
          * Install a list of index items. Add an event listener for the 
          * ModInstallEvent class to track the installation progress
+         * @warning Does not download any missing dependencies - use the 
+         * `install(IndexItemHandle)` overload if you aren't sure all the 
+         * dependencies are installed!
          * @param list List of items to install
          */
         void install(IndexInstallList const& list);

loader/src/loader/Index.cpp

@@ -8,7 +8,6 @@
 #include <hash/hash.hpp>
 #include <Geode/utils/JsonValidation.hpp>
 
-
 using namespace geode::prelude;
 
 // ModInstallEvent
@@ -26,54 +25,6 @@ ListenerResult ModInstallFilter::handle(utils::MiniFunction<Callback> fn, ModIns
 
 ModInstallFilter::ModInstallFilter(std::string const& id) : m_id(id) {}
 
-// IndexUpdateEvent implementation
-
-// The reason sources have private implementation events that are 
-// turned into the global IndexUpdateEvent is because it makes it much 
-// simpler to keep track of progress, what errors were received, etc. 
-// without having to store a ton of members
-
-struct geode::IndexSourceImpl final {
-    std::string repository;
-    bool isUpToDate = false;
-
-    std::string dirname() const {
-        return string::replace(this->repository, "/", "_");
-    }
-
-    ghc::filesystem::path path() const {
-        return dirs::getIndexDir() / this->dirname();
-    }
-
-    ghc::filesystem::path checksum() const {
-        // not storing this in the source's directory as that gets replaced by 
-        // the newly fetched index
-        return dirs::getIndexDir() / (this->dirname() + ".checksum");
-    }
-};
-
-void IndexSourceImplDeleter::operator()(IndexSourceImpl* src) {
-    delete src;
-}
-
-struct geode::SourceUpdateEvent : public Event {
-    IndexSourceImpl* source;
-    const UpdateStatus status;
-    SourceUpdateEvent(IndexSourceImpl* src, const UpdateStatus status)
-      : source(src), status(status) {}
-};
-
-class SourceUpdateFilter : public EventFilter<SourceUpdateEvent> {
-public:
-    using Callback = void(SourceUpdateEvent*);
-
-    ListenerResult handle(utils::MiniFunction<Callback> fn, SourceUpdateEvent* event) {
-        fn(event);
-        return ListenerResult::Propagate;
-    }
-    SourceUpdateFilter() {}
-};
-
 // IndexUpdateEvent
 
 IndexUpdateEvent::IndexUpdateEvent(const UpdateStatus status) : status(status) {}
@@ -90,10 +41,59 @@ IndexUpdateFilter::IndexUpdateFilter() {}
 
 // IndexItem
 
-Result<IndexItemHandle> IndexItem::createFromDir(
+class IndexItem::Impl final {
-    std::string const& sourceRepository,
+private:
+    ghc::filesystem::path m_path;
+    ModInfo m_info;
+    std::string m_downloadURL;
+    std::string m_downloadHash;
+    std::unordered_set<PlatformID> m_platforms;
+    bool m_isFeatured;
+    std::unordered_set<std::string> m_tags;
+
+    friend class IndexItem;
+
+public:
+    /**
+     * Create IndexItem from a directory
+     */
+    static Result<std::shared_ptr<IndexItem>> create(
         ghc::filesystem::path const& dir
-) {
+    );
+};
+
+IndexItem::IndexItem() : m_impl(std::make_unique<Impl>()) {}
+IndexItem::~IndexItem() = default;
+
+ghc::filesystem::path IndexItem::getPath() const {
+    return m_impl->m_path;
+}
+
+ModInfo IndexItem::getModInfo() const {
+    return m_impl->m_info;
+}
+
+std::string IndexItem::getDownloadURL() const {
+    return m_impl->m_downloadURL;
+}
+
+std::string IndexItem::getPackageHash() const {
+    return m_impl->m_downloadHash;
+}
+
+std::unordered_set<PlatformID> IndexItem::getAvailablePlatforms() const {
+    return m_impl->m_platforms;
+}
+
+bool IndexItem::isFeatured() const {
+    return m_impl->m_isFeatured;
+}
+
+std::unordered_set<std::string> IndexItem::getTags() const {
+    return m_impl->m_tags;
+}
+
+Result<IndexItemHandle> IndexItem::Impl::create(ghc::filesystem::path const& dir) {
     GEODE_UNWRAP_INTO(
         auto entry, file::readJson(dir / "entry.json")
             .expect("Unable to read entry.json")
@@ -111,18 +111,15 @@ Result<IndexItemHandle> IndexItem::createFromDir(
         platforms.insert(PlatformID::from(plat.template get<std::string>()));
     }
 
-    auto item = std::make_shared<IndexItem>(IndexItem {
+    auto item = std::make_shared<IndexItem>();
-        .sourceRepository = sourceRepository,
+    item->m_impl->m_path = dir;
-        .path = dir,
+    item->m_impl->m_info = info;
-        .info = info,
+    item->m_impl->m_downloadURL = root.has("mod").obj().has("download").template get<std::string>();
-        .download = {
+    item->m_impl->m_downloadHash = root.has("mod").obj().has("hash").template get<std::string>();
-            .url = root.has("mod").obj().has("download").template get<std::string>(),
+    item->m_impl->m_platforms = platforms;
-            .hash = root.has("mod").obj().has("hash").template get<std::string>(),
+    item->m_impl->m_isFeatured = root.has("featured").template get<bool>();
-            .platforms = platforms,
+    item->m_impl->m_tags = root.has("tags").template get<std::unordered_set<std::string>>();
-        },
+
-        .isFeatured = root.has("featured").template get<bool>(),
-        .tags = root.has("tags").template get<std::unordered_set<std::string>>()
-    });
     if (checker.isError()) {
         return Err(checker.getError());
     }
@@ -152,263 +149,53 @@ static Result<> flattenGithubRepo(ghc::filesystem::path const& dir) {
     return Ok();
 }
 
+// Index impl
+
+class Index::Impl final {
+public:
+    // for once, the fact that std::map is ordered is useful (this makes 
+    // getting the latest version of a mod as easy as items.rbegin())
+    using ItemVersions = std::map<size_t, IndexItemHandle>;
+
+private:
+    std::unordered_map<
+        IndexItemHandle,
+        utils::web::SentAsyncWebRequestHandle
+    > m_runningInstallations;
+    std::atomic<bool> m_isUpToDate = false;
+    std::atomic<bool> m_updating = false;
+    std::atomic<bool> m_triedToUpdate = false;
+    std::unordered_map<std::string, ItemVersions> m_items;
+
+    friend class Index;
+
+    void cleanupItems();
+    void downloadIndex();
+    void checkForUpdates();
+    void updateFromLocalTree();
+    void installNext(size_t index, IndexInstallList const& list);
+
+public:
+    Impl() {
+        new EventListener<IndexUpdateFilter>([this](IndexUpdateEvent* ev) {
+            m_updating = std::holds_alternative<UpdateProgress>(ev->status);
+        });
+    }
+};
+
 // Index globals
 
-Index::Index() {
+Index::Index() : m_impl(std::make_unique<Impl>()) {}
-    new EventListener(
+Index::~Index() = default;
-        std::bind(&Index::onSourceUpdate, this, std::placeholders::_1),
-        SourceUpdateFilter()
-    );
-    this->addSource("geode-sdk/mods");
-}
 
 Index* Index::get() {
     static auto inst = new Index();
     return inst;
 }
 
-// Sources
-
-void Index::addSource(std::string const& repository) {
-    m_sources.emplace_back(new IndexSourceImpl {
-        .repository = repository
-    });
-}
-
-void Index::removeSource(std::string const& repository) {
-    ranges::remove(m_sources, [repository](IndexSourcePtr const& src) {
-        return src->repository == repository;
-    });
-}
-
-std::vector<std::string> Index::getSources() const {
-    std::vector<std::string> res;
-    for (auto& src : m_sources) {
-        res.push_back(src->repository);
-    }
-    return res;
-}
-
 // Updating
 
-void Index::onSourceUpdate(SourceUpdateEvent* event) {
+void Index::Impl::cleanupItems() {
-    // save status for aggregating SourceUpdateEvents to a single global 
-    // IndexUpdateEvent
-    m_sourceStatuses[event->source->repository] = event->status;
-
-    // figure out aggregate event
-    enum { Finished, Progress, Failed, } whatToPost = Finished;
-    for (auto& [src, status] : m_sourceStatuses) {
-        // if some source is still updating, post progress
-        if (std::holds_alternative<UpdateProgress>(status)) {
-            whatToPost = Progress;
-            break;
-        }
-        // otherwise, if some source failed, then post failed
-        else if (std::holds_alternative<UpdateFailed>(status)) {
-            if (whatToPost != Progress) {
-                whatToPost = Failed;
-            }
-        }
-        // otherwise if all are finished, whatToPost is already set to that
-    }
-
-    switch (whatToPost) {
-        case Finished: {
-            log::debug("Index up-to-date");
-            // clear source statuses to allow updating index again
-            m_sourceStatuses.clear();
-            // post finish event
-            IndexUpdateEvent(UpdateFinished()).post();
-        } break;
-
-        case Progress: {
-            // get total progress
-            size_t total = 0;
-            for (auto& [src, status] : m_sourceStatuses) {
-                if (std::holds_alternative<UpdateProgress>(status)) {
-                    total += std::get<UpdateProgress>(status).first;
-                } else {
-                    total += 100;
-                }
-            }
-            IndexUpdateEvent(
-                UpdateProgress(
-                    static_cast<uint8_t>(total / m_sourceStatuses.size()),
-                    "Downloading"
-                )
-            ).post();
-        } break;
-
-        case Failed: {
-            std::string info = "";
-            for (auto& [src, status] : m_sourceStatuses) {
-                if (std::holds_alternative<UpdateFailed>(status)) {
-                    info += src + ": " + std::get<UpdateFailed>(status) + "\n";
-                }
-            }
-            log::debug("Index update failed: {}", info);
-            // clear source statuses to allow updating index again
-            m_sourceStatuses.clear();
-            // post finish event
-            IndexUpdateEvent(UpdateFailed(info)).post();
-        } break;
-    }
-}
-
-void Index::checkSourceUpdates(IndexSourceImpl* src) {
-    if (src->isUpToDate) {
-        return this->updateSourceFromLocal(src);
-    }
-
-    log::debug("Checking updates for source {}", src->repository);
-    SourceUpdateEvent(src, UpdateProgress(0, "Checking status")).post();
-
-    // read old commit SHA
-    // not using saved values for this one as we don't want to refetch 
-    // index even if the game crashes
-    auto oldSHA = file::readString(src->checksum()).unwrapOr("");
-    web::AsyncWebRequest()
-        .join(fmt::format("index-update-{}", src->repository))
-        .header(fmt::format("If-None-Match: \"{}\"", oldSHA))
-        .header("Accept: application/vnd.github.sha")
-        .fetch(fmt::format("https://api.github.com/repos/{}/commits/main", src->repository))
-        .text()
-        .then([this, src, oldSHA](std::string const& newSHA) {
-            // check if should just be updated from local cache
-            if (
-                // if no new hash was given (rate limited) or the new hash is the 
-                // same as old
-                (newSHA.empty() || oldSHA == newSHA) &&
-                // make sure the downloaded local copy actually exists
-                ghc::filesystem::exists(src->path()) &&
-                ghc::filesystem::exists(src->path() / "config.json")
-            ) {
-                this->updateSourceFromLocal(src);
-            }
-            // otherwise save hash and download source
-            else {
-                (void)file::writeString(src->checksum(), newSHA);
-                this->downloadSource(src);
-            }
-        })
-        .expect([src](std::string const& err) {
-            SourceUpdateEvent(
-                src,
-                UpdateFailed(fmt::format("Error checking for updates: {}", err))
-            ).post();
-        });
-}
-
-void Index::downloadSource(IndexSourceImpl* src) {
-    log::debug("Downloading source {}", src->repository);
-
-    SourceUpdateEvent(src, UpdateProgress(0, "Beginning download")).post();
-
-    auto targetFile = dirs::getIndexDir() / fmt::format("{}.zip", src->dirname());
-
-    web::AsyncWebRequest()
-        .join(fmt::format("index-download-{}", src->repository))
-        .fetch(fmt::format("https://github.com/{}/zipball/main", src->repository))
-        .into(targetFile)
-        .then([this, src, targetFile](auto) {
-            auto targetDir = src->path();
-            // delete old unzipped index
-            try {
-                if (ghc::filesystem::exists(targetDir)) {
-                    ghc::filesystem::remove_all(targetDir);
-                }
-            }
-            catch(...) {
-                SourceUpdateEvent(
-                    src, UpdateFailed("Unable to clear cached index")
-                ).post();
-                return;
-            }
-
-            // unzip new index
-            auto unzip = file::Unzip::intoDir(targetFile, targetDir, true)
-                .expect("Unable to unzip new index");
-            if (!unzip) {
-                SourceUpdateEvent(
-                    src, UpdateFailed(unzip.unwrapErr())
-                ).post();
-                return;
-            }
-
-            // remove the directory github adds to the root of the zip
-            (void)flattenGithubRepo(targetDir);
-
-            // update index
-            this->updateSourceFromLocal(src);
-        })
-        .expect([src](std::string const& err) {
-            SourceUpdateEvent(
-                src, UpdateFailed(fmt::format("Error downloading: {}", err))
-            ).post();
-        })
-        .progress([src](auto&, double now, double total) {
-            SourceUpdateEvent(
-                src,
-                UpdateProgress(
-                    static_cast<uint8_t>(now / total * 100.0),
-                    "Downloading"
-                )
-            ).post();
-        });
-}
-
-void Index::updateSourceFromLocal(IndexSourceImpl* src) {
-    log::debug("Updating local cache for source {}", src->repository);
-    SourceUpdateEvent(src, UpdateProgress(100, "Updating local cache")).post();
-    // delete old items from this url if such exist
-    for (auto& [_, versions] : m_items) {
-        for (auto it = versions.begin(); it != versions.end(); ) {
-            if (it->second->sourceRepository == src->repository) {
-                it = versions.erase(it);
-            } else {
-                ++it;
-            }
-        }
-    }
-    this->cleanupItems();
-
-    // read directory and add new items
-    try {
-        for (auto& dir : ghc::filesystem::directory_iterator(src->path() / "mods")) {
-            auto addRes = IndexItem::createFromDir(src->repository, dir);
-            if (!addRes) {
-                log::warn("Unable to add index item from {}: {}", dir, addRes.unwrapErr());
-                continue;
-            }
-            auto add = addRes.unwrap();
-            // check if this major version of this item has already been added 
-            if (m_items[add->info.id()].count(add->info.version().getMajor())) {
-                log::warn(
-                    "Item {}@{} has already been added, skipping",
-                    add->info.id(), add->info.version()
-                );
-                continue;
-            }
-            // add new major version of this item
-            m_items[add->info.id()].insert({
-                add->info.version().getMajor(),
-                add
-            });
-        }
-    } catch(std::exception& e) {
-        SourceUpdateEvent(src, fmt::format(
-            "Unable to read source {}", src->repository
-        )).post();
-        return;
-    }
-
-    // mark source as finished
-    src->isUpToDate = true;
-    SourceUpdateEvent(src, UpdateFinished()).post();
-}
-
-void Index::cleanupItems() {
     // delete mods with no versions
     for (auto it = m_items.begin(); it != m_items.end(); ) {
         if (!it->second.size()) {
@@ -420,49 +207,173 @@ void Index::cleanupItems() {
 }
 
 bool Index::isUpToDate() const {
-    for (auto& source : m_sources) {
+    return m_impl->m_isUpToDate;
-        if (!source->isUpToDate) {
-            return false;
-        }
-    }
-    return true;
 }
 
 bool Index::hasTriedToUpdate() const {
-    return m_triedToUpdate;
+    return m_impl->m_triedToUpdate;
+}
+
+void Index::Impl::downloadIndex() {
+    log::debug("Downloading index");
+
+    IndexUpdateEvent(UpdateProgress(0, "Beginning download")).post();
+
+    auto targetFile = dirs::getTempDir() / "updated-index.zip";
+
+    web::AsyncWebRequest()
+        .join("index-download")
+        .fetch("https://github.com/geode-sdk/mods/zipball/main")
+        .into(targetFile)
+        .then([this, targetFile](auto) {
+            auto targetDir = dirs::getIndexDir() / "v0";
+            // delete old unzipped index
+            try {
+                if (ghc::filesystem::exists(targetDir)) {
+                    ghc::filesystem::remove_all(targetDir);
+                }
+            }
+            catch(...) {
+                IndexUpdateEvent(UpdateFailed("Unable to clear cached index")).post();
+                return;
+            }
+
+            // unzip new index
+            auto unzip = file::Unzip::intoDir(targetFile, targetDir, true)
+                .expect("Unable to unzip new index");
+            if (!unzip) {
+                IndexUpdateEvent(UpdateFailed(unzip.unwrapErr())).post();
+                return;
+            }
+
+            // remove the directory github adds to the root of the zip
+            (void)flattenGithubRepo(targetDir);
+
+            // update index
+            this->updateFromLocalTree();
+        })
+        .expect([](std::string const& err) {
+            IndexUpdateEvent(UpdateFailed(fmt::format("Error downloading: {}", err))).post();
+        })
+        .progress([](auto&, double now, double total) {
+            IndexUpdateEvent(
+                UpdateProgress(
+                    static_cast<uint8_t>(now / total * 100.0),
+                    "Downloading"
+                )
+            ).post();
+        });
+}
+
+void Index::Impl::checkForUpdates() {
+    if (m_isUpToDate) {
+        return this->updateFromLocalTree();
+    }
+
+    log::debug("Checking updates for index");
+    IndexUpdateEvent(UpdateProgress(0, "Checking status")).post();
+
+    auto checksum = dirs::getIndexDir() / ".checksum";
+
+    // read old commit SHA
+    // not using saved values for this one as we don't want to refetch 
+    // index even if the game crashes
+    auto oldSHA = file::readString(checksum).unwrapOr("");
+    web::AsyncWebRequest()
+        .join("index-update")
+        .header(fmt::format("If-None-Match: \"{}\"", oldSHA))
+        .header("Accept: application/vnd.github.sha")
+        .fetch("https://api.github.com/repos/geode-sdk/mods/commits/main")
+        .text()
+        .then([this, checksum, oldSHA](std::string const& newSHA) {
+            // check if should just be updated from local cache
+            if (
+                // if no new hash was given (rate limited) or the new hash is the 
+                // same as old
+                (newSHA.empty() || oldSHA == newSHA) &&
+                // make sure the downloaded local copy actually exists
+                ghc::filesystem::exists(dirs::getIndexDir() / "v0" / "config.json")
+            ) {
+                this->updateFromLocalTree();
+            }
+            // otherwise save hash and download source
+            else {
+                (void)file::writeString(checksum, newSHA);
+                this->downloadIndex();
+            }
+        })
+        .expect([](std::string const& err) {
+            IndexUpdateEvent(
+                UpdateFailed(fmt::format("Error checking for updates: {}", err))
+            ).post();
+        });
+}
+
+void Index::Impl::updateFromLocalTree() {
+    log::debug("Updating local index cache");
+    IndexUpdateEvent(UpdateProgress(100, "Updating local cache")).post();
+    // delete old items
+    m_items.clear();
+
+    // read directory and add new items
+    try {
+        for (auto& dir : ghc::filesystem::directory_iterator(dirs::getIndexDir() / "v0" / "mods")) {
+            auto addRes = IndexItem::Impl::create(dir);
+            if (!addRes) {
+                log::warn("Unable to add index item from {}: {}", dir, addRes.unwrapErr());
+                continue;
+            }
+            auto add = addRes.unwrap();
+            auto info = add->getModInfo();
+            // check if this major version of this item has already been added 
+            if (m_items[info.id()].count(info.version().getMajor())) {
+                log::warn(
+                    "Item {}@{} has already been added, skipping",
+                    info.id(), info.version()
+                );
+                continue;
+            }
+            // add new major version of this item
+            m_items[info.id()].insert({
+                info.version().getMajor(),
+                add
+            });
+        }
+    } catch(std::exception& e) {
+        IndexUpdateEvent("Unable to read local index tree").post();
+        return;
+    }
+
+    // mark source as finished
+    m_isUpToDate = true;
+    IndexUpdateEvent(UpdateFinished()).post();
 }
 
 void Index::update(bool force) {
     // create index dir if it doesn't exist
     (void)file::createDirectoryAll(dirs::getIndexDir());
 
-    m_triedToUpdate = true;
+    m_impl->m_triedToUpdate = true;
 
-    // update all sources in GD thread for synchronization (m_sourceStatuses 
+    // check if update is already happening
-    // and every other member access happens in AsyncWebRequest callbacks 
+    if (m_impl->m_updating) {
-    // which are always run in the GD thread aswell)
-    Loader::get()->queueInGDThread([force, this]() {
-        // check if some sources are already being updated
-        if (m_sourceStatuses.size()) {
         return;
     }
+    m_impl->m_updating = true;
 
     // update sources
-        for (auto& src : m_sources) {
     if (force) {
-                this->downloadSource(src.get());
+        m_impl->downloadIndex();
     } else {
-                this->checkSourceUpdates(src.get());
+        m_impl->checkForUpdates();
     }
-        }
-    });
 }
 
 // Items
 
 std::vector<IndexItemHandle> Index::getItems() const {
     std::vector<IndexItemHandle> res;
-    for (auto& items : map::values(m_items)) {
+    for (auto& items : map::values(m_impl->m_items)) {
         for (auto& item : items) {
             res.push_back(item.second);
         }
@@ -472,9 +383,9 @@ std::vector<IndexItemHandle> Index::getItems() const {
 
 std::vector<IndexItemHandle> Index::getFeaturedItems() const {
     std::vector<IndexItemHandle> res;
-    for (auto& items : map::values(m_items)) {
+    for (auto& items : map::values(m_impl->m_items)) {
         for (auto& item : items) {
-            if (item.second->isFeatured) {
+            if (item.second->isFeatured()) {
                 res.push_back(item.second);
             }
         }
@@ -486,9 +397,9 @@ std::vector<IndexItemHandle> Index::getItemsByDeveloper(
     std::string const& name
 ) const {
     std::vector<IndexItemHandle> res;
-    for (auto& items : map::values(m_items)) {
+    for (auto& items : map::values(m_impl->m_items)) {
         for (auto& item : items) {
-            if (item.second->info.developer() == name) {
+            if (item.second->getModInfo().developer() == name) {
                 res.push_back(item.second);
             }
         }
@@ -506,8 +417,8 @@ bool Index::isKnownItem(
 IndexItemHandle Index::getMajorItem(
     std::string const& id
 ) const {
-    if (m_items.count(id)) {
+    if (m_impl->m_items.count(id)) {
-        return m_items.at(id).rbegin()->second;
+        return m_impl->m_items.at(id).rbegin()->second;
     }
     return nullptr;
 }
@@ -516,18 +427,18 @@ IndexItemHandle Index::getItem(
     std::string const& id,
     std::optional<VersionInfo> version
 ) const {
-    if (m_items.count(id)) {
+    if (m_impl->m_items.count(id)) {
-        auto versions = m_items.at(id);
+        auto versions = m_impl->m_items.at(id);
         if (version) {
             // prefer most major version
-            for (auto& [_, item] : ranges::reverse(m_items.at(id))) {
+            for (auto& [_, item] : ranges::reverse(m_impl->m_items.at(id))) {
-                if (version.value() == item->info.version()) {
+                if (version.value() == item->getModInfo().version()) {
                     return item;
                 }
             }
         } else {
             if (versions.size()) {
-                return m_items.at(id).rbegin()->second;
+                return m_impl->m_items.at(id).rbegin()->second;
             }
         }
     }
@@ -538,10 +449,10 @@ IndexItemHandle Index::getItem(
     std::string const& id,
     ComparableVersionInfo version
 ) const {
-    if (m_items.count(id)) {
+    if (m_impl->m_items.count(id)) {
         // prefer most major version
-        for (auto& [_, item] : ranges::reverse(m_items.at(id))) {
+        for (auto& [_, item] : ranges::reverse(m_impl->m_items.at(id))) {
-            if (version.compare(item->info.version())) {
+            if (version.compare(item->getModInfo().version())) {
                 return item;
             }
         }
@@ -558,17 +469,17 @@ IndexItemHandle Index::getItem(Mod* mod) const {
 }
 
 bool Index::isUpdateAvailable(IndexItemHandle item) const {
-    auto installed = Loader::get()->getInstalledMod(item->info.id());
+    auto installed = Loader::get()->getInstalledMod(item->getModInfo().id());
     if (!installed) {
         return false;
     }
-    return item->info.version() > installed->getVersion();
+    return item->getModInfo().version() > installed->getVersion();
 }
 
 bool Index::areUpdatesAvailable() const {
     for (auto& mod : Loader::get()->getAllMods()) {
         auto item = this->getMajorItem(mod->getID());
-        if (item && item->info.version() > mod->getVersion()) {
+        if (item && item->getModInfo().version() > mod->getVersion()) {
             return true;
         }
     }
@@ -578,17 +489,17 @@ bool Index::areUpdatesAvailable() const {
 // Item installation
 
 Result<IndexInstallList> Index::getInstallList(IndexItemHandle item) const {
-    if (!item->download.platforms.count(GEODE_PLATFORM_TARGET)) {
+    if (!item->getAvailablePlatforms().count(GEODE_PLATFORM_TARGET)) {
         return Err("Mod is not available on {}", GEODE_PLATFORM_NAME);
     }
     
     IndexInstallList list;
     list.target = item;
-    for (auto& dep : item->info.dependencies()) {
+    for (auto& dep : item->getModInfo().dependencies()) {
         if (!dep.isResolved()) {
             // check if this dep is available in the index
             if (auto depItem = this->getItem(dep.id, dep.version)) {
-                if (!depItem->download.platforms.count(GEODE_PLATFORM_TARGET)) {
+                if (!depItem->getAvailablePlatforms().count(GEODE_PLATFORM_TARGET)) {
                     return Err(
                         "Dependency {} is not available on {}",
                         dep.id, GEODE_PLATFORM_NAME
@@ -606,7 +517,7 @@ Result<IndexInstallList> Index::getInstallList(IndexItemHandle item) const {
                     "reason is that the version of the dependency this mod "
                     "depends on is not available. Please let the the developer "
                     "({}) of the mod know!",
-                    dep.id, dep.version.toString(), item->info.developer()
+                    dep.id, dep.version.toString(), item->getModInfo().developer()
                 );
             }
         }
@@ -618,10 +529,10 @@ Result<IndexInstallList> Index::getInstallList(IndexItemHandle item) const {
     return Ok(list);
 }
 
-void Index::installNext(size_t index, IndexInstallList const& list) {
+void Index::Impl::installNext(size_t index, IndexInstallList const& list) {
     auto postError = [this, list](std::string const& error) {
         m_runningInstallations.erase(list.target);
-        ModInstallEvent(list.target->info.id(), error).post();
+        ModInstallEvent(list.target->getModInfo().id(), error).post();
     };
 
     // If we're at the end of the list, move the downloaded items to mods
@@ -630,12 +541,12 @@ void Index::installNext(size_t index, IndexInstallList const& list) {
         // Move all downloaded files
         for (auto& item : list.list) {
             // If the mod is already installed, delete the old .geode file
-            if (auto mod = Loader::get()->getInstalledMod(item->info.id())) {
+            if (auto mod = Loader::get()->getInstalledMod(item->getModInfo().id())) {
                 auto res = mod->uninstall();
                 if (!res) {
                     return postError(fmt::format(
                         "Unable to uninstall old version of {}: {}",
-                        item->info.id(), res.unwrapErr()
+                        item->getModInfo().id(), res.unwrapErr()
                     ));
                 }
             }
@@ -643,13 +554,13 @@ void Index::installNext(size_t index, IndexInstallList const& list) {
             // Move the temp file
             try {
                 ghc::filesystem::rename(
-                    dirs::getTempDir() / (item->info.id() + ".index"),
+                    dirs::getTempDir() / (item->getModInfo().id() + ".index"),
-                    dirs::getModsDir() / (item->info.id() + ".geode")
+                    dirs::getModsDir() / (item->getModInfo().id() + ".geode")
                 );
             } catch(std::exception& e) {
                 return postError(fmt::format(
                     "Unable to install {}: {}",
-                    item->info.id(), e.what()
+                    item->getModInfo().id(), e.what()
                 ));
             }
         }
@@ -657,7 +568,7 @@ void Index::installNext(size_t index, IndexInstallList const& list) {
         // load mods
         Loader::get()->refreshModsList();
 
-        ModInstallEvent(list.target->info.id(), UpdateFinished()).post();
+        ModInstallEvent(list.target->getModInfo().id(), UpdateFinished()).post();
         return;
     }
 
@@ -668,10 +579,10 @@ void Index::installNext(size_t index, IndexInstallList const& list) {
     };
 
     auto item = list.list.at(index);
-    auto tempFile = dirs::getTempDir() / (item->info.id() + ".index");
+    auto tempFile = dirs::getTempDir() / (item->getModInfo().id() + ".index");
     m_runningInstallations[list.target] = web::AsyncWebRequest()
-        .join("install_item_" + item->info.id())
+        .join("install_item_" + item->getModInfo().id())
-        .fetch(item->download.url)
+        .fetch(item->getDownloadURL())
         .into(tempFile)
         .then([=](auto) {
             // Check for 404
@@ -680,25 +591,25 @@ void Index::installNext(size_t index, IndexInstallList const& list) {
                 return postError(fmt::format(
                     "Binary file download for {} returned \"404 Not found\". "
                     "Report this to the Geode development team.",
-                    item->info.id()
+                    item->getModInfo().id()
                 ));
             }
 
             // Verify checksum
             ModInstallEvent(
-                list.target->info.id(),
+                list.target->getModInfo().id(),
                 UpdateProgress(
                     scaledProgress(100),
-                    fmt::format("Verifying {}", item->info.id())
+                    fmt::format("Verifying {}", item->getModInfo().id())
                 )
             ).post();
             
-            if (::calculateHash(tempFile) != item->download.hash) {
+            if (::calculateHash(tempFile) != item->getPackageHash()) {
                 return postError(fmt::format(
                     "Checksum mismatch with {}! (Downloaded file did not match what "
                     "was expected. Try again, and if the download fails another time, "
                     "report this to the Geode development team.)",
-                    item->info.id()
+                    item->getModInfo().id()
                 ));
             }
 
@@ -708,15 +619,15 @@ void Index::installNext(size_t index, IndexInstallList const& list) {
         .expect([postError, list, item](std::string const& err) {
             postError(fmt::format(
                 "Unable to download {}: {}",
-                item->info.id(), err
+                item->getModInfo().id(), err
             ));
         })
         .progress([this, item, list, scaledProgress](auto&, double now, double total) {
             ModInstallEvent(
-                list.target->info.id(),
+                list.target->getModInfo().id(),
                 UpdateProgress(
                     scaledProgress(now / total * 100.0),
-                    fmt::format("Downloading {}", item->info.id())
+                    fmt::format("Downloading {}", item->getModInfo().id())
                 )
             ).post();
         })
@@ -728,22 +639,22 @@ void Index::installNext(size_t index, IndexInstallList const& list) {
 
 void Index::cancelInstall(IndexItemHandle item) {
     Loader::get()->queueInGDThread([this, item]() {
-        if (m_runningInstallations.count(item)) {
+        if (m_impl->m_runningInstallations.count(item)) {
-            m_runningInstallations.at(item)->cancel();
+            m_impl->m_runningInstallations.at(item)->cancel();
-            m_runningInstallations.erase(item);
+            m_impl->m_runningInstallations.erase(item);
         }
     });
 }
 
 void Index::install(IndexInstallList const& list) {
     Loader::get()->queueInGDThread([this, list]() {
-        this->installNext(0, list);
+        m_impl->installNext(0, list);
     });
 }
 
 void Index::install(IndexItemHandle item) {
     Loader::get()->queueInGDThread([this, item]() {
-        if (m_runningInstallations.count(item)) {
+        if (m_impl->m_runningInstallations.count(item)) {
             return;
         }
         auto list = this->getInstallList(item);
@@ -751,7 +662,7 @@ void Index::install(IndexItemHandle item) {
             this->install(list.unwrap());
         } else {
             ModInstallEvent(
-                item->info.id(),
+                item->getModInfo().id(),
                 UpdateFailed(list.unwrapErr())
             ).post();
         }
@@ -762,9 +673,9 @@ void Index::install(IndexItemHandle item) {
 
 std::unordered_set<std::string> Index::getTags() const {
     std::unordered_set<std::string> tags;
-    for (auto& [_, versions] : m_items) {
+    for (auto& [_, versions] : m_impl->m_items) {
         for (auto& [_, item] : versions) {
-            for (auto& tag : item->tags) {
+            for (auto& tag : item->getTags()) {
                 tags.insert(tag);
             }
         }

loader/src/ui/internal/GeodeUI.cpp

@@ -88,7 +88,7 @@ CCNode* geode::createModLogo(Mod* mod, CCSize const& size) {
 
 CCNode* geode::createIndexItemLogo(IndexItemHandle item, CCSize const& size) {
     CCNode* spr = nullptr;
-    auto logoPath = ghc::filesystem::absolute(item->path / "logo.png");
+    auto logoPath = ghc::filesystem::absolute(item->getPath() / "logo.png");
     spr = CCSprite::create(logoPath.string().c_str());
     if (!spr) {
         spr = CCSprite::createWithSpriteFrameName("no-logo.png"_spr);
@@ -96,7 +96,7 @@ CCNode* geode::createIndexItemLogo(IndexItemHandle item, CCSize const& size) {
     if (!spr) {
         spr = CCLabelBMFont::create("N/A", "goldFont.fnt");
     }
-    if (item->isFeatured) {
+    if (item->isFeatured()) {
         auto glowSize = size + CCSize(4.f, 4.f);
 
         auto logoGlow = CCSprite::createWithSpriteFrameName("logo-glow.png"_spr);

loader/src/ui/internal/info/DevProfilePopup.cpp

@@ -27,7 +27,7 @@ bool DevProfilePopup::setup(std::string const& developer) {
 
     // index mods
     for (auto& item : Index::get()->getItemsByDeveloper(developer)) {
-        if (Loader::get()->isModInstalled(item->info.id())) {
+        if (Loader::get()->isModInstalled(item->getModInfo().id())) {
             continue;
         }
         auto cell = IndexItemCell::create(

loader/src/ui/internal/info/ModInfoPopup.cpp

@@ -290,7 +290,7 @@ LocalModInfoPopup::LocalModInfoPopup()
 bool LocalModInfoPopup::init(Mod* mod, ModListLayer* list) {
     m_item = Index::get()->getMajorItem(mod->getModInfo().id());
     if (m_item)
-        m_installListener.setFilter(m_item->info.id());
+        m_installListener.setFilter(m_item->getModInfo().id());
     m_mod = mod;
 
     if (!ModInfoPopup::init(mod->getModInfo(), list)) return false;
@@ -382,10 +382,10 @@ bool LocalModInfoPopup::init(Mod* mod, ModListLayer* list) {
 
             // TODO: use column layout here?
 
-            if (m_item->info.version().getMajor() > minorIndexItem->info.version().getMajor()) {
+            if (m_item->getModInfo().version().getMajor() > minorIndexItem->getModInfo().version().getMajor()) {
                 // has major update
                 m_latestVersionLabel = CCLabelBMFont::create(
-                    ("Available: " + m_item->info.version().toString()).c_str(),
+                    ("Available: " + m_item->getModInfo().version().toString()).c_str(),
                     "bigFont.fnt"
                 );
                 m_latestVersionLabel->setScale(.35f);
@@ -395,10 +395,10 @@ bool LocalModInfoPopup::init(Mod* mod, ModListLayer* list) {
                 m_mainLayer->addChild(m_latestVersionLabel);
             }
 
-            if (minorIndexItem->info.version() > mod->getModInfo().version()) {
+            if (minorIndexItem->getModInfo().version() > mod->getModInfo().version()) {
                 // has minor update
                 m_minorVersionLabel = CCLabelBMFont::create(
-                    ("Available: " + minorIndexItem->info.version().toString()).c_str(),
+                    ("Available: " + minorIndexItem->getModInfo().version().toString()).c_str(),
                     "bigFont.fnt"
                 );
                 m_minorVersionLabel->setScale(.35f);
@@ -516,7 +516,8 @@ void LocalModInfoPopup::onUpdate(CCObject*) {
                     [](IndexItemHandle handle) {
                         return fmt::format(
                             " - <cr>{}</c> (<cy>{}</c>)",
-                            handle->info.name(), handle->info.id()
+                            handle->getModInfo().name(),
+                            handle->getModInfo().id()
                         );
                     }
                 ),
@@ -683,11 +684,11 @@ IndexItemInfoPopup::IndexItemInfoPopup()
 
 bool IndexItemInfoPopup::init(IndexItemHandle item, ModListLayer* list) {
     m_item = item;
-    m_installListener.setFilter(m_item->info.id());
+    m_installListener.setFilter(m_item->getModInfo().id());
 
     auto winSize = CCDirector::sharedDirector()->getWinSize();
 
-    if (!ModInfoPopup::init(item->info, list)) return false;
+    if (!ModInfoPopup::init(item->getModInfo(), list)) return false;
 
     m_installBtnSpr = IconButtonSprite::create(
         "GE_button_01.png"_spr,
@@ -770,7 +771,8 @@ void IndexItemInfoPopup::onInstall(CCObject*) {
                     [](IndexItemHandle handle) {
                         return fmt::format(
                             " - <cr>{}</c> (<cy>{}</c>)",
-                            handle->info.name(), handle->info.id()
+                            handle->getModInfo().name(),
+                            handle->getModInfo().id()
                         );
                     }
                 ),
@@ -811,7 +813,7 @@ CCNode* IndexItemInfoPopup::createLogo(CCSize const& size) {
 }
 
 ModInfo IndexItemInfoPopup::getModInfo() const {
-    return m_item->info;
+    return m_item->getModInfo();
 }
 
 IndexItemInfoPopup* IndexItemInfoPopup::create(

loader/src/ui/internal/list/ModListCell.cpp

@@ -273,7 +273,7 @@ bool ModCell::init(
                 ComparableVersionInfo(mod->getModInfo().version(), VersionCompare::MoreEq)
             );
 
-            if (latestIndexItem->info.version().getMajor() > minorIndexItem->info.version().getMajor()) {
+            if (latestIndexItem->getModInfo().version().getMajor() > minorIndexItem->getModInfo().version().getMajor()) {
                 auto updateIcon = CCSprite::createWithSpriteFrameName("updates-available.png"_spr);
                 updateIcon->setPosition(viewSpr->getContentSize() - CCSize { 2.f, 2.f });
                 updateIcon->setZOrder(99);
@@ -327,7 +327,7 @@ bool IndexItemCell::init(
 
     m_item = item;
 
-    this->setupInfo(item->info, item->tags.size(), display);
+    this->setupInfo(item->getModInfo(), item->getTags().size(), display);
    
     auto viewSpr = ButtonSprite::create(
         "View", "bigFont.fnt", "GJ_button_01.png", .8f
@@ -337,9 +337,9 @@ bool IndexItemCell::init(
     auto viewBtn = CCMenuItemSpriteExtra::create(viewSpr, this, menu_selector(IndexItemCell::onInfo));
     m_menu->addChild(viewBtn);
 
-    if (item->tags.size()) {
+    if (item->getTags().size()) {
         float x = m_height / 2 + this->getLogoSize() / 2 + 13.f;
-        for (auto& category : item->tags) {
+        for (auto& category : item->getTags()) {
             auto node = TagNode::create(category);
             node->setAnchorPoint({ .0f, .5f });
             node->setPositionX(x);
@@ -364,7 +364,7 @@ bool IndexItemCell::init(
 void IndexItemCell::updateState() {}
 
 std::string IndexItemCell::getDeveloper() const {
-    return m_item->info.developer();
+    return m_item->getModInfo().developer();
 }
 
 CCNode* IndexItemCell::createLogo(CCSize const& size) {

loader/src/ui/internal/list/ModListLayer.cpp

@@ -95,33 +95,33 @@ static std::optional<int> queryMatch(ModListQuery const& query, Mod* mod) {
 static std::optional<int> queryMatch(ModListQuery const& query, IndexItemHandle item) {
     // if no force visibility was provided and item is already installed, don't 
     // show it
-    if (!query.forceVisibility && Loader::get()->isModInstalled(item->info.id())) {
+    if (!query.forceVisibility && Loader::get()->isModInstalled(item->getModInfo().id())) {
         return std::nullopt;
     }
     // make sure all tags match
     for (auto& tag : query.tags) {
-        if (!item->tags.count(tag)) {
+        if (!item->getTags().count(tag)) {
             return std::nullopt;
         }
     }
     // make sure at least some platform matches
     if (!ranges::contains(query.platforms, [item](PlatformID id) {
-        return item->download.platforms.count(id);
+        return item->getAvailablePlatforms().count(id);
     })) {
         return std::nullopt;
     }
     // otherwise match keywords
-    if (auto match = queryMatchKeywords(query, item->info)) {
+    if (auto match = queryMatchKeywords(query, item->getModInfo())) {
         auto weighted = match.value();
         // add extra weight on tag matches
         if (query.keywords) {
-            WEIGHTED_MATCH_ADD(ranges::join(item->tags, " "), 1.4);
+            WEIGHTED_MATCH_ADD(ranges::join(item->getTags(), " "), 1.4);
         }
         // add extra weight to featured items to keep power consolidated in the 
         // hands of the rich Geode bourgeoisie
         // the number 420 is a reference to the number one bourgeois of modern 
         // society, elon musk
-        weighted += item->isFeatured ? 420 : 0;
+        weighted += item->isFeatured() ? 420 : 0;
         return static_cast<int>(weighted);
     }
     // keywords must match bruh