From 28cf749951a8f811606bb233efecfd36738c3c89 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Thu, 8 Aug 2024 16:08:46 +1000 Subject: [PATCH] Make FileGatherer shaped more like a normal iterator --- src/tangara/audio/audio_fsm.cpp | 1 + src/tangara/database/database.cpp | 34 +++++----- src/tangara/database/database.hpp | 6 +- src/tangara/database/file_gatherer.cpp | 72 --------------------- src/tangara/database/file_gatherer.hpp | 34 ---------- src/tangara/database/track_finder.cpp | 84 +++++++++++++++++++++++++ src/tangara/database/track_finder.hpp | 33 ++++++++++ src/tangara/input/lvgl_input_driver.cpp | 1 + src/tangara/system_fsm/idle.cpp | 4 +- src/tangara/system_fsm/running.cpp | 9 +-- 10 files changed, 141 insertions(+), 137 deletions(-) delete mode 100644 src/tangara/database/file_gatherer.cpp delete mode 100644 src/tangara/database/file_gatherer.hpp create mode 100644 src/tangara/database/track_finder.cpp create mode 100644 src/tangara/database/track_finder.hpp diff --git a/src/tangara/audio/audio_fsm.cpp b/src/tangara/audio/audio_fsm.cpp index a43cd932..8da11665 100644 --- a/src/tangara/audio/audio_fsm.cpp +++ b/src/tangara/audio/audio_fsm.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "audio/audio_source.hpp" diff --git a/src/tangara/database/database.cpp b/src/tangara/database/database.cpp index aec661d9..2d72fe95 100644 --- a/src/tangara/database/database.cpp +++ b/src/tangara/database/database.cpp @@ -24,6 +24,7 @@ #include "cppbor.h" #include "cppbor_parse.h" #include "database/index.hpp" +#include "database/track_finder.hpp" #include "debug.hpp" #include "esp_log.h" #include "esp_timer.h" @@ -40,7 +41,6 @@ #include "database/db_events.hpp" #include "database/env_esp.hpp" -#include "database/file_gatherer.hpp" #include "database/records.hpp" #include "database/tag_parser.hpp" #include "database/track.hpp" @@ -122,8 +122,7 @@ static auto CheckDatabase(leveldb::DB& db, locale::ICollator& col) -> bool { return true; } -auto Database::Open(IFileGatherer& gatherer, - ITagParser& parser, +auto Database::Open(ITagParser& parser, locale::ICollator& collator, tasks::WorkerPool& bg_worker) -> cpp::result { @@ -168,8 +167,7 @@ auto Database::Open(IFileGatherer& gatherer, } ESP_LOGI(kTag, "Database opened successfully"); - return new Database(db, cache.release(), gatherer, parser, - collator); + return new Database(db, cache.release(), parser, collator); }) .get(); } @@ -182,12 +180,10 @@ auto Database::Destroy() -> void { Database::Database(leveldb::DB* db, leveldb::Cache* cache, - IFileGatherer& file_gatherer, ITagParser& tag_parser, locale::ICollator& collator) : db_(db), cache_(cache), - file_gatherer_(file_gatherer), tag_parser_(tag_parser), collator_(collator), is_updating_(false) { @@ -401,7 +397,11 @@ auto Database::updateIndexes() -> void { // Stage 2: search for newly added files. ESP_LOGI(kTag, "scanning for new tracks"); uint64_t num_files = 0; - file_gatherer_.FindFiles("", [&](std::string_view path, const FILINFO& info) { + + auto track_finder = std::make_shared(""); + + FILINFO info; + while (auto path = track_finder->next(info)) { num_files++; events::Ui().Dispatch(event::UpdateProgress{ .stage = event::UpdateProgress::Stage::kScanningForNewTracks, @@ -409,15 +409,15 @@ auto Database::updateIndexes() -> void { }); std::string unused; - if (db_->Get(read_options, EncodePathKey(path), &unused).ok()) { + if (db_->Get(read_options, EncodePathKey(*path), &unused).ok()) { // This file is already in the database; skip it. - return; + continue; } - std::shared_ptr tags = tag_parser_.ReadAndParseTags(path); + std::shared_ptr tags = tag_parser_.ReadAndParseTags(*path); if (!tags || tags->encoding() == Container::kUnsupported) { // No parseable tags; skip this fiile. - return; + continue; } // Check for any existing track with the same hash. @@ -438,14 +438,14 @@ auto Database::updateIndexes() -> void { if (!data) { data = std::make_shared(); data->id = *existing_id; - } else if (data->filepath != path) { + } else if (std::string_view{data->filepath} != *path) { ESP_LOGW(kTag, "hash collision: %s, %s, %s", tags->title().value_or("no title").c_str(), tags->artist().value_or("no artist").c_str(), tags->album().value_or("no album").c_str()); // Don't commit anything if there's a hash collision, since we're // likely to make a big mess. - return; + continue; } } else { num_new_tracks++; @@ -454,7 +454,7 @@ auto Database::updateIndexes() -> void { } // Make sure the file-based metadata on the TrackData is up to date. - data->filepath = path; + data->filepath = *path; data->tags_hash = hash; data->modified_at = {info.fdate, info.ftime}; @@ -467,10 +467,10 @@ auto Database::updateIndexes() -> void { dbCreateIndexesForTrack(*data, *tags, batch); batch.Put(EncodeDataKey(data->id), EncodeDataValue(*data)); batch.Put(EncodeHashKey(data->tags_hash), EncodeHashValue(data->id)); - batch.Put(EncodePathKey(path), TrackIdToBytes(data->id)); + batch.Put(EncodePathKey(*path), TrackIdToBytes(data->id)); db_->Write(leveldb::WriteOptions(), &batch); - }); + }; uint64_t end_time = esp_timer_get_time(); diff --git a/src/tangara/database/database.hpp b/src/tangara/database/database.hpp index 39665dbf..6994d0b8 100644 --- a/src/tangara/database/database.hpp +++ b/src/tangara/database/database.hpp @@ -19,7 +19,6 @@ #include "collation.hpp" #include "cppbor.h" -#include "database/file_gatherer.hpp" #include "database/index.hpp" #include "database/records.hpp" #include "database/tag_parser.hpp" @@ -56,8 +55,7 @@ class Database { ALREADY_OPEN, FAILED_TO_OPEN, }; - static auto Open(IFileGatherer& file_gatherer, - ITagParser& tag_parser, + static auto Open(ITagParser& tag_parser, locale::ICollator& collator, tasks::WorkerPool& bg_worker) -> cpp::result; @@ -96,7 +94,6 @@ class Database { leveldb::Cache* cache_; // Not owned. - IFileGatherer& file_gatherer_; ITagParser& tag_parser_; locale::ICollator& collator_; @@ -105,7 +102,6 @@ class Database { Database(leveldb::DB* db, leveldb::Cache* cache, - IFileGatherer& file_gatherer, ITagParser& tag_parser, locale::ICollator& collator); diff --git a/src/tangara/database/file_gatherer.cpp b/src/tangara/database/file_gatherer.cpp deleted file mode 100644 index dd4b1138..00000000 --- a/src/tangara/database/file_gatherer.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2023 jacqueline - * - * SPDX-License-Identifier: GPL-3.0-only - */ - -#include "database/file_gatherer.hpp" - -#include -#include -#include -#include - -#include "ff.h" - -#include "drivers/spi.hpp" -#include "memory_resource.hpp" - -namespace database { - -static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR"); - -auto FileGathererImpl::FindFiles( - const std::string& root, - std::function cb) -> void { - std::pmr::deque to_explore{&memory::kSpiRamResource}; - to_explore.push_back({root.data(), root.size()}); - - while (!to_explore.empty()) { - auto next_path_str = to_explore.front(); - to_explore.pop_front(); - - const TCHAR* next_path = static_cast(next_path_str.c_str()); - - FF_DIR dir; - FRESULT res = f_opendir(&dir, next_path); - if (res != FR_OK) { - // TODO: log. - continue; - } - - for (;;) { - FILINFO info; - res = f_readdir(&dir, &info); - if (res != FR_OK || info.fname[0] == 0) { - // No more files in the directory. - break; - } else if (info.fattrib & (AM_HID | AM_SYS) || info.fname[0] == '.') { - // System or hidden file. Ignore it and move on. - continue; - } else { - std::pmr::string full_path{&memory::kSpiRamResource}; - full_path += next_path_str; - full_path += "/"; - full_path += info.fname; - - if (info.fattrib & AM_DIR) { - // This is a directory. Add it to the explore queue. - to_explore.push_back(full_path); - } else { - // This is a file! Let the callback know about it. - // std::invoke(cb, full_path.str(), info); - std::invoke(cb, full_path, info); - } - } - } - - f_closedir(&dir); - } -} - -} // namespace database diff --git a/src/tangara/database/file_gatherer.hpp b/src/tangara/database/file_gatherer.hpp deleted file mode 100644 index 38558b9e..00000000 --- a/src/tangara/database/file_gatherer.hpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 jacqueline - * - * SPDX-License-Identifier: GPL-3.0-only - */ - -#pragma once - -#include -#include -#include -#include - -#include "ff.h" - -namespace database { - -class IFileGatherer { - public: - virtual ~IFileGatherer() {}; - - virtual auto FindFiles( - const std::string& root, - std::function cb) -> void = 0; -}; - -class FileGathererImpl : public IFileGatherer { - public: - virtual auto FindFiles(const std::string& root, - std::function - cb) -> void override; -}; - -} // namespace database diff --git a/src/tangara/database/track_finder.cpp b/src/tangara/database/track_finder.cpp new file mode 100644 index 00000000..86948e70 --- /dev/null +++ b/src/tangara/database/track_finder.cpp @@ -0,0 +1,84 @@ +/* + * Copyright 2023 jacqueline + * + * SPDX-License-Identifier: GPL-3.0-only + */ + +#include "database/track_finder.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "database/track_finder.hpp" +#include "ff.h" + +#include "drivers/spi.hpp" +#include "memory_resource.hpp" + +namespace database { + +static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR"); + +TrackFinder::TrackFinder(std::string_view root) + : to_explore_(&memory::kSpiRamResource) { + to_explore_.push_back({root.data(), root.size()}); +} + +auto TrackFinder::next(FILINFO& out_info) -> std::optional { + std::scoped_lock lock{mut_}; + while (!to_explore_.empty() || current_) { + if (!current_) { + current_.emplace(); + + // Get the next directory to iterate through. + current_->first = to_explore_.front(); + to_explore_.pop_front(); + const TCHAR* next_path = + static_cast(current_->first.data()); + + // Open it for iterating. + FRESULT res = f_opendir(¤t_->second, next_path); + if (res != FR_OK) { + current_.reset(); + continue; + } + } + + FILINFO info; + FRESULT res = f_readdir(¤t_->second, &info); + if (res != FR_OK || info.fname[0] == 0) { + // No more files in the directory. + f_closedir(¤t_->second); + current_.reset(); + continue; + } else if (info.fattrib & (AM_HID | AM_SYS) || info.fname[0] == '.') { + // System or hidden file. Ignore it and move on. + continue; + } else { + // A valid file or folder. + std::pmr::string full_path{&memory::kSpiRamResource}; + full_path += current_->first; + full_path += "/"; + full_path += info.fname; + + if (info.fattrib & AM_DIR) { + // This is a directory. Add it to the explore queue. + to_explore_.push_back(full_path); + } else { + // This is a file! We can return now. + out_info = info; + return {{full_path.data(), full_path.size()}}; + } + } + } + + // Out of things to explore. + return {}; +} + +} // namespace database diff --git a/src/tangara/database/track_finder.hpp b/src/tangara/database/track_finder.hpp new file mode 100644 index 00000000..aba208e9 --- /dev/null +++ b/src/tangara/database/track_finder.hpp @@ -0,0 +1,33 @@ +/* + * Copyright 2023 jacqueline + * + * SPDX-License-Identifier: GPL-3.0-only + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "ff.h" + +namespace database { + +class TrackFinder { + public: + TrackFinder(std::string_view root); + + auto next(FILINFO&) -> std::optional; + + private: + std::mutex mut_; + std::pmr::deque to_explore_; + std::optional> current_; +}; + +} // namespace database diff --git a/src/tangara/input/lvgl_input_driver.cpp b/src/tangara/input/lvgl_input_driver.cpp index f6beabda..824e49cc 100644 --- a/src/tangara/input/lvgl_input_driver.cpp +++ b/src/tangara/input/lvgl_input_driver.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include "core/lv_group.h" diff --git a/src/tangara/system_fsm/idle.cpp b/src/tangara/system_fsm/idle.cpp index d233f603..2d66b01d 100644 --- a/src/tangara/system_fsm/idle.cpp +++ b/src/tangara/system_fsm/idle.cpp @@ -4,8 +4,9 @@ * SPDX-License-Identifier: GPL-3.0-only */ +#include "ui/ui_fsm.hpp" + #include "app_console/app_console.hpp" -#include "database/file_gatherer.hpp" #include "drivers/gpios.hpp" #include "freertos/portmacro.h" #include "freertos/projdefs.h" @@ -17,7 +18,6 @@ #include "events/event_queue.hpp" #include "system_fsm/system_events.hpp" #include "system_fsm/system_fsm.hpp" -#include "ui/ui_fsm.hpp" namespace system_fsm { namespace states { diff --git a/src/tangara/system_fsm/running.cpp b/src/tangara/system_fsm/running.cpp index c808e9da..f9bca074 100644 --- a/src/tangara/system_fsm/running.cpp +++ b/src/tangara/system_fsm/running.cpp @@ -8,7 +8,6 @@ #include "audio/audio_events.hpp" #include "database/database.hpp" #include "database/db_events.hpp" -#include "database/file_gatherer.hpp" #include "drivers/gpios.hpp" #include "drivers/spi.hpp" #include "ff.h" @@ -36,8 +35,6 @@ static void timer_callback(TimerHandle_t timer) { events::System().Dispatch(internal::UnmountTimeout{}); } -static database::IFileGatherer* sFileGatherer; - void Running::entry() { if (!sUnmountTimer) { sUnmountTimer = xTimerCreate("unmount_timeout", kTicksBeforeUnmount, false, @@ -174,10 +171,8 @@ auto Running::mountStorage() -> void { sStorage.reset(storage_res.value()); ESP_LOGI(kTag, "opening database"); - sFileGatherer = new database::FileGathererImpl(); - auto database_res = - database::Database::Open(*sFileGatherer, sServices->tag_parser(), - sServices->collator(), sServices->bg_worker()); + auto database_res = database::Database::Open( + sServices->tag_parser(), sServices->collator(), sServices->bg_worker()); if (database_res.has_error()) { unmountStorage(); return;