Make FileGatherer shaped more like a normal iterator

src/tangara/audio/audio_fsm.cpp

@@ -9,6 +9,7 @@
 #include <cstdint>
 #include <future>
 #include <memory>
+#include <sstream>
 #include <variant>
 
 #include "audio/audio_source.hpp"

src/tangara/database/database.cpp

@@ -24,6 +24,7 @@
 #include "cppbor.h"
 #include "cppbor_parse.h"
 #include "database/index.hpp"
+#include "database/track_finder.hpp"
 #include "debug.hpp"
 #include "esp_log.h"
 #include "esp_timer.h"
@@ -40,7 +41,6 @@
 
 #include "database/db_events.hpp"
 #include "database/env_esp.hpp"
-#include "database/file_gatherer.hpp"
 #include "database/records.hpp"
 #include "database/tag_parser.hpp"
 #include "database/track.hpp"
@@ -122,8 +122,7 @@ static auto CheckDatabase(leveldb::DB& db, locale::ICollator& col) -> bool {
   return true;
 }
 
-auto Database::Open(IFileGatherer& gatherer,
+auto Database::Open(ITagParser& parser,
-                    ITagParser& parser,
                     locale::ICollator& collator,
                     tasks::WorkerPool& bg_worker)
     -> cpp::result<Database*, DatabaseError> {
@@ -168,8 +167,7 @@ auto Database::Open(IFileGatherer& gatherer,
             }
 
             ESP_LOGI(kTag, "Database opened successfully");
-            return new Database(db, cache.release(), gatherer, parser,
+            return new Database(db, cache.release(), parser, collator);
-                                collator);
           })
       .get();
 }
@@ -182,12 +180,10 @@ auto Database::Destroy() -> void {
 
 Database::Database(leveldb::DB* db,
                    leveldb::Cache* cache,
-                   IFileGatherer& file_gatherer,
                    ITagParser& tag_parser,
                    locale::ICollator& collator)
     : db_(db),
       cache_(cache),
-      file_gatherer_(file_gatherer),
       tag_parser_(tag_parser),
       collator_(collator),
       is_updating_(false) {
@@ -401,7 +397,11 @@ auto Database::updateIndexes() -> void {
   // Stage 2: search for newly added files.
   ESP_LOGI(kTag, "scanning for new tracks");
   uint64_t num_files = 0;
-  file_gatherer_.FindFiles("", [&](std::string_view path, const FILINFO& info) {
+
+  auto track_finder = std::make_shared<TrackFinder>("");
+
+  FILINFO info;
+  while (auto path = track_finder->next(info)) {
     num_files++;
     events::Ui().Dispatch(event::UpdateProgress{
         .stage = event::UpdateProgress::Stage::kScanningForNewTracks,
@@ -409,15 +409,15 @@ auto Database::updateIndexes() -> void {
     });
 
     std::string unused;
-    if (db_->Get(read_options, EncodePathKey(path), &unused).ok()) {
+    if (db_->Get(read_options, EncodePathKey(*path), &unused).ok()) {
       // This file is already in the database; skip it.
-      return;
+      continue;
     }
 
-    std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags(path);
+    std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags(*path);
     if (!tags || tags->encoding() == Container::kUnsupported) {
       // No parseable tags; skip this fiile.
-      return;
+      continue;
     }
 
     // Check for any existing track with the same hash.
@@ -438,14 +438,14 @@ auto Database::updateIndexes() -> void {
       if (!data) {
         data = std::make_shared<TrackData>();
         data->id = *existing_id;
-      } else if (data->filepath != path) {
+      } else if (std::string_view{data->filepath} != *path) {
         ESP_LOGW(kTag, "hash collision: %s, %s, %s",
                  tags->title().value_or("no title").c_str(),
                  tags->artist().value_or("no artist").c_str(),
                  tags->album().value_or("no album").c_str());
         // Don't commit anything if there's a hash collision, since we're
         // likely to make a big mess.
-        return;
+        continue;
       }
     } else {
       num_new_tracks++;
@@ -454,7 +454,7 @@ auto Database::updateIndexes() -> void {
     }
 
     // Make sure the file-based metadata on the TrackData is up to date.
-    data->filepath = path;
+    data->filepath = *path;
     data->tags_hash = hash;
     data->modified_at = {info.fdate, info.ftime};
 
@@ -467,10 +467,10 @@ auto Database::updateIndexes() -> void {
     dbCreateIndexesForTrack(*data, *tags, batch);
     batch.Put(EncodeDataKey(data->id), EncodeDataValue(*data));
     batch.Put(EncodeHashKey(data->tags_hash), EncodeHashValue(data->id));
-    batch.Put(EncodePathKey(path), TrackIdToBytes(data->id));
+    batch.Put(EncodePathKey(*path), TrackIdToBytes(data->id));
 
     db_->Write(leveldb::WriteOptions(), &batch);
-  });
+  };
 
   uint64_t end_time = esp_timer_get_time();
 

src/tangara/database/database.hpp

@@ -19,7 +19,6 @@
 
 #include "collation.hpp"
 #include "cppbor.h"
-#include "database/file_gatherer.hpp"
 #include "database/index.hpp"
 #include "database/records.hpp"
 #include "database/tag_parser.hpp"
@@ -56,8 +55,7 @@ class Database {
     ALREADY_OPEN,
     FAILED_TO_OPEN,
   };
-  static auto Open(IFileGatherer& file_gatherer,
+  static auto Open(ITagParser& tag_parser,
-                   ITagParser& tag_parser,
                    locale::ICollator& collator,
                    tasks::WorkerPool& bg_worker)
       -> cpp::result<Database*, DatabaseError>;
@@ -96,7 +94,6 @@ class Database {
   leveldb::Cache* cache_;
 
   // Not owned.
-  IFileGatherer& file_gatherer_;
   ITagParser& tag_parser_;
   locale::ICollator& collator_;
 
@@ -105,7 +102,6 @@ class Database {
 
   Database(leveldb::DB* db,
            leveldb::Cache* cache,
-           IFileGatherer& file_gatherer,
            ITagParser& tag_parser,
            locale::ICollator& collator);
 

src/tangara/database/file_gatherer.cpp

@@ -1,72 +0,0 @@
-/*
- * Copyright 2023 jacqueline <me@jacqueline.id.au>
- *
- * SPDX-License-Identifier: GPL-3.0-only
- */
-
-#include "database/file_gatherer.hpp"
-
-#include <deque>
-#include <functional>
-#include <sstream>
-#include <string>
-
-#include "ff.h"
-
-#include "drivers/spi.hpp"
-#include "memory_resource.hpp"
-
-namespace database {
-
-static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR");
-
-auto FileGathererImpl::FindFiles(
-    const std::string& root,
-    std::function<void(std::string_view, const FILINFO&)> cb) -> void {
-  std::pmr::deque<std::pmr::string> to_explore{&memory::kSpiRamResource};
-  to_explore.push_back({root.data(), root.size()});
-
-  while (!to_explore.empty()) {
-    auto next_path_str = to_explore.front();
-    to_explore.pop_front();
-
-    const TCHAR* next_path = static_cast<const TCHAR*>(next_path_str.c_str());
-
-    FF_DIR dir;
-    FRESULT res = f_opendir(&dir, next_path);
-    if (res != FR_OK) {
-      // TODO: log.
-      continue;
-    }
-
-    for (;;) {
-      FILINFO info;
-      res = f_readdir(&dir, &info);
-      if (res != FR_OK || info.fname[0] == 0) {
-        // No more files in the directory.
-        break;
-      } else if (info.fattrib & (AM_HID | AM_SYS) || info.fname[0] == '.') {
-        // System or hidden file. Ignore it and move on.
-        continue;
-      } else {
-        std::pmr::string full_path{&memory::kSpiRamResource};
-        full_path += next_path_str;
-        full_path += "/";
-        full_path += info.fname;
-
-        if (info.fattrib & AM_DIR) {
-          // This is a directory. Add it to the explore queue.
-          to_explore.push_back(full_path);
-        } else {
-          // This is a file! Let the callback know about it.
-          // std::invoke(cb, full_path.str(), info);
-          std::invoke(cb, full_path, info);
-        }
-      }
-    }
-
-    f_closedir(&dir);
-  }
-}
-
-}  // namespace database

src/tangara/database/file_gatherer.hpp

@@ -1,34 +0,0 @@
-/*
- * Copyright 2023 jacqueline <me@jacqueline.id.au>
- *
- * SPDX-License-Identifier: GPL-3.0-only
- */
-
-#pragma once
-
-#include <deque>
-#include <functional>
-#include <sstream>
-#include <string>
-
-#include "ff.h"
-
-namespace database {
-
-class IFileGatherer {
- public:
-  virtual ~IFileGatherer() {};
-
-  virtual auto FindFiles(
-      const std::string& root,
-      std::function<void(std::string_view, const FILINFO&)> cb) -> void = 0;
-};
-
-class FileGathererImpl : public IFileGatherer {
- public:
-  virtual auto FindFiles(const std::string& root,
-                         std::function<void(std::string_view, const FILINFO&)>
-                             cb) -> void override;
-};
-
-}  // namespace database

src/tangara/database/track_finder.cpp

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2023 jacqueline <me@jacqueline.id.au>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#include "database/track_finder.hpp"
+
+#include <deque>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <string_view>
+
+#include "database/track_finder.hpp"
+#include "ff.h"
+
+#include "drivers/spi.hpp"
+#include "memory_resource.hpp"
+
+namespace database {
+
+static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR");
+
+TrackFinder::TrackFinder(std::string_view root)
+    : to_explore_(&memory::kSpiRamResource) {
+  to_explore_.push_back({root.data(), root.size()});
+}
+
+auto TrackFinder::next(FILINFO& out_info) -> std::optional<std::string> {
+  std::scoped_lock<std::mutex> lock{mut_};
+  while (!to_explore_.empty() || current_) {
+    if (!current_) {
+      current_.emplace();
+
+      // Get the next directory to iterate through.
+      current_->first = to_explore_.front();
+      to_explore_.pop_front();
+      const TCHAR* next_path =
+          static_cast<const TCHAR*>(current_->first.data());
+
+      // Open it for iterating.
+      FRESULT res = f_opendir(&current_->second, next_path);
+      if (res != FR_OK) {
+        current_.reset();
+        continue;
+      }
+    }
+
+    FILINFO info;
+    FRESULT res = f_readdir(&current_->second, &info);
+    if (res != FR_OK || info.fname[0] == 0) {
+      // No more files in the directory.
+      f_closedir(&current_->second);
+      current_.reset();
+      continue;
+    } else if (info.fattrib & (AM_HID | AM_SYS) || info.fname[0] == '.') {
+      // System or hidden file. Ignore it and move on.
+      continue;
+    } else {
+      // A valid file or folder.
+      std::pmr::string full_path{&memory::kSpiRamResource};
+      full_path += current_->first;
+      full_path += "/";
+      full_path += info.fname;
+
+      if (info.fattrib & AM_DIR) {
+        // This is a directory. Add it to the explore queue.
+        to_explore_.push_back(full_path);
+      } else {
+        // This is a file! We can return now.
+        out_info = info;
+        return {{full_path.data(), full_path.size()}};
+      }
+    }
+  }
+
+  // Out of things to explore.
+  return {};
+}
+
+}  // namespace database

src/tangara/database/track_finder.hpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2023 jacqueline <me@jacqueline.id.au>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#pragma once
+
+#include <deque>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <sstream>
+#include <string>
+
+#include "ff.h"
+
+namespace database {
+
+class TrackFinder {
+ public:
+  TrackFinder(std::string_view root);
+
+  auto next(FILINFO&) -> std::optional<std::string>;
+
+ private:
+  std::mutex mut_;
+  std::pmr::deque<std::pmr::string> to_explore_;
+  std::optional<std::pair<std::pmr::string, FF_DIR>> current_;
+};
+
+}  // namespace database

src/tangara/input/lvgl_input_driver.cpp

@@ -8,6 +8,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <sstream>
 #include <variant>
 
 #include "core/lv_group.h"

src/tangara/system_fsm/idle.cpp

@@ -4,8 +4,9 @@
  * SPDX-License-Identifier: GPL-3.0-only
  */
 
+#include "ui/ui_fsm.hpp"
+
 #include "app_console/app_console.hpp"
-#include "database/file_gatherer.hpp"
 #include "drivers/gpios.hpp"
 #include "freertos/portmacro.h"
 #include "freertos/projdefs.h"
@@ -17,7 +18,6 @@
 #include "events/event_queue.hpp"
 #include "system_fsm/system_events.hpp"
 #include "system_fsm/system_fsm.hpp"
-#include "ui/ui_fsm.hpp"
 
 namespace system_fsm {
 namespace states {

src/tangara/system_fsm/running.cpp

@@ -8,7 +8,6 @@
 #include "audio/audio_events.hpp"
 #include "database/database.hpp"
 #include "database/db_events.hpp"
-#include "database/file_gatherer.hpp"
 #include "drivers/gpios.hpp"
 #include "drivers/spi.hpp"
 #include "ff.h"
@@ -36,8 +35,6 @@ static void timer_callback(TimerHandle_t timer) {
   events::System().Dispatch(internal::UnmountTimeout{});
 }
 
-static database::IFileGatherer* sFileGatherer;
-
 void Running::entry() {
   if (!sUnmountTimer) {
     sUnmountTimer = xTimerCreate("unmount_timeout", kTicksBeforeUnmount, false,
@@ -174,10 +171,8 @@ auto Running::mountStorage() -> void {
   sStorage.reset(storage_res.value());
 
   ESP_LOGI(kTag, "opening database");
-  sFileGatherer = new database::FileGathererImpl();
+  auto database_res = database::Database::Open(
-  auto database_res =
+      sServices->tag_parser(), sServices->collator(), sServices->bg_worker());
-      database::Database::Open(*sFileGatherer, sServices->tag_parser(),
-                               sServices->collator(), sServices->bg_worker());
   if (database_res.has_error()) {
     unmountStorage();
     return;