Remove pre-iterator concepts

- No more IndexRecord/Result/dbGetPage nonsense
 - Queue is just track ids
 - i am so tired and have so much to do

lib/leveldb/include/leveldb/slice.h

@@ -35,6 +35,9 @@ class LEVELDB_EXPORT Slice {
   // Create a slice that refers to the contents of "s"
   Slice(const std::string& s) : data_(s.data()), size_(s.size()) {}
 
+  // Create a slice that refers to the contents of "s"
+  Slice(std::string_view s) : data_(s.data()), size_(s.size()) {}
+
   // Create a slice that refers to s[0,strlen(s)-1]
   Slice(const char* s) : data_(s), size_(strlen(s)) {}
 

lua/browser.lua

@@ -115,13 +115,15 @@ function browser.create(opts)
     btn:onevent(lvgl.EVENT.FOCUSED, function()
       screen.focused_item = this_item
       if screen.last_item - 5 < this_item then
-        opts.iterator:next(screen.add_item)
+        screen.add_item(opts.iterator())
       end
     end)
   end
 
   for _ = 1, 8 do
-    opts.iterator:next(screen.add_item)
+    local val = opts.iterator()
+    if not val then break end
+    screen.add_item(val)
   end
 
   return screen

lua/main_menu.lua

@@ -34,7 +34,7 @@ return function()
   end)
 
   local indexes = database.indexes()
-  for id, idx in ipairs(indexes) do
+  for _, idx in ipairs(indexes) do
     local btn = menu.list:add_btn(nil, tostring(idx))
     btn:onClicked(function()
       backstack.push(function()

src/app_console/app_console.cpp

@@ -123,8 +123,7 @@ int CmdPlayFile(int argc, char** argv) {
 
   if (is_id) {
     database::TrackId id = std::atoi(argv[1]);
-    auto editor = AppConsole::sServices->track_queue().Edit();
+    AppConsole::sServices->track_queue().append(id);
-    AppConsole::sServices->track_queue().Append(editor, id);
   } else {
     std::pmr::string path{&memory::kSpiRamResource};
     path += '/';
@@ -134,7 +133,8 @@ int CmdPlayFile(int argc, char** argv) {
       path += argv[i];
     }
 
-    events::Audio().Dispatch(audio::PlayFile{.filename = path});
+    events::Audio().Dispatch(
+        audio::PlayFile{.filename = {path.data(), path.size()}});
   }
 
   return 0;
@@ -161,7 +161,7 @@ int CmdDbInit(int argc, char** argv) {
     std::cout << "no database open" << std::endl;
     return 1;
   }
-  db->Update();
+  db->updateIndexes();
 
   return 0;
 }
@@ -176,87 +176,6 @@ void RegisterDbInit() {
   esp_console_cmd_register(&cmd);
 }
 
-int CmdDbTracks(int argc, char** argv) {
-  static const std::pmr::string usage = "usage: db_tracks";
-  if (argc != 1) {
-    std::cout << usage << std::endl;
-    return 1;
-  }
-
-  auto db = AppConsole::sServices->database().lock();
-  if (!db) {
-    std::cout << "no database open" << std::endl;
-    return 1;
-  }
-  std::unique_ptr<database::Result<database::Track>> res(
-      db->GetTracks(20).get());
-  while (true) {
-    for (const auto& s : res->values()) {
-      std::cout << s->tags()[database::Tag::kTitle].value_or("[BLANK]")
-                << std::endl;
-    }
-    if (res->next_page()) {
-      auto continuation = res->next_page().value();
-      res.reset(db->GetPage<database::Track>(&continuation).get());
-    } else {
-      break;
-    }
-  }
-
-  return 0;
-}
-
-void RegisterDbTracks() {
-  esp_console_cmd_t cmd{.command = "db_tracks",
-                        .help = "lists titles of ALL tracks in the database",
-                        .hint = NULL,
-                        .func = &CmdDbTracks,
-                        .argtable = NULL};
-  esp_console_cmd_register(&cmd);
-}
-
-int CmdDbDump(int argc, char** argv) {
-  static const std::pmr::string usage = "usage: db_dump";
-  if (argc != 1) {
-    std::cout << usage << std::endl;
-    return 1;
-  }
-
-  auto db = AppConsole::sServices->database().lock();
-  if (!db) {
-    std::cout << "no database open" << std::endl;
-    return 1;
-  }
-
-  std::cout << "=== BEGIN DUMP ===" << std::endl;
-
-  std::unique_ptr<database::Result<std::pmr::string>> res(db->GetDump(5).get());
-  while (true) {
-    for (const auto& s : res->values()) {
-      std::cout << *s << std::endl;
-    }
-    if (res->next_page()) {
-      auto continuation = res->next_page().value();
-      res.reset(db->GetPage<std::pmr::string>(&continuation).get());
-    } else {
-      break;
-    }
-  }
-
-  std::cout << "=== END DUMP ===" << std::endl;
-
-  return 0;
-}
-
-void RegisterDbDump() {
-  esp_console_cmd_t cmd{.command = "db_dump",
-                        .help = "prints every key/value pair in the db",
-                        .hint = NULL,
-                        .func = &CmdDbDump,
-                        .argtable = NULL};
-  esp_console_cmd_register(&cmd);
-}
-
 int CmdTasks(int argc, char** argv) {
 #if (configUSE_TRACE_FACILITY == 0)
   std::cout << "configUSE_TRACE_FACILITY must be enabled" << std::endl;
@@ -270,8 +189,8 @@ int CmdTasks(int argc, char** argv) {
     return 1;
   }
 
-  // Pad the number of tasks so that uxTaskGetSystemState still returns info if
+  // Pad the number of tasks so that uxTaskGetSystemState still returns info
-  // new tasks are started during measurement.
+  // if new tasks are started during measurement.
   size_t num_tasks = uxTaskGetNumberOfTasks() + 4;
   TaskStatus_t* start_status = new TaskStatus_t[num_tasks];
   TaskStatus_t* end_status = new TaskStatus_t[num_tasks];
@@ -576,9 +495,11 @@ int CmdHaptics(int argc, char** argv) {
       "  haptic_effect from-effect to-effect\n"
       "  haptic_effect from-effect to-effect library\n"
       "eg.\n"
-      "  haptic_effect        (plays from 1 to 123 with the default library)\n"
+      "  haptic_effect        (plays from 1 to 123 with the default "
+      "library)\n"
       "  haptic_effect 3      (plays from 1 to 123 with library 3\n"
-      "  haptic_effect 1 100  (plays from 1 to 100 with the default library)\n"
+      "  haptic_effect 1 100  (plays from 1 to 100 with the default "
+      "library)\n"
       "  haptic_effect 1 10 4 (plays from 1 to 10 with library 4)";
 
   auto& haptics = AppConsole::sServices->haptics();
@@ -643,8 +564,6 @@ auto AppConsole::RegisterExtraComponents() -> void {
   RegisterAudioStatus();
   */
   RegisterDbInit();
-  RegisterDbTracks();
-  RegisterDbDump();
   RegisterTasks();
 
   RegisterHeaps();

src/audio/CMakeLists.txt

@@ -9,6 +9,6 @@ idf_component_register(
   "audio_source.cpp"
   INCLUDE_DIRS "include"
   REQUIRES "codecs" "drivers" "cbor" "result" "tasks" "span" "memory" "tinyfsm"
-  "database" "system_fsm" "playlist" "speexdsp")
+  "database" "system_fsm" "speexdsp")
 
 target_compile_options(${COMPONENT_LIB} PRIVATE ${EXTRA_WARNINGS})

src/audio/audio_fsm.cpp

@@ -93,6 +93,17 @@ void AudioState::react(const OutputModeChanged& ev) {
   sOutput->SetMode(IAudioOutput::Modes::kOnPaused);
 }
 
+auto AudioState::playTrack(database::TrackId id) -> void {
+  sCurrentTrack = id;
+  sServices->bg_worker().Dispatch<void>([=]() {
+    auto db = sServices->database().lock();
+    if (!db) {
+      return;
+    }
+    sFileSource->SetPath(db->getTrackPath(id));
+  });
+}
+
 namespace states {
 
 void Uninitialised::react(const system_fsm::BootComplete& ev) {
@@ -143,19 +154,11 @@ void Standby::react(const internal::InputFileOpened& ev) {
 }
 
 void Standby::react(const QueueUpdate& ev) {
-  auto current_track = sServices->track_queue().Current();
+  auto current_track = sServices->track_queue().current();
   if (!current_track || (sCurrentTrack && *sCurrentTrack == *current_track)) {
     return;
   }
-
+  playTrack(*current_track);
-  sCurrentTrack = current_track;
-
-  auto db = sServices->database().lock();
-  if (!db) {
-    ESP_LOGW(kTag, "database not open; ignoring play request");
-    return;
-  }
-  sFileSource->SetPath(db->GetTrackPath(*current_track));
 }
 
 void Standby::react(const TogglePlayPause& ev) {
@@ -187,22 +190,14 @@ void Playback::react(const QueueUpdate& ev) {
   if (!ev.current_changed) {
     return;
   }
-  auto current_track = sServices->track_queue().Current();
+  auto current_track = sServices->track_queue().current();
   if (!current_track) {
     sFileSource->SetPath();
     sCurrentTrack.reset();
     transit<Standby>();
     return;
   }
-
+  playTrack(*current_track);
-  sCurrentTrack = current_track;
-
-  auto db = sServices->database().lock();
-  if (!db) {
-    return;
-  }
-
-  sFileSource->SetPath(db->GetTrackPath(*current_track));
 }
 
 void Playback::react(const TogglePlayPause& ev) {
@@ -220,9 +215,8 @@ void Playback::react(const internal::InputFileClosed& ev) {}
 
 void Playback::react(const internal::InputFileFinished& ev) {
   ESP_LOGI(kTag, "finished playing file");
-  auto editor = sServices->track_queue().Edit();
+  sServices->track_queue().next();
-  sServices->track_queue().Next(editor);
+  if (!sServices->track_queue().current()) {
-  if (!sServices->track_queue().Current()) {
     transit<Standby>();
   }
 }

src/audio/fatfs_audio_input.cpp

@@ -50,23 +50,19 @@ FatfsAudioInput::FatfsAudioInput(database::ITagParser& tag_parser,
       bg_worker_(bg_worker),
       new_stream_mutex_(),
       new_stream_(),
-      has_new_stream_(false),
+      has_new_stream_(false) {}
-      pending_path_() {}
 
 FatfsAudioInput::~FatfsAudioInput() {}
 
-auto FatfsAudioInput::SetPath(std::future<std::optional<std::pmr::string>> fut)
+auto FatfsAudioInput::SetPath(std::optional<std::string> path) -> void {
-    -> void {
+  if (path) {
-  std::lock_guard<std::mutex> guard{new_stream_mutex_};
+    SetPath(*path);
-  pending_path_.reset(
+  } else {
-      new database::FutureFetcher<std::optional<std::pmr::string>>(
+    SetPath();
-          std::move(fut)));
+  }
-
-  has_new_stream_ = true;
-  has_new_stream_.notify_one();
 }
 
-auto FatfsAudioInput::SetPath(const std::pmr::string& path) -> void {
+auto FatfsAudioInput::SetPath(const std::string& path) -> void {
   std::lock_guard<std::mutex> guard{new_stream_mutex_};
   if (OpenFile(path)) {
     has_new_stream_ = true;
@@ -96,16 +92,6 @@ auto FatfsAudioInput::NextStream() -> std::shared_ptr<TaggedStream> {
         continue;
       }
 
-      // If the path is a future, then wait for it to complete.
-      if (pending_path_) {
-        auto res = pending_path_->Result();
-        pending_path_.reset();
-
-        if (res && *res) {
-          OpenFile(**res);
-        }
-      }
-
       if (new_stream_ == nullptr) {
         continue;
       }
@@ -117,7 +103,7 @@ auto FatfsAudioInput::NextStream() -> std::shared_ptr<TaggedStream> {
   }
 }
 
-auto FatfsAudioInput::OpenFile(const std::pmr::string& path) -> bool {
+auto FatfsAudioInput::OpenFile(const std::string& path) -> bool {
   ESP_LOGI(kTag, "opening file %s", path.c_str());
 
   auto tags = tag_parser_.ReadAndParseTags(path);

src/audio/include/audio_events.hpp

@@ -42,7 +42,7 @@ struct QueueUpdate : tinyfsm::Event {
 };
 
 struct PlayFile : tinyfsm::Event {
-  std::pmr::string filename;
+  std::string filename;
 };
 
 struct StepUpVolume : tinyfsm::Event {};

src/audio/include/audio_fsm.hpp

@@ -60,6 +60,8 @@ class AudioState : public tinyfsm::Fsm<AudioState> {
   virtual void react(const internal::AudioPipelineIdle&) {}
 
  protected:
+  auto playTrack(database::TrackId id) -> void;
+
   static std::shared_ptr<system_fsm::ServiceLocator> sServices;
 
   static std::shared_ptr<FatfsAudioInput> sFileSource;

src/audio/include/fatfs_audio_input.hpp

@@ -38,8 +38,8 @@ class FatfsAudioInput : public IAudioSource {
    * Immediately cease reading any current source, and begin reading from the
    * given file path.
    */
-  auto SetPath(std::future<std::optional<std::pmr::string>>) -> void;
+  auto SetPath(std::optional<std::string>) -> void;
-  auto SetPath(const std::pmr::string&) -> void;
+  auto SetPath(const std::string&) -> void;
   auto SetPath() -> void;
 
   auto HasNewStream() -> bool override;
@@ -49,7 +49,7 @@ class FatfsAudioInput : public IAudioSource {
   FatfsAudioInput& operator=(const FatfsAudioInput&) = delete;
 
  private:
-  auto OpenFile(const std::pmr::string& path) -> bool;
+  auto OpenFile(const std::string& path) -> bool;
 
   auto ContainerToStreamType(database::Container)
       -> std::optional<codecs::StreamType>;
@@ -61,9 +61,6 @@ class FatfsAudioInput : public IAudioSource {
   std::shared_ptr<TaggedStream> new_stream_;
 
   std::atomic<bool> has_new_stream_;
-
-  std::unique_ptr<database::FutureFetcher<std::optional<std::pmr::string>>>
-      pending_path_;
 };
 
 }  // namespace audio

src/audio/include/track_queue.hpp

@@ -9,10 +9,11 @@
 #include <list>
 #include <memory>
 #include <mutex>
+#include <shared_mutex>
 #include <vector>
 
 #include "database.hpp"
-#include "source.hpp"
+#include "tasks.hpp"
 #include "track.hpp"
 
 namespace audio {
@@ -32,77 +33,52 @@ namespace audio {
  */
 class TrackQueue {
  public:
-  TrackQueue();
+  TrackQueue(tasks::Worker& bg_worker);
-
-  class Editor {
-   public:
-    ~Editor();
-
-    // Cannot be copied or moved.
-    Editor(const Editor&) = delete;
-    Editor& operator=(const Editor&) = delete;
-
-   private:
-    friend TrackQueue;
-
-    Editor(TrackQueue&);
-
-    std::lock_guard<std::recursive_mutex> lock_;
-    bool has_current_changed_;
-  };
-
-  auto Edit() -> Editor;
 
   /* Returns the currently playing track. */
-  auto Current() const -> std::optional<database::TrackId>;
+  auto current() const -> std::optional<database::TrackId>;
 
   /* Returns, in order, tracks that have been queued to be played next. */
-  auto PeekNext(std::size_t limit) const -> std::vector<database::TrackId>;
+  auto peekNext(std::size_t limit) const -> std::vector<database::TrackId>;
 
   /*
    * Returns the tracks in the queue that have already been played, ordered
    * most recently played first.
    */
-  auto PeekPrevious(std::size_t limit) const -> std::vector<database::TrackId>;
+  auto peekPrevious(std::size_t limit) const -> std::vector<database::TrackId>;
 
-  auto GetCurrentPosition() const -> size_t;
+  auto currentPosition() const -> size_t;
-  auto GetTotalSize() const -> size_t;
+  auto totalSize() const -> size_t;
 
   using Item = std::variant<database::TrackId, database::TrackIterator>;
-  auto Insert(Editor&, Item, size_t) -> void;
+  auto insert(Item) -> void;
-  auto Append(Editor&, Item i) -> void;
+  auto append(Item i) -> void;
 
   /*
    * Advances to the next track in the queue, placing the current track at the
    * front of the 'played' queue.
    */
-  auto Next(Editor&) -> std::optional<database::TrackId>;
+  auto next() -> void;
-  auto Previous(Editor&) -> std::optional<database::TrackId>;
+  auto previous() -> void;
 
-  auto SkipTo(Editor&, database::TrackId) -> void;
+  auto skipTo(database::TrackId) -> void;
 
   /*
    * Removes all tracks from all queues, and stops any currently playing track.
    */
-  auto Clear(Editor&) -> void;
+  auto clear() -> void;
-
-  auto Save(std::weak_ptr<database::Database>) -> void;
-  auto Load(std::weak_ptr<database::Database>) -> void;
 
   // Cannot be copied or moved.
   TrackQueue(const TrackQueue&) = delete;
   TrackQueue& operator=(const TrackQueue&) = delete;
 
  private:
-  // FIXME: Make this a shared_mutex so that multithread reads don't block.
+  mutable std::shared_mutex mutex_;
-  mutable std::recursive_mutex mutex_;
 
-  std::optional<database::TrackId> current_;
+  tasks::Worker& bg_worker_;
 
-  // Note: stored in reverse order, i.e. most recent played it at the *back* of
+  size_t pos_;
-  // this vector.
+  std::pmr::vector<database::TrackId> tracks_;
-  std::pmr::vector<database::TrackId> played_;
-  std::pmr::vector<Item> enqueued_;
 };
 
 }  // namespace audio

src/audio/track_queue.cpp

@@ -8,6 +8,7 @@
 #include <stdint.h>
 
 #include <algorithm>
+#include <memory>
 #include <mutex>
 #include <optional>
 #include <variant>
@@ -19,7 +20,7 @@
 #include "database.hpp"
 #include "event_queue.hpp"
 #include "memory_resource.hpp"
-#include "source.hpp"
+#include "tasks.hpp"
 #include "track.hpp"
 #include "ui_fsm.hpp"
 
@@ -27,385 +28,137 @@ namespace audio {
 
 [[maybe_unused]] static constexpr char kTag[] = "tracks";
 
-static const std::string kSerialiseKey = "queue";
+auto notifyChanged(bool current_changed) -> void {
-static const std::string kCurrentKey = "cur";
+  QueueUpdate ev{.current_changed = current_changed};
-static const std::string kPlayedKey = "prev";
-static const std::string kEnqueuedKey = "next";
-
-TrackQueue::Editor::Editor(TrackQueue& queue)
-    : lock_(queue.mutex_), has_current_changed_(false) {}
-
-TrackQueue::Editor::~Editor() {
-  QueueUpdate ev{.current_changed = has_current_changed_};
-  events::Audio().Dispatch(ev);
   events::Ui().Dispatch(ev);
+  events::Audio().Dispatch(ev);
 }
 
-TrackQueue::TrackQueue()
+TrackQueue::TrackQueue(tasks::Worker& bg_worker)
     : mutex_(),
-      current_(),
+      bg_worker_(bg_worker),
-      played_(&memory::kSpiRamResource),
+      pos_(0),
-      enqueued_(&memory::kSpiRamResource) {}
+      tracks_(&memory::kSpiRamResource) {}
 
-auto TrackQueue::Edit() -> Editor {
+auto TrackQueue::current() const -> std::optional<database::TrackId> {
-  return Editor(*this);
+  const std::shared_lock<std::shared_mutex> lock(mutex_);
+  if (pos_ >= tracks_.size()) {
+    return {};
   }
-
+  return tracks_[pos_];
-auto TrackQueue::Current() const -> std::optional<database::TrackId> {
-  const std::lock_guard<std::recursive_mutex> lock(mutex_);
-  return current_;
 }
 
-auto TrackQueue::PeekNext(std::size_t limit) const
+auto TrackQueue::peekNext(std::size_t limit) const
     -> std::vector<database::TrackId> {
-  const std::lock_guard<std::recursive_mutex> lock(mutex_);
+  const std::shared_lock<std::shared_mutex> lock(mutex_);
-  std::vector<database::TrackId> ret;
+  std::vector<database::TrackId> out;
-
+  for (size_t i = pos_ + 1; i < pos_ + limit + 1 && i < tracks_.size(); i++) {
-  for (auto it = enqueued_.begin(); it != enqueued_.end() && limit > 0; it++) {
+    out.push_back(i);
-    std::visit(
-        [&](auto&& arg) {
-          using T = std::decay_t<decltype(arg)>;
-          if constexpr (std::is_same_v<T, database::TrackId>) {
-            ret.push_back(arg);
-            limit--;
-          } else if constexpr (std::is_same_v<T, database::TrackIterator>) {
-            auto copy = arg;
-            while (limit > 0) {
-              auto next = copy.Next();
-              if (!next) {
-                break;
-              }
-              ret.push_back(*next);
-              limit--;
-            }
   }
-        },
+  return out;
-        *it);
 }
 
-  return ret;
+auto TrackQueue::peekPrevious(std::size_t limit) const
-}
-
-auto TrackQueue::PeekPrevious(std::size_t limit) const
     -> std::vector<database::TrackId> {
-  const std::lock_guard<std::recursive_mutex> lock(mutex_);
+  const std::shared_lock<std::shared_mutex> lock(mutex_);
-  std::vector<database::TrackId> ret;
+  std::vector<database::TrackId> out;
-  ret.reserve(limit);
+  for (size_t i = pos_ - 1; i < pos_ - limit - 1 && i >= tracks_.size(); i--) {
-
+    out.push_back(i);
-  for (auto it = played_.rbegin(); it != played_.rend(); it++, limit--) {
-    ret.push_back(*it);
-  }
-
-  return ret;
   }
-
+  return out;
-auto TrackQueue::GetCurrentPosition() const -> size_t {
-  const std::lock_guard<std::recursive_mutex> lock(mutex_);
-  size_t played = played_.size();
-  if (current_) {
-    played += 1;
-  }
-  return played;
 }
 
-auto TrackQueue::GetTotalSize() const -> size_t {
+auto TrackQueue::currentPosition() const -> size_t {
-  const std::lock_guard<std::recursive_mutex> lock(mutex_);
+  const std::shared_lock<std::shared_mutex> lock(mutex_);
-  size_t total = GetCurrentPosition();
+  return pos_;
-
-  for (const auto& item : enqueued_) {
-    std::visit(
-        [&](auto&& arg) {
-          using T = std::decay_t<decltype(arg)>;
-          if constexpr (std::is_same_v<T, database::TrackId>) {
-            total++;
-          } else if constexpr (std::is_same_v<T, database::TrackIterator>) {
-            total += arg.Size();
-          }
-        },
-        item);
 }
 
-  return total;
+auto TrackQueue::totalSize() const -> size_t {
+  const std::shared_lock<std::shared_mutex> lock(mutex_);
+  return tracks_.size();
 }
 
-auto TrackQueue::Insert(Editor& ed, Item i, size_t index) -> void {
+auto TrackQueue::insert(Item i) -> void {
-  if (index == 0) {
+  bool current_changed = pos_ == tracks_.size();
-    enqueued_.insert(enqueued_.begin(), i);
+  if (std::holds_alternative<database::TrackId>(i)) {
-  }
+    const std::unique_lock<std::shared_mutex> lock(mutex_);
-
+    tracks_.push_back(std::get<database::TrackId>(i));
-  // We can't insert halfway through an iterator, so we need to ensure that the
+    notifyChanged(current_changed);
-  // first `index` items in the queue are reified into track ids.
+  } else if (std::holds_alternative<database::TrackIterator>(i)) {
-  size_t current_index = 0;
+    bg_worker_.Dispatch<void>([=, this]() {
-  while (current_index < index && current_index < enqueued_.size()) {
+      database::TrackIterator it = std::get<database::TrackIterator>(i);
-    std::visit(
+      size_t working_pos = pos_;
-        [&](auto&& arg) {
+      while (true) {
-          using T = std::decay_t<decltype(arg)>;
+        auto next = *it;
-          if constexpr (std::is_same_v<T, database::TrackId>) {
-            // This item is already a track id; nothing to do.
-            current_index++;
-          } else if constexpr (std::is_same_v<T, database::TrackIterator>) {
-            // This item is an iterator. Push it back one, replacing its old
-            // index with the next value from it.
-            auto next = arg.Next();
-            auto iterator_index = enqueued_.begin() + current_index;
         if (!next) {
-              // Out of values. Remove the iterator completely.
+          break;
-              enqueued_.erase(iterator_index);
-              // Don't increment current_index, since the next item in the
-              // queue will have been moved down.
-            } else {
-              enqueued_.insert(iterator_index, *next);
-              current_index++;
-            }
-          }
-        },
-        enqueued_[current_index]);
-  }
-
-  // Double check the previous loop didn't run out of items.
-  if (index > enqueued_.size()) {
-    ESP_LOGE(kTag, "insert index was out of bounds");
-    return;
         }
-
+        const std::unique_lock<std::shared_mutex> lock(mutex_);
-  // Finally, we can now do the actual insertion.
+        tracks_.insert(tracks_.begin() + working_pos, *next);
-  enqueued_.insert(enqueued_.begin() + index, i);
+        working_pos++;
+        it++;
       }
-
+      notifyChanged(current_changed);
-auto TrackQueue::Append(Editor& ed, Item i) -> void {
+    });
-  enqueued_.push_back(i);
-  if (!current_) {
-    Next(ed);
   }
 }
 
-auto TrackQueue::Next(Editor& ed) -> std::optional<database::TrackId> {
+auto TrackQueue::append(Item i) -> void {
-  if (current_) {
+  bool current_changed = pos_ == tracks_.size();
-    ed.has_current_changed_ = true;
+  if (std::holds_alternative<database::TrackId>(i)) {
-    played_.push_back(*current_);
+    const std::unique_lock<std::shared_mutex> lock(mutex_);
-  }
+    tracks_.push_back(std::get<database::TrackId>(i));
-  current_.reset();
+    notifyChanged(current_changed);
-
+  } else if (std::holds_alternative<database::TrackIterator>(i)) {
-  while (!current_ && !enqueued_.empty()) {
+    bg_worker_.Dispatch<void>([=, this]() {
-    ed.has_current_changed_ = true;
+      database::TrackIterator it = std::get<database::TrackIterator>(i);
-    std::visit(
+      while (true) {
-        [&](auto&& arg) {
+        auto next = *it;
-          using T = std::decay_t<decltype(arg)>;
-          if constexpr (std::is_same_v<T, database::TrackId>) {
-            current_ = arg;
-            enqueued_.erase(enqueued_.begin());
-          } else if constexpr (std::is_same_v<T, database::TrackIterator>) {
-            auto next = arg.Next();
         if (!next) {
-              enqueued_.erase(enqueued_.begin());
+          break;
-            } else {
-              current_ = *next;
-            }
-          }
-        },
-        enqueued_.front());
-  }
-
-  return current_;
-}
-
-auto TrackQueue::Previous(Editor& ed) -> std::optional<database::TrackId> {
-  if (played_.empty()) {
-    return current_;
-  }
-  ed.has_current_changed_ = true;
-  if (current_) {
-    enqueued_.insert(enqueued_.begin(), *current_);
-  }
-  current_ = played_.back();
-  played_.pop_back();
-  return current_;
-}
-
-auto TrackQueue::SkipTo(Editor& ed, database::TrackId id) -> void {
-  while ((!current_ || *current_ != id) && !enqueued_.empty()) {
-    Next(ed);
         }
+        const std::unique_lock<std::shared_mutex> lock(mutex_);
+        tracks_.push_back(*next);
+        it++;
       }
-
+      notifyChanged(current_changed);
-auto TrackQueue::Clear(Editor& ed) -> void {
+    });
-  ed.has_current_changed_ = current_.has_value();
-  current_.reset();
-  played_.clear();
-  enqueued_.clear();
   }
-
-auto TrackQueue::Save(std::weak_ptr<database::Database> db) -> void {
-  cppbor::Map root{};
-
-  if (current_) {
-    root.add(cppbor::Bstr{kCurrentKey}, cppbor::Uint{*current_});
 }
 
-  cppbor::Array played{};
+auto TrackQueue::next() -> void {
-  for (const auto& id : played_) {
+  const std::unique_lock<std::shared_mutex> lock(mutex_);
-    played.add(cppbor::Uint{id});
+  pos_ = std::min<size_t>(pos_ + 1, tracks_.size());
-  }
-  root.add(cppbor::Bstr{kPlayedKey}, std::move(played));
 
-  cppbor::Array enqueued{};
+  notifyChanged(true);
-  for (const auto& item : enqueued_) {
-    std::visit(
-        [&](auto&& arg) {
-          using T = std::decay_t<decltype(arg)>;
-          if constexpr (std::is_same_v<T, database::TrackId>) {
-            enqueued.add(cppbor::Uint{arg});
-          } else if constexpr (std::is_same_v<T, database::TrackIterator>) {
-            enqueued.add(arg.cbor());
-          }
-        },
-        item);
 }
-  root.add(cppbor::Bstr{kEnqueuedKey}, std::move(enqueued));
 
-  auto db_lock = db.lock();
+auto TrackQueue::previous() -> void {
-  if (!db_lock) {
+  const std::unique_lock<std::shared_mutex> lock(mutex_);
-    return;
+  if (pos_ > 0) {
-  }
+    pos_--;
-  db_lock->Put(kSerialiseKey, root.toString());
   }
 
-class Parser : public cppbor::ParseClient {
+  notifyChanged(true);
- public:
-  Parser(std::weak_ptr<database::Database> db,
-         std::optional<database::TrackId>& current,
-         std::pmr::vector<database::TrackId>& played,
-         std::pmr::vector<TrackQueue::Item>& enqueued)
-      : state_(State::kInit),
-        db_(db),
-        current_(current),
-        played_(played),
-        enqueued_(enqueued) {}
-
-  virtual ParseClient* item(std::unique_ptr<cppbor::Item>& item,
-                            const uint8_t* hdrBegin,
-                            const uint8_t* valueBegin,
-                            const uint8_t* end) override {
-    switch (state_) {
-      case State::kInit:
-        if (item->type() == cppbor::MAP) {
-          state_ = State::kRoot;
-        }
-        break;
-      case State::kRoot:
-        if (item->type() != cppbor::TSTR) {
-          break;
-        }
-        if (item->asTstr()->value() == kCurrentKey) {
-          state_ = State::kCurrent;
-        } else if (item->asTstr()->value() == kPlayedKey) {
-          state_ = State::kPlayed;
-        } else if (item->asTstr()->value() == kEnqueuedKey) {
-          state_ = State::kEnqueued;
-        }
-        break;
-      case State::kCurrent:
-        if (item->type() == cppbor::UINT) {
-          current_ = item->asUint()->value();
-        }
-        state_ = State::kRoot;
-        break;
-      case State::kPlayed:
-        if (item->type() == cppbor::UINT) {
-          played_.push_back(item->asUint()->value());
-        }
-        break;
-      case State::kEnqueued:
-        if (item->type() == cppbor::UINT) {
-          played_.push_back(item->asUint()->value());
-        } else if (item->type() == cppbor::ARRAY) {
-          queue_depth_ = 1;
-          state_ = State::kEnqueuedIterator;
-        }
-        break;
-      case State::kEnqueuedIterator:
-        if (item->type() == cppbor::MAP || item->type() == cppbor::ARRAY) {
-          queue_depth_++;
-        }
-        break;
-      case State::kFinished:
-        break;
-    }
-
-    return this;
 }
 
-  ParseClient* itemEnd(std::unique_ptr<cppbor::Item>& item,
+auto TrackQueue::skipTo(database::TrackId id) -> void {
-                       const uint8_t* hdrBegin,
+  const std::unique_lock<std::shared_mutex> lock(mutex_);
-                       const uint8_t* valueBegin,
+  for (size_t i = pos_; i < tracks_.size(); i++) {
-                       const uint8_t* end) override {
+    if (tracks_[i] == id) {
-    std::optional<database::TrackIterator> parsed_it;
+      pos_ = i;
-    switch (state_) {
-      case State::kInit:
-      case State::kRoot:
-      case State::kCurrent:
-        state_ = State::kFinished;
-        break;
-      case State::kEnqueued:
-      case State::kPlayed:
-        state_ = State::kRoot;
-        break;
-      case State::kEnqueuedIterator:
-        if (item->type() == cppbor::MAP || item->type() == cppbor::ARRAY) {
-          queue_depth_++;
     }
-        if (queue_depth_ == 0) {
-          parsed_it = database::TrackIterator::Parse(db_, *item->asArray());
-          if (parsed_it) {
-            enqueued_.push_back(std::move(*parsed_it));
-          }
-        }
-        state_ = State::kEnqueued;
-        break;
-      case State::kFinished:
-        break;
-    }
-    return this;
   }
 
-  void error(const uint8_t* position,
+  notifyChanged(true);
-             const std::string& errorMessage) override {
-    ESP_LOGE(kTag, "restoring saved queue failed: %s", errorMessage.c_str());
 }
 
- private:
+auto TrackQueue::clear() -> void {
-  enum class State {
+  const std::unique_lock<std::shared_mutex> lock(mutex_);
-    kInit,
+  pos_ = 0;
-    kRoot,
+  tracks_.clear();
-    kCurrent,
-    kPlayed,
-    kEnqueued,
-    kEnqueuedIterator,
-    kFinished,
-  } state_;
-
-  std::weak_ptr<database::Database> db_;
-
-  int queue_depth_;
-
-  std::optional<database::TrackId>& current_;
-  std::pmr::vector<database::TrackId>& played_;
-  std::pmr::vector<TrackQueue::Item>& enqueued_;
-};
-
-auto TrackQueue::Load(std::weak_ptr<database::Database> db) -> void {
-  auto db_lock = db.lock();
-  if (!db_lock) {
-    return;
-  }
-  auto raw = db_lock->Get(kSerialiseKey);
-  if (!raw) {
-    return;
-  }
 
-  Parser p{db, current_, played_, enqueued_};
+  notifyChanged(true);
-  const uint8_t* data = reinterpret_cast<const uint8_t*>(raw->data());
-  cppbor::parse(data, data + raw->size(), &p);
 }
 
 }  // namespace audio

src/database/database.cpp

@@ -16,6 +16,7 @@
 #include <memory>
 #include <optional>
 #include <sstream>
+#include <variant>
 
 #include "collation.hpp"
 #include "cppbor.h"
@@ -200,11 +201,11 @@ Database::~Database() {
   sIsDbOpen.store(false);
 }
 
-auto Database::Put(const std::string& key, const std::string& val) -> void {
+auto Database::put(const std::string& key, const std::string& val) -> void {
   db_->Put(leveldb::WriteOptions{}, kKeyCustom + key, val);
 }
 
-auto Database::Get(const std::string& key) -> std::optional<std::string> {
+auto Database::get(const std::string& key) -> std::optional<std::string> {
   std::string val;
   auto res = db_->Get(leveldb::ReadOptions{}, kKeyCustom + key, &val);
   if (!res.ok()) {
@@ -213,9 +214,40 @@ auto Database::Get(const std::string& key) -> std::optional<std::string> {
   return val;
 }
 
-auto Database::Update() -> std::future<void> {
+auto Database::getTrackPath(TrackId id) -> std::optional<std::string> {
+  auto track_data = dbGetTrackData(id);
+  if (!track_data) {
+    return {};
+  }
+  return std::string{track_data->filepath.data(), track_data->filepath.size()};
+}
+
+auto Database::getTrack(TrackId id) -> std::shared_ptr<Track> {
+  std::shared_ptr<TrackData> data = dbGetTrackData(id);
+  if (!data || data->is_tombstoned) {
+    return {};
+  }
+  std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags(
+      {data->filepath.data(), data->filepath.size()});
+  if (!tags) {
+    return {};
+  }
+  return std::make_shared<Track>(data, tags);
+}
+
+auto Database::getIndexes() -> std::vector<IndexInfo> {
+  // TODO(jacqueline): This probably needs to be async? When we have runtime
+  // configurable indexes, they will need to come from somewhere.
+  return {
+      kAllTracks,
+      kAllAlbums,
+      kAlbumsByArtist,
+      kTracksByGenre,
+  };
+}
+
+auto Database::updateIndexes() -> void {
   events::Ui().Dispatch(event::UpdateStarted{});
-  return worker_task_->Dispatch<void>([&]() -> void {
   leveldb::ReadOptions read_options;
   read_options.fill_cache = false;
 
@@ -266,8 +298,8 @@ auto Database::Update() -> std::future<void> {
         track->modified_at = modified_at;
       }
 
-        std::shared_ptr<TrackTags> tags =
+      std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags(
-            tag_parser_.ReadAndParseTags(track->filepath);
+          {track->filepath.data(), track->filepath.size()});
       if (!tags || tags->encoding() == Container::kUnsupported) {
         // We couldn't read the tags for this track. Either they were
         // malformed, or perhaps the file is missing. Either way, tombstone
@@ -305,7 +337,7 @@ auto Database::Update() -> std::future<void> {
   // Stage 2: search for newly added files.
   ESP_LOGI(kTag, "scanning for new tracks");
   uint64_t num_processed = 0;
-    file_gatherer_.FindFiles("", [&](const std::pmr::string& path,
+  file_gatherer_.FindFiles("", [&](const std::string& path,
                                    const FILINFO& info) {
     num_processed++;
     events::Ui().Dispatch(event::UpdateProgress{
@@ -375,7 +407,8 @@ auto Database::Update() -> std::future<void> {
       dbPutTrackData(*existing_data);
       auto t = std::make_shared<Track>(existing_data, tags);
       dbCreateIndexesForTrack(*t);
-      } else if (existing_data->filepath != path) {
+    } else if (existing_data->filepath !=
+               std::pmr::string{path.data(), path.size()}) {
       ESP_LOGW(kTag, "tag hash collision for %s and %s",
                existing_data->filepath.c_str(), path.c_str());
       ESP_LOGI(kTag, "hash components: %s, %s, %s",
@@ -385,148 +418,8 @@ auto Database::Update() -> std::future<void> {
     }
   });
   events::Ui().Dispatch(event::UpdateFinished{});
-  });
-}
-
-auto Database::GetTrackPath(TrackId id)
-    -> std::future<std::optional<std::pmr::string>> {
-  return worker_task_->Dispatch<std::optional<std::pmr::string>>(
-      [=, this]() -> std::optional<std::pmr::string> {
-        auto track_data = dbGetTrackData(id);
-        if (track_data) {
-          return track_data->filepath;
-        }
-        return {};
-      });
-}
-
-auto Database::GetTrack(TrackId id) -> std::future<std::shared_ptr<Track>> {
-  return worker_task_->Dispatch<std::shared_ptr<Track>>(
-      [=, this]() -> std::shared_ptr<Track> {
-        std::shared_ptr<TrackData> data = dbGetTrackData(id);
-        if (!data || data->is_tombstoned) {
-          return {};
-        }
-        std::shared_ptr<TrackTags> tags =
-            tag_parser_.ReadAndParseTags(data->filepath);
-        if (!tags) {
-          return {};
-        }
-        return std::make_shared<Track>(data, tags);
-      });
-}
-
-auto Database::GetBulkTracks(std::vector<TrackId> ids)
-    -> std::future<std::vector<std::shared_ptr<Track>>> {
-  return worker_task_->Dispatch<std::vector<std::shared_ptr<Track>>>(
-      [=, this]() -> std::vector<std::shared_ptr<Track>> {
-        std::map<TrackId, std::shared_ptr<Track>> id_to_track{};
-
-        // Sort the list of ids so that we can retrieve them all in a single
-        // iteration through the database, without re-seeking.
-        std::vector<TrackId> sorted_ids = ids;
-        std::sort(sorted_ids.begin(), sorted_ids.end());
-
-        std::unique_ptr<leveldb::Iterator> it{
-            db_->NewIterator(leveldb::ReadOptions{})};
-        for (const TrackId& id : sorted_ids) {
-          std::string key = EncodeDataKey(id);
-          it->Seek(key);
-          if (!it->Valid() || it->key() != key) {
-            // This id wasn't found at all. Skip it.
-            continue;
-          }
-          std::shared_ptr<Track> track =
-              ParseRecord<Track>(it->key(), it->value());
-          if (track) {
-            id_to_track.insert({id, track});
-          }
-        }
-
-        // We've fetched all of the ids in the request, so now just put them
-        // back into the order they were asked for in.
-        std::vector<std::shared_ptr<Track>> results;
-        for (const TrackId& id : ids) {
-          if (id_to_track.contains(id)) {
-            results.push_back(id_to_track.at(id));
-          } else {
-            // This lookup failed.
-            results.push_back({});
-          }
-        }
-        return results;
-      });
-}
-
-auto Database::GetIndexes() -> std::vector<IndexInfo> {
-  // TODO(jacqueline): This probably needs to be async? When we have runtime
-  // configurable indexes, they will need to come from somewhere.
-  return {
-      kAllTracks,
-      kAllAlbums,
-      kAlbumsByArtist,
-      kTracksByGenre,
-  };
-}
-
-auto Database::GetTracksByIndex(IndexId index, std::size_t page_size)
-    -> std::future<Result<IndexRecord>*> {
-  return worker_task_->Dispatch<Result<IndexRecord>*>(
-      [=, this]() -> Result<IndexRecord>* {
-        IndexKey::Header header{
-            .id = index,
-            .depth = 0,
-            .components_hash = 0,
-        };
-        std::string prefix = EncodeIndexPrefix(header);
-        Continuation c{.prefix = {prefix.data(), prefix.size()},
-                       .start_key = {prefix.data(), prefix.size()},
-                       .forward = true,
-                       .was_prev_forward = true,
-                       .page_size = page_size};
-        return dbGetPage<IndexRecord>(c);
-      });
-}
-
-auto Database::GetTracks(std::size_t page_size) -> std::future<Result<Track>*> {
-  return worker_task_->Dispatch<Result<Track>*>([=, this]() -> Result<Track>* {
-    std::string prefix = EncodeDataPrefix();
-    Continuation c{.prefix = {prefix.data(), prefix.size()},
-                   .start_key = {prefix.data(), prefix.size()},
-                   .forward = true,
-                   .was_prev_forward = true,
-                   .page_size = page_size};
-    return dbGetPage<Track>(c);
-  });
-}
-
-auto Database::GetDump(std::size_t page_size)
-    -> std::future<Result<std::pmr::string>*> {
-  return worker_task_->Dispatch<Result<std::pmr::string>*>(
-      [=, this]() -> Result<std::pmr::string>* {
-        Continuation c{.prefix = "",
-                       .start_key = "",
-                       .forward = true,
-                       .was_prev_forward = true,
-                       .page_size = page_size};
-        return dbGetPage<std::pmr::string>(c);
-      });
 }
 
-template <typename T>
-auto Database::GetPage(Continuation* c) -> std::future<Result<T>*> {
-  Continuation copy = *c;
-  return worker_task_->Dispatch<Result<T>*>(
-      [=, this]() -> Result<T>* { return dbGetPage<T>(copy); });
-}
-
-template auto Database::GetPage<Track>(Continuation* c)
-    -> std::future<Result<Track>*>;
-template auto Database::GetPage<IndexRecord>(Continuation* c)
-    -> std::future<Result<IndexRecord>*>;
-template auto Database::GetPage<std::pmr::string>(Continuation* c)
-    -> std::future<Result<std::pmr::string>*>;
-
 auto Database::dbMintNewTrackId() -> TrackId {
   TrackId next_id = 1;
   std::string val;
@@ -592,7 +485,7 @@ auto Database::dbGetHash(const uint64_t& hash) -> std::optional<TrackId> {
 }
 
 auto Database::dbCreateIndexesForTrack(const Track& track) -> void {
-  for (const IndexInfo& index : GetIndexes()) {
+  for (const IndexInfo& index : getIndexes()) {
     leveldb::WriteBatch writes;
     auto entries = Index(collator_, index, track);
     for (const auto& it : entries) {
@@ -609,7 +502,7 @@ auto Database::dbRemoveIndexes(std::shared_ptr<TrackData> data) -> void {
     return;
   }
   Track track{data, tags};
-  for (const IndexInfo& index : GetIndexes()) {
+  for (const IndexInfo& index : getIndexes()) {
     auto entries = Index(collator_, index, track);
     for (auto it = entries.rbegin(); it != entries.rend(); it++) {
       auto key = EncodeIndexKey(it->first);
@@ -666,512 +559,209 @@ auto Database::dbRecoverTagsFromHashes(
   return out;
 }
 
-template <typename T>
+auto seekToOffset(leveldb::Iterator* it, int offset) {
-auto Database::dbGetPage(const Continuation& c) -> Result<T>* {
+  while (it->Valid() && offset != 0) {
-  // Work out our starting point. Sometimes this will already done.
+    if (offset < 0) {
-  std::unique_ptr<leveldb::Iterator> it{
-      db_->NewIterator(leveldb::ReadOptions{})};
-  it->Seek({c.start_key.data(), c.start_key.size()});
-
-  // Fix off-by-one if we just changed direction.
-  if (c.forward != c.was_prev_forward) {
-    if (c.forward) {
-      it->Next();
-    } else {
       it->Prev();
-    }
+      offset++;
-  }
-
-  // Grab results.
-  std::optional<std::pmr::string> first_key;
-  std::vector<std::shared_ptr<T>> records;
-  while (records.size() < c.page_size && it->Valid()) {
-    if (!it->key().starts_with({c.prefix.data(), c.prefix.size()})) {
-      break;
-    }
-    if (!first_key) {
-      first_key = it->key().ToString();
-    }
-    std::shared_ptr<T> parsed = ParseRecord<T>(it->key(), it->value());
-    if (parsed) {
-      records.push_back(parsed);
-    }
-    if (c.forward) {
-      it->Next();
     } else {
-      it->Prev();
+      it->Next();
+      offset--;
     }
   }
-
-  if (!it->Valid() ||
-      !it->key().starts_with({c.prefix.data(), c.prefix.size()})) {
-    it.reset();
 }
 
-  // Put results into canonical order if we were iterating backwards.
+auto Database::getRecord(const SearchKey& c)
-  if (!c.forward) {
+    -> std::optional<std::pair<std::pmr::string, Record>> {
-    std::reverse(records.begin(), records.end());
+  std::unique_ptr<leveldb::Iterator> it{
-  }
+      db_->NewIterator(leveldb::ReadOptions{})};
 
-  // Work out the new continuations.
+  it->Seek(c.startKey());
-  std::optional<Continuation> next_page;
+  seekToOffset(it.get(), c.offset);
-  if (c.forward) {
+  if (!it->Valid() || !it->key().starts_with(std::string_view{c.prefix})) {
-    if (it != nullptr) {
+    return {};
-      // We were going forward, and now we want the next page.
-      std::pmr::string key{it->key().data(), it->key().size(),
-                           &memory::kSpiRamResource};
-      next_page = Continuation{
-          .prefix = c.prefix,
-          .start_key = key,
-          .forward = true,
-          .was_prev_forward = true,
-          .page_size = c.page_size,
-      };
-    }
-    // No iterator means we ran out of results in this direction.
-  } else {
-    // We were going backwards, and now we want the next page. This is a
-    // reversal, to set the start key to the first record we saw and mark that
-    // it's off by one.
-    next_page = Continuation{
-        .prefix = c.prefix,
-        .start_key = *first_key,
-        .forward = true,
-        .was_prev_forward = false,
-        .page_size = c.page_size,
-    };
   }
 
-  std::optional<Continuation> prev_page;
+  std::optional<IndexKey> key = ParseIndexKey(it->key());
-  if (c.forward) {
+  if (!key) {
-    // We were going forwards, and now we want the previous page. Set the
+    ESP_LOGW(kTag, "parsing index key failed");
-    // search key to the first result we saw, and mark that it's off by one.
+    return {};
-    prev_page = Continuation{
-        .prefix = c.prefix,
-        .start_key = *first_key,
-        .forward = false,
-        .was_prev_forward = true,
-        .page_size = c.page_size,
-    };
-  } else {
-    if (it != nullptr) {
-      // We were going backwards, and we still want to go backwards.
-      std::pmr::string key{it->key().data(), it->key().size(),
-                           &memory::kSpiRamResource};
-      prev_page = Continuation{
-          .prefix = c.prefix,
-          .start_key = key,
-          .forward = false,
-          .was_prev_forward = false,
-          .page_size = c.page_size,
-      };
-    }
-    // No iterator means we ran out of results in this direction.
   }
 
-  return new Result<T>(std::move(records), next_page, prev_page);
+  return std::make_pair(std::pmr::string{it->key().data(), it->key().size(),
+                                         &memory::kSpiRamResource},
+                        Record{*key, it->value()});
 }
 
-auto Database::dbCount(const Continuation& c) -> size_t {
+auto Database::countRecords(const SearchKey& c) -> size_t {
   std::unique_ptr<leveldb::Iterator> it{
       db_->NewIterator(leveldb::ReadOptions{})};
-  size_t count = 0;
-  for (it->Seek({c.start_key.data(), c.start_key.size()});
-       it->Valid() && it->key().starts_with({c.prefix.data(), c.prefix.size()});
-       it->Next()) {
-    count++;
-  }
-  return count;
-}
-
-template auto Database::dbGetPage<Track>(const Continuation& c)
-    -> Result<Track>*;
-template auto Database::dbGetPage<std::pmr::string>(const Continuation& c)
-    -> Result<std::pmr::string>*;
 
-template <>
+  it->Seek(c.startKey());
-auto Database::ParseRecord<IndexRecord>(const leveldb::Slice& key,
+  seekToOffset(it.get(), c.offset);
-                                        const leveldb::Slice& val)
+  if (!it->Valid() || !it->key().starts_with(std::string_view{c.prefix})) {
-    -> std::shared_ptr<IndexRecord> {
-  std::optional<IndexKey> data = ParseIndexKey(key);
-  if (!data) {
     return {};
   }
 
-  std::optional<std::pmr::string> title;
+  size_t count = 0;
-  if (!val.empty()) {
+  while (it->Valid() && it->key().starts_with(std::string_view{c.prefix})) {
-    title = val.ToString();
+    it->Next();
+    count++;
   }
 
-  return std::make_shared<IndexRecord>(*data, title, data->track);
+  return count;
 }
 
-template <>
+auto SearchKey::startKey() const -> std::string_view {
-auto Database::ParseRecord<Track>(const leveldb::Slice& key,
+  if (key) {
-                                  const leveldb::Slice& val)
+    return *key;
-    -> std::shared_ptr<Track> {
-  std::shared_ptr<TrackData> data = ParseDataValue(val);
-  if (!data || data->is_tombstoned) {
-    return {};
   }
-  std::shared_ptr<TrackTags> tags =
+  return prefix;
-      tag_parser_.ReadAndParseTags(data->filepath);
-  if (!tags) {
-    return {};
-  }
-  return std::make_shared<Track>(data, tags);
 }
 
-template <>
+Record::Record(const IndexKey& key, const leveldb::Slice& t)
-auto Database::ParseRecord<std::pmr::string>(const leveldb::Slice& key,
+    : text_(t.data(), t.size(), &memory::kSpiRamResource) {
-                                             const leveldb::Slice& val)
+  if (key.track) {
-    -> std::shared_ptr<std::pmr::string> {
+    contents_ = *key.track;
-  std::ostringstream stream;
-  stream << "key: ";
-  if (key.size() < 3 || key.data()[1] != '\0') {
-    stream << key.ToString().c_str();
   } else {
-    for (size_t i = 0; i < key.size(); i++) {
+    contents_ = ExpandHeader(key.header, key.item);
-      if (i == 0) {
-        stream << key.data()[i];
-      } else if (i == 1) {
-        stream << " / 0x";
-      } else {
-        stream << std::hex << std::setfill('0') << std::setw(2)
-               << static_cast<int>(key.data()[i]);
-      }
-    }
   }
-  if (!val.empty()) {
-    stream << "\tval: 0x";
-    for (int i = 0; i < val.size(); i++) {
-      stream << std::hex << std::setfill('0') << std::setw(2)
-             << static_cast<int>(val.data()[i]);
-    }
-  }
-  std::pmr::string res{stream.str(), &memory::kSpiRamResource};
-  return std::make_shared<std::pmr::string>(res);
 }
 
-IndexRecord::IndexRecord(const IndexKey& key,
+auto Record::text() const -> std::string_view {
-                         std::optional<std::pmr::string> title,
+  return text_;
-                         std::optional<TrackId> track)
-    : key_(key), override_text_(title), track_(track) {}
-
-auto IndexRecord::text() const -> std::optional<std::pmr::string> {
-  if (override_text_) {
-    return override_text_;
-  }
-  return key_.item;
 }
 
-auto IndexRecord::track() const -> std::optional<TrackId> {
+auto Record::contents() const
-  return track_;
+    -> const std::variant<TrackId, IndexKey::Header>& {
+  return contents_;
 }
 
-auto IndexRecord::Expand(std::size_t page_size) const
+Iterator::Iterator(std::shared_ptr<Database> db, IndexId idx)
-    -> std::optional<Continuation> {
+    : Iterator(db,
-  if (track_) {
+               IndexKey::Header{
-    return {};
+                   .id = idx,
-  }
+                   .depth = 0,
-  std::string new_prefix = EncodeIndexPrefix(ExpandHeader());
+                   .components_hash = 0,
-  return Continuation{
+               }) {}
-      .prefix = {new_prefix.data(), new_prefix.size()},
-      .start_key = {new_prefix.data(), new_prefix.size()},
-      .forward = true,
-      .was_prev_forward = true,
-      .page_size = page_size,
-  };
-}
-
-auto IndexRecord::ExpandHeader() const -> IndexKey::Header {
-  return ::database::ExpandHeader(key_.header, key_.item);
-}
-
-Iterator::Iterator(std::weak_ptr<Database> db, const IndexInfo& idx)
-    : db_(db), pos_mutex_(), current_pos_(), prev_pos_() {
-  std::string prefix = EncodeIndexPrefix(
-      IndexKey::Header{.id = idx.id, .depth = 0, .components_hash = 0});
-  current_pos_ = Continuation{.prefix = {prefix.data(), prefix.size()},
-                              .start_key = {prefix.data(), prefix.size()},
-                              .forward = true,
-                              .was_prev_forward = true,
-                              .page_size = 1};
-}
-
-auto Iterator::Parse(std::weak_ptr<Database> db, const cppbor::Array& encoded)
-    -> std::optional<Iterator> {
-  // Ensure the input looks reasonable.
-  if (encoded.size() != 3) {
-    return {};
-  }
 
-  if (encoded[0]->type() != cppbor::TSTR) {
+Iterator::Iterator(std::shared_ptr<Database> db, const IndexKey::Header& header)
-    return {};
+    : db_(db), key_{}, current_() {
-  }
+  std::string prefix = EncodeIndexPrefix(header);
-  const std::string& prefix = encoded[0]->asTstr()->value();
+  key_ = {
-
+      .prefix = {prefix.data(), prefix.size(), &memory::kSpiRamResource},
-  std::optional<Continuation> current_pos{};
+      .key = {},
-  if (encoded[1]->type() == cppbor::TSTR) {
+      .offset = 0,
-    const std::string& key = encoded[1]->asTstr()->value();
-    current_pos = Continuation{
-        .prefix = {prefix.data(), prefix.size()},
-        .start_key = {key.data(), key.size()},
-        .forward = true,
-        .was_prev_forward = true,
-        .page_size = 1,
   };
+  iterate(key_);
 }
 
-  std::optional<Continuation> prev_pos{};
+auto Iterator::value() const -> const std::optional<Record>& {
-  if (encoded[2]->type() == cppbor::TSTR) {
+  return current_;
-    const std::string& key = encoded[2]->asTstr()->value();
-    current_pos = Continuation{
-        .prefix = {prefix.data(), prefix.size()},
-        .start_key = {key.data(), key.size()},
-        .forward = false,
-        .was_prev_forward = false,
-        .page_size = 1,
-    };
 }
 
-  return Iterator{db, std::move(current_pos), std::move(prev_pos)};
+auto Iterator::next() -> void {
+  SearchKey new_key = key_;
+  new_key.offset = 1;
+  iterate(new_key);
 }
 
-Iterator::Iterator(std::weak_ptr<Database> db, const Continuation& c)
+auto Iterator::prev() -> void {
-    : db_(db), pos_mutex_(), current_pos_(c), prev_pos_() {}
+  SearchKey new_key = key_;
-
+  new_key.offset = -1;
-Iterator::Iterator(const Iterator& other)
+  iterate(new_key);
-    : db_(other.db_),
-      pos_mutex_(),
-      current_pos_(other.current_pos_),
-      prev_pos_(other.prev_pos_) {}
-
-Iterator::Iterator(std::weak_ptr<Database> db,
-                   std::optional<Continuation>&& cur,
-                   std::optional<Continuation>&& prev)
-    : db_(db), current_pos_(cur), prev_pos_(prev) {}
-
-Iterator& Iterator::operator=(const Iterator& other) {
-  current_pos_ = other.current_pos_;
-  prev_pos_ = other.prev_pos_;
-  return *this;
 }
 
-auto Iterator::Next(Callback cb) -> void {
+auto Iterator::iterate(const SearchKey& key) -> void {
   auto db = db_.lock();
   if (!db) {
-    InvokeNull(cb);
+    ESP_LOGW(kTag, "iterate with dead db");
-    return;
-  }
-  db->worker_task_->Dispatch<void>([=]() {
-    std::lock_guard lock{pos_mutex_};
-    if (!current_pos_) {
-      InvokeNull(cb);
     return;
   }
-    std::unique_ptr<Result<IndexRecord>> res{
+  auto res = db->getRecord(key);
-        db->dbGetPage<IndexRecord>(*current_pos_)};
+  if (res) {
-    prev_pos_ = current_pos_;
+    key_ = {
-    current_pos_ = res->next_page();
+        .prefix = key_.prefix,
-    if (!res || res->values().empty() || !res->values()[0]) {
+        .key = res->first,
-      ESP_LOGI(kTag, "dropping empty result");
+        .offset = 0,
-      InvokeNull(cb);
+    };
-      return;
+    current_ = res->second;
+  } else {
+    key_ = key;
+    current_.reset();
   }
-    std::invoke(cb, *res->values()[0]);
-  });
 }
 
-auto Iterator::NextSync() -> std::optional<IndexRecord> {
+auto Iterator::count() const -> size_t {
   auto db = db_.lock();
   if (!db) {
-    return {};
+    ESP_LOGW(kTag, "count with dead db");
-  }
+    return 0;
-  std::lock_guard lock{pos_mutex_};
-  if (!current_pos_) {
-    return {};
-  }
-  std::unique_ptr<Result<IndexRecord>> res{
-      db->dbGetPage<IndexRecord>(*current_pos_)};
-  prev_pos_ = current_pos_;
-  current_pos_ = res->next_page();
-  if (!res || res->values().empty() || !res->values()[0]) {
-    ESP_LOGI(kTag, "dropping empty result");
-    return {};
   }
-  return *res->values()[0];
+  return db->countRecords(key_);
 }
 
-auto Iterator::PeekSync() -> std::optional<IndexRecord> {
+TrackIterator::TrackIterator(const Iterator& it) : db_(it.db_), levels_() {
-  auto db = db_.lock();
+  levels_.push_back(it);
-  if (!db) {
+  next(false);
-    return {};
-  }
-  auto pos = current_pos_;
-  if (!pos) {
-    return {};
-  }
-  std::unique_ptr<Result<IndexRecord>> res{db->dbGetPage<IndexRecord>(*pos)};
-  if (!res || res->values().empty() || !res->values()[0]) {
-    return {};
-  }
-  return *res->values()[0];
 }
 
-auto Iterator::Prev(Callback cb) -> void {
+auto TrackIterator::next() -> void {
-  auto db = db_.lock();
+  next(true);
-  if (!db) {
-    InvokeNull(cb);
-    return;
 }
-  db->worker_task_->Dispatch<void>([=]() {
+
-    std::lock_guard lock{pos_mutex_};
+auto TrackIterator::next(bool advance) -> void {
-    if (!prev_pos_) {
+  while (!levels_.empty()) {
-      InvokeNull(cb);
+    if (advance) {
-      return;
+      levels_.back().next();
-    }
-    std::unique_ptr<Result<IndexRecord>> res{
-        db->dbGetPage<IndexRecord>(*current_pos_)};
-    current_pos_ = prev_pos_;
-    prev_pos_ = res->prev_page();
-    std::invoke(cb, *res->values()[0]);
-  });
     }
 
-auto Iterator::Size() const -> size_t {
+    auto& cur = levels_.back().value();
+    if (!cur) {
+      // The current top iterator is out of tracks. Pop it, and move the parent
+      // to the next item.
+      levels_.pop_back();
+      advance = true;
+    } else if (std::holds_alternative<IndexKey::Header>(cur->contents())) {
+      // This record is a branch. Push a new iterator.
+      auto key = std::get<IndexKey::Header>(cur->contents());
       auto db = db_.lock();
       if (!db) {
-    return {};
+        return;
-  }
-  std::optional<Continuation> pos = current_pos_;
-  if (!pos) {
-    return 0;
-  }
-  return db->dbCount(*pos);
-}
-
-auto Iterator::InvokeNull(Callback cb) -> void {
-  std::invoke(cb, std::optional<IndexRecord>{});
-}
-
-auto Iterator::cbor() const -> cppbor::Array&& {
-  cppbor::Array res;
-
-  std::pmr::string prefix;
-  if (current_pos_) {
-    prefix = current_pos_->prefix;
-  } else if (prev_pos_) {
-    prefix = prev_pos_->prefix;
-  } else {
-    ESP_LOGW(kTag, "iterator has no prefix");
-    return std::move(res);
       }
-
+      levels_.emplace_back(db, key);
-  if (current_pos_) {
+      // Don't skip the first value of the new level.
-    res.add(cppbor::Tstr(current_pos_->start_key));
+      advance = false;
-  } else {
+    } else if (std::holds_alternative<TrackId>(cur->contents())) {
-    res.add(cppbor::Null());
+      // New record is a leaf.
+      break;
     }
-
-  if (prev_pos_) {
-    res.add(cppbor::Tstr(prev_pos_->start_key));
-  } else {
-    res.add(cppbor::Null());
   }
-
-  return std::move(res);
 }
 
-auto TrackIterator::Parse(std::weak_ptr<Database> db,
+auto TrackIterator::value() const -> std::optional<TrackId> {
-                          const cppbor::Array& encoded)
+  if (levels_.empty()) {
-    -> std::optional<TrackIterator> {
-  TrackIterator ret{db};
-
-  for (const auto& item : encoded) {
-    if (item->type() == cppbor::ARRAY) {
-      auto it = Iterator::Parse(db, *item->asArray());
-      if (it) {
-        ret.levels_.push_back(std::move(*it));
-      } else {
     return {};
   }
+  auto cur = levels_.back().value();
+  if (!cur) {
+    return {};
   }
+  if (std::holds_alternative<TrackId>(cur->contents())) {
+    return std::get<TrackId>(cur->contents());
   }
-
+  return {};
-  return ret;
-}
-
-TrackIterator::TrackIterator(const Iterator& it) : db_(it.db_), levels_() {
-  if (it.current_pos_) {
-    levels_.push_back(it);
-  }
-  NextLeaf();
-}
-
-TrackIterator::TrackIterator(const TrackIterator& other)
-    : db_(other.db_), levels_(other.levels_) {}
-
-TrackIterator::TrackIterator(std::weak_ptr<Database> db) : db_(db), levels_() {}
-
-TrackIterator& TrackIterator::operator=(TrackIterator&& other) {
-  levels_ = std::move(other.levels_);
-  return *this;
-}
-
-auto TrackIterator::Next() -> std::optional<TrackId> {
-  std::optional<TrackId> next{};
-  while (!next && !levels_.empty()) {
-    auto next_record = levels_.back().NextSync();
-    if (!next_record) {
-      levels_.pop_back();
-      NextLeaf();
-      continue;
-    }
-    // May still be nullopt_t; hence the loop.
-    next = next_record->track();
-  }
-  return next;
 }
 
-auto TrackIterator::Size() const -> size_t {
+auto TrackIterator::count() const -> size_t {
   size_t size = 0;
   TrackIterator copy{*this};
   while (!copy.levels_.empty()) {
-    size += copy.levels_.back().Size();
+    size += copy.levels_.back().count();
     copy.levels_.pop_back();
-    copy.NextLeaf();
+    copy.next();
   }
   return size;
 }
 
-auto TrackIterator::NextLeaf() -> void {
-  while (!levels_.empty()) {
-    ESP_LOGI(kTag, "check next candidate");
-    Iterator& candidate = levels_.back();
-    auto next = candidate.PeekSync();
-    if (!next) {
-      ESP_LOGI(kTag, "candidate is empty");
-      levels_.pop_back();
-      continue;
-    }
-    if (!next->track()) {
-      ESP_LOGI(kTag, "candidate is a branch");
-      candidate.NextSync();
-      levels_.push_back(Iterator{db_, next->Expand(1).value()});
-      continue;
-    }
-    ESP_LOGI(kTag, "candidate is a leaf");
-    break;
-  }
-}
-
-auto TrackIterator::cbor() const -> cppbor::Array&& {
-  cppbor::Array res;
-  for (const auto& i : levels_) {
-    res.add(i.cbor());
-  }
-  return std::move(res);
-}
-
 }  // namespace database

src/database/file_gatherer.cpp

@@ -21,13 +21,13 @@ namespace database {
 static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR");
 
 auto FileGathererImpl::FindFiles(
-    const std::pmr::string& root,
+    const std::string& root,
-    std::function<void(const std::pmr::string&, const FILINFO&)> cb) -> void {
+    std::function<void(const std::string&, const FILINFO&)> cb) -> void {
-  std::pmr::deque<std::pmr::string> to_explore(&memory::kSpiRamResource);
+  std::deque<std::string> to_explore;
   to_explore.push_back(root);
 
   while (!to_explore.empty()) {
-    std::pmr::string next_path_str = to_explore.front();
+    std::string next_path_str = to_explore.front();
     const TCHAR* next_path = static_cast<const TCHAR*>(next_path_str.c_str());
 
     FF_DIR dir;
@@ -54,7 +54,7 @@ auto FileGathererImpl::FindFiles(
         // System or hidden file. Ignore it and move on.
         continue;
       } else {
-        std::pmr::string full_path{&memory::kSpiRamResource};
+        std::string full_path;
         full_path += next_path_str;
         full_path += "/";
         full_path += info.fname;

src/database/include/database.hpp

@@ -35,61 +35,18 @@
 
 namespace database {
 
-struct Continuation {
+struct SearchKey;
-  std::pmr::string prefix;
+class Record;
-  std::pmr::string start_key;
+class Iterator;
-  bool forward;
-  bool was_prev_forward;
-  size_t page_size;
-};
 
 /*
- * Wrapper for a set of results from the database. Owns the list of results, as
+ * Handle to an open database. This can be used to store large amounts of
- * well as a continuation token that can be used to continue fetching more
+ * persistent data on the SD card, in a manner that can be retrieved later very
- * results if they were paginated.
+ * quickly.
+ *
+ * A database includes a number of 'indexes'. Each index is a sorted,
+ * hierarchical view of all the playable tracks on the device.
  */
-template <typename T>
-class Result {
- public:
-  auto values() const -> const std::vector<std::shared_ptr<T>>& {
-    return values_;
-  }
-
-  auto next_page() -> std::optional<Continuation>& { return next_page_; }
-  auto prev_page() -> std::optional<Continuation>& { return prev_page_; }
-
-  Result(const std::vector<std::shared_ptr<T>>&& values,
-         std::optional<Continuation> next,
-         std::optional<Continuation> prev)
-      : values_(values), next_page_(next), prev_page_(prev) {}
-
-  Result(const Result&) = delete;
-  Result& operator=(const Result&) = delete;
-
- private:
-  std::vector<std::shared_ptr<T>> values_;
-  std::optional<Continuation> next_page_;
-  std::optional<Continuation> prev_page_;
-};
-
-class IndexRecord {
- public:
-  explicit IndexRecord(const IndexKey&,
-                       std::optional<std::pmr::string>,
-                       std::optional<TrackId>);
-
-  auto text() const -> std::optional<std::pmr::string>;
-  auto track() const -> std::optional<TrackId>;
-
-  auto Expand(std::size_t) const -> std::optional<Continuation>;
-  auto ExpandHeader() const -> IndexKey::Header;
-
- private:
-  IndexKey key_;
-  std::optional<std::pmr::string> override_text_;
-  std::optional<TrackId> track_;
-};
-
 class Database {
  public:
   enum DatabaseError {
@@ -106,31 +63,19 @@ class Database {
 
   ~Database();
 
-  auto Put(const std::string& key, const std::string& val) -> void;
+  /* Adds an arbitrary record to the database. */
-  auto Get(const std::string& key) -> std::optional<std::string>;
+  auto put(const std::string& key, const std::string& val) -> void;
 
-  auto Update() -> std::future<void>;
+  /* Retrives a value previously stored with `put`. */
+  auto get(const std::string& key) -> std::optional<std::string>;
 
-  auto GetTrackPath(TrackId id) -> std::future<std::optional<std::pmr::string>>;
+  auto getTrackPath(TrackId id) -> std::optional<std::string>;
+  auto getTrack(TrackId id) -> std::shared_ptr<Track>;
 
-  auto GetTrack(TrackId id) -> std::future<std::shared_ptr<Track>>;
+  auto getIndexes() -> std::vector<IndexInfo>;
-
+  auto updateIndexes() -> void;
-  /*
-   * Fetches data for multiple tracks more efficiently than multiple calls to
-   * GetTrack.
-   */
-  auto GetBulkTracks(std::vector<TrackId> id)
-      -> std::future<std::vector<std::shared_ptr<Track>>>;
-
-  auto GetIndexes() -> std::vector<IndexInfo>;
-  auto GetTracksByIndex(IndexId index, std::size_t page_size)
-      -> std::future<Result<IndexRecord>*>;
-  auto GetTracks(std::size_t page_size) -> std::future<Result<Track>*>;
-  auto GetDump(std::size_t page_size) -> std::future<Result<std::pmr::string>*>;
-
-  template <typename T>
-  auto GetPage(Continuation* c) -> std::future<Result<T>*>;
 
+  // Cannot be copied or moved.
   Database(const Database&) = delete;
   Database& operator=(const Database&) = delete;
 
@@ -157,106 +102,134 @@ class Database {
            std::shared_ptr<tasks::Worker> worker);
 
   auto dbMintNewTrackId() -> TrackId;
-  auto dbEntomb(TrackId track, uint64_t hash) -> void;
 
+  auto dbEntomb(TrackId track, uint64_t hash) -> void;
   auto dbPutTrackData(const TrackData& s) -> void;
   auto dbGetTrackData(TrackId id) -> std::shared_ptr<TrackData>;
   auto dbPutHash(const uint64_t& hash, TrackId i) -> void;
   auto dbGetHash(const uint64_t& hash) -> std::optional<TrackId>;
+
   auto dbCreateIndexesForTrack(const Track& track) -> void;
   auto dbRemoveIndexes(std::shared_ptr<TrackData>) -> void;
+
   auto dbIngestTagHashes(const TrackTags&,
                          std::pmr::unordered_map<Tag, uint64_t>&) -> void;
   auto dbRecoverTagsFromHashes(const std::pmr::unordered_map<Tag, uint64_t>&)
       -> std::shared_ptr<TrackTags>;
 
-  template <typename T>
+  auto getRecord(const SearchKey& c)
-  auto dbGetPage(const Continuation& c) -> Result<T>*;
+      -> std::optional<std::pair<std::pmr::string, Record>>;
+  auto countRecords(const SearchKey& c) -> size_t;
+};
 
-  auto dbCount(const Continuation& c) -> size_t;
+/*
+ * Container for the data needed to iterate through database records. This is a
+ * lower-level type that the higher-level iterators are built from; most users
+ * outside this namespace shouldn't need to work with continuations.
+ */
+struct SearchKey {
+  std::pmr::string prefix;
+  /* If not given, then iteration starts from `prefix`. */
+  std::optional<std::pmr::string> key;
+  int offset;
 
-  template <typename T>
+  auto startKey() const -> std::string_view;
-  auto ParseRecord(const leveldb::Slice& key, const leveldb::Slice& val)
-      -> std::shared_ptr<T>;
 };
 
-template <>
-auto Database::ParseRecord<IndexRecord>(const leveldb::Slice& key,
-                                        const leveldb::Slice& val)
-    -> std::shared_ptr<IndexRecord>;
-template <>
-auto Database::ParseRecord<Track>(const leveldb::Slice& key,
-                                  const leveldb::Slice& val)
-    -> std::shared_ptr<Track>;
-template <>
-auto Database::ParseRecord<std::pmr::string>(const leveldb::Slice& key,
-                                             const leveldb::Slice& val)
-    -> std::shared_ptr<std::pmr::string>;
-
 /*
- * Utility for accessing a large set of database records, one record at a time.
+ * A record belonging to one of the database's indexes. This may either be a
+ * leaf record, containing a track id, or a branch record, containing a new
+ * Header to retrieve results at the next level of the index.
  */
-class Iterator {
+class Record {
  public:
-  static auto Parse(std::weak_ptr<Database>, const cppbor::Array&)
+  Record(const IndexKey&, const leveldb::Slice&);
-      -> std::optional<Iterator>;
 
-  Iterator(std::weak_ptr<Database>, const IndexInfo&);
+  Record(const Record&) = default;
-  Iterator(std::weak_ptr<Database>, const Continuation&);
+  Record& operator=(const Record& other) = default;
-  Iterator(const Iterator&);
 
-  Iterator& operator=(const Iterator& other);
+  auto text() const -> std::string_view;
+  auto contents() const -> const std::variant<TrackId, IndexKey::Header>&;
 
-  auto database() const { return db_; }
+ private:
+  std::pmr::string text_;
+  std::variant<TrackId, IndexKey::Header> contents_;
+};
 
-  using Callback = std::function<void(std::optional<IndexRecord>)>;
+/*
+ * Utility for accessing a large set of database records, one record at a time.
+ */
+class Iterator {
+ public:
+  Iterator(std::shared_ptr<Database>, IndexId);
+  Iterator(std::shared_ptr<Database>, const IndexKey::Header&);
 
-  auto Next(Callback) -> void;
+  Iterator(const Iterator&) = default;
-  auto NextSync() -> std::optional<IndexRecord>;
+  Iterator& operator=(const Iterator& other) = default;
 
-  auto Prev(Callback) -> void;
+  auto value() const -> const std::optional<Record>&;
+  std::optional<Record> operator*() const { return value(); }
 
-  auto PeekSync() -> std::optional<IndexRecord>;
+  auto next() -> void;
+  std::optional<Record> operator++() {
+    next();
+    return value();
+  }
+  std::optional<Record> operator++(int) {
+    auto val = value();
+    next();
+    return val;
+  }
 
-  auto Size() const -> size_t;
+  auto prev() -> void;
+  std::optional<Record> operator--() {
+    prev();
+    return value();
+  }
+  std::optional<Record> operator--(int) {
+    auto val = value();
+    prev();
+    return val;
+  }
 
-  auto cbor() const -> cppbor::Array&&;
+  auto count() const -> size_t;
 
  private:
-  Iterator(std::weak_ptr<Database>,
+  auto iterate(const SearchKey& key) -> void;
-           std::optional<Continuation>&&,
-           std::optional<Continuation>&&);
 
   friend class TrackIterator;
 
-  auto InvokeNull(Callback) -> void;
-
   std::weak_ptr<Database> db_;
-
+  SearchKey key_;
-  std::mutex pos_mutex_;
+  std::optional<Record> current_;
-  std::optional<Continuation> current_pos_;
-  std::optional<Continuation> prev_pos_;
 };
 
 class TrackIterator {
  public:
-  static auto Parse(std::weak_ptr<Database>, const cppbor::Array&)
-      -> std::optional<TrackIterator>;
-
   TrackIterator(const Iterator&);
-  TrackIterator(const TrackIterator&);
 
-  TrackIterator& operator=(TrackIterator&& other);
+  TrackIterator(const TrackIterator&) = default;
+  TrackIterator& operator=(TrackIterator&& other) = default;
+
+  auto value() const -> std::optional<TrackId>;
+  std::optional<TrackId> operator*() const { return value(); }
 
-  auto Next() -> std::optional<TrackId>;
+  auto next() -> void;
-  auto Size() const -> size_t;
+  std::optional<TrackId> operator++() {
+    next();
+    return value();
+  }
+  std::optional<TrackId> operator++(int) {
+    auto val = value();
+    next();
+    return val;
+  }
 
-  auto cbor() const -> cppbor::Array&&;
+  auto count() const -> size_t;
 
  private:
   TrackIterator(std::weak_ptr<Database>);
-
+  auto next(bool advance) -> void;
-  auto NextLeaf() -> void;
 
   std::weak_ptr<Database> db_;
   std::vector<Iterator> levels_;

src/database/include/file_gatherer.hpp

@@ -20,16 +20,16 @@ class IFileGatherer {
   virtual ~IFileGatherer(){};
 
   virtual auto FindFiles(
-      const std::pmr::string& root,
+      const std::string& root,
-      std::function<void(const std::pmr::string&, const FILINFO&)> cb)
+      std::function<void(const std::string&, const FILINFO&)> cb)
       -> void = 0;
 };
 
 class FileGathererImpl : public IFileGatherer {
  public:
   virtual auto FindFiles(
-      const std::pmr::string& root,
+      const std::string& root,
-      std::function<void(const std::pmr::string&, const FILINFO&)> cb)
+      std::function<void(const std::string&, const FILINFO&)> cb)
       -> void override;
 };
 

src/database/include/tag_parser.hpp

@@ -16,24 +16,24 @@ namespace database {
 class ITagParser {
  public:
   virtual ~ITagParser() {}
-  virtual auto ReadAndParseTags(const std::pmr::string& path)
+  virtual auto ReadAndParseTags(const std::string& path)
       -> std::shared_ptr<TrackTags> = 0;
 };
 
 class GenericTagParser : public ITagParser {
  public:
-  auto ReadAndParseTags(const std::pmr::string& path)
+  auto ReadAndParseTags(const std::string& path)
       -> std::shared_ptr<TrackTags> override;
 };
 
 class TagParserImpl : public ITagParser {
  public:
   TagParserImpl();
-  auto ReadAndParseTags(const std::pmr::string& path)
+  auto ReadAndParseTags(const std::string& path)
       -> std::shared_ptr<TrackTags> override;
 
  private:
-  std::map<std::pmr::string, std::unique_ptr<ITagParser>> extension_to_parser_;
+  std::map<std::string, std::unique_ptr<ITagParser>> extension_to_parser_;
   GenericTagParser generic_parser_;
 
   /*
@@ -43,14 +43,14 @@ class TagParserImpl : public ITagParser {
   std::mutex cache_mutex_;
   util::LruCache<16, std::pmr::string, std::shared_ptr<TrackTags>> cache_;
 
-  // We could also consider keeping caches of artist name -> std::pmr::string
+  // We could also consider keeping caches of artist name -> std::string and
-  // and similar. This hasn't been done yet, as this isn't a common workload in
+  // similar. This hasn't been done yet, as this isn't a common workload in
   // any of our UI.
 };
 
 class OpusTagParser : public ITagParser {
  public:
-  auto ReadAndParseTags(const std::pmr::string& path)
+  auto ReadAndParseTags(const std::string& path)
       -> std::shared_ptr<TrackTags> override;
 };
 

src/database/include/track.hpp

@@ -123,7 +123,7 @@ struct TrackData {
  public:
   TrackData()
       : id(0),
-        filepath(&memory::kSpiRamResource),
+        filepath(),
         tags_hash(0),
         individual_tag_hashes(&memory::kSpiRamResource),
         is_tombstoned(false),

src/database/tag_parser.cpp

@@ -32,7 +32,7 @@ const static std::array<std::pair<const char*, Tag>, 5> kVorbisIdToTag = {{
 static auto convert_track_number(int number) -> std::pmr::string {
   std::ostringstream oss;
   oss << std::setw(4) << std::setfill('0') << number;
-  return std::pmr::string(oss.str());
+  return std::pmr::string(oss.str(), &memory::kSpiRamResource);
 }
 
 static auto convert_track_number(const std::pmr::string& raw)
@@ -131,11 +131,12 @@ TagParserImpl::TagParserImpl() {
   extension_to_parser_["opus"] = std::make_unique<OpusTagParser>();
 }
 
-auto TagParserImpl::ReadAndParseTags(const std::pmr::string& path)
+auto TagParserImpl::ReadAndParseTags(const std::string& path)
     -> std::shared_ptr<TrackTags> {
   {
     std::lock_guard<std::mutex> lock{cache_mutex_};
-    std::optional<std::shared_ptr<TrackTags>> cached = cache_.Get(path);
+    std::optional<std::shared_ptr<TrackTags>> cached =
+        cache_.Get({path.data(), path.size()});
     if (cached) {
       return *cached;
     }
@@ -143,8 +144,8 @@ auto TagParserImpl::ReadAndParseTags(const std::pmr::string& path)
 
   ITagParser* parser = &generic_parser_;
   auto dot_pos = path.find_last_of(".");
-  if (dot_pos != std::pmr::string::npos && path.size() - dot_pos > 1) {
+  if (dot_pos != std::string::npos && path.size() - dot_pos > 1) {
-    std::pmr::string extension = path.substr(dot_pos + 1);
+    std::string extension = path.substr(dot_pos + 1);
     std::transform(extension.begin(), extension.end(), extension.begin(),
                    [](unsigned char c) { return std::tolower(c); });
     if (extension_to_parser_.contains(extension)) {
@@ -162,8 +163,9 @@ auto TagParserImpl::ReadAndParseTags(const std::pmr::string& path)
   // start.
   if (!tags->at(Tag::kAlbumTrack)) {
     auto slash_pos = path.find_last_of("/");
-    if (slash_pos != std::pmr::string::npos && path.size() - slash_pos > 1) {
+    if (slash_pos != std::string::npos && path.size() - slash_pos > 1) {
-      tags->set(Tag::kAlbumTrack, path.substr(slash_pos + 1));
+      std::string trunc = path.substr(slash_pos + 1);
+      tags->set(Tag::kAlbumTrack, {trunc.data(), trunc.size()});
     }
   }
 
@@ -174,13 +176,13 @@ auto TagParserImpl::ReadAndParseTags(const std::pmr::string& path)
 
   {
     std::lock_guard<std::mutex> lock{cache_mutex_};
-    cache_.Put(path, tags);
+    cache_.Put({path.data(), path.size(), &memory::kSpiRamResource}, tags);
   }
 
   return tags;
 }
 
-auto GenericTagParser::ReadAndParseTags(const std::pmr::string& path)
+auto GenericTagParser::ReadAndParseTags(const std::string& path)
     -> std::shared_ptr<TrackTags> {
   libtags::Aux aux;
   auto out = std::make_shared<TrackTags>();
@@ -254,10 +256,10 @@ auto GenericTagParser::ReadAndParseTags(const std::pmr::string& path)
   return out;
 }
 
-auto OpusTagParser::ReadAndParseTags(const std::pmr::string& path)
+auto OpusTagParser::ReadAndParseTags(const std::string& path)
     -> std::shared_ptr<TrackTags> {
   auto lock = drivers::acquire_spi();
-  std::pmr::string vfs_path = "/sdcard" + path;
+  std::string vfs_path = "/sdcard" + path;
   int err;
   OggOpusFile* f = op_test_file(vfs_path.c_str(), &err);
   if (f == NULL) {

src/lua/bridge.cpp

@@ -37,23 +37,8 @@ static auto open_settings_fn(lua_State* state) -> int {
   return 0;
 }
 
-static auto open_now_playing_fn(lua_State* state) -> int {
-  events::Ui().Dispatch(ui::internal::ShowNowPlaying{});
-  return 0;
-}
-
-static auto open_browse_fn(lua_State* state) -> int {
-  int index = luaL_checkinteger(state, 1);
-  events::Ui().Dispatch(ui::internal::IndexSelected{
-      .id = static_cast<uint8_t>(index),
-  });
-  return 0;
-}
-
 static const struct luaL_Reg kLegacyUiFuncs[] = {
     {"open_settings", open_settings_fn},
-    {"open_now_playing", open_now_playing_fn},
-    {"open_browse", open_browse_fn},
     {NULL, NULL}};
 
 static auto lua_legacy_ui(lua_State* state) -> int {

src/lua/lua_database.cpp

@@ -8,7 +8,10 @@
 
 #include <memory>
 #include <string>
+#include <type_traits>
+#include <variant>
 
+#include "bridge.hpp"
 #include "lua.hpp"
 
 #include "esp_log.h"
@@ -34,6 +37,17 @@ static constexpr char kDbIndexMetatable[] = "db_index";
 static constexpr char kDbRecordMetatable[] = "db_record";
 static constexpr char kDbIteratorMetatable[] = "db_iterator";
 
+struct LuaIndexInfo {
+  database::IndexId id;
+  size_t name_size;
+  char name_data[];
+
+  auto name() -> std::string_view { return {name_data, name_size}; }
+};
+
+static_assert(std::is_trivially_destructible<LuaIndexInfo>());
+static_assert(std::is_trivially_copy_assignable<LuaIndexInfo>());
+
 static auto indexes(lua_State* state) -> int {
   Bridge* instance = Bridge::Get(state);
 
@@ -44,11 +58,15 @@ static auto indexes(lua_State* state) -> int {
     return 1;
   }
 
-  for (const auto& i : db->GetIndexes()) {
+  for (const auto& i : db->getIndexes()) {
-    database::IndexInfo** data = reinterpret_cast<database::IndexInfo**>(
+    LuaIndexInfo* data = reinterpret_cast<LuaIndexInfo*>(
-        lua_newuserdata(state, sizeof(uintptr_t)));
+        lua_newuserdata(state, sizeof(LuaIndexInfo) + i.name.size()));
     luaL_setmetatable(state, kDbIndexMetatable);
-    *data = new database::IndexInfo{i};
+    *data = LuaIndexInfo{
+        .id = i.id,
+        .name_size = i.name.size(),
+    };
+    std::memcpy(data->name_data, i.name.data(), i.name.size());
     lua_rawseti(state, -2, i.id);
   }
 
@@ -65,33 +83,28 @@ static const struct luaL_Reg kDatabaseFuncs[] = {{"indexes", indexes},
  * trivially copyable.
  */
 struct LuaRecord {
-  database::TrackId id_or_zero;
+  std::variant<database::TrackId, database::IndexKey::Header> contents;
-  database::IndexKey::Header header_at_next_depth;
   size_t text_size;
   char text[];
 };
 
-static_assert(std::is_trivially_copyable_v<LuaRecord> == true);
+static_assert(std::is_trivially_destructible<LuaRecord>());
-
+static_assert(std::is_trivially_copy_assignable<LuaRecord>());
-static auto push_lua_record(lua_State* L, const database::IndexRecord& r)
-    -> void {
-  // Bake out the text into something concrete.
-  auto text = r.text().value_or("");
 
+static auto push_lua_record(lua_State* L, const database::Record& r) -> void {
   // Create and init the userdata.
   LuaRecord* record = reinterpret_cast<LuaRecord*>(
-      lua_newuserdata(L, sizeof(LuaRecord) + text.size()));
+      lua_newuserdata(L, sizeof(LuaRecord) + r.text().size()));
   luaL_setmetatable(L, kDbRecordMetatable);
 
   // Init all the fields
   *record = {
-      .id_or_zero = r.track().value_or(0),
+      .contents = r.contents(),
-      .header_at_next_depth = r.ExpandHeader(),
+      .text_size = r.text().size(),
-      .text_size = text.size(),
   };
 
   // Copy the string data across.
-  std::memcpy(record->text, text.data(), text.size());
+  std::memcpy(record->text, r.text().data(), r.text().size());
 }
 
 auto db_check_iterator(lua_State* L, int stack_pos) -> database::Iterator* {
@@ -100,49 +113,30 @@ auto db_check_iterator(lua_State* L, int stack_pos) -> database::Iterator* {
   return it;
 }
 
-static auto push_iterator(lua_State* state,
+static auto push_iterator(lua_State* state, const database::Iterator& it)
-                          std::variant<database::Iterator*,
+    -> void {
-                                       database::Continuation,
-                                       database::IndexInfo> val) -> void {
-  Bridge* instance = Bridge::Get(state);
   database::Iterator** data = reinterpret_cast<database::Iterator**>(
       lua_newuserdata(state, sizeof(uintptr_t)));
-  std::visit(
+  *data = new database::Iterator(it);
-      [&](auto&& arg) {
-        using T = std::decay_t<decltype(arg)>;
-        if constexpr (std::is_same_v<T, database::Iterator*>) {
-          *data = new database::Iterator(*arg);
-        } else {
-          *data = new database::Iterator(instance->services().database(), arg);
-        }
-      },
-      val);
   luaL_setmetatable(state, kDbIteratorMetatable);
 }
 
 static auto db_iterate(lua_State* state) -> int {
   database::Iterator* it = db_check_iterator(state, 1);
-  luaL_checktype(state, 2, LUA_TFUNCTION);
+  std::optional<database::Record> res = (*it)++;
-  int callback_ref = luaL_ref(state, LUA_REGISTRYINDEX);
 
-  it->Next([=](std::optional<database::IndexRecord> res) {
-    events::Ui().RunOnTask([=]() {
-      lua_rawgeti(state, LUA_REGISTRYINDEX, callback_ref);
   if (res) {
     push_lua_record(state, *res);
   } else {
     lua_pushnil(state);
   }
-      CallProtected(state, 1, 0);
+
-      luaL_unref(state, LUA_REGISTRYINDEX, callback_ref);
+  return 1;
-    });
-  });
-  return 0;
 }
 
 static auto db_iterator_clone(lua_State* state) -> int {
   database::Iterator* it = db_check_iterator(state, 1);
-  push_iterator(state, it);
+  push_iterator(state, *it);
   return 1;
 }
 
@@ -154,6 +148,7 @@ static auto db_iterator_gc(lua_State* state) -> int {
 
 static const struct luaL_Reg kDbIteratorFuncs[] = {{"next", db_iterate},
                                                    {"clone", db_iterator_clone},
+                                                   {"__call", db_iterate},
                                                    {"__gc", db_iterator_gc},
                                                    {NULL, NULL}};
 
@@ -168,18 +163,22 @@ static auto record_contents(lua_State* state) -> int {
   LuaRecord* data = reinterpret_cast<LuaRecord*>(
       luaL_checkudata(state, 1, kDbRecordMetatable));
 
-  if (data->id_or_zero) {
+  std::visit(
-    lua_pushinteger(state, data->id_or_zero);
+      [&](auto&& arg) {
+        using T = std::decay_t<decltype(arg)>;
+        if constexpr (std::is_same_v<T, database::TrackId>) {
+          lua_pushinteger(state, arg);
+        } else if constexpr (std::is_same_v<T, database::IndexKey::Header>) {
+          Bridge* bridge = Bridge::Get(state);
+          auto db = bridge->services().database().lock();
+          if (!db) {
+            lua_pushnil(state);
           } else {
-    std::string p = database::EncodeIndexPrefix(data->header_at_next_depth);
+            push_iterator(state, database::Iterator{db, arg});
-    push_iterator(state, database::Continuation{
-                             .prefix = {p.data(), p.size()},
-                             .start_key = {p.data(), p.size()},
-                             .forward = true,
-                             .was_prev_forward = true,
-                             .page_size = 1,
-                         });
           }
+        }
+      },
+      data->contents);
 
   return 1;
 }
@@ -190,38 +189,33 @@ static const struct luaL_Reg kDbRecordFuncs[] = {{"title", record_text},
                                                  {NULL, NULL}};
 
 static auto index_name(lua_State* state) -> int {
-  database::IndexInfo** data = reinterpret_cast<database::IndexInfo**>(
+  LuaIndexInfo* data = reinterpret_cast<LuaIndexInfo*>(
       luaL_checkudata(state, 1, kDbIndexMetatable));
   if (data == NULL) {
     return 0;
   }
-  lua_pushstring(state, (*data)->name.c_str());
+  lua_pushlstring(state, data->name_data, data->name_size);
   return 1;
 }
 
 static auto index_iter(lua_State* state) -> int {
-  database::IndexInfo** data = reinterpret_cast<database::IndexInfo**>(
+  LuaIndexInfo* data = reinterpret_cast<LuaIndexInfo*>(
       luaL_checkudata(state, 1, kDbIndexMetatable));
   if (data == NULL) {
     return 0;
   }
-  push_iterator(state, **data);
+  Bridge* bridge = Bridge::Get(state);
-  return 1;
+  auto db = bridge->services().database().lock();
-}
+  if (!db) {
-
+    lua_pushnil(state);
-static auto index_gc(lua_State* state) -> int {
-  database::IndexInfo** data = reinterpret_cast<database::IndexInfo**>(
-      luaL_checkudata(state, 1, kDbIndexMetatable));
-  if (data != NULL) {
-    delete *data;
   }
-  return 0;
+  push_iterator(state, database::Iterator{db, data->id});
+  return 1;
 }
 
 static const struct luaL_Reg kDbIndexFuncs[] = {{"name", index_name},
                                                 {"iter", index_iter},
                                                 {"__tostring", index_name},
-                                                {"__gc", index_gc},
                                                 {NULL, NULL}};
 
 static auto lua_database(lua_State* state) -> int {

src/lua/lua_queue.cpp

@@ -21,7 +21,6 @@
 #include "index.hpp"
 #include "property.hpp"
 #include "service_locator.hpp"
-#include "source.hpp"
 #include "track.hpp"
 #include "track_queue.hpp"
 #include "ui_events.hpp"
@@ -37,15 +36,13 @@ static auto queue_add(lua_State* state) -> int {
     database::TrackId id = luaL_checkinteger(state, 1);
     instance->services().bg_worker().Dispatch<void>([=]() {
       audio::TrackQueue& queue = instance->services().track_queue();
-      auto editor = queue.Edit();
+      queue.append(id);
-      queue.Append(editor, id);
     });
   } else {
-    database::Iterator it = *db_check_iterator(state, 1);
+    database::Iterator* it = db_check_iterator(state, 1);
     instance->services().bg_worker().Dispatch<void>([=]() {
       audio::TrackQueue& queue = instance->services().track_queue();
-      auto editor = queue.Edit();
+      queue.append(database::TrackIterator{*it});
-      queue.Append(editor, database::TrackIterator{it});
     });
   }
 
@@ -55,8 +52,7 @@ static auto queue_add(lua_State* state) -> int {
 static auto queue_clear(lua_State* state) -> int {
   Bridge* instance = Bridge::Get(state);
   audio::TrackQueue& queue = instance->services().track_queue();
-  auto editor = queue.Edit();
+  queue.clear();
-  queue.Clear(editor);
   return 0;
 }
 

src/playlist/CMakeLists.txt

@@ -1,10 +0,0 @@
-# Copyright 2023 jacqueline <me@jacqueline.id.au>
-#
-# SPDX-License-Identifier: GPL-3.0-only
-
-idf_component_register(
-  SRCS "source.cpp" "shuffler.cpp"
-  INCLUDE_DIRS "include"
-  REQUIRES "database" "util")
-
-target_compile_options(${COMPONENT_LIB} PRIVATE ${EXTRA_WARNINGS})

src/playlist/include/shuffler.hpp

@@ -1,68 +0,0 @@
-/*
- * Copyright 2023 jacqueline <me@jacqueline.id.au>
- *
- * SPDX-License-Identifier: GPL-3.0-only
- */
-
-#pragma once
-
-#include <deque>
-#include <memory>
-#include <mutex>
-#include <variant>
-#include <vector>
-
-#include "bloom_filter.hpp"
-#include "database.hpp"
-#include "future_fetcher.hpp"
-#include "random.hpp"
-#include "source.hpp"
-#include "track.hpp"
-
-namespace playlist {
-
-/*
- * A source composes of other sources and/or specific extra tracks. Supports
- * iteration over its contents in a random order.
- */
-class Shuffler : public ISource {
- public:
-  static auto Create() -> Shuffler*;
-
-  explicit Shuffler(
-      util::IRandom* random,
-      std::unique_ptr<util::BloomFilter<database::TrackId>> filter);
-
-  auto Current() -> std::optional<database::TrackId> override;
-  auto Advance() -> std::optional<database::TrackId> override;
-  auto Peek(std::size_t, std::vector<database::TrackId>*)
-      -> std::size_t override;
-
-  typedef std::variant<database::TrackId, std::shared_ptr<IResetableSource>>
-      Item;
-  auto Add(Item) -> void;
-
-  /*
-   * Returns the enqueued items, starting from the current item, in their
-   * original insertion order.
-   */
-  auto Unshuffle() -> std::vector<Item>;
-
-  // Not copyable or movable.
-
-  Shuffler(const Shuffler&) = delete;
-  Shuffler& operator=(const Shuffler&) = delete;
-
- private:
-  auto RefillBuffer() -> void;
-
-  util::IRandom* random_;
-
-  std::unique_ptr<util::BloomFilter<database::TrackId>> already_played_;
-  bool out_of_items_;
-
-  std::deque<Item> ordered_items_;
-  std::deque<database::TrackId> shuffled_items_buffer_;
-};
-
-}  // namespace playlist

src/playlist/include/source.hpp

@@ -1,157 +0,0 @@
-/*
- * Copyright 2023 jacqueline <me@jacqueline.id.au>
- *
- * SPDX-License-Identifier: GPL-3.0-only
- */
-
-#pragma once
-
-#include <deque>
-#include <memory>
-#include <mutex>
-#include <stack>
-#include <variant>
-#include <vector>
-
-#include "bloom_filter.hpp"
-#include "database.hpp"
-#include "future_fetcher.hpp"
-#include "random.hpp"
-#include "track.hpp"
-
-namespace playlist {
-
-/*
- * Stateful interface for iterating over a collection of tracks by id.
- */
-class ISource {
- public:
-  virtual ~ISource() {}
-
-  virtual auto Current() -> std::optional<database::TrackId> = 0;
-
-  /*
-   * Discards the current track id and continues to the next in this source.
-   * Returns the new current track id.
-   */
-  virtual auto Advance() -> std::optional<database::TrackId> = 0;
-
-  /*
-   * Repeatedly advances until a track with the given id is the current track.
-   * Returns false if this source ran out of tracks before the requested id
-   * was encounted, true otherwise.
-   */
-  virtual auto AdvanceTo(database::TrackId id) -> bool {
-    for (auto t = Current(); t.has_value(); t = Advance()) {
-      if (*t == id) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /*
-   * Places the next n tracks into the given vector, in order. Does not change
-   * the value returned by Current().
-   */
-  virtual auto Peek(std::size_t n, std::vector<database::TrackId>*)
-      -> std::size_t = 0;
-};
-
-/*
- * A Source that supports restarting iteration from its original initial
- * value.
- */
-class IResetableSource : public ISource {
- public:
-  virtual ~IResetableSource() {}
-
-  virtual auto Previous() -> std::optional<database::TrackId> = 0;
-
-  /*
-   * Restarts iteration from this source's initial value.
-   */
-  virtual auto Reset() -> void = 0;
-};
-
-class IteratorSource : public IResetableSource {
- public:
-  IteratorSource(const database::Iterator&);
-
-  auto Current() -> std::optional<database::TrackId> override;
-  auto Advance() -> std::optional<database::TrackId> override;
-  auto Peek(std::size_t n, std::vector<database::TrackId>*)
-      -> std::size_t override;
-
-  auto Previous() -> std::optional<database::TrackId> override;
-  auto Reset() -> void override;
-
- private:
-  const database::Iterator& start_;
-  std::optional<database::TrackId> current_;
-  std::stack<database::Iterator, std::vector<database::Iterator>> next_;
-};
-
-auto CreateSourceFromResults(
-    std::weak_ptr<database::Database>,
-    std::shared_ptr<database::Result<database::IndexRecord>>)
-    -> std::shared_ptr<IResetableSource>;
-
-class IndexRecordSource : public IResetableSource {
- public:
-  IndexRecordSource(std::weak_ptr<database::Database> db,
-                    std::shared_ptr<database::Result<database::IndexRecord>>);
-
-  IndexRecordSource(std::weak_ptr<database::Database> db,
-                    std::shared_ptr<database::Result<database::IndexRecord>>,
-                    std::size_t,
-                    std::shared_ptr<database::Result<database::IndexRecord>>,
-                    std::size_t);
-
-  auto Current() -> std::optional<database::TrackId> override;
-  auto Advance() -> std::optional<database::TrackId> override;
-  auto Peek(std::size_t n, std::vector<database::TrackId>*)
-      -> std::size_t override;
-
-  auto Previous() -> std::optional<database::TrackId> override;
-  auto Reset() -> void override;
-
- private:
-  std::weak_ptr<database::Database> db_;
-
-  std::shared_ptr<database::Result<database::IndexRecord>> initial_page_;
-  ssize_t initial_item_;
-
-  std::shared_ptr<database::Result<database::IndexRecord>> current_page_;
-  ssize_t current_item_;
-};
-
-class NestedSource : public IResetableSource {
- public:
-  NestedSource(std::weak_ptr<database::Database> db,
-               std::shared_ptr<database::Result<database::IndexRecord>>);
-
-  auto Current() -> std::optional<database::TrackId> override;
-  auto Advance() -> std::optional<database::TrackId> override;
-  auto Peek(std::size_t n, std::vector<database::TrackId>*)
-      -> std::size_t override;
-
-  auto Previous() -> std::optional<database::TrackId> override;
-  auto Reset() -> void override;
-
- private:
-  auto CreateChild(std::shared_ptr<database::IndexRecord> page)
-      -> std::shared_ptr<IResetableSource>;
-
-  std::weak_ptr<database::Database> db_;
-
-  std::shared_ptr<database::Result<database::IndexRecord>> initial_page_;
-  ssize_t initial_item_;
-
-  std::shared_ptr<database::Result<database::IndexRecord>> current_page_;
-  ssize_t current_item_;
-
-  std::shared_ptr<IResetableSource> current_child_;
-};
-
-}  // namespace playlist

src/playlist/shuffler.cpp

@@ -1,166 +0,0 @@
-/*
- * Copyright 2023 jacqueline <me@jacqueline.id.au>
- *
- * SPDX-License-Identifier: GPL-3.0-only
- */
-
-#include "shuffler.hpp"
-
-#include <algorithm>
-#include <functional>
-#include <memory>
-#include <set>
-#include <variant>
-
-#include "bloom_filter.hpp"
-#include "database.hpp"
-#include "komihash.h"
-#include "random.hpp"
-#include "track.hpp"
-
-static constexpr std::size_t kShufflerBufferSize = 32;
-
-namespace playlist {
-
-auto Shuffler::Create() -> Shuffler* {
-  return new Shuffler(util::sRandom,
-                      std::make_unique<util::BloomFilter<database::TrackId>>(
-                          [](database::TrackId id) {
-                            return komihash(&id, sizeof(database::TrackId), 0);
-                          }));
-}
-
-Shuffler::Shuffler(util::IRandom* random,
-                   std::unique_ptr<util::BloomFilter<database::TrackId>> filter)
-    : random_(random), already_played_(std::move(filter)) {}
-
-auto Shuffler::Current() -> std::optional<database::TrackId> {
-  if (shuffled_items_buffer_.empty()) {
-    return {};
-  }
-  return shuffled_items_buffer_.front();
-}
-
-auto Shuffler::Advance() -> std::optional<database::TrackId> {
-  if (shuffled_items_buffer_.empty() && !out_of_items_) {
-    RefillBuffer();
-  }
-
-  auto res = Current();
-  if (res) {
-    // Mark tracks off in the bloom filter only *after* they've been advanced
-    // past. This gives us the most flexibility for reshuffling when adding new
-    // items.
-    already_played_->Insert(*res);
-    shuffled_items_buffer_.pop_front();
-  }
-  return res;
-}
-
-auto Shuffler::Peek(std::size_t num, std::vector<database::TrackId>* out)
-    -> std::size_t {
-  if (shuffled_items_buffer_.size() < num) {
-    RefillBuffer();
-  }
-  for (int i = 0; i < num; i++) {
-    if (i >= shuffled_items_buffer_.size()) {
-      // We must be out of data, since the buffer didn't fill up.
-      return i;
-    }
-    out->push_back(shuffled_items_buffer_.at(i));
-  }
-  return num;
-}
-
-auto Shuffler::Add(Item item) -> void {
-  ordered_items_.push_back(item);
-  out_of_items_ = false;
-
-  // Empty out the buffer of already shuffled items, since we will need to
-  // shuffle again in order to incorporate the newly added item(s). We keep the
-  // current item however because we wouldn't want Add() to change the value of
-  // Current() unless we're completely out of items.
-  if (shuffled_items_buffer_.size() > 1) {
-    shuffled_items_buffer_.erase(shuffled_items_buffer_.begin() + 1,
-                                 shuffled_items_buffer_.end());
-  }
-  RefillBuffer();
-}
-
-auto Shuffler::Unshuffle() -> std::vector<Item> {
-  std::vector<Item> ret;
-  database::TrackId current = shuffled_items_buffer_.front();
-  bool has_found_current = false;
-
-  for (const Item& item : ordered_items_) {
-    if (!has_found_current) {
-      // TODO(jacqueline): *Should* this include previous items? What is the
-      // 'previous' button meant to do after unshuffling?
-      if (std::holds_alternative<database::TrackId>(item)) {
-        has_found_current = current == std::get<database::TrackId>(item);
-      } else {
-        auto source = std::get<std::shared_ptr<IResetableSource>>(item);
-        source->Reset();
-        has_found_current =
-            std::get<std::shared_ptr<IResetableSource>>(item)->AdvanceTo(
-                current);
-      }
-    } else {
-      ret.push_back(item);
-    }
-  }
-
-  return ret;
-}
-
-auto Shuffler::RefillBuffer() -> void {
-  // Don't waste time iterating if we know there's nothing new.
-  if (out_of_items_) {
-    return;
-  }
-
-  int num_to_sample = kShufflerBufferSize - shuffled_items_buffer_.size();
-  int resovoir_offset = shuffled_items_buffer_.size();
-
-  std::set<database::TrackId> in_buffer;
-  for (const database::TrackId& id : shuffled_items_buffer_) {
-    in_buffer.insert(id);
-  }
-
-  uint32_t i = 0;
-  auto consider_item = [&, this](const database::TrackId& item) {
-    if (already_played_->Contains(item) || in_buffer.contains(item)) {
-      return;
-    }
-    if (i < num_to_sample) {
-      shuffled_items_buffer_.push_back(item);
-    } else {
-      uint32_t index_to_replace = random_->RangeInclusive(0, i);
-      if (index_to_replace < num_to_sample) {
-        shuffled_items_buffer_[resovoir_offset + index_to_replace] = item;
-      }
-    }
-    i++;
-  };
-
-  for (const Item& item : ordered_items_) {
-    if (std::holds_alternative<database::TrackId>(item)) {
-      std::invoke(consider_item, std::get<database::TrackId>(item));
-    } else {
-      auto source = std::get<std::shared_ptr<IResetableSource>>(item);
-      source->Reset();
-      while (source->Advance()) {
-        std::invoke(consider_item, *source->Current());
-      }
-    }
-  }
-
-  out_of_items_ = i > num_to_sample;
-  // We've now got a random *selection*, but the order might be predictable
-  // (e.g. if there were only `num_to_sample` new items). Do a final in-memory
-  // shuffle.
-  std::random_shuffle(shuffled_items_buffer_.begin() + resovoir_offset,
-                      shuffled_items_buffer_.end());
-}
-
-}  // namespace playlist

src/playlist/source.cpp

@@ -1,360 +0,0 @@
-/*
- * Copyright 2023 jacqueline <me@jacqueline.id.au>
- *
- * SPDX-License-Identifier: GPL-3.0-only
- */
-
-#include "source.hpp"
-
-#include <algorithm>
-#include <functional>
-#include <memory>
-#include <set>
-#include <variant>
-
-#include "esp_log.h"
-
-#include "bloom_filter.hpp"
-#include "database.hpp"
-#include "komihash.h"
-#include "random.hpp"
-#include "track.hpp"
-
-namespace playlist {
-
-[[maybe_unused]] static constexpr char kTag[] = "queue_src";
-
-IteratorSource::IteratorSource(const database::Iterator& it)
-    : start_(it), current_(), next_() {
-  Reset();
-  Advance();
-}
-
-auto IteratorSource::Current() -> std::optional<database::TrackId> {
-  return current_;
-}
-
-auto IteratorSource::Advance() -> std::optional<database::TrackId> {
-  ESP_LOGI(kTag, "advancing");
-  while (!next_.empty()) {
-    auto next = next_.top().NextSync();
-    if (!next) {
-      ESP_LOGI(kTag, "top was empty");
-      next_.pop();
-      continue;
-    }
-    if (next->track()) {
-      ESP_LOGI(kTag, "top held track %lu", next->track().value_or(0));
-      current_ = next->track();
-      return current_;
-    }
-    ESP_LOGI(kTag, "top held records");
-    next_.push(database::Iterator(start_.database(), next->Expand(1).value()));
-  }
-  ESP_LOGI(kTag, "exhausted");
-  return {};
-}
-
-auto IteratorSource::Peek(std::size_t n, std::vector<database::TrackId>*)
-    -> std::size_t {
-  return 0;
-}
-
-auto IteratorSource::Previous() -> std::optional<database::TrackId> {
-  return {};
-}
-
-auto IteratorSource::Reset() -> void {
-  while (!next_.empty()) {
-    next_.pop();
-  }
-  next_.push(start_);
-}
-
-auto CreateSourceFromResults(
-    std::weak_ptr<database::Database> db,
-    std::shared_ptr<database::Result<database::IndexRecord>> results)
-    -> std::shared_ptr<IResetableSource> {
-  if (results->values()[0]->track()) {
-    return std::make_shared<IndexRecordSource>(db, results);
-  } else {
-    return std::make_shared<NestedSource>(db, results);
-  }
-}
-
-IndexRecordSource::IndexRecordSource(
-    std::weak_ptr<database::Database> db,
-    std::shared_ptr<database::Result<database::IndexRecord>> initial)
-    : db_(db),
-      initial_page_(initial),
-      initial_item_(0),
-      current_page_(initial_page_),
-      current_item_(initial_item_) {}
-
-IndexRecordSource::IndexRecordSource(
-    std::weak_ptr<database::Database> db,
-    std::shared_ptr<database::Result<database::IndexRecord>> initial,
-    std::size_t initial_index,
-    std::shared_ptr<database::Result<database::IndexRecord>> current,
-    std::size_t current_index)
-    : db_(db),
-      initial_page_(initial),
-      initial_item_(initial_index),
-      current_page_(current),
-      current_item_(current_index) {}
-
-auto IndexRecordSource::Current() -> std::optional<database::TrackId> {
-  if (current_page_->values().size() <= current_item_) {
-    return {};
-  }
-  if (current_page_ == initial_page_ && current_item_ < initial_item_) {
-    return {};
-  }
-
-  return current_page_->values().at(current_item_)->track();
-}
-
-auto IndexRecordSource::Advance() -> std::optional<database::TrackId> {
-  current_item_++;
-  if (current_item_ >= current_page_->values().size()) {
-    auto next_page = current_page_->next_page();
-    if (!next_page) {
-      current_item_--;
-      return {};
-    }
-
-    auto db = db_.lock();
-    if (!db) {
-      return {};
-    }
-
-    current_page_.reset(db->GetPage<database::IndexRecord>(&*next_page).get());
-    current_item_ = 0;
-  }
-
-  return Current();
-}
-
-auto IndexRecordSource::Previous() -> std::optional<database::TrackId> {
-  if (current_page_ == initial_page_ && current_item_ <= initial_item_) {
-    return {};
-  }
-
-  current_item_--;
-  if (current_item_ < 0) {
-    auto prev_page = current_page_->prev_page();
-    if (!prev_page) {
-      return {};
-    }
-
-    auto db = db_.lock();
-    if (!db) {
-      return {};
-    }
-
-    current_page_.reset(db->GetPage<database::IndexRecord>(&*prev_page).get());
-    current_item_ = current_page_->values().size() - 1;
-  }
-
-  return Current();
-}
-
-auto IndexRecordSource::Peek(std::size_t n, std::vector<database::TrackId>* out)
-    -> std::size_t {
-  if (current_page_->values().size() <= current_item_) {
-    return {};
-  }
-
-  auto db = db_.lock();
-  if (!db) {
-    return 0;
-  }
-
-  std::size_t items_added = 0;
-
-  std::shared_ptr<database::Result<database::IndexRecord>> working_page =
-      current_page_;
-  std::size_t working_item = current_item_ + 1;
-
-  while (n > 0) {
-    if (working_item >= working_page->values().size()) {
-      auto next_page = current_page_->next_page();
-      if (!next_page) {
-        break;
-      }
-      // TODO(jacqueline): It would probably be a good idea to hold onto these
-      // peeked pages, to avoid needing to look them up again later.
-      working_page.reset(db->GetPage<database::IndexRecord>(&*next_page).get());
-      working_item = 0;
-    }
-
-    auto record = working_page->values().at(working_item);
-    if (record->track()) {
-      out->push_back(record->track().value());
-      n--;
-      items_added++;
-    }
-    working_item++;
-  }
-
-  return items_added;
-}
-
-auto IndexRecordSource::Reset() -> void {
-  current_page_ = initial_page_;
-  current_item_ = initial_item_;
-}
-
-NestedSource::NestedSource(
-    std::weak_ptr<database::Database> db,
-    std::shared_ptr<database::Result<database::IndexRecord>> initial)
-    : db_(db),
-      initial_page_(initial),
-      initial_item_(0),
-      current_page_(initial_page_),
-      current_item_(initial_item_),
-      current_child_(CreateChild(initial->values()[0])) {}
-
-auto NestedSource::Current() -> std::optional<database::TrackId> {
-  if (current_child_) {
-    return current_child_->Current();
-  }
-  return {};
-}
-
-auto NestedSource::Advance() -> std::optional<database::TrackId> {
-  if (!current_child_) {
-    return {};
-  }
-
-  auto child_next = current_child_->Advance();
-  if (child_next) {
-    return child_next;
-  }
-  // Our current child has run out of tracks. Move on to the next child.
-  current_item_++;
-  current_child_.reset();
-
-  if (current_item_ >= current_page_->values().size()) {
-    // We're even out of items in this page!
-    auto next_page = current_page_->next_page();
-    if (!next_page) {
-      current_item_--;
-      return {};
-    }
-
-    auto db = db_.lock();
-    if (!db) {
-      return {};
-    }
-
-    current_page_.reset(db->GetPage<database::IndexRecord>(&*next_page).get());
-    current_item_ = 0;
-  }
-  current_child_ = CreateChild(current_page_->values()[current_item_]);
-
-  return Current();
-}
-
-auto NestedSource::Previous() -> std::optional<database::TrackId> {
-  if (current_page_ == initial_page_ && current_item_ <= initial_item_) {
-    return {};
-  }
-
-  current_item_--;
-  current_child_.reset();
-
-  if (current_item_ < 0) {
-    auto prev_page = current_page_->prev_page();
-    if (!prev_page) {
-      return {};
-    }
-
-    auto db = db_.lock();
-    if (!db) {
-      return {};
-    }
-
-    current_page_.reset(db->GetPage<database::IndexRecord>(&*prev_page).get());
-    current_item_ = current_page_->values().size() - 1;
-  }
-  current_child_ = CreateChild(current_page_->values()[current_item_]);
-
-  return Current();
-}
-
-auto NestedSource::Peek(std::size_t n, std::vector<database::TrackId>* out)
-    -> std::size_t {
-  if (current_page_->values().size() <= current_item_) {
-    return {};
-  }
-
-  auto db = db_.lock();
-  if (!db) {
-    return 0;
-  }
-
-  std::size_t items_added = 0;
-
-  std::shared_ptr<database::Result<database::IndexRecord>> working_page =
-      current_page_;
-  std::size_t working_item = current_item_;
-  std::shared_ptr<IResetableSource> working_child = current_child_;
-
-  while (working_child) {
-    auto res = working_child->Peek(n, out);
-    n -= res;
-    items_added += res;
-
-    if (n == 0) {
-      break;
-    } else {
-      working_item++;
-      if (working_item < working_page->values().size()) {
-        working_child = CreateChild(working_page->values()[working_item]);
-      } else {
-        auto next_page = current_page_->next_page();
-        if (!next_page) {
-          break;
-        }
-        working_page.reset(
-            db->GetPage<database::IndexRecord>(&*next_page).get());
-        working_item = 0;
-        working_child = CreateChild(working_page->values()[0]);
-      }
-    }
-  }
-
-  return items_added;
-}
-
-auto NestedSource::Reset() -> void {
-  current_page_ = initial_page_;
-  current_item_ = initial_item_;
-  current_child_ = CreateChild(initial_page_->values()[initial_item_]);
-}
-
-auto NestedSource::CreateChild(std::shared_ptr<database::IndexRecord> record)
-    -> std::shared_ptr<IResetableSource> {
-  auto cont = record->Expand(10);
-  if (!cont) {
-    return {};
-  }
-  auto db = db_.lock();
-  if (!db) {
-    return {};
-  }
-  std::shared_ptr<database::Result<database::IndexRecord>> next_level{
-      db->GetPage<database::IndexRecord>(&*cont).get()};
-  if (!next_level) {
-    return {};
-  }
-  auto next_level_record = next_level->values()[0];
-  if (next_level_record->track()) {
-    return std::make_shared<IndexRecordSource>(db_, next_level);
-  } else {
-    return std::make_shared<NestedSource>(db_, next_level);
-  }
-}
-
-}  // namespace playlist

src/system_fsm/booting.cpp

@@ -86,7 +86,8 @@ auto Booting::entry() -> void {
   sServices->battery(std::make_unique<battery::Battery>(
       sServices->samd(), std::unique_ptr<drivers::AdcBattery>(adc)));
 
-  sServices->track_queue(std::make_unique<audio::TrackQueue>());
+  sServices->track_queue(
+      std::make_unique<audio::TrackQueue>(sServices->bg_worker()));
   sServices->tag_parser(std::make_unique<database::TagParserImpl>());
   sServices->collator(locale::CreateCollator());
 

src/ui/include/ui_events.hpp

@@ -30,18 +30,6 @@ struct OnSystemError : tinyfsm::Event {};
 
 namespace internal {
 
-struct RecordSelected : tinyfsm::Event {
-  bool show_menu;
-  std::pmr::vector<std::pmr::string> new_crumbs;
-  std::shared_ptr<database::Result<database::IndexRecord>> initial_page;
-  std::shared_ptr<database::Result<database::IndexRecord>> page;
-  std::size_t record;
-};
-
-struct IndexSelected : tinyfsm::Event {
-  database::IndexId id;
-};
-
 struct ControlSchemeChanged : tinyfsm::Event {};
 struct ReindexDatabase : tinyfsm::Event {};
 

src/ui/include/ui_fsm.hpp

@@ -64,7 +64,6 @@ class UiState : public tinyfsm::Fsm<UiState> {
   virtual void react(const system_fsm::KeyLockChanged&);
   virtual void react(const OnLuaError&) {}
 
-  virtual void react(const internal::RecordSelected&) {}
   virtual void react(const internal::BackPressed&) {}
   virtual void react(const internal::ShowSettingsPage&){};
   virtual void react(const internal::ModalCancelPressed&) {

src/ui/ui_fsm.cpp

@@ -41,7 +41,6 @@
 #include "screen_lua.hpp"
 #include "screen_settings.hpp"
 #include "screen_splash.hpp"
-#include "source.hpp"
 #include "spiffs.hpp"
 #include "storage.hpp"
 #include "system_events.hpp"
@@ -118,7 +117,7 @@ void UiState::react(const audio::PlaybackUpdate& ev) {}
 
 void UiState::react(const audio::QueueUpdate&) {
   auto& queue = sServices->track_queue();
-  sPlaybackModel.current_track.set(queue.Current());
+  sPlaybackModel.current_track.set(queue.current());
 }
 
 void UiState::react(const internal::ControlSchemeChanged&) {
@@ -281,15 +280,14 @@ void Lua::react(const system_fsm::BatteryStateChanged& ev) {
 }
 
 void Lua::react(const audio::QueueUpdate&) {
-  sServices->bg_worker().Dispatch<void>([=]() {
   auto& queue = sServices->track_queue();
-    size_t total_size = queue.GetTotalSize();
+  queue_size_->Update(static_cast<int>(queue.totalSize()));
-    size_t current_pos = queue.GetCurrentPosition();
+
-    events::Ui().RunOnTask([=]() {
+  int current_pos = queue.currentPosition();
-      queue_size_->Update(static_cast<int>(total_size));
+  if (queue.current()) {
-      queue_position_->Update(static_cast<int>(current_pos));
+    current_pos++;
-    });
+  }
-  });
+  queue_position_->Update(current_pos);
 }
 
 void Lua::react(const audio::PlaybackStarted& ev) {
@@ -373,7 +371,7 @@ void Indexing::entry() {
     // TODO: Hmm.
     return;
   }
-  db->Update();
+  db->updateIndexes();
 }
 
 void Indexing::exit() {