Database init is now stable!

src/database/CMakeLists.txt

@@ -1,7 +1,7 @@
 idf_component_register(
-  SRCS "env_esp.cpp" "database.cpp" "tag_processor.cpp" "db_task.cpp"
+  SRCS "env_esp.cpp" "database.cpp" "song.cpp" "db_task.cpp" "records.cpp"
   INCLUDE_DIRS "include"
-  REQUIRES "result" "span" "esp_psram" "fatfs" "libtags" "komihash")
+  REQUIRES "result" "span" "esp_psram" "fatfs" "libtags" "komihash" "cbor")
 
 target_compile_options(${COMPONENT_LIB} PRIVATE ${EXTRA_WARNINGS})
 

src/database/database.cpp

@@ -1,5 +1,9 @@
 #include "database.hpp"
+#include <stdint.h>
+#include <cstdint>
 #include <functional>
+#include <iomanip>
+#include <sstream>
 
 #include "esp_log.h"
 #include "ff.h"
@@ -8,19 +12,37 @@
 #include "db_task.hpp"
 #include "env_esp.hpp"
 #include "file_gatherer.hpp"
+#include "leveldb/db.h"
 #include "leveldb/iterator.h"
 #include "leveldb/options.h"
 #include "leveldb/slice.h"
+#include "leveldb/write_batch.h"
+#include "records.hpp"
 #include "result.hpp"
-#include "tag_processor.hpp"
+#include "song.hpp"
 
 namespace database {
 
 static SingletonEnv<leveldb::EspEnv> sEnv;
 static const char* kTag = "DB";
 
+static const std::string kSongIdKey("next_song_id");
+
 static std::atomic<bool> sIsDbOpen(false);
 
+template <typename Parser>
+auto IterateAndParse(leveldb::Iterator* it, std::size_t limit, Parser p)
+    -> void {
+  for (int i = 0; i < limit; i++) {
+    if (!it->Valid()) {
+      delete it;
+      break;
+    }
+    std::invoke(p, it->key(), it->value());
+    it->Next();
+  }
+}
+
 auto Database::Open() -> cpp::result<Database*, DatabaseError> {
   // TODO(jacqueline): Why isn't compare_and_exchange_* available?
   if (sIsDbOpen.exchange(true)) {
@@ -65,44 +87,205 @@ Database::~Database() {
   sIsDbOpen.store(false);
 }
 
-template <typename Parser>
-auto IterateAndParse(leveldb::Iterator* it, std::size_t limit, Parser p)
-    -> void {
-  for (int i = 0; i < limit; i++) {
-    if (!it->Valid()) {
-      break;
+auto Database::Update() -> std::future<void> {
+  return RunOnDbTask<void>([&]() -> void {
+    // Stage 1: verify all existing songs are still valid.
+    ESP_LOGI(kTag, "verifying existing songs");
+    const leveldb::Snapshot* snapshot = db_->GetSnapshot();
+    leveldb::ReadOptions read_options;
+    read_options.fill_cache = false;
+    read_options.snapshot = snapshot;
+    leveldb::Iterator* it = db_->NewIterator(read_options);
+    OwningSlice prefix = CreateDataPrefix();
+    it->Seek(prefix.slice);
+    while (it->Valid() && it->key().starts_with(prefix.slice)) {
+      std::optional<SongData> song = ParseDataValue(it->value());
+      if (!song) {
+        // The value was malformed. Drop this record.
+        ESP_LOGW(kTag, "dropping malformed metadata");
+        db_->Delete(leveldb::WriteOptions(), it->key());
+        it->Next();
+        continue;
+      }
+
+      if (song->is_tombstoned()) {
+        ESP_LOGW(kTag, "skipping tombstoned %lx", song->id());
+        it->Next();
+        continue;
+      }
+
+      SongTags tags;
+      if (!ReadAndParseTags(song->filepath(), &tags)) {
+        // We couldn't read the tags for this song. Either they were
+        // malformed, or perhaps the file is missing. Either way, tombstone
+        // this record.
+        ESP_LOGW(kTag, "entombing missing #%lx", song->id());
+        dbPutSongData(song->Entomb());
+        it->Next();
+        continue;
+      }
+
+      uint64_t new_hash = tags.Hash();
+      if (new_hash != song->tags_hash()) {
+        // This song's tags have changed. Since the filepath is exactly the
+        // same, we assume this is a legitimate correction. Update the
+        // database.
+        ESP_LOGI(kTag, "updating hash (%llx -> %llx)", song->tags_hash(),
+                 new_hash);
+        dbPutSongData(song->UpdateHash(new_hash));
+        dbPutHash(new_hash, song->id());
+      }
+
+      it->Next();
     }
-    std::invoke(p, it->key(), it->value());
-    it->Next();
-  }
-}
+    delete it;
+    db_->ReleaseSnapshot(snapshot);
 
-auto Database::Populate() -> std::future<void> {
-  return RunOnDbTask<void>([&]() -> void {
-    leveldb::WriteOptions opt;
-    opt.sync = true;
+    // Stage 2: search for newly added files.
+    ESP_LOGI(kTag, "scanning for new songs");
     FindFiles("", [&](const std::string& path) {
-      ESP_LOGI(kTag, "considering %s", path.c_str());
-      FileInfo info;
-      if (GetInfo(path, &info)) {
-        ESP_LOGI(kTag, "added as '%s'", info.title.c_str());
-        db_->Put(opt, "title:" + info.title, path);
+      SongTags tags;
+      if (!ReadAndParseTags(path, &tags)) {
+        // No parseable tags; skip this fiile.
+        return;
+      }
+
+      // Check for any existing record with the same hash.
+      uint64_t hash = tags.Hash();
+      OwningSlice key = CreateHashKey(hash);
+      std::optional<SongId> existing_hash;
+      std::string raw_entry;
+      if (db_->Get(leveldb::ReadOptions(), key.slice, &raw_entry).ok()) {
+        existing_hash = ParseHashValue(raw_entry);
+      }
+
+      if (!existing_hash) {
+        // We've never met this song before! Or we have, but the entry is
+        // malformed. Either way, record this as a new song.
+        SongId id = dbMintNewSongId();
+        ESP_LOGI(kTag, "recording new 0x%lx", id);
+        dbPutSong(id, path, hash);
+        return;
+      }
+
+      std::optional<SongData> existing_data = dbGetSongData(*existing_hash);
+      if (!existing_data) {
+        // We found a hash that matches, but there's no data record? Weird.
+        SongData new_data(*existing_hash, path, hash);
+        dbPutSongData(new_data);
+        return;
+      }
+
+      if (existing_data->is_tombstoned()) {
+        ESP_LOGI(kTag, "exhuming song %lu", existing_data->id());
+        dbPutSongData(existing_data->Exhume(path));
+      } else if (existing_data->filepath() != path) {
+        ESP_LOGW(kTag, "tag hash collision");
       }
     });
-    db_->Put(opt, "title:coolkeywithoutval", leveldb::Slice());
   });
 }
 
-auto parse_song(const leveldb::Slice& key, const leveldb::Slice& val)
+auto Database::Destroy() -> std::future<void> {
+  return RunOnDbTask<void>([&]() -> void {
+    const leveldb::Snapshot* snap = db_->GetSnapshot();
+    leveldb::ReadOptions options;
+    options.snapshot = snap;
+    leveldb::Iterator* it = db_->NewIterator(options);
+    it->SeekToFirst();
+    while (it->Valid()) {
+      db_->Delete(leveldb::WriteOptions(), it->key());
+      it->Next();
+    }
+    db_->ReleaseSnapshot(snap);
+  });
+}
+
+auto Database::dbMintNewSongId() -> SongId {
+  std::string val;
+  auto status = db_->Get(leveldb::ReadOptions(), kSongIdKey, &val);
+  if (!status.ok()) {
+    // TODO(jacqueline): check the db is actually empty.
+    ESP_LOGW(kTag, "error getting next id: %s", status.ToString().c_str());
+  }
+  SongId next_id = BytesToSongId(val);
+
+  if (!db_->Put(leveldb::WriteOptions(), kSongIdKey,
+                SongIdToBytes(next_id + 1).slice)
+           .ok()) {
+    ESP_LOGE(kTag, "failed to write next song id");
+  }
+
+  return next_id;
+}
+
+auto Database::dbEntomb(SongId id, uint64_t hash) -> void {
+  OwningSlice key = CreateHashKey(hash);
+  OwningSlice val = CreateHashValue(id);
+  if (!db_->Put(leveldb::WriteOptions(), key.slice, val.slice).ok()) {
+    ESP_LOGE(kTag, "failed to entomb #%llx (id #%lx)", hash, id);
+  }
+}
+
+auto Database::dbPutSongData(const SongData& s) -> void {
+  OwningSlice key = CreateDataKey(s.id());
+  OwningSlice val = CreateDataValue(s);
+  if (!db_->Put(leveldb::WriteOptions(), key.slice, val.slice).ok()) {
+    ESP_LOGE(kTag, "failed to write data for #%lx", s.id());
+  }
+}
+
+auto Database::dbGetSongData(SongId id) -> std::optional<SongData> {
+  OwningSlice key = CreateDataKey(id);
+  std::string raw_val;
+  if (!db_->Get(leveldb::ReadOptions(), key.slice, &raw_val).ok()) {
+    ESP_LOGW(kTag, "no key found for #%lx", id);
+    return {};
+  }
+  return ParseDataValue(raw_val);
+}
+
+auto Database::dbPutHash(const uint64_t& hash, SongId i) -> void {
+  OwningSlice key = CreateHashKey(hash);
+  OwningSlice val = CreateHashValue(i);
+  if (!db_->Put(leveldb::WriteOptions(), key.slice, val.slice).ok()) {
+    ESP_LOGE(kTag, "failed to write hash for #%lx", i);
+  }
+}
+
+auto Database::dbGetHash(const uint64_t& hash) -> std::optional<SongId> {
+  OwningSlice key = CreateHashKey(hash);
+  std::string raw_val;
+  if (!db_->Get(leveldb::ReadOptions(), key.slice, &raw_val).ok()) {
+    ESP_LOGW(kTag, "no key found for hash #%llx", hash);
+    return {};
+  }
+  return ParseHashValue(raw_val);
+}
+
+auto Database::dbPutSong(SongId id,
+                         const std::string& path,
+                         const uint64_t& hash) -> void {
+  dbPutSongData(SongData(id, path, hash));
+  dbPutHash(hash, id);
+}
+
+auto parse_song(const leveldb::Slice& key, const leveldb::Slice& value)
     -> std::optional<Song> {
-  Song s;
-  s.title = key.ToString();
-  return s;
+  std::optional<SongData> data = ParseDataValue(value);
+  if (!data) {
+    return {};
+  }
+  SongTags tags;
+  if (!ReadAndParseTags(data->filepath(), &tags)) {
+    return {};
+  }
+  return Song(*data, tags);
 }
 
 auto Database::GetSongs(std::size_t page_size) -> std::future<Result<Song>> {
   return RunOnDbTask<Result<Song>>([=, this]() -> Result<Song> {
-    return Query<Song>("title:", page_size, &parse_song);
+    return Query<Song>(CreateDataPrefix().slice, page_size, &parse_song);
   });
 }
 
@@ -114,4 +297,49 @@ auto Database::GetMoreSongs(std::size_t page_size, Continuation c)
   });
 }
 
+auto parse_dump(const leveldb::Slice& key, const leveldb::Slice& value)
+    -> std::optional<std::string> {
+  std::ostringstream stream;
+  stream << "key: ";
+  if (key.size() < 3 || key.data()[1] != '\0') {
+    stream << key.ToString().c_str();
+  } else {
+    std::string str = key.ToString();
+    for (size_t i = 0; i < str.size(); i++) {
+      if (i == 0) {
+        stream << str[i];
+      } else if (i == 1) {
+        stream << " / 0x";
+      } else {
+        stream << std::hex << std::setfill('0') << std::setw(2) << static_cast<int>(str[i]);
+      }
+    }
+    for (std::size_t i = 2; i < str.size(); i++) {
+    }
+  }
+  stream << "\tval: 0x";
+  std::string str = value.ToString();
+  for (int i = 0; i < value.size(); i++) {
+    stream << std::hex << std::setfill('0') << std::setw(2) << static_cast<int>(str[i]);
+  }
+  return stream.str();
+}
+
+auto Database::GetDump(std::size_t page_size)
+    -> std::future<Result<std::string>> {
+  leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
+  it->SeekToFirst();
+  return RunOnDbTask<Result<std::string>>([=, this]() -> Result<std::string> {
+    return Query<std::string>(it, page_size, &parse_dump);
+  });
+}
+
+auto Database::GetMoreDump(std::size_t page_size, Continuation c)
+    -> std::future<Result<std::string>> {
+  leveldb::Iterator* it = c.release();
+  return RunOnDbTask<Result<std::string>>([=, this]() -> Result<std::string> {
+    return Query<std::string>(it, page_size, &parse_dump);
+  });
+}
+
 }  // namespace database

src/database/include/database.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <stdint.h>
 #include <cstdint>
 #include <future>
 #include <memory>
@@ -13,46 +14,37 @@
 #include "leveldb/iterator.h"
 #include "leveldb/options.h"
 #include "leveldb/slice.h"
+#include "records.hpp"
 #include "result.hpp"
+#include "song.hpp"
 
 namespace database {
 
-struct Artist {
-  std::string name;
-};
-
-struct Album {
-  std::string name;
-};
-
-typedef uint64_t SongId_t;
-
-struct Song {
-  std::string title;
-  uint64_t id;
-};
-
-struct SongMetadata {};
-
 typedef std::unique_ptr<leveldb::Iterator> Continuation;
 
+/*
+ * Wrapper for a set of results from the database. Owns the list of results, as
+ * well as a continuation token that can be used to continue fetching more
+ * results if they were paginated.
+ */
 template <typename T>
 class Result {
  public:
-  auto values() -> std::unique_ptr<std::vector<T>> {
-    return std::move(values_);
-  }
+  auto values() -> std::vector<T>* { return values_.release(); }
   auto continuation() -> Continuation { return std::move(c_); }
   auto HasMore() -> bool { return c_->Valid(); }
 
+  Result(std::vector<T>* values, Continuation c)
+      : values_(values), c_(std::move(c)) {}
+
   Result(std::unique_ptr<std::vector<T>> values, Continuation c)
       : values_(std::move(values)), c_(std::move(c)) {}
 
   Result(Result&& other)
-      : values_(std::move(other.values_)), c_(std::move(other.c_)) {}
+      : values_(move(other.values_)), c_(std::move(other.c_)) {}
 
   Result operator=(Result&& other) {
-    return Result(other.values(), other.continuation());
+    return Result(other.values(), std::move(other.continuation()));
   }
 
   Result(const Result&) = delete;
@@ -73,30 +65,16 @@ class Database {
 
   ~Database();
 
-  auto Populate() -> std::future<void>;
-
-  auto GetArtists(std::size_t page_size) -> std::future<Result<Artist>>;
-  auto GetMoreArtists(std::size_t page_size, Continuation c)
-      -> std::future<Result<Artist>>;
-
-  auto GetAlbums(std::size_t page_size, std::optional<Artist> artist)
-      -> std::future<Result<Album>>;
-  auto GetMoreAlbums(std::size_t page_size, Continuation c)
-      -> std::future<Result<Album>>;
+  auto Update() -> std::future<void>;
+  auto Destroy() -> std::future<void>;
 
   auto GetSongs(std::size_t page_size) -> std::future<Result<Song>>;
-  auto GetSongs(std::size_t page_size, std::optional<Artist> artist)
-      -> std::future<Result<Song>>;
-  auto GetSongs(std::size_t page_size,
-                std::optional<Artist> artist,
-                std::optional<Album> album) -> std::future<Result<Song>>;
   auto GetMoreSongs(std::size_t page_size, Continuation c)
       -> std::future<Result<Song>>;
 
-  auto GetSongIds(std::optional<Artist> artist, std::optional<Album> album)
-      -> std::future<std::vector<SongId_t>>;
-  auto GetSongFilePath(SongId_t id) -> std::future<std::optional<std::string>>;
-  auto GetSongMetadata(SongId_t id) -> std::future<std::optional<SongMetadata>>;
+  auto GetDump(std::size_t page_size) -> std::future<Result<std::string>>;
+  auto GetMoreDump(std::size_t page_size, Continuation c)
+      -> std::future<Result<std::string>>;
 
   Database(const Database&) = delete;
   Database& operator=(const Database&) = delete;
@@ -107,6 +85,16 @@ class Database {
 
   Database(leveldb::DB* db, leveldb::Cache* cache);
 
+  auto dbMintNewSongId() -> SongId;
+  auto dbEntomb(SongId song, uint64_t hash) -> void;
+
+  auto dbPutSongData(const SongData& s) -> void;
+  auto dbGetSongData(SongId id) -> std::optional<SongData>;
+  auto dbPutHash(const uint64_t& hash, SongId i) -> void;
+  auto dbGetHash(const uint64_t& hash) -> std::optional<SongId>;
+  auto dbPutSong(SongId id, const std::string& path, const uint64_t& hash)
+      -> void;
+
   template <typename T>
   using Parser = std::function<std::optional<T>(const leveldb::Slice& key,
                                                 const leveldb::Slice& value)>;

src/database/include/records.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <leveldb/db.h>
+#include <stdint.h>
+#include <string>
+
+#include "leveldb/slice.h"
+#include "song.hpp"
+
+namespace database {
+
+class OwningSlice {
+ public:
+  std::string data;
+  leveldb::Slice slice;
+
+  explicit OwningSlice(std::string d);
+};
+
+auto CreateDataPrefix() -> OwningSlice;
+auto CreateDataKey(const SongId& id) -> OwningSlice;
+auto CreateDataValue(const SongData& song) -> OwningSlice;
+auto ParseDataValue(const leveldb::Slice& slice) -> std::optional<SongData>;
+
+auto CreateHashKey(const uint64_t& hash) -> OwningSlice;
+auto ParseHashValue(const leveldb::Slice&) -> std::optional<SongId>;
+auto CreateHashValue(SongId id) -> OwningSlice;
+
+auto SongIdToBytes(SongId id) -> OwningSlice;
+auto BytesToSongId(const std::string& bytes) -> SongId;
+
+}  // namespace database

src/database/include/song.hpp

@@ -0,0 +1,76 @@
+#pragma once
+
+#include <stdint.h>
+#include <cstdint>
+#include <optional>
+#include <string>
+
+#include "leveldb/db.h"
+#include "span.hpp"
+
+namespace database {
+
+typedef uint32_t SongId;
+
+enum Encoding { ENC_UNSUPPORTED, ENC_MP3 };
+
+struct SongTags {
+  Encoding encoding;
+  std::optional<std::string> title;
+  std::optional<std::string> artist;
+  std::optional<std::string> album;
+  auto Hash() const -> uint64_t;
+};
+
+auto ReadAndParseTags(const std::string& path, SongTags* out) -> bool;
+
+class SongData {
+ private:
+  const SongId id_;
+  const std::string filepath_;
+  const uint64_t tags_hash_;
+  const uint32_t play_count_;
+  const bool is_tombstoned_;
+
+ public:
+  SongData(SongId id, const std::string& path, uint64_t hash)
+      : id_(id),
+        filepath_(path),
+        tags_hash_(hash),
+        play_count_(0),
+        is_tombstoned_(false) {}
+  SongData(SongId id,
+           const std::string& path,
+           uint64_t hash,
+           uint32_t play_count,
+           bool is_tombstoned)
+      : id_(id),
+        filepath_(path),
+        tags_hash_(hash),
+        play_count_(play_count),
+        is_tombstoned_(is_tombstoned) {}
+
+  auto id() const -> SongId { return id_; }
+  auto filepath() const -> std::string { return filepath_; }
+  auto play_count() const -> uint32_t { return play_count_; }
+  auto tags_hash() const -> uint64_t { return tags_hash_; }
+  auto is_tombstoned() const -> bool { return is_tombstoned_; }
+
+  auto UpdateHash(uint64_t new_hash) const -> SongData;
+  auto Entomb() const -> SongData;
+  auto Exhume(const std::string& new_path) const -> SongData;
+};
+
+class Song {
+ public:
+  Song(SongData data, SongTags tags) : data_(data), tags_(tags) {}
+
+  auto data() -> const SongData& { return data_; }
+  auto tags() -> const SongTags& { return tags_; }
+
+ private:
+  const SongData data_;
+  const SongTags tags_;
+};
+
+}  // namespace database

src/database/include/tag_processor.hpp

@@ -1,16 +0,0 @@
-#pragma once
-
-#include <string>
-
-namespace database {
-
-struct FileInfo {
-  bool is_playable;
-  std::string artist;
-  std::string album;
-  std::string title;
-};
-
-auto GetInfo(const std::string& path, FileInfo* out) -> bool;
-
-}  // namespace database

src/database/records.cpp

@@ -0,0 +1,203 @@
+#include "records.hpp"
+
+#include <cbor.h>
+#include <esp_log.h>
+#include <stdint.h>
+
+#include <sstream>
+#include <vector>
+
+#include "song.hpp"
+
+namespace database {
+
+static const char* kTag = "RECORDS";
+
+static const char kDataPrefix = 'D';
+static const char kHashPrefix = 'H';
+static const char kFieldSeparator = '\0';
+
+template <typename T>
+auto cbor_encode(uint8_t** out_buf, T fn) -> std::size_t {
+  CborEncoder size_encoder;
+  cbor_encoder_init(&size_encoder, NULL, 0, 0);
+  std::invoke(fn, &size_encoder);
+  std::size_t buf_size = cbor_encoder_get_extra_bytes_needed(&size_encoder);
+  *out_buf = new uint8_t[buf_size];
+
+  CborEncoder encoder;
+  cbor_encoder_init(&encoder, *out_buf, buf_size, 0);
+  std::invoke(fn, &encoder);
+
+  return buf_size;
+}
+
+OwningSlice::OwningSlice(std::string d) : data(d), slice(data) {}
+
+auto CreateDataPrefix() -> OwningSlice {
+  char data[2] = {kDataPrefix, kFieldSeparator};
+  return OwningSlice({data, 2});
+}
+
+auto CreateDataKey(const SongId& id) -> OwningSlice {
+  std::ostringstream output;
+  output.put(kDataPrefix).put(kFieldSeparator);
+  output << SongIdToBytes(id).data;
+  return OwningSlice(output.str());
+}
+
+auto CreateDataValue(const SongData& song) -> OwningSlice {
+  uint8_t* buf;
+  std::size_t buf_len = cbor_encode(&buf, [&](CborEncoder* enc) {
+    CborEncoder array_encoder;
+    CborError err;
+    err = cbor_encoder_create_array(enc, &array_encoder, 5);
+    if (err != CborNoError && err != CborErrorOutOfMemory) {
+      ESP_LOGE(kTag, "encoding err %u", err);
+      return;
+    }
+    err = cbor_encode_int(&array_encoder, song.id());
+    if (err != CborNoError && err != CborErrorOutOfMemory) {
+      ESP_LOGE(kTag, "encoding err %u", err);
+      return;
+    }
+    err = cbor_encode_text_string(&array_encoder, song.filepath().c_str(),
+                                  song.filepath().size());
+    if (err != CborNoError && err != CborErrorOutOfMemory) {
+      ESP_LOGE(kTag, "encoding err %u", err);
+      return;
+    }
+    err = cbor_encode_uint(&array_encoder, song.tags_hash());
+    if (err != CborNoError && err != CborErrorOutOfMemory) {
+      ESP_LOGE(kTag, "encoding err %u", err);
+      return;
+    }
+    err = cbor_encode_int(&array_encoder, song.play_count());
+    if (err != CborNoError && err != CborErrorOutOfMemory) {
+      ESP_LOGE(kTag, "encoding err %u", err);
+      return;
+    }
+    err = cbor_encode_boolean(&array_encoder, song.is_tombstoned());
+    if (err != CborNoError && err != CborErrorOutOfMemory) {
+      ESP_LOGE(kTag, "encoding err %u", err);
+      return;
+    }
+    err = cbor_encoder_close_container(enc, &array_encoder);
+    if (err != CborNoError && err != CborErrorOutOfMemory) {
+      ESP_LOGE(kTag, "encoding err %u", err);
+      return;
+    }
+  });
+  std::string as_str(reinterpret_cast<char*>(buf), buf_len);
+  delete buf;
+  return OwningSlice(as_str);
+}
+
+auto ParseDataValue(const leveldb::Slice& slice) -> std::optional<SongData> {
+  CborParser parser;
+  CborValue container;
+  CborError err;
+  err = cbor_parser_init(reinterpret_cast<const uint8_t*>(slice.data()),
+                         slice.size(), 0, &parser, &container);
+  if (err != CborNoError) {
+    return {};
+  }
+
+  CborValue val;
+  err = cbor_value_enter_container(&container, &val);
+  if (err != CborNoError) {
+    return {};
+  }
+
+  uint64_t raw_int;
+  err = cbor_value_get_uint64(&val, &raw_int);
+  if (err != CborNoError) {
+    return {};
+  }
+  SongId id = raw_int;
+  err = cbor_value_advance(&val);
+  if (err != CborNoError) {
+    return {};
+  }
+
+  char* raw_path;
+  std::size_t len;
+  err = cbor_value_dup_text_string(&val, &raw_path, &len, &val);
+  if (err != CborNoError) {
+    return {};
+  }
+  std::string path(raw_path, len);
+  delete raw_path;
+
+  err = cbor_value_get_uint64(&val, &raw_int);
+  if (err != CborNoError) {
+    return {};
+  }
+  uint64_t hash = raw_int;
+  err = cbor_value_advance(&val);
+  if (err != CborNoError) {
+    return {};
+  }
+
+  err = cbor_value_get_uint64(&val, &raw_int);
+  if (err != CborNoError) {
+    return {};
+  }
+  uint32_t play_count = raw_int;
+  err = cbor_value_advance(&val);
+  if (err != CborNoError) {
+    return {};
+  }
+
+  bool is_tombstoned;
+  err = cbor_value_get_boolean(&val, &is_tombstoned);
+  if (err != CborNoError) {
+    return {};
+  }
+
+  return SongData(id, path, hash, play_count, is_tombstoned);
+}
+
+auto CreateHashKey(const uint64_t& hash) -> OwningSlice {
+  std::ostringstream output;
+  output.put(kHashPrefix).put(kFieldSeparator);
+
+  uint8_t buf[16];
+  CborEncoder enc;
+  cbor_encoder_init(&enc, buf, sizeof(buf), 0);
+  cbor_encode_uint(&enc, hash);
+  std::size_t len = cbor_encoder_get_buffer_size(&enc, buf);
+  output.write(reinterpret_cast<char*>(buf), len);
+
+  return OwningSlice(output.str());
+}
+
+auto ParseHashValue(const leveldb::Slice& slice) -> std::optional<SongId> {
+  return BytesToSongId(slice.ToString());
+}
+
+auto CreateHashValue(SongId id) -> OwningSlice {
+  return SongIdToBytes(id);
+}
+
+auto SongIdToBytes(SongId id) -> OwningSlice {
+  uint8_t buf[8];
+  CborEncoder enc;
+  cbor_encoder_init(&enc, buf, sizeof(buf), 0);
+  cbor_encode_uint(&enc, id);
+  std::size_t len = cbor_encoder_get_buffer_size(&enc, buf);
+  std::string as_str(reinterpret_cast<char*>(buf), len);
+  return OwningSlice(as_str);
+}
+
+auto BytesToSongId(const std::string& bytes) -> SongId {
+  CborParser parser;
+  CborValue val;
+  cbor_parser_init(reinterpret_cast<const uint8_t*>(bytes.data()), bytes.size(),
+                   0, &parser, &val);
+  uint64_t raw_id;
+  cbor_value_get_uint64(&val, &raw_id);
+  return raw_id;
+}
+
+}  // namespace database

src/database/song.cpp

@@ -1,9 +1,8 @@
-#include "tag_processor.hpp"
+#include "song.hpp"
 
 #include <esp_log.h>
 #include <ff.h>
 #include <komihash.h>
-#include <stdint.h>
 #include <tags.h>
 
 namespace database {
@@ -13,9 +12,7 @@ namespace libtags {
 struct Aux {
   FIL file;
   FILINFO info;
-  std::string artist;
-  std::string album;
-  std::string title;
+  SongTags* tags;
 };
 
 static int read(Tagctx* ctx, void* buf, int cnt) {
@@ -54,11 +51,11 @@ static void tag(Tagctx* ctx,
                 Tagread f) {
   Aux* aux = reinterpret_cast<Aux*>(ctx->aux);
   if (t == Ttitle) {
-    aux->title = v;
+    aux->tags->title = v;
   } else if (t == Tartist) {
-    aux->artist = v;
+    aux->tags->artist = v;
   } else if (t == Talbum) {
-    aux->album = v;
+    aux->tags->album = v;
   }
 }
 
@@ -69,11 +66,12 @@ static void toc(Tagctx* ctx, int ms, int offset) {}
 static const std::size_t kBufSize = 1024;
 static const char* kTag = "TAGS";
 
-auto GetInfo(const std::string& path, FileInfo* out) -> bool {
+auto ReadAndParseTags(const std::string& path, SongTags* out) -> bool {
   libtags::Aux aux;
+  aux.tags = out;
   if (f_stat(path.c_str(), &aux.info) != FR_OK ||
       f_open(&aux.file, path.c_str(), FA_READ) != FR_OK) {
-    ESP_LOGI(kTag, "failed to open file");
+    ESP_LOGW(kTag, "failed to open file %s", path.c_str());
     return false;
   }
   // Fine to have this on the stack; this is only called on the leveldb task.
@@ -90,28 +88,44 @@ auto GetInfo(const std::string& path, FileInfo* out) -> bool {
   f_close(&aux.file);
 
   if (res != 0) {
-    ESP_LOGI(kTag, "failed to parse tags");
+    // Parsing failed.
     return false;
   }
 
-  if (ctx.format == Fmp3) {
-    ESP_LOGI(kTag, "file is mp3");
-    ESP_LOGI(kTag, "artist: %s", aux.artist.c_str());
-    ESP_LOGI(kTag, "album: %s", aux.album.c_str());
-    ESP_LOGI(kTag, "title: %s", aux.title.c_str());
-    komihash_stream_t hash;
-    komihash_stream_init(&hash, 0);
-    komihash_stream_update(&hash, aux.artist.c_str(), aux.artist.length());
-    komihash_stream_update(&hash, aux.album.c_str(), aux.album.length());
-    komihash_stream_update(&hash, aux.title.c_str(), aux.title.length());
-    uint64_t final_hash = komihash_stream_final(&hash);
-    ESP_LOGI(kTag, "hash: %#llx", final_hash);
-    out->is_playable = true;
-    out->title = aux.title;
-    return true;
+  switch (ctx.format) {
+    case Fmp3:
+      out->encoding = ENC_MP3;
+      break;
+    default:
+      out->encoding = ENC_UNSUPPORTED;
   }
 
-  return false;
+  return true;
+}
+
+auto HashString(komihash_stream_t* stream, std::string str) -> void {
+  komihash_stream_update(stream, str.c_str(), str.length());
+}
+
+auto SongTags::Hash() const -> uint64_t {
+  komihash_stream_t stream;
+  komihash_stream_init(&stream, 0);
+  HashString(&stream, title.value_or(""));
+  HashString(&stream, artist.value_or(""));
+  HashString(&stream, album.value_or(""));
+  return komihash_stream_final(&stream);
+}
+
+auto SongData::UpdateHash(uint64_t new_hash) const -> SongData {
+  return SongData(id_, filepath_, new_hash, play_count_, is_tombstoned_);
+}
+
+auto SongData::Entomb() const -> SongData {
+  return SongData(id_, filepath_, tags_hash_, play_count_, true);
+}
+
+auto SongData::Exhume(const std::string& new_path) const -> SongData {
+  return SongData(id_, new_path, tags_hash_, play_count_, false);
 }
 
 }  // namespace database

src/dev_console/include/console.hpp

@@ -12,7 +12,7 @@ class Console {
   auto Launch() -> void;
 
  protected:
-  virtual auto GetStackSizeKiB() -> uint16_t { return 4; }
+  virtual auto GetStackSizeKiB() -> uint16_t { return 8; }
   virtual auto RegisterExtraComponents() -> void {}
 
  private:

src/main/app_console.cpp

@@ -154,7 +154,7 @@ int CmdDbInit(int argc, char** argv) {
     return 1;
   }
 
-  sInstance->database_->Populate().get();
+  sInstance->database_->Update();
 
   return 0;
 }
@@ -177,11 +177,11 @@ int CmdDbSongs(int argc, char** argv) {
   }
 
   database::Result<database::Song> res =
-      sInstance->database_->GetSongs(10).get();
+      sInstance->database_->GetSongs(20).get();
   while (true) {
-    std::unique_ptr<std::vector<database::Song>> r = res.values();
+    std::unique_ptr<std::vector<database::Song>> r(res.values());
     for (database::Song s : *r) {
-      std::cout << s.title << std::endl;
+      std::cout << s.tags().title.value_or("[BLANK]") << std::endl;
     }
     if (res.HasMore()) {
       res = sInstance->database_->GetMoreSongs(10, res.continuation()).get();
@@ -202,6 +202,43 @@ void RegisterDbSongs() {
   esp_console_cmd_register(&cmd);
 }
 
+int CmdDbDump(int argc, char** argv) {
+  static const std::string usage = "usage: db_dump";
+  if (argc != 1) {
+    std::cout << usage << std::endl;
+    return 1;
+  }
+
+  std::cout << "=== BEGIN DUMP ===" << std::endl;
+
+  database::Result<std::string> res = sInstance->database_->GetDump(20).get();
+  while (true) {
+    std::unique_ptr<std::vector<std::string>> r(res.values());
+    if (r == nullptr) {
+      break;
+    }
+    for (std::string s : *r) {
+      std::cout << s << std::endl;
+    }
+    if (res.HasMore()) {
+      res = sInstance->database_->GetMoreDump(20, res.continuation()).get();
+    }
+  }
+
+  std::cout << "=== END DUMP ===" << std::endl;
+
+  return 0;
+}
+
+void RegisterDbDump() {
+  esp_console_cmd_t cmd{.command = "db_dump",
+                        .help = "prints every key/value pair in the db",
+                        .hint = NULL,
+                        .func = &CmdDbDump,
+                        .argtable = NULL};
+  esp_console_cmd_register(&cmd);
+}
+
 AppConsole::AppConsole(audio::AudioPlayback* playback,
                        database::Database* database)
     : playback_(playback), database_(database) {
@@ -219,6 +256,7 @@ auto AppConsole::RegisterExtraComponents() -> void {
   RegisterAudioStatus();
   RegisterDbInit();
   RegisterDbSongs();
+  RegisterDbDump();
 }
 
 }  // namespace console