Make FileGatherer shaped more like a normal iterator

custom
jacqueline 9 months ago
parent 30aaefca64
commit 28cf749951
  1. 1
      src/tangara/audio/audio_fsm.cpp
  2. 34
      src/tangara/database/database.cpp
  3. 6
      src/tangara/database/database.hpp
  4. 72
      src/tangara/database/file_gatherer.cpp
  5. 34
      src/tangara/database/file_gatherer.hpp
  6. 84
      src/tangara/database/track_finder.cpp
  7. 33
      src/tangara/database/track_finder.hpp
  8. 1
      src/tangara/input/lvgl_input_driver.cpp
  9. 4
      src/tangara/system_fsm/idle.cpp
  10. 9
      src/tangara/system_fsm/running.cpp

@ -9,6 +9,7 @@
#include <cstdint>
#include <future>
#include <memory>
#include <sstream>
#include <variant>
#include "audio/audio_source.hpp"

@ -24,6 +24,7 @@
#include "cppbor.h"
#include "cppbor_parse.h"
#include "database/index.hpp"
#include "database/track_finder.hpp"
#include "debug.hpp"
#include "esp_log.h"
#include "esp_timer.h"
@ -40,7 +41,6 @@
#include "database/db_events.hpp"
#include "database/env_esp.hpp"
#include "database/file_gatherer.hpp"
#include "database/records.hpp"
#include "database/tag_parser.hpp"
#include "database/track.hpp"
@ -122,8 +122,7 @@ static auto CheckDatabase(leveldb::DB& db, locale::ICollator& col) -> bool {
return true;
}
auto Database::Open(IFileGatherer& gatherer,
ITagParser& parser,
auto Database::Open(ITagParser& parser,
locale::ICollator& collator,
tasks::WorkerPool& bg_worker)
-> cpp::result<Database*, DatabaseError> {
@ -168,8 +167,7 @@ auto Database::Open(IFileGatherer& gatherer,
}
ESP_LOGI(kTag, "Database opened successfully");
return new Database(db, cache.release(), gatherer, parser,
collator);
return new Database(db, cache.release(), parser, collator);
})
.get();
}
@ -182,12 +180,10 @@ auto Database::Destroy() -> void {
Database::Database(leveldb::DB* db,
leveldb::Cache* cache,
IFileGatherer& file_gatherer,
ITagParser& tag_parser,
locale::ICollator& collator)
: db_(db),
cache_(cache),
file_gatherer_(file_gatherer),
tag_parser_(tag_parser),
collator_(collator),
is_updating_(false) {
@ -401,7 +397,11 @@ auto Database::updateIndexes() -> void {
// Stage 2: search for newly added files.
ESP_LOGI(kTag, "scanning for new tracks");
uint64_t num_files = 0;
file_gatherer_.FindFiles("", [&](std::string_view path, const FILINFO& info) {
auto track_finder = std::make_shared<TrackFinder>("");
FILINFO info;
while (auto path = track_finder->next(info)) {
num_files++;
events::Ui().Dispatch(event::UpdateProgress{
.stage = event::UpdateProgress::Stage::kScanningForNewTracks,
@ -409,15 +409,15 @@ auto Database::updateIndexes() -> void {
});
std::string unused;
if (db_->Get(read_options, EncodePathKey(path), &unused).ok()) {
if (db_->Get(read_options, EncodePathKey(*path), &unused).ok()) {
// This file is already in the database; skip it.
return;
continue;
}
std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags(path);
std::shared_ptr<TrackTags> tags = tag_parser_.ReadAndParseTags(*path);
if (!tags || tags->encoding() == Container::kUnsupported) {
// No parseable tags; skip this fiile.
return;
continue;
}
// Check for any existing track with the same hash.
@ -438,14 +438,14 @@ auto Database::updateIndexes() -> void {
if (!data) {
data = std::make_shared<TrackData>();
data->id = *existing_id;
} else if (data->filepath != path) {
} else if (std::string_view{data->filepath} != *path) {
ESP_LOGW(kTag, "hash collision: %s, %s, %s",
tags->title().value_or("no title").c_str(),
tags->artist().value_or("no artist").c_str(),
tags->album().value_or("no album").c_str());
// Don't commit anything if there's a hash collision, since we're
// likely to make a big mess.
return;
continue;
}
} else {
num_new_tracks++;
@ -454,7 +454,7 @@ auto Database::updateIndexes() -> void {
}
// Make sure the file-based metadata on the TrackData is up to date.
data->filepath = path;
data->filepath = *path;
data->tags_hash = hash;
data->modified_at = {info.fdate, info.ftime};
@ -467,10 +467,10 @@ auto Database::updateIndexes() -> void {
dbCreateIndexesForTrack(*data, *tags, batch);
batch.Put(EncodeDataKey(data->id), EncodeDataValue(*data));
batch.Put(EncodeHashKey(data->tags_hash), EncodeHashValue(data->id));
batch.Put(EncodePathKey(path), TrackIdToBytes(data->id));
batch.Put(EncodePathKey(*path), TrackIdToBytes(data->id));
db_->Write(leveldb::WriteOptions(), &batch);
});
};
uint64_t end_time = esp_timer_get_time();

@ -19,7 +19,6 @@
#include "collation.hpp"
#include "cppbor.h"
#include "database/file_gatherer.hpp"
#include "database/index.hpp"
#include "database/records.hpp"
#include "database/tag_parser.hpp"
@ -56,8 +55,7 @@ class Database {
ALREADY_OPEN,
FAILED_TO_OPEN,
};
static auto Open(IFileGatherer& file_gatherer,
ITagParser& tag_parser,
static auto Open(ITagParser& tag_parser,
locale::ICollator& collator,
tasks::WorkerPool& bg_worker)
-> cpp::result<Database*, DatabaseError>;
@ -96,7 +94,6 @@ class Database {
leveldb::Cache* cache_;
// Not owned.
IFileGatherer& file_gatherer_;
ITagParser& tag_parser_;
locale::ICollator& collator_;
@ -105,7 +102,6 @@ class Database {
Database(leveldb::DB* db,
leveldb::Cache* cache,
IFileGatherer& file_gatherer,
ITagParser& tag_parser,
locale::ICollator& collator);

@ -1,72 +0,0 @@
/*
* Copyright 2023 jacqueline <me@jacqueline.id.au>
*
* SPDX-License-Identifier: GPL-3.0-only
*/
#include "database/file_gatherer.hpp"
#include <deque>
#include <functional>
#include <sstream>
#include <string>
#include "ff.h"
#include "drivers/spi.hpp"
#include "memory_resource.hpp"
namespace database {
static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR");
auto FileGathererImpl::FindFiles(
const std::string& root,
std::function<void(std::string_view, const FILINFO&)> cb) -> void {
std::pmr::deque<std::pmr::string> to_explore{&memory::kSpiRamResource};
to_explore.push_back({root.data(), root.size()});
while (!to_explore.empty()) {
auto next_path_str = to_explore.front();
to_explore.pop_front();
const TCHAR* next_path = static_cast<const TCHAR*>(next_path_str.c_str());
FF_DIR dir;
FRESULT res = f_opendir(&dir, next_path);
if (res != FR_OK) {
// TODO: log.
continue;
}
for (;;) {
FILINFO info;
res = f_readdir(&dir, &info);
if (res != FR_OK || info.fname[0] == 0) {
// No more files in the directory.
break;
} else if (info.fattrib & (AM_HID | AM_SYS) || info.fname[0] == '.') {
// System or hidden file. Ignore it and move on.
continue;
} else {
std::pmr::string full_path{&memory::kSpiRamResource};
full_path += next_path_str;
full_path += "/";
full_path += info.fname;
if (info.fattrib & AM_DIR) {
// This is a directory. Add it to the explore queue.
to_explore.push_back(full_path);
} else {
// This is a file! Let the callback know about it.
// std::invoke(cb, full_path.str(), info);
std::invoke(cb, full_path, info);
}
}
}
f_closedir(&dir);
}
}
} // namespace database

@ -1,34 +0,0 @@
/*
* Copyright 2023 jacqueline <me@jacqueline.id.au>
*
* SPDX-License-Identifier: GPL-3.0-only
*/
#pragma once
#include <deque>
#include <functional>
#include <sstream>
#include <string>
#include "ff.h"
namespace database {
class IFileGatherer {
public:
virtual ~IFileGatherer() {};
virtual auto FindFiles(
const std::string& root,
std::function<void(std::string_view, const FILINFO&)> cb) -> void = 0;
};
class FileGathererImpl : public IFileGatherer {
public:
virtual auto FindFiles(const std::string& root,
std::function<void(std::string_view, const FILINFO&)>
cb) -> void override;
};
} // namespace database

@ -0,0 +1,84 @@
/*
* Copyright 2023 jacqueline <me@jacqueline.id.au>
*
* SPDX-License-Identifier: GPL-3.0-only
*/
#include "database/track_finder.hpp"
#include <deque>
#include <functional>
#include <memory>
#include <mutex>
#include <sstream>
#include <string>
#include <string_view>
#include "database/track_finder.hpp"
#include "ff.h"
#include "drivers/spi.hpp"
#include "memory_resource.hpp"
namespace database {
static_assert(sizeof(TCHAR) == sizeof(char), "TCHAR must be CHAR");
TrackFinder::TrackFinder(std::string_view root)
: to_explore_(&memory::kSpiRamResource) {
to_explore_.push_back({root.data(), root.size()});
}
auto TrackFinder::next(FILINFO& out_info) -> std::optional<std::string> {
std::scoped_lock<std::mutex> lock{mut_};
while (!to_explore_.empty() || current_) {
if (!current_) {
current_.emplace();
// Get the next directory to iterate through.
current_->first = to_explore_.front();
to_explore_.pop_front();
const TCHAR* next_path =
static_cast<const TCHAR*>(current_->first.data());
// Open it for iterating.
FRESULT res = f_opendir(&current_->second, next_path);
if (res != FR_OK) {
current_.reset();
continue;
}
}
FILINFO info;
FRESULT res = f_readdir(&current_->second, &info);
if (res != FR_OK || info.fname[0] == 0) {
// No more files in the directory.
f_closedir(&current_->second);
current_.reset();
continue;
} else if (info.fattrib & (AM_HID | AM_SYS) || info.fname[0] == '.') {
// System or hidden file. Ignore it and move on.
continue;
} else {
// A valid file or folder.
std::pmr::string full_path{&memory::kSpiRamResource};
full_path += current_->first;
full_path += "/";
full_path += info.fname;
if (info.fattrib & AM_DIR) {
// This is a directory. Add it to the explore queue.
to_explore_.push_back(full_path);
} else {
// This is a file! We can return now.
out_info = info;
return {{full_path.data(), full_path.size()}};
}
}
}
// Out of things to explore.
return {};
}
} // namespace database

@ -0,0 +1,33 @@
/*
* Copyright 2023 jacqueline <me@jacqueline.id.au>
*
* SPDX-License-Identifier: GPL-3.0-only
*/
#pragma once
#include <deque>
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <sstream>
#include <string>
#include "ff.h"
namespace database {
class TrackFinder {
public:
TrackFinder(std::string_view root);
auto next(FILINFO&) -> std::optional<std::string>;
private:
std::mutex mut_;
std::pmr::deque<std::pmr::string> to_explore_;
std::optional<std::pair<std::pmr::string, FF_DIR>> current_;
};
} // namespace database

@ -8,6 +8,7 @@
#include <cstdint>
#include <memory>
#include <sstream>
#include <variant>
#include "core/lv_group.h"

@ -4,8 +4,9 @@
* SPDX-License-Identifier: GPL-3.0-only
*/
#include "ui/ui_fsm.hpp"
#include "app_console/app_console.hpp"
#include "database/file_gatherer.hpp"
#include "drivers/gpios.hpp"
#include "freertos/portmacro.h"
#include "freertos/projdefs.h"
@ -17,7 +18,6 @@
#include "events/event_queue.hpp"
#include "system_fsm/system_events.hpp"
#include "system_fsm/system_fsm.hpp"
#include "ui/ui_fsm.hpp"
namespace system_fsm {
namespace states {

@ -8,7 +8,6 @@
#include "audio/audio_events.hpp"
#include "database/database.hpp"
#include "database/db_events.hpp"
#include "database/file_gatherer.hpp"
#include "drivers/gpios.hpp"
#include "drivers/spi.hpp"
#include "ff.h"
@ -36,8 +35,6 @@ static void timer_callback(TimerHandle_t timer) {
events::System().Dispatch(internal::UnmountTimeout{});
}
static database::IFileGatherer* sFileGatherer;
void Running::entry() {
if (!sUnmountTimer) {
sUnmountTimer = xTimerCreate("unmount_timeout", kTicksBeforeUnmount, false,
@ -174,10 +171,8 @@ auto Running::mountStorage() -> void {
sStorage.reset(storage_res.value());
ESP_LOGI(kTag, "opening database");
sFileGatherer = new database::FileGathererImpl();
auto database_res =
database::Database::Open(*sFileGatherer, sServices->tag_parser(),
sServices->collator(), sServices->bg_worker());
auto database_res = database::Database::Open(
sServices->tag_parser(), sServices->collator(), sServices->bg_worker());
if (database_res.has_error()) {
unmountStorage();
return;

Loading…
Cancel
Save