From eacea59e8a3f9602ed06834a8edc4e6ab18a4bb9 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Tue, 30 Jan 2024 11:03:31 +1100 Subject: [PATCH] Do more to avoid and recover from partial db updates - do not power off in an update is in progress - explicitly store last update time, rather than deriving it from unchanged tracks. --- src/database/database.cpp | 59 +++++++++++++++++++++++---- src/database/include/database.hpp | 6 +++ src/system_fsm/include/system_fsm.hpp | 3 ++ src/system_fsm/running.cpp | 7 ++++ src/system_fsm/system_fsm.cpp | 3 +- 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/database/database.cpp b/src/database/database.cpp index fa4f621e..bb26bc96 100644 --- a/src/database/database.cpp +++ b/src/database/database.cpp @@ -22,6 +22,7 @@ #include "collation.hpp" #include "cppbor.h" +#include "cppbor_parse.h" #include "esp_log.h" #include "ff.h" #include "freertos/projdefs.h" @@ -60,6 +61,7 @@ static const uint8_t kCurrentDbVersion = 5; static const char kKeyCustom[] = "U\0"; static const char kKeyCollator[] = "collator"; static const char kKeyTrackId[] = "next_track_id"; +static const char kKeyLastUpdate[] = "last_update"; static std::atomic sIsDbOpen(false); @@ -188,7 +190,8 @@ Database::Database(leveldb::DB* db, cache_(cache), file_gatherer_(file_gatherer), tag_parser_(tag_parser), - collator_(collator) {} + collator_(collator), + is_updating_(false) {} Database::~Database() { // Delete db_ first so that any outstanding background work finishes before @@ -272,11 +275,18 @@ auto Database::getIndexes() -> std::vector { } auto Database::updateIndexes() -> void { + if (is_updating_.exchange(true)) { + return; + } events::Ui().Dispatch(event::UpdateStarted{}); + events::System().Dispatch(event::UpdateStarted{}); + leveldb::ReadOptions read_options; read_options.fill_cache = false; - std::pair newest_track{0, 0}; + std::pair last_update = dbGetLastUpdate(); + ESP_LOGI(kTag, "last update was at %u,%u", last_update.first, + last_update.second); // Stage 1: verify all existing tracks are still valid. ESP_LOGI(kTag, "verifying existing tracks"); @@ -317,7 +327,6 @@ auto Database::updateIndexes() -> void { modified_at = {info.fdate, info.ftime}; } if (modified_at == track->modified_at) { - newest_track = std::max(modified_at, newest_track); continue; } else { track->modified_at = modified_at; @@ -356,12 +365,10 @@ auto Database::updateIndexes() -> void { } } - ESP_LOGI(kTag, "newest unmodified was at %u,%u", newest_track.first, - newest_track.second); - // Stage 2: search for newly added files. ESP_LOGI(kTag, "scanning for new tracks"); uint64_t num_processed = 0; + std::pair newest_track = last_update; file_gatherer_.FindFiles("", [&](const std::string& path, const FILINFO& info) { num_processed++; @@ -371,9 +378,10 @@ auto Database::updateIndexes() -> void { }); std::pair modified{info.fdate, info.ftime}; - if (modified < newest_track) { + if (modified < last_update) { return; } + newest_track = std::max(modified, newest_track); std::shared_ptr tags = tag_parser_.ReadAndParseTags(path); if (!tags || tags->encoding() == Container::kUnsupported) { @@ -442,7 +450,44 @@ auto Database::updateIndexes() -> void { tags->album().value_or("no album").c_str()); } }); + dbSetLastUpdate(newest_track); + ESP_LOGI(kTag, "newest track was at %u,%u", newest_track.first, + newest_track.second); + + is_updating_ = false; events::Ui().Dispatch(event::UpdateFinished{}); + events::System().Dispatch(event::UpdateFinished{}); +} + +auto Database::isUpdating() -> bool { + return is_updating_; +} + +auto Database::dbGetLastUpdate() -> std::pair { + std::string raw; + if (!db_->Get(leveldb::ReadOptions{}, kKeyLastUpdate, &raw).ok()) { + return {0, 0}; + } + auto [res, unused, err] = cppbor::parseWithViews( + reinterpret_cast(raw.data()), raw.size()); + if (!res || res->type() != cppbor::ARRAY) { + return {0, 0}; + } + auto as_arr = res->asArray(); + if (as_arr->size() != 2 || as_arr->get(0)->type() != cppbor::UINT || + as_arr->get(1)->type() != cppbor::UINT) { + return {0, 0}; + } + return {as_arr->get(0)->asUint()->unsignedValue(), + as_arr->get(1)->asUint()->unsignedValue()}; +} + +auto Database::dbSetLastUpdate(std::pair time) -> void { + auto encoding = cppbor::Array{ + cppbor::Uint{time.first}, + cppbor::Uint{time.second}, + }; + db_->Put(leveldb::WriteOptions{}, kKeyLastUpdate, encoding.toString()); } auto Database::dbMintNewTrackId() -> TrackId { diff --git a/src/database/include/database.hpp b/src/database/include/database.hpp index cb4064fb..c4da3dc1 100644 --- a/src/database/include/database.hpp +++ b/src/database/include/database.hpp @@ -78,6 +78,7 @@ class Database { auto getIndexes() -> std::vector; auto updateIndexes() -> void; + auto isUpdating() -> bool; // Cannot be copied or moved. Database(const Database&) = delete; @@ -96,12 +97,17 @@ class Database { ITagParser& tag_parser_; locale::ICollator& collator_; + std::atomic is_updating_; + Database(leveldb::DB* db, leveldb::Cache* cache, IFileGatherer& file_gatherer, ITagParser& tag_parser, locale::ICollator& collator); + auto dbGetLastUpdate() -> std::pair; + auto dbSetLastUpdate(std::pair) -> void; + auto dbMintNewTrackId() -> TrackId; auto dbEntomb(TrackId track, uint64_t hash) -> void; diff --git a/src/system_fsm/include/system_fsm.hpp b/src/system_fsm/include/system_fsm.hpp index 2db1cddc..1e340711 100644 --- a/src/system_fsm/include/system_fsm.hpp +++ b/src/system_fsm/include/system_fsm.hpp @@ -13,6 +13,7 @@ #include "battery.hpp" #include "bluetooth.hpp" #include "database.hpp" +#include "db_events.hpp" #include "display.hpp" #include "gpios.hpp" #include "nvs.hpp" @@ -60,6 +61,7 @@ class SystemState : public tinyfsm::Fsm { virtual void react(const StorageError&) {} virtual void react(const KeyLockChanged&) {} virtual void react(const SdDetectChanged&) {} + virtual void react(const database::event::UpdateFinished&) {} virtual void react(const audio::PlaybackFinished&) {} virtual void react(const internal::IdleTimeout&) {} @@ -98,6 +100,7 @@ class Running : public SystemState { void react(const KeyLockChanged&) override; void react(const SdDetectChanged&) override; void react(const audio::PlaybackFinished&) override; + void react(const database::event::UpdateFinished&) override; using SystemState::react; diff --git a/src/system_fsm/running.cpp b/src/system_fsm/running.cpp index 9e10f9ec..ec448657 100644 --- a/src/system_fsm/running.cpp +++ b/src/system_fsm/running.cpp @@ -7,6 +7,7 @@ #include "app_console.hpp" #include "audio_events.hpp" #include "database.hpp" +#include "db_events.hpp" #include "file_gatherer.hpp" #include "freertos/projdefs.h" #include "result.hpp" @@ -47,6 +48,12 @@ void Running::react(const audio::PlaybackFinished& ev) { } } +void Running::react(const database::event::UpdateFinished&) { + if (IdleCondition()) { + transit(); + } +} + void Running::react(const SdDetectChanged& ev) { if (ev.has_sd_card) { if (!sStorage && mountStorage()) { diff --git a/src/system_fsm/system_fsm.cpp b/src/system_fsm/system_fsm.cpp index df750e80..977f4a6d 100644 --- a/src/system_fsm/system_fsm.cpp +++ b/src/system_fsm/system_fsm.cpp @@ -95,7 +95,8 @@ void SystemState::react(const internal::SamdInterrupt&) { } auto SystemState::IdleCondition() -> bool { - return sServices->gpios().IsLocked() && + auto db = sServices->database().lock(); + return sServices->gpios().IsLocked() && !(db && db->isUpdating()) && audio::AudioState::is_in_state(); }