From 083f4011aa740d492d9a9ceb07c7228003f5ad39 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Wed, 26 Apr 2023 12:27:11 +1000 Subject: [PATCH] removed unused raw db stuff --- src/database/CMakeLists.txt | 2 +- src/database/database.cpp | 39 +++++++- src/database/include/database.hpp | 3 + src/database/include/table.hpp | 87 ---------------- src/database/include/table_reader.hpp | 53 ---------- src/database/include/table_writer.hpp | 5 - src/database/table.cpp | 137 -------------------------- src/main/main.cpp | 8 +- 8 files changed, 49 insertions(+), 285 deletions(-) delete mode 100644 src/database/include/table.hpp delete mode 100644 src/database/include/table_reader.hpp delete mode 100644 src/database/include/table_writer.hpp delete mode 100644 src/database/table.cpp diff --git a/src/database/CMakeLists.txt b/src/database/CMakeLists.txt index f66578cb..01365bf7 100644 --- a/src/database/CMakeLists.txt +++ b/src/database/CMakeLists.txt @@ -1,5 +1,5 @@ idf_component_register( - SRCS "table.cpp" "env_esp.cpp" "database.cpp" + SRCS "env_esp.cpp" "database.cpp" INCLUDE_DIRS "include" REQUIRES "result" "span" "esp_psram" "fatfs") diff --git a/src/database/database.cpp b/src/database/database.cpp index 558f63ce..cfac77af 100644 --- a/src/database/database.cpp +++ b/src/database/database.cpp @@ -1,13 +1,16 @@ #include "database.hpp" #include "esp_log.h" +#include "ff.h" #include "leveldb/cache.h" #include "env_esp.hpp" +#include "leveldb/options.h" namespace database { static SingletonEnv sEnv; +static const char *kTag = "DB"; auto Database::Open() -> cpp::result { leveldb::DB* db; @@ -23,7 +26,7 @@ auto Database::Open() -> cpp::result { auto status = leveldb::DB::Open(options, "/.db", &db); if (!status.ok()) { delete cache; - ESP_LOGE("DB", "failed to open db, status %s", status.ToString().c_str()); + ESP_LOGE(kTag, "failed to open db, status %s", status.ToString().c_str()); return cpp::fail(FAILED_TO_OPEN); } @@ -35,4 +38,38 @@ Database::Database(leveldb::DB* db, leveldb::Cache* cache) Database::~Database() {} +FRESULT scan_files(const std::string &path) { + FRESULT res; + FF_DIR dir; + static FILINFO fno; + + res = f_opendir(&dir, path.c_str()); + if (res == FR_OK) { + for (;;) { + res = f_readdir(&dir, &fno); + if (res != FR_OK || fno.fname[0] == 0) break; + if (fno.fname[0] == '.') continue; + if (fno.fattrib & AM_DIR) { + std::string new_path = path + "/" + fno.fname; + res = scan_files(new_path); + if (res != FR_OK) break; + } else { + ESP_LOGI(kTag, "found %s", fno.fname); + } + } + f_closedir(&dir); + } + + return res; +} + +auto Database::Initialise() -> void { + // TODO(jacqueline): Abstractions lol + scan_files("/"); +} + +auto Database::Update() -> void { + // TODO(jacqueline): Incremental updates! +} + } // namespace database diff --git a/src/database/include/database.hpp b/src/database/include/database.hpp index cfef0a7d..b9df5fd4 100644 --- a/src/database/include/database.hpp +++ b/src/database/include/database.hpp @@ -17,6 +17,9 @@ class Database { ~Database(); + auto Initialise() -> void; + auto Update() -> void; + private: std::unique_ptr db_; std::unique_ptr cache_; diff --git a/src/database/include/table.hpp b/src/database/include/table.hpp deleted file mode 100644 index 438c23b6..00000000 --- a/src/database/include/table.hpp +++ /dev/null @@ -1,87 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "esp32/himem.h" -#include "ff.h" -#include "span.hpp" -#include "sys/_stdint.h" - -namespace database { - -// Types used for indexing into files on disk. These should, at minimum, match -// the size of the types that the underlying filesystem uses to address within -// files. FAT32 uses 32 bit address. If we drop this and just support exFAT, we -// can change these to 64 bit types. -typedef uint32_t Index_t; -typedef Index_t IndexOffset_t; - -// The amount of memory that will be used to page database columns in from disk. -// Currently we only use a single 'page' in PSRAM per column, but with some -// refactoring we could easily page more. -// Keep this value 32KiB-aligned for himem compatibility. -extern const std::size_t kRamBlockSize; - -struct DatabaseHeader { - uint32_t magic_number; - uint16_t db_version; - Index_t num_indices; -}; - -struct DatabaseEntry { - uint8_t type; - std::string path; - - std::string title; - std::string album; - std::string artist; - std::string album_artist; -}; - -struct IndexEntry { - uint8_t type; - IndexOffset_t path; - - IndexOffset_t title; - IndexOffset_t album; - IndexOffset_t artist; - IndexOffset_t album_artist; -}; - -struct RowData { - std::unique_ptr arr; - std::size_t length; -}; - -// Representation of a single column of data. Each column is simply a tightly -// packed list of [size, [bytes, ...]] pairs. Callers are responsible for -// parsing and encoding the actual bytes themselves. -class Column { - public: - static auto Open(std::string) -> std::optional; - - Column(FIL file, std::size_t file_size); - ~Column(); - - auto ReadDataAtOffset(esp_himem_rangehandle_t, IndexOffset_t) -> RowData; - auto AppendRow(cpp::span row) -> bool; - auto FlushChanges() -> void; - - private: - FIL file_; - IndexOffset_t length_; - - esp_himem_handle_t block_; - std::pair loaded_range_; - - auto IsOffsetLoaded(IndexOffset_t offset) -> bool; - auto LoadOffsetFromDisk(cpp::span dest, IndexOffset_t offset) - -> bool; -}; - -} // namespace database diff --git a/src/database/include/table_reader.hpp b/src/database/include/table_reader.hpp deleted file mode 100644 index 9f7cc4ee..00000000 --- a/src/database/include/table_reader.hpp +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include - -#include "result.hpp" -#include "span.hpp" - -#include "table.hpp" - -namespace database { - -class TableReader { - public: - enum ReadError { - OUT_OF_RANGE, - IO_ERROR, - PARSE_ERROR, - }; - - auto ReadEntryAtIndex(Index_t index) -> cpp::result; - - template - auto ReadColumnOffsetAtIndex(Column col, Index_t index) - -> cpp::result; - - template - auto ParseColumnAtIndex(Column col, Index_t index) - -> cpp::result { - return ReadColumnOffsetAtIndex(col, index).map([&](IndexOffset_t offset) { - return ReadColumnAtOffset(col, offset); - }); - } - - template - auto ParseColumnAtOffset(Column col, IndexOffset_t offset) - -> cpp::result { - return ReadDataAtOffset(col.Filename(), offset) - .flat_map([&](cpp::span data) { - auto res = = col.ParseValue(data); - if (res) { - return *res; - } else { - return cpp::fail(PARSE_ERROR); - } - }); - } - - private: - auto ReadDataAtOffset(std::string filename, IndexOffset_t offset) - -> cpp::span; -}; - -} // namespace database diff --git a/src/database/include/table_writer.hpp b/src/database/include/table_writer.hpp deleted file mode 100644 index 9e01dd9d..00000000 --- a/src/database/include/table_writer.hpp +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -#include "table.hpp" - -namespace database {} // namespace database diff --git a/src/database/table.cpp b/src/database/table.cpp deleted file mode 100644 index de425837..00000000 --- a/src/database/table.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#include "table.hpp" - -#include -#include - -#include "esp32/himem.h" -#include "esp_err.h" -#include "ff.h" - -namespace database { - -const std::size_t kRamBlockSize = 32 * 1024; - -auto Column::Open(std::string path) -> std::optional { - FILINFO info; - FRESULT res = f_stat(path.c_str(), &info); - if (res != FR_OK) { - return {}; - } - - FIL file; - res = f_open(&file, path.c_str(), FA_READ | FA_WRITE); - if (res != FR_OK) { - return {}; - } - - return std::make_optional(file, info.fsize); -} - -Column::Column(FIL file, std::size_t file_size) - : file_(file), length_(file_size), loaded_range_(0, 0) { - ESP_ERROR_CHECK(esp_himem_alloc(kRamBlockSize, &block_)); -} - -Column::~Column() { - f_close(&file_); - esp_himem_free(block_); -} - -auto Column::ReadDataAtOffset(esp_himem_rangehandle_t range, - IndexOffset_t offset) -> RowData { - // To start, we always need to map our address space. - std::byte* paged_block; - esp_himem_map(block_, range, 0, 0, kRamBlockSize, 0, - reinterpret_cast(&paged_block)); - - // Next, we need to see how long the data we're returning is. This might - // already exist in memory. - if (!IsOffsetLoaded(offset) || - !IsOffsetLoaded(offset + sizeof(std::size_t))) { - LoadOffsetFromDisk({paged_block, kRamBlockSize}, offset); - } - - IndexOffset_t paged_offset = offset - loaded_range_.first; - std::size_t data_size = - *(reinterpret_cast(paged_block + paged_offset)); - - // Now that we have the size, we need to do the same thing again to get the - // real data. Hopefully this doesn't require an actual second disk read, since - // LoadOffsetFromDisk should load a generous amount after the offset we - // previously gave. - if (!IsOffsetLoaded(offset) || !IsOffsetLoaded(offset + data_size)) { - LoadOffsetFromDisk({paged_block, kRamBlockSize}, offset); - } - - paged_offset = offset - loaded_range_.first + sizeof(IndexOffset_t); - cpp::span src(paged_block + paged_offset, data_size); - - auto res = std::make_unique(data_size); - cpp::span dest(res.get(), data_size); - - std::copy(src.begin(), src.end(), dest.begin()); - - // Finally, unmap from the range we were given to return it to its initial - // state. - esp_himem_unmap(range, paged_block, kRamBlockSize); - - return {std::move(res), data_size}; -} - -auto Column::AppendRow(cpp::span row) -> bool { - FRESULT res = f_lseek(&file_, length_); - if (res != FR_OK) { - // TODO(jacqueline): Handle errors. - return false; - } - - std::size_t bytes_written = 0; - std::size_t length = row.size_bytes(); - res = f_write(&file_, &length, sizeof(std::size_t), &bytes_written); - if (res != FR_OK || bytes_written != sizeof(std::size_t)) { - // TODO(jacqueline): Handle errors. - return false; - } - - res = f_write(&file_, row.data(), length, &bytes_written); - if (res != FR_OK || bytes_written != length) { - // TODO(jacqueline): Handle errors. - return false; - } - - length_ += sizeof(std::size_t) + row.size_bytes(); - - return true; -} - -auto Column::FlushChanges() -> void { - f_sync(&file_); -} - -auto Column::IsOffsetLoaded(IndexOffset_t offset) -> bool { - return loaded_range_.first <= offset && - loaded_range_.second > offset + sizeof(std::size_t); -} - -auto Column::LoadOffsetFromDisk(cpp::span dest, IndexOffset_t offset) - -> bool { - FRESULT res = f_lseek(&file_, offset); - if (res != FR_OK) { - // TODO(jacqueline): Handle errors. - return false; - } - - UINT bytes_read = 0; - res = f_read(&file_, dest.data(), dest.size(), &bytes_read); - if (res != FR_OK) { - // TODO(jacqueline): Handle errors. - return false; - } - - loaded_range_.first = offset; - loaded_range_.second = offset + bytes_read; - - return true; -} - -} // namespace database diff --git a/src/main/main.cpp b/src/main/main.cpp index d58b07b6..312018cb 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -48,7 +48,13 @@ void db_main(void* whatever) { ESP_LOGI(TAG, "database good :)"); } - vTaskDelay(pdMS_TO_TICKS(10000)); + vTaskDelay(pdMS_TO_TICKS(2000)); + + db->Initialise(); + + vTaskDelay(pdMS_TO_TICKS(2000)); + + db.reset(); vTaskDelete(NULL); }