MightyPork
/
tangara-fw
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add pagination to database queries

Browse Source
custom
jacqueline 2 years ago
parent 785349eb5b
commit d71f726c42
6 changed files with 325 additions and 150 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 219
      src/database/database.cpp
  2. 81
      src/database/include/database.hpp
  3. 11
      src/database/include/song.hpp
  4. 6
      src/database/song.cpp
  5. 129
      src/database/test/test_database.cpp
  6. 29
      src/main/app_console.cpp

219
src/database/database.cpp
Unescape View File

@ -2,9 +2,11 @@
#include <stdint.h> #include <stdint.h>
#include <algorithm>
#include <cstdint> #include <cstdint>
#include <functional> #include <functional>
#include <iomanip> #include <iomanip>
#include <memory>
#include <sstream> #include <sstream>
#include "esp_log.h" #include "esp_log.h"
@ -216,6 +218,43 @@ auto Database::Update() -> std::future<void> {
}); });
} }
auto Database::GetSongs(std::size_t page_size) -> std::future<Result<Song>*> {
return RunOnDbTask<Result<Song>*>([=, this]() -> Result<Song>* {
Continuation<Song> c{.iterator = nullptr,
.prefix = CreateDataPrefix().data,
.start_key = CreateDataPrefix().data,
.forward = true,
.was_prev_forward = true,
.page_size = page_size};
return dbGetPage(c);
});
}
auto Database::GetDump(std::size_t page_size)
-> std::future<Result<std::string>*> {
return RunOnDbTask<Result<std::string>*>([=, this]() -> Result<std::string>* {
Continuation<std::string> c{.iterator = nullptr,
.prefix = "",
.start_key = "",
.forward = true,
.was_prev_forward = true,
.page_size = page_size};
return dbGetPage(c);
});
}
template <typename T>
auto Database::GetPage(Continuation<T>* c) -> std::future<Result<T>*> {
Continuation<T> copy = *c;
return RunOnDbTask<Result<T>*>(
[=, this]() -> Result<T>* { return dbGetPage(copy); });
}
template auto Database::GetPage<Song>(Continuation<Song>* c)
-> std::future<Result<Song>*>;
template auto Database::GetPage<std::string>(Continuation<std::string>* c)
-> std::future<Result<std::string>*>;
auto Database::dbMintNewSongId() -> SongId { auto Database::dbMintNewSongId() -> SongId {
SongId next_id = 1; SongId next_id = 1;
std::string val; std::string val;
@ -287,37 +326,148 @@ auto Database::dbPutSong(SongId id,
dbPutHash(hash, id); dbPutHash(hash, id);
} }
auto parse_song(ITagParser* parser, template <typename T>
const leveldb::Slice& key, auto Database::dbGetPage(const Continuation<T>& c) -> Result<T>* {
const leveldb::Slice& value) -> std::optional<Song> { // Work out our starting point. Sometimes this will already done.
std::optional<SongData> data = ParseDataValue(value); leveldb::Iterator* it = nullptr;
if (!data) { if (c.iterator != nullptr) {
it = c.iterator->release();
}
if (it == nullptr) {
it = db_->NewIterator(leveldb::ReadOptions());
it->Seek(c.start_key);
}
// Fix off-by-one if we just changed direction.
if (c.forward != c.was_prev_forward) {
if (c.forward) {
it->Next();
} else {
it->Prev();
}
}
// Grab results.
std::optional<std::string> first_key;
std::vector<T> records;
while (records.size() < c.page_size && it->Valid()) {
if (!it->key().starts_with(c.prefix)) {
break;
}
if (!first_key) {
first_key = it->key().ToString();
}
std::optional<T> parsed = ParseRecord<T>(it->key(), it->value());
if (parsed) {
records.push_back(*parsed);
}
if (c.forward) {
it->Next();
} else {
it->Prev();
}
}
std::unique_ptr<leveldb::Iterator> iterator(it);
if (iterator != nullptr) {
if (!iterator->Valid() || !it->key().starts_with(c.prefix)) {
iterator.reset();
}
}
// Put results into canonical order if we were iterating backwards.
if (!c.forward) {
std::reverse(records.begin(), records.end());
}
// Work out the new continuations.
std::optional<Continuation<T>> next_page;
if (c.forward) {
if (iterator != nullptr) {
// We were going forward, and now we want the next page. Re-use the
// existing iterator, and point the start key at it.
std::string key = iterator->key().ToString();
next_page = Continuation<T>{
.iterator = std::make_shared<std::unique_ptr<leveldb::Iterator>>(
std::move(iterator)),
.prefix = c.prefix,
.start_key = key,
.forward = true,
.was_prev_forward = true,
.page_size = c.page_size,
};
}
// No iterator means we ran out of results in this direction.
} else {
// We were going backwards, and now we want the next page. This is a
// reversal, to set the start key to the first record we saw and mark that
// it's off by one.
next_page = Continuation<T>{
.iterator = nullptr,
.prefix = c.prefix,
.start_key = *first_key,
.forward = true,
.was_prev_forward = false,
.page_size = c.page_size,
};
}
std::optional<Continuation<T>> prev_page;
if (c.forward) {
// We were going forwards, and now we want the previous page. Set the search
// key to the first result we saw, and mark that it's off by one.
prev_page = Continuation<T>{
.iterator = nullptr,
.prefix = c.prefix,
.start_key = *first_key,
.forward = false,
.was_prev_forward = true,
.page_size = c.page_size,
};
} else {
if (iterator != nullptr) {
// We were going backwards, and we still want to go backwards. The
// iterator is still valid.
std::string key = iterator->key().ToString();
prev_page = Continuation<T>{
.iterator = std::make_shared<std::unique_ptr<leveldb::Iterator>>(
std::move(iterator)),
.prefix = c.prefix,
.start_key = key,
.forward = false,
.was_prev_forward = false,
.page_size = c.page_size,
};
}
// No iterator means we ran out of results in this direction.
}
return new Result<T>(std::move(records), next_page, prev_page);
}
template auto Database::dbGetPage<Song>(const Continuation<Song>& c)
-> Result<Song>*;
template auto Database::dbGetPage<std::string>(
const Continuation<std::string>& c) -> Result<std::string>*;
template <>
auto Database::ParseRecord<Song>(const leveldb::Slice& key,
const leveldb::Slice& val)
-> std::optional<Song> {
std::optional<SongData> data = ParseDataValue(val);
if (!data || data->is_tombstoned()) {
return {}; return {};
} }
SongTags tags; SongTags tags;
if (!parser->ReadAndParseTags(data->filepath(), &tags)) { if (!tag_parser_->ReadAndParseTags(data->filepath(), &tags)) {
return {}; return {};
} }
return Song(*data, tags); return Song(*data, tags);
} }
auto Database::GetSongs(std::size_t page_size) -> std::future<Result<Song>> { template <>
return RunOnDbTask<Result<Song>>([=, this]() -> Result<Song> { auto Database::ParseRecord<std::string>(const leveldb::Slice& key,
return Query<Song>(CreateDataPrefix().slice, page_size, const leveldb::Slice& val)
std::bind_front(&parse_song, tag_parser_));
});
}
auto Database::GetMoreSongs(std::size_t page_size, Continuation c)
-> std::future<Result<Song>> {
leveldb::Iterator* it = c.release();
return RunOnDbTask<Result<Song>>([=, this]() -> Result<Song> {
return Query<Song>(it, page_size,
std::bind_front(&parse_song, tag_parser_));
});
}
auto parse_dump(const leveldb::Slice& key, const leveldb::Slice& value)
-> std::optional<std::string> { -> std::optional<std::string> {
std::ostringstream stream; std::ostringstream stream;
stream << "key: "; stream << "key: ";
@ -335,33 +485,14 @@ auto parse_dump(const leveldb::Slice& key, const leveldb::Slice& value)
<< static_cast<int>(str[i]); << static_cast<int>(str[i]);
} }
} }
for (std::size_t i = 2; i < str.size(); i++) {
}
} }
stream << "\tval: 0x"; stream << "\tval: 0x";
std::string str = value.ToString(); std::string str = val.ToString();
for (int i = 0; i < value.size(); i++) { for (int i = 0; i < val.size(); i++) {
stream << std::hex << std::setfill('0') << std::setw(2) stream << std::hex << std::setfill('0') << std::setw(2)
<< static_cast<int>(str[i]); << static_cast<int>(str[i]);
} }
return stream.str(); return stream.str();
} }
auto Database::GetDump(std::size_t page_size)
-> std::future<Result<std::string>> {
leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
it->SeekToFirst();
return RunOnDbTask<Result<std::string>>([=, this]() -> Result<std::string> {
return Query<std::string>(it, page_size, &parse_dump);
});
}
auto Database::GetMoreDump(std::size_t page_size, Continuation c)
-> std::future<Result<std::string>> {
leveldb::Iterator* it = c.release();
return RunOnDbTask<Result<std::string>>([=, this]() -> Result<std::string> {
return Query<std::string>(it, page_size, &parse_dump);
});
}
} // namespace database } // namespace database

81
src/database/include/database.hpp
Unescape View File

@ -22,7 +22,15 @@
namespace database { namespace database {
typedef std::unique_ptr<leveldb::Iterator> Continuation; template <typename T>
struct Continuation {
std::shared_ptr<std::unique_ptr<leveldb::Iterator>> iterator;
std::string prefix;
std::string start_key;
bool forward;
bool was_prev_forward;
size_t page_size;
};
/* /*
* Wrapper for a set of results from the database. Owns the list of results, as * Wrapper for a set of results from the database. Owns the list of results, as
@ -32,29 +40,23 @@ typedef std::unique_ptr<leveldb::Iterator> Continuation;
template <typename T> template <typename T>
class Result { class Result {
public: public:
auto values() -> std::vector<T>* { return values_.release(); } auto values() const -> const std::vector<T>& { return values_; }
auto continuation() -> Continuation { return std::move(c_); }
auto HasMore() -> bool { return c_->Valid(); }
Result(std::vector<T>* values, Continuation c)
: values_(values), c_(std::move(c)) {}
Result(std::unique_ptr<std::vector<T>> values, Continuation c)
: values_(std::move(values)), c_(std::move(c)) {}
Result(Result&& other) auto next_page() -> std::optional<Continuation<T>>& { return next_page_; }
: values_(move(other.values_)), c_(std::move(other.c_)) {} auto prev_page() -> std::optional<Continuation<T>>& { return prev_page_; }
Result operator=(Result&& other) { Result(const std::vector<T>&& values,
return Result(other.values(), std::move(other.continuation())); std::optional<Continuation<T>> next,
} std::optional<Continuation<T>> prev)
: values_(values), next_page_(next), prev_page_(prev) {}
Result(const Result&) = delete; Result(const Result&) = delete;
Result& operator=(const Result&) = delete; Result& operator=(const Result&) = delete;
private: private:
std::unique_ptr<std::vector<T>> values_; std::vector<T> values_;
Continuation c_; std::optional<Continuation<T>> next_page_;
std::optional<Continuation<T>> prev_page_;
}; };
class Database { class Database {
@ -73,13 +75,11 @@ class Database {
auto Update() -> std::future<void>; auto Update() -> std::future<void>;
auto GetSongs(std::size_t page_size) -> std::future<Result<Song>>; auto GetSongs(std::size_t page_size) -> std::future<Result<Song>*>;
auto GetMoreSongs(std::size_t page_size, Continuation c) auto GetDump(std::size_t page_size) -> std::future<Result<std::string>*>;
-> std::future<Result<Song>>;
auto GetDump(std::size_t page_size) -> std::future<Result<std::string>>; template <typename T>
auto GetMoreDump(std::size_t page_size, Continuation c) auto GetPage(Continuation<T>* c) -> std::future<Result<T>*>;
-> std::future<Result<std::string>>;
Database(const Database&) = delete; Database(const Database&) = delete;
Database& operator=(const Database&) = delete; Database& operator=(const Database&) = delete;
@ -110,31 +110,20 @@ class Database {
-> void; -> void;
template <typename T> template <typename T>
using Parser = std::function<std::optional<T>(const leveldb::Slice& key, auto dbGetPage(const Continuation<T>& c) -> Result<T>*;
const leveldb::Slice& value)>;
template <typename T>
auto Query(const leveldb::Slice& prefix,
std::size_t max_results,
Parser<T> parser) -> Result<T> {
leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
it->Seek(prefix);
return Query(it, max_results, parser);
}
template <typename T> template <typename T>
auto Query(leveldb::Iterator* it, std::size_t max_results, Parser<T> parser) auto ParseRecord(const leveldb::Slice& key, const leveldb::Slice& val)
-> Result<T> { -> std::optional<T>;
auto results = std::make_unique<std::vector<T>>();
for (std::size_t i = 0; i < max_results && it->Valid(); i++) {
std::optional<T> r = std::invoke(parser, it->key(), it->value());
if (r) {
results->push_back(*r);
}
it->Next();
}
return {std::move(results), std::unique_ptr<leveldb::Iterator>(it)};
}
}; };
template <>
auto Database::ParseRecord<Song>(const leveldb::Slice& key,
const leveldb::Slice& val)
-> std::optional<Song>;
template <>
auto Database::ParseRecord<std::string>(const leveldb::Slice& key,
const leveldb::Slice& val)
-> std::optional<std::string>;
} // namespace database } // namespace database

11
src/database/include/song.hpp
Unescape View File

@ -4,6 +4,7 @@
#include <optional> #include <optional>
#include <string> #include <string>
#include <utility>
#include "leveldb/db.h" #include "leveldb/db.h"
#include "span.hpp" #include "span.hpp"
@ -133,16 +134,20 @@ class SongData {
*/ */
class Song { class Song {
public: public:
Song(SongData data, SongTags tags) : data_(data), tags_(tags) {} Song(const SongData& data, const SongTags& tags) : data_(data), tags_(tags) {}
Song(const Song& other) = default;
auto data() -> const SongData& { return data_; } auto data() const -> const SongData& { return data_; }
auto tags() -> const SongTags& { return tags_; } auto tags() const -> const SongTags& { return tags_; }
bool operator==(const Song&) const = default; bool operator==(const Song&) const = default;
Song operator=(const Song& other) const { return Song(other); }
private: private:
const SongData data_; const SongData data_;
const SongTags tags_; const SongTags tags_;
}; };
void swap(Song& first, Song& second);
} // namespace database } // namespace database

6
src/database/song.cpp
Unescape View File

@ -36,4 +36,10 @@ auto SongData::Exhume(const std::string& new_path) const -> SongData {
return SongData(id_, new_path, tags_hash_, play_count_, false); return SongData(id_, new_path, tags_hash_, play_count_, false);
} }
void swap(Song& first, Song& second) {
Song temp = first;
first = second;
second = temp;
}
} // namespace database } // namespace database

129
src/database/test/test_database.cpp
Unescape View File

@ -60,8 +60,8 @@ TEST_CASE("song database", "[integration]") {
std::unique_ptr<Database> db(open_res.value()); std::unique_ptr<Database> db(open_res.value());
SECTION("empty database") { SECTION("empty database") {
std::unique_ptr<std::vector<Song>> res(db->GetSongs(10).get().values()); std::unique_ptr<Result<Song>> res(db->GetSongs(10).get());
REQUIRE(res->size() == 0); REQUIRE(res->values().size() == 0);
} }
SECTION("add new songs") { SECTION("add new songs") {
@ -71,24 +71,23 @@ TEST_CASE("song database", "[integration]") {
db->Update(); db->Update();
std::unique_ptr<std::vector<Song>> res(db->GetSongs(10).get().values()); std::unique_ptr<Result<Song>> res(db->GetSongs(10).get());
REQUIRE(res->size() == 3); REQUIRE(res->values().size() == 3);
CHECK(*res->at(0).tags().title == "Song 1"); CHECK(*res->values().at(0).tags().title == "Song 1");
CHECK(res->at(0).data().id() == 1); CHECK(res->values().at(0).data().id() == 1);
CHECK(*res->at(1).tags().title == "Song 2"); CHECK(*res->values().at(1).tags().title == "Song 2");
CHECK(res->at(1).data().id() == 2); CHECK(res->values().at(1).data().id() == 2);
CHECK(*res->at(2).tags().title == "Song 3"); CHECK(*res->values().at(2).tags().title == "Song 3");
CHECK(res->at(2).data().id() == 3); CHECK(res->values().at(2).data().id() == 3);
SECTION("update with no filesystem changes") { SECTION("update with no filesystem changes") {
db->Update(); db->Update();
std::unique_ptr<std::vector<Song>> new_res( std::unique_ptr<Result<Song>> new_res(db->GetSongs(10).get());
db->GetSongs(10).get().values()); REQUIRE(new_res->values().size() == 3);
REQUIRE(new_res->size() == 3); CHECK(res->values().at(0) == new_res->values().at(0));
CHECK(res->at(0) == new_res->at(0)); CHECK(res->values().at(1) == new_res->values().at(1));
CHECK(res->at(1) == new_res->at(1)); CHECK(res->values().at(2) == new_res->values().at(2));
CHECK(res->at(2) == new_res->at(2));
} }
SECTION("update with all songs gone") { SECTION("update with all songs gone") {
@ -96,19 +95,17 @@ TEST_CASE("song database", "[integration]") {
db->Update(); db->Update();
std::unique_ptr<std::vector<Song>> new_res( std::unique_ptr<Result<Song>> new_res(db->GetSongs(10).get());
db->GetSongs(10).get().values()); CHECK(new_res->values().size() == 0);
CHECK(new_res->size() == 0);
SECTION("update with one song returned") { SECTION("update with one song returned") {
songs.MakeSong("song2.wav", "Song 2"); songs.MakeSong("song2.wav", "Song 2");
db->Update(); db->Update();
std::unique_ptr<std::vector<Song>> new_res( std::unique_ptr<Result<Song>> new_res(db->GetSongs(10).get());
db->GetSongs(10).get().values()); REQUIRE(new_res->values().size() == 1);
REQUIRE(new_res->size() == 1); CHECK(res->values().at(1) == new_res->values().at(0));
CHECK(res->at(1) == new_res->at(0));
} }
} }
@ -117,11 +114,10 @@ TEST_CASE("song database", "[integration]") {
db->Update(); db->Update();
std::unique_ptr<std::vector<Song>> new_res( std::unique_ptr<Result<Song>> new_res(db->GetSongs(10).get());
db->GetSongs(10).get().values()); REQUIRE(new_res->values().size() == 2);
REQUIRE(new_res->size() == 2); CHECK(res->values().at(0) == new_res->values().at(0));
CHECK(res->at(0) == new_res->at(0)); CHECK(res->values().at(2) == new_res->values().at(1));
CHECK(res->at(2) == new_res->at(1));
} }
SECTION("update with tags changed") { SECTION("update with tags changed") {
@ -129,14 +125,14 @@ TEST_CASE("song database", "[integration]") {
db->Update(); db->Update();
std::unique_ptr<std::vector<Song>> new_res( std::unique_ptr<Result<Song>> new_res(db->GetSongs(10).get());
db->GetSongs(10).get().values()); REQUIRE(new_res->values().size() == 3);
REQUIRE(new_res->size() == 3); CHECK(res->values().at(0) == new_res->values().at(0));
CHECK(res->at(0) == new_res->at(0)); CHECK(res->values().at(1) == new_res->values().at(1));
CHECK(res->at(1) == new_res->at(1)); CHECK(*new_res->values().at(2).tags().title == "The Song 3");
CHECK(*new_res->at(2).tags().title == "The Song 3");
// The id should not have changed, since this was just a tag update. // The id should not have changed, since this was just a tag update.
CHECK(res->at(2).data().id() == new_res->at(2).data().id()); CHECK(res->values().at(2).data().id() ==
new_res->values().at(2).data().id());
} }
SECTION("update with one new song") { SECTION("update with one new song") {
@ -144,14 +140,61 @@ TEST_CASE("song database", "[integration]") {
db->Update(); db->Update();
std::unique_ptr<std::vector<Song>> new_res( std::unique_ptr<Result<Song>> new_res(db->GetSongs(10).get());
db->GetSongs(10).get().values()); REQUIRE(new_res->values().size() == 4);
REQUIRE(new_res->size() == 4); CHECK(res->values().at(0) == new_res->values().at(0));
CHECK(res->at(0) == new_res->at(0)); CHECK(res->values().at(1) == new_res->values().at(1));
CHECK(res->at(1) == new_res->at(1)); CHECK(res->values().at(2) == new_res->values().at(2));
CHECK(res->at(2) == new_res->at(2)); CHECK(*new_res->values().at(3).tags().title ==
CHECK(*new_res->at(3).tags().title == "Song 1 (nightcore remix)"); "Song 1 (nightcore remix)");
CHECK(new_res->at(3).data().id() == 4); CHECK(new_res->values().at(3).data().id() == 4);
}
SECTION("get songs with pagination") {
std::unique_ptr<Result<Song>> res(db->GetSongs(1).get());
REQUIRE(res->values().size() == 1);
CHECK(res->values().at(0).data().id() == 1);
REQUIRE(res->next_page());
res.reset(db->GetPage(&res->next_page().value()).get());
REQUIRE(res->values().size() == 1);
CHECK(res->values().at(0).data().id() == 2);
REQUIRE(res->next_page());
res.reset(db->GetPage(&res->next_page().value()).get());
REQUIRE(res->values().size() == 1);
CHECK(res->values().at(0).data().id() == 3);
REQUIRE(!res->next_page());
SECTION("page backwards") {
REQUIRE(res->prev_page());
res.reset(db->GetPage(&res->prev_page().value()).get());
REQUIRE(res->values().size() == 1);
CHECK(res->values().at(0).data().id() == 2);
REQUIRE(res->prev_page());
res.reset(db->GetPage(&res->prev_page().value()).get());
REQUIRE(res->values().size() == 1);
CHECK(res->values().at(0).data().id() == 1);
REQUIRE(!res->prev_page());
SECTION("page forwards again") {
REQUIRE(res->next_page());
res.reset(db->GetPage(&res->next_page().value()).get());
REQUIRE(res->values().size() == 1);
CHECK(res->values().at(0).data().id() == 2);
CHECK(res->next_page());
CHECK(res->prev_page());
}
}
} }
} }
} }

29
src/main/app_console.cpp
Unescape View File

@ -176,15 +176,15 @@ int CmdDbSongs(int argc, char** argv) {
return 1; return 1;
} }
database::Result<database::Song> res = std::unique_ptr<database::Result<database::Song>> res(
sInstance->database_->GetSongs(20).get(); sInstance->database_->GetSongs(5).get());
while (true) { while (true) {
std::unique_ptr<std::vector<database::Song>> r(res.values()); for (database::Song s : res->values()) {
for (database::Song s : *r) {
std::cout << s.tags().title.value_or("[BLANK]") << std::endl; std::cout << s.tags().title.value_or("[BLANK]") << std::endl;
} }
if (res.HasMore()) { if (res->next_page()) {
res = sInstance->database_->GetMoreSongs(10, res.continuation()).get(); auto continuation = res->next_page().value();
res.reset(sInstance->database_->GetPage(&continuation).get());
} else { } else {
break; break;
} }
@ -211,17 +211,18 @@ int CmdDbDump(int argc, char** argv) {
std::cout << "=== BEGIN DUMP ===" << std::endl; std::cout << "=== BEGIN DUMP ===" << std::endl;
database::Result<std::string> res = sInstance->database_->GetDump(20).get(); std::unique_ptr<database::Result<std::string>> res(
sInstance->database_->GetDump(5).get());
while (true) { while (true) {
std::unique_ptr<std::vector<std::string>> r(res.values()); for (std::string s : res->values()) {
if (r == nullptr) {
break;
}
for (std::string s : *r) {
std::cout << s << std::endl; std::cout << s << std::endl;
} }
if (res.HasMore()) { if (res->next_page()) {
res = sInstance->database_->GetMoreDump(20, res.continuation()).get(); auto continuation = res->next_page().value();
res.reset(
sInstance->database_->GetPage<std::string>(&continuation).get());
} else {
break;
} }
} }

Write Preview
Loading…
Cancel
Save