From afbf3c31f4d1a605c264f719531f4183ee5a3022 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Fri, 13 Oct 2023 15:05:49 +1100 Subject: [PATCH] Use libcppbor for much much nicer db encoding --- lib/libcppbor/CMakeLists.txt | 7 + lib/libcppbor/LICENSE | 202 ++++ lib/libcppbor/cppbor.cpp | 599 ++++++++++ lib/libcppbor/cppbor_parse.cpp | 423 +++++++ lib/libcppbor/include/cppbor/cppbor.h | 1141 +++++++++++++++++++ lib/libcppbor/include/cppbor/cppbor_parse.h | 195 ++++ src/database/CMakeLists.txt | 2 +- src/database/database.cpp | 67 +- src/database/include/records.hpp | 31 +- src/database/include/track.hpp | 5 - src/database/index.cpp | 2 +- src/database/records.cpp | 372 ++---- src/database/track.cpp | 6 +- tools/cmake/common.cmake | 11 +- 14 files changed, 2698 insertions(+), 365 deletions(-) create mode 100644 lib/libcppbor/CMakeLists.txt create mode 100644 lib/libcppbor/LICENSE create mode 100644 lib/libcppbor/cppbor.cpp create mode 100644 lib/libcppbor/cppbor_parse.cpp create mode 100644 lib/libcppbor/include/cppbor/cppbor.h create mode 100644 lib/libcppbor/include/cppbor/cppbor_parse.h diff --git a/lib/libcppbor/CMakeLists.txt b/lib/libcppbor/CMakeLists.txt new file mode 100644 index 00000000..f68a820c --- /dev/null +++ b/lib/libcppbor/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2023 jacqueline +# +# SPDX-License-Identifier: GPL-3.0-only +idf_component_register( + SRCS cppbor.cpp cppbor_parse.cpp + INCLUDE_DIRS "include/cppbor" +) diff --git a/lib/libcppbor/LICENSE b/lib/libcppbor/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/lib/libcppbor/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/libcppbor/cppbor.cpp b/lib/libcppbor/cppbor.cpp new file mode 100644 index 00000000..c66ca752 --- /dev/null +++ b/lib/libcppbor/cppbor.cpp @@ -0,0 +1,599 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cppbor.h" + +#include +#include + +#include "cppbor_parse.h" + +using std::string; +using std::vector; + +#define CHECK(x) (void)(x) + +namespace cppbor { + +namespace { + +template ::value>> +Iterator writeBigEndian(T value, Iterator pos) { + for (unsigned i = 0; i < sizeof(value); ++i) { + *pos++ = static_cast(value >> (8 * (sizeof(value) - 1))); + value = static_cast(value << 8); + } + return pos; +} + +template ::value>> +void writeBigEndian(T value, std::function& cb) { + for (unsigned i = 0; i < sizeof(value); ++i) { + cb(static_cast(value >> (8 * (sizeof(value) - 1)))); + value = static_cast(value << 8); + } +} + +bool cborAreAllElementsNonCompound(const Item* compoundItem) { + if (compoundItem->type() == ARRAY) { + const Array* array = compoundItem->asArray(); + for (size_t n = 0; n < array->size(); n++) { + const Item* entry = (*array)[n].get(); + switch (entry->type()) { + case ARRAY: + case MAP: + return false; + default: + break; + } + } + } else { + const Map* map = compoundItem->asMap(); + for (auto& [keyEntry, valueEntry] : *map) { + switch (keyEntry->type()) { + case ARRAY: + case MAP: + return false; + default: + break; + } + switch (valueEntry->type()) { + case ARRAY: + case MAP: + return false; + default: + break; + } + } + } + return true; +} + +bool prettyPrintInternal(const Item* item, string& out, size_t indent, size_t maxBStrSize, + const vector& mapKeysToNotPrint) { + if (!item) { + out.append(""); + return false; + } + + char buf[80]; + + string indentString(indent, ' '); + + size_t tagCount = item->semanticTagCount(); + while (tagCount > 0) { + --tagCount; + snprintf(buf, sizeof(buf), "tag %" PRIu64 " ", item->semanticTag(tagCount)); + out.append(buf); + } + + switch (item->type()) { + case SEMANTIC: + // Handled above. + break; + + case UINT: + snprintf(buf, sizeof(buf), "%" PRIu64, item->asUint()->unsignedValue()); + out.append(buf); + break; + + case NINT: + snprintf(buf, sizeof(buf), "%" PRId64, item->asNint()->value()); + out.append(buf); + break; + + case BSTR: { + const uint8_t* valueData; + size_t valueSize; + const Bstr* bstr = item->asBstr(); + if (bstr != nullptr) { + const vector& value = bstr->value(); + valueData = value.data(); + valueSize = value.size(); + } else { + const ViewBstr* viewBstr = item->asViewBstr(); + assert(viewBstr != nullptr); + + valueData = viewBstr->view().data(); + valueSize = viewBstr->view().size(); + } + + if (valueSize > maxBStrSize) { + snprintf(buf, sizeof(buf), "", valueSize); + out.append(buf); + } else { + out.append("{"); + for (size_t n = 0; n < valueSize; n++) { + if (n > 0) { + out.append(", "); + } + snprintf(buf, sizeof(buf), "0x%02x", valueData[n]); + out.append(buf); + } + out.append("}"); + } + } break; + + case TSTR: + out.append("'"); + { + // TODO: escape "'" characters + if (item->asTstr() != nullptr) { + out.append(item->asTstr()->value().c_str()); + } else { + const ViewTstr* viewTstr = item->asViewTstr(); + assert(viewTstr != nullptr); + out.append(viewTstr->view()); + } + } + out.append("'"); + break; + + case ARRAY: { + const Array* array = item->asArray(); + if (array->size() == 0) { + out.append("[]"); + } else if (cborAreAllElementsNonCompound(array)) { + out.append("["); + for (size_t n = 0; n < array->size(); n++) { + if (!prettyPrintInternal((*array)[n].get(), out, indent + 2, maxBStrSize, + mapKeysToNotPrint)) { + return false; + } + out.append(", "); + } + out.append("]"); + } else { + out.append("[\n" + indentString); + for (size_t n = 0; n < array->size(); n++) { + out.append(" "); + if (!prettyPrintInternal((*array)[n].get(), out, indent + 2, maxBStrSize, + mapKeysToNotPrint)) { + return false; + } + out.append(",\n" + indentString); + } + out.append("]"); + } + } break; + + case MAP: { + const Map* map = item->asMap(); + + if (map->size() == 0) { + out.append("{}"); + } else { + out.append("{\n" + indentString); + for (auto& [map_key, map_value] : *map) { + out.append(" "); + + if (!prettyPrintInternal(map_key.get(), out, indent + 2, maxBStrSize, + mapKeysToNotPrint)) { + return false; + } + out.append(" : "); + if (map_key->type() == TSTR && + std::find(mapKeysToNotPrint.begin(), mapKeysToNotPrint.end(), + map_key->asTstr()->value()) != mapKeysToNotPrint.end()) { + out.append(""); + } else { + if (!prettyPrintInternal(map_value.get(), out, indent + 2, maxBStrSize, + mapKeysToNotPrint)) { + return false; + } + } + out.append(",\n" + indentString); + } + out.append("}"); + } + } break; + + case SIMPLE: + const Bool* asBool = item->asSimple()->asBool(); + const Null* asNull = item->asSimple()->asNull(); + if (asBool != nullptr) { + out.append(asBool->value() ? "true" : "false"); + } else if (asNull != nullptr) { + out.append("null"); + } else { + return false; + } + break; + } + + return true; +} + +} // namespace + +size_t headerSize(uint64_t addlInfo) { + if (addlInfo < ONE_BYTE_LENGTH) return 1; + if (addlInfo <= std::numeric_limits::max()) return 2; + if (addlInfo <= std::numeric_limits::max()) return 3; + if (addlInfo <= std::numeric_limits::max()) return 5; + return 9; +} + +uint8_t* encodeHeader(MajorType type, uint64_t addlInfo, uint8_t* pos, const uint8_t* end) { + size_t sz = headerSize(addlInfo); + if (end - pos < static_cast(sz)) return nullptr; + switch (sz) { + case 1: + *pos++ = type | static_cast(addlInfo); + return pos; + case 2: + *pos++ = type | ONE_BYTE_LENGTH; + *pos++ = static_cast(addlInfo); + return pos; + case 3: + *pos++ = type | TWO_BYTE_LENGTH; + return writeBigEndian(static_cast(addlInfo), pos); + case 5: + *pos++ = type | FOUR_BYTE_LENGTH; + return writeBigEndian(static_cast(addlInfo), pos); + case 9: + *pos++ = type | EIGHT_BYTE_LENGTH; + return writeBigEndian(addlInfo, pos); + default: + CHECK(false); // Impossible to get here. + return nullptr; + } +} + +void encodeHeader(MajorType type, uint64_t addlInfo, EncodeCallback encodeCallback) { + size_t sz = headerSize(addlInfo); + switch (sz) { + case 1: + encodeCallback(type | static_cast(addlInfo)); + break; + case 2: + encodeCallback(type | ONE_BYTE_LENGTH); + encodeCallback(static_cast(addlInfo)); + break; + case 3: + encodeCallback(type | TWO_BYTE_LENGTH); + writeBigEndian(static_cast(addlInfo), encodeCallback); + break; + case 5: + encodeCallback(type | FOUR_BYTE_LENGTH); + writeBigEndian(static_cast(addlInfo), encodeCallback); + break; + case 9: + encodeCallback(type | EIGHT_BYTE_LENGTH); + writeBigEndian(addlInfo, encodeCallback); + break; + default: + CHECK(false); // Impossible to get here. + } +} + +bool Item::operator==(const Item& other) const& { + if (type() != other.type()) return false; + switch (type()) { + case UINT: + return *asUint() == *(other.asUint()); + case NINT: + return *asNint() == *(other.asNint()); + case BSTR: + if (asBstr() != nullptr && other.asBstr() != nullptr) { + return *asBstr() == *(other.asBstr()); + } + if (asViewBstr() != nullptr && other.asViewBstr() != nullptr) { + return *asViewBstr() == *(other.asViewBstr()); + } + // Interesting corner case: comparing a Bstr and ViewBstr with + // identical contents. The function currently returns false for + // this case. + // TODO: if it should return true, this needs a deep comparison + return false; + case TSTR: + if (asTstr() != nullptr && other.asTstr() != nullptr) { + return *asTstr() == *(other.asTstr()); + } + if (asViewTstr() != nullptr && other.asViewTstr() != nullptr) { + return *asViewTstr() == *(other.asViewTstr()); + } + // Same corner case as Bstr + return false; + case ARRAY: + return *asArray() == *(other.asArray()); + case MAP: + return *asMap() == *(other.asMap()); + case SIMPLE: + return *asSimple() == *(other.asSimple()); + case SEMANTIC: + return *asSemanticTag() == *(other.asSemanticTag()); + default: + CHECK(false); // Impossible to get here. + return false; + } +} + +Nint::Nint(int64_t v) : mValue(v) { + CHECK(v < 0); +} + +bool Simple::operator==(const Simple& other) const& { + if (simpleType() != other.simpleType()) return false; + + switch (simpleType()) { + case BOOLEAN: + return *asBool() == *(other.asBool()); + case NULL_T: + return true; + default: + CHECK(false); // Impossible to get here. + return false; + } +} + +uint8_t* Bstr::encode(uint8_t* pos, const uint8_t* end) const { + pos = encodeHeader(mValue.size(), pos, end); + if (!pos || end - pos < static_cast(mValue.size())) return nullptr; + return std::copy(mValue.begin(), mValue.end(), pos); +} + +void Bstr::encodeValue(EncodeCallback encodeCallback) const { + for (auto c : mValue) { + encodeCallback(c); + } +} + +uint8_t* ViewBstr::encode(uint8_t* pos, const uint8_t* end) const { + pos = encodeHeader(mView.size(), pos, end); + if (!pos || end - pos < static_cast(mView.size())) return nullptr; + return std::copy(mView.begin(), mView.end(), pos); +} + +void ViewBstr::encodeValue(EncodeCallback encodeCallback) const { + for (auto c : mView) { + encodeCallback(static_cast(c)); + } +} + +uint8_t* Tstr::encode(uint8_t* pos, const uint8_t* end) const { + pos = encodeHeader(mValue.size(), pos, end); + if (!pos || end - pos < static_cast(mValue.size())) return nullptr; + return std::copy(mValue.begin(), mValue.end(), pos); +} + +void Tstr::encodeValue(EncodeCallback encodeCallback) const { + for (auto c : mValue) { + encodeCallback(static_cast(c)); + } +} + +uint8_t* ViewTstr::encode(uint8_t* pos, const uint8_t* end) const { + pos = encodeHeader(mView.size(), pos, end); + if (!pos || end - pos < static_cast(mView.size())) return nullptr; + return std::copy(mView.begin(), mView.end(), pos); +} + +void ViewTstr::encodeValue(EncodeCallback encodeCallback) const { + for (auto c : mView) { + encodeCallback(static_cast(c)); + } +} + +bool Array::operator==(const Array& other) const& { + return size() == other.size() + // Can't use vector::operator== because the contents are pointers. std::equal lets us + // provide a predicate that does the dereferencing. + && std::equal(mEntries.begin(), mEntries.end(), other.mEntries.begin(), + [](auto& a, auto& b) -> bool { return *a == *b; }); +} + +uint8_t* Array::encode(uint8_t* pos, const uint8_t* end) const { + pos = encodeHeader(size(), pos, end); + if (!pos) return nullptr; + for (auto& entry : mEntries) { + pos = entry->encode(pos, end); + if (!pos) return nullptr; + } + return pos; +} + +void Array::encode(EncodeCallback encodeCallback) const { + encodeHeader(size(), encodeCallback); + for (auto& entry : mEntries) { + entry->encode(encodeCallback); + } +} + +std::unique_ptr Array::clone() const { + auto res = std::make_unique(); + for (size_t i = 0; i < mEntries.size(); i++) { + res->add(mEntries[i]->clone()); + } + return res; +} + +bool Map::operator==(const Map& other) const& { + return size() == other.size() + // Can't use vector::operator== because the contents are pairs of pointers. std::equal + // lets us provide a predicate that does the dereferencing. + && std::equal(begin(), end(), other.begin(), [](auto& a, auto& b) { + return *a.first == *b.first && *a.second == *b.second; + }); +} + +uint8_t* Map::encode(uint8_t* pos, const uint8_t* end) const { + pos = encodeHeader(size(), pos, end); + if (!pos) return nullptr; + for (auto& entry : mEntries) { + pos = entry.first->encode(pos, end); + if (!pos) return nullptr; + pos = entry.second->encode(pos, end); + if (!pos) return nullptr; + } + return pos; +} + +void Map::encode(EncodeCallback encodeCallback) const { + encodeHeader(size(), encodeCallback); + for (auto& entry : mEntries) { + entry.first->encode(encodeCallback); + entry.second->encode(encodeCallback); + } +} + +bool Map::keyLess(const Item* a, const Item* b) { + // CBOR map canonicalization rules are: + + // 1. If two keys have different lengths, the shorter one sorts earlier. + if (a->encodedSize() < b->encodedSize()) return true; + if (a->encodedSize() > b->encodedSize()) return false; + + // 2. If two keys have the same length, the one with the lower value in (byte-wise) lexical + // order sorts earlier. This requires encoding both items. + auto encodedA = a->encode(); + auto encodedB = b->encode(); + + return std::lexicographical_compare(encodedA.begin(), encodedA.end(), // + encodedB.begin(), encodedB.end()); +} + +void recursivelyCanonicalize(std::unique_ptr& item) { + switch (item->type()) { + case UINT: + case NINT: + case BSTR: + case TSTR: + case SIMPLE: + return; + + case ARRAY: + std::for_each(item->asArray()->begin(), item->asArray()->end(), + recursivelyCanonicalize); + return; + + case MAP: + item->asMap()->canonicalize(true /* recurse */); + return; + + case SEMANTIC: + // This can't happen. SemanticTags delegate their type() method to the contained Item's + // type. + assert(false); + return; + } +} + +Map& Map::canonicalize(bool recurse) & { + if (recurse) { + for (auto& entry : mEntries) { + recursivelyCanonicalize(entry.first); + recursivelyCanonicalize(entry.second); + } + } + + if (size() < 2 || mCanonicalized) { + // Trivially or already canonical; do nothing. + return *this; + } + + std::sort(begin(), end(), + [](auto& a, auto& b) { return keyLess(a.first.get(), b.first.get()); }); + mCanonicalized = true; + return *this; +} + +std::unique_ptr Map::clone() const { + auto res = std::make_unique(); + for (auto& [key, value] : *this) { + res->add(key->clone(), value->clone()); + } + res->mCanonicalized = mCanonicalized; + return res; +} + +std::unique_ptr SemanticTag::clone() const { + return std::make_unique(mValue, mTaggedItem->clone()); +} + +uint8_t* SemanticTag::encode(uint8_t* pos, const uint8_t* end) const { + // Can't use the encodeHeader() method that calls type() to get the major type, since that will + // return the tagged Item's type. + pos = ::cppbor::encodeHeader(kMajorType, mValue, pos, end); + if (!pos) return nullptr; + return mTaggedItem->encode(pos, end); +} + +void SemanticTag::encode(EncodeCallback encodeCallback) const { + // Can't use the encodeHeader() method that calls type() to get the major type, since that will + // return the tagged Item's type. + ::cppbor::encodeHeader(kMajorType, mValue, encodeCallback); + mTaggedItem->encode(encodeCallback); +} + +size_t SemanticTag::semanticTagCount() const { + size_t levelCount = 1; // Count this level. + const SemanticTag* cur = this; + while (cur->mTaggedItem && (cur = cur->mTaggedItem->asSemanticTag()) != nullptr) ++levelCount; + return levelCount; +} + +uint64_t SemanticTag::semanticTag(size_t nesting) const { + // Getting the value of a specific nested tag is a bit tricky, because we start with the outer + // tag and don't know how many are inside. We count the number of nesting levels to find out + // how many there are in total, then to get the one we want we have to walk down levelCount - + // nesting steps. + size_t levelCount = semanticTagCount(); + if (nesting >= levelCount) return 0; + + levelCount -= nesting; + const SemanticTag* cur = this; + while (--levelCount > 0) cur = cur->mTaggedItem->asSemanticTag(); + + return cur->mValue; +} + +string prettyPrint(const Item* item, size_t maxBStrSize, const vector& mapKeysToNotPrint) { + string out; + prettyPrintInternal(item, out, 0, maxBStrSize, mapKeysToNotPrint); + return out; +} +string prettyPrint(const vector& encodedCbor, size_t maxBStrSize, + const vector& mapKeysToNotPrint) { + auto [item, _, message] = parse(encodedCbor); + if (item == nullptr) { + return ""; + } + + return prettyPrint(item.get(), maxBStrSize, mapKeysToNotPrint); +} + +} // namespace cppbor diff --git a/lib/libcppbor/cppbor_parse.cpp b/lib/libcppbor/cppbor_parse.cpp new file mode 100644 index 00000000..6d38b622 --- /dev/null +++ b/lib/libcppbor/cppbor_parse.cpp @@ -0,0 +1,423 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cppbor_parse.h" + +#include +#include +#include +#include +#include "cppbor.h" + +#define CHECK(x) (void)(x) + +namespace cppbor { + +namespace { + +std::string insufficientLengthString(size_t bytesNeeded, size_t bytesAvail, + const std::string& type) { + char buf[1024]; + snprintf(buf, sizeof(buf), "Need %zu byte(s) for %s, have %zu.", bytesNeeded, type.c_str(), + bytesAvail); + return std::string(buf); +} + +template >> +std::tuple parseLength(const uint8_t* pos, const uint8_t* end, + ParseClient* parseClient) { + if (pos + sizeof(T) > end) { + parseClient->error(pos - 1, insufficientLengthString(sizeof(T), end - pos, "length field")); + return {false, 0, pos}; + } + + const uint8_t* intEnd = pos + sizeof(T); + T result = 0; + do { + result = static_cast((result << 8) | *pos++); + } while (pos < intEnd); + return {true, result, pos}; +} + +std::tuple parseRecursively(const uint8_t* begin, const uint8_t* end, + bool emitViews, ParseClient* parseClient); + +std::tuple handleUint(uint64_t value, const uint8_t* hdrBegin, + const uint8_t* hdrEnd, + ParseClient* parseClient) { + std::unique_ptr item = std::make_unique(value); + return {hdrEnd, + parseClient->item(item, hdrBegin, hdrEnd /* valueBegin */, hdrEnd /* itemEnd */)}; +} + +std::tuple handleNint(uint64_t value, const uint8_t* hdrBegin, + const uint8_t* hdrEnd, + ParseClient* parseClient) { + if (value > std::numeric_limits::max()) { + parseClient->error(hdrBegin, "NINT values that don't fit in int64_t are not supported."); + return {hdrBegin, nullptr /* end parsing */}; + } + std::unique_ptr item = std::make_unique(-1 - static_cast(value)); + return {hdrEnd, + parseClient->item(item, hdrBegin, hdrEnd /* valueBegin */, hdrEnd /* itemEnd */)}; +} + +std::tuple handleBool(uint64_t value, const uint8_t* hdrBegin, + const uint8_t* hdrEnd, + ParseClient* parseClient) { + std::unique_ptr item = std::make_unique(value == TRUE); + return {hdrEnd, + parseClient->item(item, hdrBegin, hdrEnd /* valueBegin */, hdrEnd /* itemEnd */)}; +} + +std::tuple handleNull(const uint8_t* hdrBegin, const uint8_t* hdrEnd, + ParseClient* parseClient) { + std::unique_ptr item = std::make_unique(); + return {hdrEnd, + parseClient->item(item, hdrBegin, hdrEnd /* valueBegin */, hdrEnd /* itemEnd */)}; +} + +template +std::tuple handleString(uint64_t length, const uint8_t* hdrBegin, + const uint8_t* valueBegin, const uint8_t* end, + const std::string& errLabel, + ParseClient* parseClient) { + ssize_t signed_length = static_cast(length); + if (end - valueBegin < signed_length || signed_length < 0) { + parseClient->error(hdrBegin, insufficientLengthString(length, end - valueBegin, errLabel)); + return {hdrBegin, nullptr /* end parsing */}; + } + + std::unique_ptr item = std::make_unique(valueBegin, valueBegin + length); + return {valueBegin + length, + parseClient->item(item, hdrBegin, valueBegin, valueBegin + length)}; +} + +class IncompleteItem { + public: + static IncompleteItem* cast(Item* item); + + virtual ~IncompleteItem() {} + virtual void add(std::unique_ptr item) = 0; + virtual std::unique_ptr finalize() && = 0; +}; + +class IncompleteArray : public Array, public IncompleteItem { + public: + explicit IncompleteArray(size_t size) : mSize(size) {} + + // We return the "complete" size, rather than the actual size. + size_t size() const override { return mSize; } + + void add(std::unique_ptr item) override { + mEntries.push_back(std::move(item)); + } + + virtual std::unique_ptr finalize() && override { + // Use Array explicitly so the compiler picks the correct ctor overload + Array* thisArray = this; + return std::make_unique(std::move(*thisArray)); + } + + private: + size_t mSize; +}; + +class IncompleteMap : public Map, public IncompleteItem { + public: + explicit IncompleteMap(size_t size) : mSize(size) {} + + // We return the "complete" size, rather than the actual size. + size_t size() const override { return mSize; } + + void add(std::unique_ptr item) override { + if (mKeyHeldForAdding) { + mEntries.push_back({std::move(mKeyHeldForAdding), std::move(item)}); + } else { + mKeyHeldForAdding = std::move(item); + } + } + + virtual std::unique_ptr finalize() && override { + return std::make_unique(std::move(*this)); + } + + private: + std::unique_ptr mKeyHeldForAdding; + size_t mSize; +}; + +class IncompleteSemanticTag : public SemanticTag, public IncompleteItem { + public: + explicit IncompleteSemanticTag(uint64_t value) : SemanticTag(value) {} + + // We return the "complete" size, rather than the actual size. + size_t size() const override { return 1; } + + void add(std::unique_ptr item) override { mTaggedItem = std::move(item); } + + virtual std::unique_ptr finalize() && override { + return std::make_unique(std::move(*this)); + } +}; + +IncompleteItem* IncompleteItem::cast(Item* item) { + CHECK(item->isCompound()); + // Semantic tag must be check first, because SemanticTag::type returns the wrapped item's type. + if (item->asSemanticTag()) { +#if __has_feature(cxx_rtti) + CHECK(dynamic_cast(item)); +#endif + return static_cast(item); + } else if (item->type() == ARRAY) { +#if __has_feature(cxx_rtti) + CHECK(dynamic_cast(item)); +#endif + return static_cast(item); + } else if (item->type() == MAP) { +#if __has_feature(cxx_rtti) + CHECK(dynamic_cast(item)); +#endif + return static_cast(item); + } else { + CHECK(false); // Impossible to get here. + } + return nullptr; +} + +std::tuple handleEntries(size_t entryCount, const uint8_t* hdrBegin, + const uint8_t* pos, const uint8_t* end, + const std::string& typeName, + bool emitViews, + ParseClient* parseClient) { + while (entryCount > 0) { + --entryCount; + if (pos == end) { + parseClient->error(hdrBegin, "Not enough entries for " + typeName + "."); + return {hdrBegin, nullptr /* end parsing */}; + } + std::tie(pos, parseClient) = parseRecursively(pos, end, emitViews, parseClient); + if (!parseClient) return {hdrBegin, nullptr}; + } + return {pos, parseClient}; +} + +std::tuple handleCompound( + std::unique_ptr item, uint64_t entryCount, const uint8_t* hdrBegin, + const uint8_t* valueBegin, const uint8_t* end, const std::string& typeName, + bool emitViews, ParseClient* parseClient) { + parseClient = + parseClient->item(item, hdrBegin, valueBegin, valueBegin /* don't know the end yet */); + if (!parseClient) return {hdrBegin, nullptr}; + + const uint8_t* pos; + std::tie(pos, parseClient) = + handleEntries(entryCount, hdrBegin, valueBegin, end, typeName, emitViews, parseClient); + if (!parseClient) return {hdrBegin, nullptr}; + + return {pos, parseClient->itemEnd(item, hdrBegin, valueBegin, pos)}; +} + +std::tuple parseRecursively(const uint8_t* begin, const uint8_t* end, + bool emitViews, ParseClient* parseClient) { + if (begin == end) { + parseClient->error( + begin, + "Input buffer is empty. Begin and end cannot point to the same location."); + return {begin, nullptr}; + } + + const uint8_t* pos = begin; + + MajorType type = static_cast(*pos & 0xE0); + uint8_t tagInt = *pos & 0x1F; + ++pos; + + bool success = true; + uint64_t addlData; + if (tagInt < ONE_BYTE_LENGTH) { + addlData = tagInt; + } else if (tagInt > EIGHT_BYTE_LENGTH) { + parseClient->error( + begin, + "Reserved additional information value or unsupported indefinite length item."); + return {begin, nullptr}; + } else { + switch (tagInt) { + case ONE_BYTE_LENGTH: + std::tie(success, addlData, pos) = parseLength(pos, end, parseClient); + break; + + case TWO_BYTE_LENGTH: + std::tie(success, addlData, pos) = parseLength(pos, end, parseClient); + break; + + case FOUR_BYTE_LENGTH: + std::tie(success, addlData, pos) = parseLength(pos, end, parseClient); + break; + + case EIGHT_BYTE_LENGTH: + std::tie(success, addlData, pos) = parseLength(pos, end, parseClient); + break; + + default: + CHECK(false); // It's impossible to get here + break; + } + } + + if (!success) return {begin, nullptr}; + + switch (type) { + case UINT: + return handleUint(addlData, begin, pos, parseClient); + + case NINT: + return handleNint(addlData, begin, pos, parseClient); + + case BSTR: + if (emitViews) { + return handleString(addlData, begin, pos, end, "byte string", parseClient); + } else { + return handleString(addlData, begin, pos, end, "byte string", parseClient); + } + + case TSTR: + if (emitViews) { + return handleString(addlData, begin, pos, end, "text string", parseClient); + } else { + return handleString(addlData, begin, pos, end, "text string", parseClient); + } + + case ARRAY: + return handleCompound(std::make_unique(addlData), addlData, begin, pos, + end, "array", emitViews, parseClient); + + case MAP: + return handleCompound(std::make_unique(addlData), addlData * 2, begin, + pos, end, "map", emitViews, parseClient); + + case SEMANTIC: + return handleCompound(std::make_unique(addlData), 1, begin, pos, + end, "semantic", emitViews, parseClient); + + case SIMPLE: + switch (addlData) { + case TRUE: + case FALSE: + return handleBool(addlData, begin, pos, parseClient); + case NULL_V: + return handleNull(begin, pos, parseClient); + default: + parseClient->error(begin, "Unsupported floating-point or simple value."); + return {begin, nullptr}; + } + } + CHECK(false); // Impossible to get here. + return {}; +} + +class FullParseClient : public ParseClient { + public: + virtual ParseClient* item(std::unique_ptr& item, const uint8_t*, const uint8_t*, + const uint8_t* end) override { + if (mParentStack.empty() && !item->isCompound()) { + // This is the first and only item. + mTheItem = std::move(item); + mPosition = end; + return nullptr; // We're done. + } + + if (item->isCompound()) { + // Starting a new compound data item, i.e. a new parent. Save it on the parent stack. + // It's safe to save a raw pointer because the unique_ptr is guaranteed to stay in + // existence until the corresponding itemEnd() call. + mParentStack.push(item.get()); + return this; + } else { + appendToLastParent(std::move(item)); + return this; + } + } + + virtual ParseClient* itemEnd(std::unique_ptr& item, const uint8_t*, const uint8_t*, + const uint8_t* end) override { + CHECK(item->isCompound() && item.get() == mParentStack.top()); + mParentStack.pop(); + IncompleteItem* incompleteItem = IncompleteItem::cast(item.get()); + std::unique_ptr finalizedItem = std::move(*incompleteItem).finalize(); + + if (mParentStack.empty()) { + mTheItem = std::move(finalizedItem); + mPosition = end; + return nullptr; // We're done + } else { + appendToLastParent(std::move(finalizedItem)); + return this; + } + } + + virtual void error(const uint8_t* position, const std::string& errorMessage) override { + mPosition = position; + mErrorMessage = errorMessage; + } + + std::tuple /* result */, const uint8_t* /* newPos */, + std::string /* errMsg */> + parseResult() { + std::unique_ptr p = std::move(mTheItem); + return {std::move(p), mPosition, std::move(mErrorMessage)}; + } + + private: + void appendToLastParent(std::unique_ptr item) { + auto parent = mParentStack.top(); + IncompleteItem::cast(parent)->add(std::move(item)); + } + + std::unique_ptr mTheItem; + std::stack mParentStack; + const uint8_t* mPosition = nullptr; + std::string mErrorMessage; +}; + +} // anonymous namespace + +void parse(const uint8_t* begin, const uint8_t* end, ParseClient* parseClient) { + parseRecursively(begin, end, false, parseClient); +} + +std::tuple /* result */, const uint8_t* /* newPos */, + std::string /* errMsg */> +parse(const uint8_t* begin, const uint8_t* end) { + FullParseClient parseClient; + parse(begin, end, &parseClient); + return parseClient.parseResult(); +} + +void parseWithViews(const uint8_t* begin, const uint8_t* end, ParseClient* parseClient) { + parseRecursively(begin, end, true, parseClient); +} + +std::tuple /* result */, const uint8_t* /* newPos */, + std::string /* errMsg */> +parseWithViews(const uint8_t* begin, const uint8_t* end) { + FullParseClient parseClient; + parseWithViews(begin, end, &parseClient); + return parseClient.parseResult(); +} + +} // namespace cppbor diff --git a/lib/libcppbor/include/cppbor/cppbor.h b/lib/libcppbor/include/cppbor/cppbor.h new file mode 100644 index 00000000..f7a2af0c --- /dev/null +++ b/lib/libcppbor/include/cppbor/cppbor.h @@ -0,0 +1,1141 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef OS_WINDOWS +#include + +#define ssize_t SSIZE_T +#endif // OS_WINDOWS + +#ifdef TRUE +#undef TRUE +#endif // TRUE +#ifdef FALSE +#undef FALSE +#endif // FALSE + +namespace cppbor { + +enum MajorType : uint8_t { + UINT = 0 << 5, + NINT = 1 << 5, + BSTR = 2 << 5, + TSTR = 3 << 5, + ARRAY = 4 << 5, + MAP = 5 << 5, + SEMANTIC = 6 << 5, + SIMPLE = 7 << 5, +}; + +enum SimpleType { + BOOLEAN, + NULL_T, // Only two supported, as yet. +}; + +enum SpecialAddlInfoValues : uint8_t { + FALSE = 20, + TRUE = 21, + NULL_V = 22, + ONE_BYTE_LENGTH = 24, + TWO_BYTE_LENGTH = 25, + FOUR_BYTE_LENGTH = 26, + EIGHT_BYTE_LENGTH = 27, +}; + +class Item; +class Uint; +class Nint; +class Int; +class Tstr; +class Bstr; +class Simple; +class Bool; +class Array; +class Map; +class Null; +class SemanticTag; +class EncodedItem; +class ViewTstr; +class ViewBstr; + +/** + * Returns the size of a CBOR header that contains the additional info value addlInfo. + */ +size_t headerSize(uint64_t addlInfo); + +/** + * Encodes a CBOR header with the specified type and additional info into the range [pos, end). + * Returns a pointer to one past the last byte written, or nullptr if there isn't sufficient space + * to write the header. + */ +uint8_t* encodeHeader(MajorType type, uint64_t addlInfo, uint8_t* pos, const uint8_t* end); + +using EncodeCallback = std::function; + +/** + * Encodes a CBOR header with the specified type and additional info, passing each byte in turn to + * encodeCallback. + */ +void encodeHeader(MajorType type, uint64_t addlInfo, EncodeCallback encodeCallback); + +/** + * Encodes a CBOR header witht he specified type and additional info, writing each byte to the + * provided OutputIterator. + */ +template ::iterator_category>>> +void encodeHeader(MajorType type, uint64_t addlInfo, OutputIterator iter) { + return encodeHeader(type, addlInfo, [&](uint8_t v) { *iter++ = v; }); +} + +/** + * Item represents a CBOR-encodeable data item. Item is an abstract interface with a set of virtual + * methods that allow encoding of the item or conversion to the appropriate derived type. + */ +class Item { + public: + virtual ~Item() {} + + /** + * Returns the CBOR type of the item. + */ + virtual MajorType type() const = 0; + + // These methods safely downcast an Item to the appropriate subclass. + virtual Int* asInt() { return nullptr; } + const Int* asInt() const { return const_cast(this)->asInt(); } + virtual Uint* asUint() { return nullptr; } + const Uint* asUint() const { return const_cast(this)->asUint(); } + virtual Nint* asNint() { return nullptr; } + const Nint* asNint() const { return const_cast(this)->asNint(); } + virtual Tstr* asTstr() { return nullptr; } + const Tstr* asTstr() const { return const_cast(this)->asTstr(); } + virtual Bstr* asBstr() { return nullptr; } + const Bstr* asBstr() const { return const_cast(this)->asBstr(); } + virtual Simple* asSimple() { return nullptr; } + const Simple* asSimple() const { return const_cast(this)->asSimple(); } + virtual Bool* asBool() { return nullptr; } + const Bool* asBool() const { return const_cast(this)->asBool(); } + virtual Null* asNull() { return nullptr; } + const Null* asNull() const { return const_cast(this)->asNull(); } + + virtual Map* asMap() { return nullptr; } + const Map* asMap() const { return const_cast(this)->asMap(); } + virtual Array* asArray() { return nullptr; } + const Array* asArray() const { return const_cast(this)->asArray(); } + + virtual ViewTstr* asViewTstr() { return nullptr; } + const ViewTstr* asViewTstr() const { return const_cast(this)->asViewTstr(); } + virtual ViewBstr* asViewBstr() { return nullptr; } + const ViewBstr* asViewBstr() const { return const_cast(this)->asViewBstr(); } + + // Like those above, these methods safely downcast an Item when it's actually a SemanticTag. + // However, if you think you want to use these methods, you probably don't. Typically, the way + // you should handle tagged Items is by calling the appropriate method above (e.g. asInt()) + // which will return a pointer to the tagged Item, rather than the tag itself. If you want to + // find out if the Item* you're holding is to something with one or more tags applied, see + // semanticTagCount() and semanticTag() below. + virtual SemanticTag* asSemanticTag() { return nullptr; } + const SemanticTag* asSemanticTag() const { return const_cast(this)->asSemanticTag(); } + + /** + * Returns the number of semantic tags prefixed to this Item. + */ + virtual size_t semanticTagCount() const { return 0; } + + /** + * Returns the semantic tag at the specified nesting level `nesting`, iff `nesting` is less than + * the value returned by semanticTagCount(). + * + * CBOR tags are "nested" by applying them in sequence. The "rightmost" tag is the "inner" tag. + * That is, given: + * + * 4(5(6("AES"))) which encodes as C1 C2 C3 63 414553 + * + * The tstr "AES" is tagged with 6. The combined entity ("AES" tagged with 6) is tagged with 5, + * etc. So in this example, semanticTagCount() would return 3, and semanticTag(0) would return + * 5 semanticTag(1) would return 5 and semanticTag(2) would return 4. For values of n > 2, + * semanticTag(n) will return 0, but this is a meaningless value. + * + * If this layering is confusing, you probably don't have to worry about it. Nested tagging does + * not appear to be common, so semanticTag(0) is the only one you'll use. + */ + virtual uint64_t semanticTag(size_t /* nesting */ = 0) const { return 0; } + + /** + * Returns true if this is a "compound" item, i.e. one that contains one or more other items. + */ + virtual bool isCompound() const { return false; } + + bool operator==(const Item& other) const&; + bool operator!=(const Item& other) const& { return !(*this == other); } + + /** + * Returns the number of bytes required to encode this Item into CBOR. Note that if this is a + * complex Item, calling this method will require walking the whole tree. + */ + virtual size_t encodedSize() const = 0; + + /** + * Encodes the Item into buffer referenced by range [*pos, end). Returns a pointer to one past + * the last position written. Returns nullptr if there isn't enough space to encode. + */ + virtual uint8_t* encode(uint8_t* pos, const uint8_t* end) const = 0; + + /** + * Encodes the Item by passing each encoded byte to encodeCallback. + */ + virtual void encode(EncodeCallback encodeCallback) const = 0; + + /** + * Clones the Item + */ + virtual std::unique_ptr clone() const = 0; + + /** + * Encodes the Item into the provided OutputIterator. + */ + template ::iterator_category> + void encode(OutputIterator i) const { + return encode([&](uint8_t v) { *i++ = v; }); + } + + /** + * Encodes the Item into a new std::vector. + */ + std::vector encode() const { + std::vector retval; + retval.reserve(encodedSize()); + encode(std::back_inserter(retval)); + return retval; + } + + /** + * Encodes the Item into a new std::string. + */ + std::string toString() const { + std::string retval; + retval.reserve(encodedSize()); + encode([&](uint8_t v) { retval.push_back(v); }); + return retval; + } + + /** + * Encodes only the header of the Item. + */ + inline uint8_t* encodeHeader(uint64_t addlInfo, uint8_t* pos, const uint8_t* end) const { + return ::cppbor::encodeHeader(type(), addlInfo, pos, end); + } + + /** + * Encodes only the header of the Item. + */ + inline void encodeHeader(uint64_t addlInfo, EncodeCallback encodeCallback) const { + ::cppbor::encodeHeader(type(), addlInfo, encodeCallback); + } +}; + +/** + * EncodedItem represents a bit of already-encoded CBOR. Caveat emptor: It does no checking to + * ensure that the provided data is a valid encoding, cannot be meaninfully-compared with other + * kinds of items and you cannot use the as*() methods to find out what's inside it. + */ +class EncodedItem : public Item { + public: + explicit EncodedItem(std::vector value) : mValue(std::move(value)) {} + + bool operator==(const EncodedItem& other) const& { return mValue == other.mValue; } + + // Type can't be meaningfully-obtained. We could extract the type from the first byte and return + // it, but you can't do any of the normal things with an EncodedItem so there's no point. + MajorType type() const override { + assert(false); + return static_cast(-1); + } + size_t encodedSize() const override { return mValue.size(); } + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override { + if (end - pos < static_cast(mValue.size())) return nullptr; + return std::copy(mValue.begin(), mValue.end(), pos); + } + void encode(EncodeCallback encodeCallback) const override { + std::for_each(mValue.begin(), mValue.end(), encodeCallback); + } + std::unique_ptr clone() const override { return std::make_unique(mValue); } + + private: + std::vector mValue; +}; + +/** + * Int is an abstraction that allows Uint and Nint objects to be manipulated without caring about + * the sign. + */ +class Int : public Item { + public: + bool operator==(const Int& other) const& { return value() == other.value(); } + + virtual int64_t value() const = 0; + using Item::asInt; + Int* asInt() override { return this; } +}; + +/** + * Uint is a concrete Item that implements CBOR major type 0. + */ +class Uint : public Int { + public: + static constexpr MajorType kMajorType = UINT; + + explicit Uint(uint64_t v) : mValue(v) {} + + bool operator==(const Uint& other) const& { return mValue == other.mValue; } + + MajorType type() const override { return kMajorType; } + using Item::asUint; + Uint* asUint() override { return this; } + + size_t encodedSize() const override { return headerSize(mValue); } + + int64_t value() const override { return mValue; } + uint64_t unsignedValue() const { return mValue; } + + using Item::encode; + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override { + return encodeHeader(mValue, pos, end); + } + void encode(EncodeCallback encodeCallback) const override { + encodeHeader(mValue, encodeCallback); + } + + std::unique_ptr clone() const override { return std::make_unique(mValue); } + + private: + uint64_t mValue; +}; + +/** + * Nint is a concrete Item that implements CBOR major type 1. + + * Note that it is incapable of expressing the full range of major type 1 values, becaue it can only + * express values that fall into the range [std::numeric_limits::min(), -1]. It cannot + * express values in the range [std::numeric_limits::min() - 1, + * -std::numeric_limits::max()]. + */ +class Nint : public Int { + public: + static constexpr MajorType kMajorType = NINT; + + explicit Nint(int64_t v); + + bool operator==(const Nint& other) const& { return mValue == other.mValue; } + + MajorType type() const override { return kMajorType; } + using Item::asNint; + Nint* asNint() override { return this; } + size_t encodedSize() const override { return headerSize(addlInfo()); } + + int64_t value() const override { return mValue; } + + using Item::encode; + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override { + return encodeHeader(addlInfo(), pos, end); + } + void encode(EncodeCallback encodeCallback) const override { + encodeHeader(addlInfo(), encodeCallback); + } + + std::unique_ptr clone() const override { return std::make_unique(mValue); } + + private: + uint64_t addlInfo() const { return -1ll - mValue; } + + int64_t mValue; +}; + +/** + * Bstr is a concrete Item that implements major type 2. + */ +class Bstr : public Item { + public: + static constexpr MajorType kMajorType = BSTR; + + // Construct an empty Bstr + explicit Bstr() {} + + // Construct from a vector + explicit Bstr(std::vector v) : mValue(std::move(v)) {} + + // Construct from a string + explicit Bstr(const std::string& v) + : mValue(reinterpret_cast(v.data()), + reinterpret_cast(v.data()) + v.size()) {} + + // Construct from a pointer/size pair + explicit Bstr(const std::pair& buf) + : mValue(buf.first, buf.first + buf.second) {} + + // Construct from a pair of iterators + template ::iterator_category, + typename = typename std::iterator_traits::iterator_category> + explicit Bstr(const std::pair& pair) : mValue(pair.first, pair.second) {} + + // Construct from an iterator range. + template ::iterator_category, + typename = typename std::iterator_traits::iterator_category> + Bstr(I1 begin, I2 end) : mValue(begin, end) {} + + bool operator==(const Bstr& other) const& { return mValue == other.mValue; } + + MajorType type() const override { return kMajorType; } + using Item::asBstr; + Bstr* asBstr() override { return this; } + size_t encodedSize() const override { return headerSize(mValue.size()) + mValue.size(); } + using Item::encode; + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override; + void encode(EncodeCallback encodeCallback) const override { + encodeHeader(mValue.size(), encodeCallback); + encodeValue(encodeCallback); + } + + const std::vector& value() const { return mValue; } + std::vector&& moveValue() { return std::move(mValue); } + + std::unique_ptr clone() const override { return std::make_unique(mValue); } + + private: + void encodeValue(EncodeCallback encodeCallback) const; + + std::vector mValue; +}; + +/** + * ViewBstr is a read-only version of Bstr backed by std::string_view + */ +class ViewBstr : public Item { + public: + static constexpr MajorType kMajorType = BSTR; + + // Construct an empty ViewBstr + explicit ViewBstr() {} + + // Construct from a string_view of uint8_t values + explicit ViewBstr(std::basic_string_view v) : mView(std::move(v)) {} + + // Construct from a string_view + explicit ViewBstr(std::string_view v) + : mView(reinterpret_cast(v.data()), v.size()) {} + + // Construct from an iterator range + template ::iterator_category, + typename = typename std::iterator_traits::iterator_category> + ViewBstr(I1 begin, I2 end) : mView(begin, end) {} + + // Construct from a uint8_t pointer pair + ViewBstr(const uint8_t* begin, const uint8_t* end) + : mView(begin, std::distance(begin, end)) {} + + bool operator==(const ViewBstr& other) const& { return mView == other.mView; } + + MajorType type() const override { return kMajorType; } + using Item::asViewBstr; + ViewBstr* asViewBstr() override { return this; } + size_t encodedSize() const override { return headerSize(mView.size()) + mView.size(); } + using Item::encode; + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override; + void encode(EncodeCallback encodeCallback) const override { + encodeHeader(mView.size(), encodeCallback); + encodeValue(encodeCallback); + } + + const std::basic_string_view& view() const { return mView; } + + std::unique_ptr clone() const override { return std::make_unique(mView); } + + private: + void encodeValue(EncodeCallback encodeCallback) const; + + std::basic_string_view mView; +}; + +/** + * Tstr is a concrete Item that implements major type 3. + */ +class Tstr : public Item { + public: + static constexpr MajorType kMajorType = TSTR; + + // Construct from a string + explicit Tstr(std::string v) : mValue(std::move(v)) {} + + // Construct from a string_view + explicit Tstr(const std::string_view& v) : mValue(v) {} + + // Construct from a C string + explicit Tstr(const char* v) : mValue(std::string(v)) {} + + // Construct from a pair of iterators + template ::iterator_category, + typename = typename std::iterator_traits::iterator_category> + explicit Tstr(const std::pair& pair) : mValue(pair.first, pair.second) {} + + // Construct from an iterator range + template ::iterator_category, + typename = typename std::iterator_traits::iterator_category> + Tstr(I1 begin, I2 end) : mValue(begin, end) {} + + bool operator==(const Tstr& other) const& { return mValue == other.mValue; } + + MajorType type() const override { return kMajorType; } + using Item::asTstr; + Tstr* asTstr() override { return this; } + size_t encodedSize() const override { return headerSize(mValue.size()) + mValue.size(); } + using Item::encode; + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override; + void encode(EncodeCallback encodeCallback) const override { + encodeHeader(mValue.size(), encodeCallback); + encodeValue(encodeCallback); + } + + const std::string& value() const { return mValue; } + std::string&& moveValue() { return std::move(mValue); } + + std::unique_ptr clone() const override { return std::make_unique(mValue); } + + private: + void encodeValue(EncodeCallback encodeCallback) const; + + std::string mValue; +}; + +/** + * ViewTstr is a read-only version of Tstr backed by std::string_view + */ +class ViewTstr : public Item { + public: + static constexpr MajorType kMajorType = TSTR; + + // Construct an empty ViewTstr + explicit ViewTstr() {} + + // Construct from a string_view + explicit ViewTstr(std::string_view v) : mView(std::move(v)) {} + + // Construct from an iterator range + template ::iterator_category, + typename = typename std::iterator_traits::iterator_category> + ViewTstr(I1 begin, I2 end) : mView(begin, end) {} + + // Construct from a uint8_t pointer pair + ViewTstr(const uint8_t* begin, const uint8_t* end) + : mView(reinterpret_cast(begin), + std::distance(begin, end)) {} + + bool operator==(const ViewTstr& other) const& { return mView == other.mView; } + + MajorType type() const override { return kMajorType; } + using Item::asViewTstr; + ViewTstr* asViewTstr() override { return this; } + size_t encodedSize() const override { return headerSize(mView.size()) + mView.size(); } + using Item::encode; + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override; + void encode(EncodeCallback encodeCallback) const override { + encodeHeader(mView.size(), encodeCallback); + encodeValue(encodeCallback); + } + + const std::string_view& view() const { return mView; } + + std::unique_ptr clone() const override { return std::make_unique(mView); } + + private: + void encodeValue(EncodeCallback encodeCallback) const; + + std::string_view mView; +}; + +/* + * Array is a concrete Item that implements CBOR major type 4. + * + * Note that Arrays are not copyable. This is because copying them is expensive and making them + * move-only ensures that they're never copied accidentally. If you actually want to copy an Array, + * use the clone() method. + */ +class Array : public Item { + public: + static constexpr MajorType kMajorType = ARRAY; + + Array() = default; + Array(const Array& other) = delete; + Array(Array&&) = default; + Array& operator=(const Array&) = delete; + Array& operator=(Array&&) = default; + + bool operator==(const Array& other) const&; + + /** + * Construct an Array from a variable number of arguments of different types. See + * details::makeItem below for details on what types may be provided. In general, this accepts + * all of the types you'd expect and doest the things you'd expect (integral values are addes as + * Uint or Nint, std::string and char* are added as Tstr, bools are added as Bool, etc.). + */ + template + Array(Args&&... args); + + /** + * The above variadic constructor is disabled if sizeof(Args) != 1, so special + * case an explicit Array constructor for creating an Array with one Item. + */ + template + explicit Array(T&& v); + + /** + * Append a single element to the Array, of any compatible type. + */ + template + Array& add(T&& v) &; + template + Array&& add(T&& v) &&; + + bool isCompound() const override { return true; } + + virtual size_t size() const { return mEntries.size(); } + + size_t encodedSize() const override { + return std::accumulate(mEntries.begin(), mEntries.end(), headerSize(size()), + [](size_t sum, auto& entry) { return sum + entry->encodedSize(); }); + } + + using Item::encode; // Make base versions visible. + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override; + void encode(EncodeCallback encodeCallback) const override; + + const std::unique_ptr& operator[](size_t index) const { return get(index); } + std::unique_ptr& operator[](size_t index) { return get(index); } + + const std::unique_ptr& get(size_t index) const { return mEntries[index]; } + std::unique_ptr& get(size_t index) { return mEntries[index]; } + + MajorType type() const override { return kMajorType; } + using Item::asArray; + Array* asArray() override { return this; } + + std::unique_ptr clone() const override; + + auto begin() { return mEntries.begin(); } + auto begin() const { return mEntries.begin(); } + auto end() { return mEntries.end(); } + auto end() const { return mEntries.end(); } + + protected: + std::vector> mEntries; +}; + +/* + * Map is a concrete Item that implements CBOR major type 5. + * + * Note that Maps are not copyable. This is because copying them is expensive and making them + * move-only ensures that they're never copied accidentally. If you actually want to copy a + * Map, use the clone() method. + */ +class Map : public Item { + public: + static constexpr MajorType kMajorType = MAP; + + using entry_type = std::pair, std::unique_ptr>; + + Map() = default; + Map(const Map& other) = delete; + Map(Map&&) = default; + Map& operator=(const Map& other) = delete; + Map& operator=(Map&&) = default; + + bool operator==(const Map& other) const&; + + /** + * Construct a Map from a variable number of arguments of different types. An even number of + * arguments must be provided (this is verified statically). See details::makeItem below for + * details on what types may be provided. In general, this accepts all of the types you'd + * expect and doest the things you'd expect (integral values are addes as Uint or Nint, + * std::string and char* are added as Tstr, bools are added as Bool, etc.). + */ + template + Map(Args&&... args); + + /** + * Append a key/value pair to the Map, of any compatible types. + */ + template + Map& add(Key&& key, Value&& value) &; + template + Map&& add(Key&& key, Value&& value) &&; + + bool isCompound() const override { return true; } + + virtual size_t size() const { return mEntries.size(); } + + size_t encodedSize() const override { + return std::accumulate( + mEntries.begin(), mEntries.end(), headerSize(size()), [](size_t sum, auto& entry) { + return sum + entry.first->encodedSize() + entry.second->encodedSize(); + }); + } + + using Item::encode; // Make base versions visible. + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override; + void encode(EncodeCallback encodeCallback) const override; + + /** + * Find and return the value associated with `key`, if any. + * + * If the searched-for `key` is not present, returns `nullptr`. + * + * Note that if the map is canonicalized (sorted), Map::get() performs a binary search. If your + * map is large and you're searching in it many times, it may be worthwhile to canonicalize it + * to make Map::get() faster. Any use of a method that might modify the map disables the + * speedup. + */ + template + const std::unique_ptr& get(Key key) const; + + // Note that use of non-const operator[] marks the map as not canonicalized. + auto& operator[](size_t index) { + mCanonicalized = false; + return mEntries[index]; + } + const auto& operator[](size_t index) const { return mEntries[index]; } + + MajorType type() const override { return kMajorType; } + using Item::asMap; + Map* asMap() override { return this; } + + /** + * Sorts the map in canonical order, as defined in RFC 7049. Use this before encoding if you + * want canonicalization; cppbor does not canonicalize by default, though the integer encodings + * are always canonical and cppbor does not support indefinite-length encodings, so map order + * canonicalization is the only thing that needs to be done. + * + * @param recurse If set to true, canonicalize() will also walk the contents of the map and + * canonicalize any contained maps as well. + */ + Map& canonicalize(bool recurse = false) &; + Map&& canonicalize(bool recurse = false) && { + canonicalize(recurse); + return std::move(*this); + } + + bool isCanonical() { return mCanonicalized; } + + std::unique_ptr clone() const override; + + auto begin() { + mCanonicalized = false; + return mEntries.begin(); + } + auto begin() const { return mEntries.begin(); } + auto end() { + mCanonicalized = false; + return mEntries.end(); + } + auto end() const { return mEntries.end(); } + + // Returns true if a < b, per CBOR map key canonicalization rules. + static bool keyLess(const Item* a, const Item* b); + + protected: + std::vector mEntries; + + private: + bool mCanonicalized = false; +}; + +class SemanticTag : public Item { + public: + static constexpr MajorType kMajorType = SEMANTIC; + + template + SemanticTag(uint64_t tagValue, T&& taggedItem); + SemanticTag(const SemanticTag& other) = delete; + SemanticTag(SemanticTag&&) = default; + SemanticTag& operator=(const SemanticTag& other) = delete; + SemanticTag& operator=(SemanticTag&&) = default; + + bool operator==(const SemanticTag& other) const& { + return mValue == other.mValue && *mTaggedItem == *other.mTaggedItem; + } + + bool isCompound() const override { return true; } + + virtual size_t size() const { return 1; } + + // Encoding returns the tag + enclosed Item. + size_t encodedSize() const override { return headerSize(mValue) + mTaggedItem->encodedSize(); } + + using Item::encode; // Make base versions visible. + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override; + void encode(EncodeCallback encodeCallback) const override; + + // type() is a bit special. In normal usage it should return the wrapped type, but during + // parsing when we haven't yet parsed the tagged item, it needs to return SEMANTIC. + MajorType type() const override { return mTaggedItem ? mTaggedItem->type() : SEMANTIC; } + using Item::asSemanticTag; + SemanticTag* asSemanticTag() override { return this; } + + // Type information reflects the enclosed Item. Note that if the immediately-enclosed Item is + // another tag, these methods will recurse down to the non-tag Item. + using Item::asInt; + Int* asInt() override { return mTaggedItem->asInt(); } + using Item::asUint; + Uint* asUint() override { return mTaggedItem->asUint(); } + using Item::asNint; + Nint* asNint() override { return mTaggedItem->asNint(); } + using Item::asTstr; + Tstr* asTstr() override { return mTaggedItem->asTstr(); } + using Item::asBstr; + Bstr* asBstr() override { return mTaggedItem->asBstr(); } + using Item::asSimple; + Simple* asSimple() override { return mTaggedItem->asSimple(); } + using Item::asMap; + Map* asMap() override { return mTaggedItem->asMap(); } + using Item::asArray; + Array* asArray() override { return mTaggedItem->asArray(); } + using Item::asViewTstr; + ViewTstr* asViewTstr() override { return mTaggedItem->asViewTstr(); } + using Item::asViewBstr; + ViewBstr* asViewBstr() override { return mTaggedItem->asViewBstr(); } + + std::unique_ptr clone() const override; + + size_t semanticTagCount() const override; + uint64_t semanticTag(size_t nesting = 0) const override; + + protected: + SemanticTag() = default; + SemanticTag(uint64_t value) : mValue(value) {} + uint64_t mValue; + std::unique_ptr mTaggedItem; +}; + +/** + * Simple is abstract Item that implements CBOR major type 7. It is intended to be subclassed to + * create concrete Simple types. At present only Bool is provided. + */ +class Simple : public Item { + public: + static constexpr MajorType kMajorType = SIMPLE; + + bool operator==(const Simple& other) const&; + + virtual SimpleType simpleType() const = 0; + MajorType type() const override { return kMajorType; } + + Simple* asSimple() override { return this; } +}; + +/** + * Bool is a concrete type that implements CBOR major type 7, with additional item values for TRUE + * and FALSE. + */ +class Bool : public Simple { + public: + static constexpr SimpleType kSimpleType = BOOLEAN; + + explicit Bool(bool v) : mValue(v) {} + + bool operator==(const Bool& other) const& { return mValue == other.mValue; } + + SimpleType simpleType() const override { return kSimpleType; } + Bool* asBool() override { return this; } + + size_t encodedSize() const override { return 1; } + + using Item::encode; + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override { + return encodeHeader(mValue ? TRUE : FALSE, pos, end); + } + void encode(EncodeCallback encodeCallback) const override { + encodeHeader(mValue ? TRUE : FALSE, encodeCallback); + } + + bool value() const { return mValue; } + + std::unique_ptr clone() const override { return std::make_unique(mValue); } + + private: + bool mValue; +}; + +/** + * Null is a concrete type that implements CBOR major type 7, with additional item value for NULL + */ +class Null : public Simple { + public: + static constexpr SimpleType kSimpleType = NULL_T; + + explicit Null() {} + + SimpleType simpleType() const override { return kSimpleType; } + Null* asNull() override { return this; } + + size_t encodedSize() const override { return 1; } + + using Item::encode; + uint8_t* encode(uint8_t* pos, const uint8_t* end) const override { + return encodeHeader(NULL_V, pos, end); + } + void encode(EncodeCallback encodeCallback) const override { + encodeHeader(NULL_V, encodeCallback); + } + + std::unique_ptr clone() const override { return std::make_unique(); } +}; + +/** + * Returns pretty-printed CBOR for |item| + * + * If a byte-string is larger than |maxBStrSize| its contents will not be printed, instead the value + * of the form "" will be + * printed. Pass zero for |maxBStrSize| to disable this. + * + * The |mapKeysToNotPrint| parameter specifies the name of map values to not print. This is useful + * for unit tests. + */ +std::string prettyPrint(const Item* item, size_t maxBStrSize = 32, + const std::vector& mapKeysToNotPrint = {}); + +/** + * Returns pretty-printed CBOR for |value|. + * + * Only valid CBOR should be passed to this function. + * + * If a byte-string is larger than |maxBStrSize| its contents will not be printed, instead the value + * of the form "" will be + * printed. Pass zero for |maxBStrSize| to disable this. + * + * The |mapKeysToNotPrint| parameter specifies the name of map values to not print. This is useful + * for unit tests. + */ +std::string prettyPrint(const std::vector& encodedCbor, size_t maxBStrSize = 32, + const std::vector& mapKeysToNotPrint = {}); + +/** + * Details. Mostly you shouldn't have to look below, except perhaps at the docstring for makeItem. + */ +namespace details { + +template +struct is_iterator_pair_over : public std::false_type {}; + +template +struct is_iterator_pair_over< + std::pair, V, + typename std::enable_if_t::value_type>>> + : public std::true_type {}; + +template +struct is_unique_ptr_of_subclass_of_v : public std::false_type {}; + +template +struct is_unique_ptr_of_subclass_of_v, + typename std::enable_if_t>> + : public std::true_type {}; + +/* check if type is one of std::string (1), std::string_view (2), null-terminated char* (3) or pair + * of iterators (4)*/ +template +struct is_text_type_v : public std::false_type {}; + +template +struct is_text_type_v< + T, typename std::enable_if_t< + /* case 1 */ // + std::is_same_v>, std::string> + /* case 2 */ // + || std::is_same_v>, std::string_view> + /* case 3 */ // + || std::is_same_v>, char*> // + || std::is_same_v>, const char*> + /* case 4 */ + || details::is_iterator_pair_over::value>> : public std::true_type {}; + +/** + * Construct a unique_ptr from many argument types. Accepts: + * + * (a) booleans; + * (b) integers, all sizes and signs; + * (c) text strings, as defined by is_text_type_v above; + * (d) byte strings, as std::vector(d1), pair of iterators (d2) or pair + * (d3); and + * (e) Item subclass instances, including Array and Map. Items may be provided by naked pointer + * (e1), unique_ptr (e2), reference (e3) or value (e3). If provided by reference or value, will + * be moved if possible. If provided by pointer, ownership is taken. + * (f) null pointer; + * (g) enums, using the underlying integer value. + */ +template +std::unique_ptr makeItem(T v) { + Item* p = nullptr; + if constexpr (/* case a */ std::is_same_v) { + p = new Bool(v); + } else if constexpr (/* case b */ std::is_integral_v) { // b + if (v < 0) { + p = new Nint(v); + } else { + p = new Uint(static_cast(v)); + } + } else if constexpr (/* case c */ // + details::is_text_type_v::value) { + p = new Tstr(v); + } else if constexpr (/* case d1 */ // + std::is_same_v>, + std::vector> + /* case d2 */ // + || details::is_iterator_pair_over::value + /* case d3 */ // + || std::is_same_v>, + std::pair>) { + p = new Bstr(v); + } else if constexpr (/* case e1 */ // + std::is_pointer_v && + std::is_base_of_v>) { + p = v; + } else if constexpr (/* case e2 */ // + details::is_unique_ptr_of_subclass_of_v::value) { + p = v.release(); + } else if constexpr (/* case e3 */ // + std::is_base_of_v) { + p = new T(std::move(v)); + } else if constexpr (/* case f */ std::is_null_pointer_v) { + p = new Null(); + } else if constexpr (/* case g */ std::is_enum_v) { + return makeItem(static_cast>(v)); + } else { + // It's odd that this can't be static_assert(false), since it shouldn't be evaluated if one + // of the above ifs matches. But static_assert(false) always triggers. + static_assert(std::is_same_v, "makeItem called with unsupported type"); + } + return std::unique_ptr(p); +} + +inline void map_helper(Map& /* map */) {} + +template +inline void map_helper(Map& map, Key&& key, Value&& value, Rest&&... rest) { + map.add(std::forward(key), std::forward(value)); + map_helper(map, std::forward(rest)...); +} + +} // namespace details + +template > +Array::Array(Args&&... args) { + mEntries.reserve(sizeof...(args)); + (mEntries.push_back(details::makeItem(std::forward(args))), ...); +} + +template >>>> +Array::Array(T&& v) { + mEntries.push_back(details::makeItem(std::forward(v))); +} + +template +Array& Array::add(T&& v) & { + mEntries.push_back(details::makeItem(std::forward(v))); + return *this; +} + +template +Array&& Array::add(T&& v) && { + mEntries.push_back(details::makeItem(std::forward(v))); + return std::move(*this); +} + +template > +Map::Map(Args&&... args) { + static_assert((sizeof...(Args)) % 2 == 0, "Map must have an even number of entries"); + mEntries.reserve(sizeof...(args) / 2); + details::map_helper(*this, std::forward(args)...); +} + +template +Map& Map::add(Key&& key, Value&& value) & { + mEntries.push_back({details::makeItem(std::forward(key)), + details::makeItem(std::forward(value))}); + mCanonicalized = false; + return *this; +} + +template +Map&& Map::add(Key&& key, Value&& value) && { + this->add(std::forward(key), std::forward(value)); + return std::move(*this); +} + +static const std::unique_ptr kEmptyItemPtr; + +template || std::is_enum_v || + details::is_text_type_v::value>> +const std::unique_ptr& Map::get(Key key) const { + auto keyItem = details::makeItem(key); + + if (mCanonicalized) { + // It's sorted, so binary-search it. + auto found = std::lower_bound(begin(), end(), keyItem.get(), + [](const entry_type& entry, const Item* key) { + return keyLess(entry.first.get(), key); + }); + return (found == end() || *found->first != *keyItem) ? kEmptyItemPtr : found->second; + } else { + // Unsorted, do a linear search. + auto found = std::find_if( + begin(), end(), [&](const entry_type& entry) { return *entry.first == *keyItem; }); + return found == end() ? kEmptyItemPtr : found->second; + } +} + +template +SemanticTag::SemanticTag(uint64_t value, T&& taggedItem) + : mValue(value), mTaggedItem(details::makeItem(std::forward(taggedItem))) {} + +} // namespace cppbor diff --git a/lib/libcppbor/include/cppbor/cppbor_parse.h b/lib/libcppbor/include/cppbor/cppbor_parse.h new file mode 100644 index 00000000..22cd18d0 --- /dev/null +++ b/lib/libcppbor/include/cppbor/cppbor_parse.h @@ -0,0 +1,195 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cppbor.h" + +namespace cppbor { + +using ParseResult = std::tuple /* result */, const uint8_t* /* newPos */, + std::string /* errMsg */>; + +/** + * Parse the first CBOR data item (possibly compound) from the range [begin, end). + * + * Returns a tuple of Item pointer, buffer pointer and error message. If parsing is successful, the + * Item pointer is non-null, the buffer pointer points to the first byte after the + * successfully-parsed item and the error message string is empty. If parsing fails, the Item + * pointer is null, the buffer pointer points to the first byte that was unparseable (the first byte + * of a data item header that is malformed in some way, e.g. an invalid value, or a length that is + * too large for the remaining buffer, etc.) and the string contains an error message describing the + * problem encountered. + */ +ParseResult parse(const uint8_t* begin, const uint8_t* end); + +/** + * Parse the first CBOR data item (possibly compound) from the range [begin, end). + * + * Returns a tuple of Item pointer, buffer pointer and error message. If parsing is successful, the + * Item pointer is non-null, the buffer pointer points to the first byte after the + * successfully-parsed item and the error message string is empty. If parsing fails, the Item + * pointer is null, the buffer pointer points to the first byte that was unparseable (the first byte + * of a data item header that is malformed in some way, e.g. an invalid value, or a length that is + * too large for the remaining buffer, etc.) and the string contains an error message describing the + * problem encountered. + * + * The returned CBOR data item will contain View* items backed by + * std::string_view types over the input range. + * WARNING! If the input range changes underneath, the corresponding views will + * carry the same change. + */ +ParseResult parseWithViews(const uint8_t* begin, const uint8_t* end); + +/** + * Parse the first CBOR data item (possibly compound) from the byte vector. + * + * Returns a tuple of Item pointer, buffer pointer and error message. If parsing is successful, the + * Item pointer is non-null, the buffer pointer points to the first byte after the + * successfully-parsed item and the error message string is empty. If parsing fails, the Item + * pointer is null, the buffer pointer points to the first byte that was unparseable (the first byte + * of a data item header that is malformed in some way, e.g. an invalid value, or a length that is + * too large for the remaining buffer, etc.) and the string contains an error message describing the + * problem encountered. + */ +inline ParseResult parse(const std::vector& encoding) { + return parse(encoding.data(), encoding.data() + encoding.size()); +} + +/** + * Parse the first CBOR data item (possibly compound) from the range [begin, begin + size). + * + * Returns a tuple of Item pointer, buffer pointer and error message. If parsing is successful, the + * Item pointer is non-null, the buffer pointer points to the first byte after the + * successfully-parsed item and the error message string is empty. If parsing fails, the Item + * pointer is null, the buffer pointer points to the first byte that was unparseable (the first byte + * of a data item header that is malformed in some way, e.g. an invalid value, or a length that is + * too large for the remaining buffer, etc.) and the string contains an error message describing the + * problem encountered. + */ +inline ParseResult parse(const uint8_t* begin, size_t size) { + return parse(begin, begin + size); +} + +/** + * Parse the first CBOR data item (possibly compound) from the range [begin, begin + size). + * + * Returns a tuple of Item pointer, buffer pointer and error message. If parsing is successful, the + * Item pointer is non-null, the buffer pointer points to the first byte after the + * successfully-parsed item and the error message string is empty. If parsing fails, the Item + * pointer is null, the buffer pointer points to the first byte that was unparseable (the first byte + * of a data item header that is malformed in some way, e.g. an invalid value, or a length that is + * too large for the remaining buffer, etc.) and the string contains an error message describing the + * problem encountered. + * + * The returned CBOR data item will contain View* items backed by + * std::string_view types over the input range. + * WARNING! If the input range changes underneath, the corresponding views will + * carry the same change. + */ +inline ParseResult parseWithViews(const uint8_t* begin, size_t size) { + return parseWithViews(begin, begin + size); +} + +/** + * Parse the first CBOR data item (possibly compound) from the value contained in a Bstr. + * + * Returns a tuple of Item pointer, buffer pointer and error message. If parsing is successful, the + * Item pointer is non-null, the buffer pointer points to the first byte after the + * successfully-parsed item and the error message string is empty. If parsing fails, the Item + * pointer is null, the buffer pointer points to the first byte that was unparseable (the first byte + * of a data item header that is malformed in some way, e.g. an invalid value, or a length that is + * too large for the remaining buffer, etc.) and the string contains an error message describing the + * problem encountered. + */ +inline ParseResult parse(const Bstr* bstr) { + if (!bstr) + return ParseResult(nullptr, nullptr, "Null Bstr pointer"); + return parse(bstr->value()); +} + +class ParseClient; + +/** + * Parse the CBOR data in the range [begin, end) in streaming fashion, calling methods on the + * provided ParseClient when elements are found. + */ +void parse(const uint8_t* begin, const uint8_t* end, ParseClient* parseClient); + +/** + * Parse the CBOR data in the range [begin, end) in streaming fashion, calling methods on the + * provided ParseClient when elements are found. Uses the View* item types + * instead of the copying ones. + */ +void parseWithViews(const uint8_t* begin, const uint8_t* end, ParseClient* parseClient); + +/** + * Parse the CBOR data in the vector in streaming fashion, calling methods on the + * provided ParseClient when elements are found. + */ +inline void parse(const std::vector& encoding, ParseClient* parseClient) { + return parse(encoding.data(), encoding.data() + encoding.size(), parseClient); +} + +/** + * A pure interface that callers of the streaming parse functions must implement. + */ +class ParseClient { + public: + virtual ~ParseClient() {} + + /** + * Called when an item is found. The Item pointer points to the found item; use type() and + * the appropriate as*() method to examine the value. hdrBegin points to the first byte of the + * header, valueBegin points to the first byte of the value and end points one past the end of + * the item. In the case of header-only items, such as integers, and compound items (ARRAY, + * MAP or SEMANTIC) whose end has not yet been found, valueBegin and end are equal and point to + * the byte past the header. + * + * Note that for compound types (ARRAY, MAP, and SEMANTIC), the Item will have no content. For + * Map and Array items, the size() method will return a correct value, but the index operators + * are unsafe, and the object cannot be safely compared with another Array/Map. + * + * The method returns a ParseClient*. In most cases "return this;" will be the right answer, + * but a different ParseClient may be returned, which the parser will begin using. If the method + * returns nullptr, parsing will be aborted immediately. + */ + virtual ParseClient* item(std::unique_ptr& item, const uint8_t* hdrBegin, + const uint8_t* valueBegin, const uint8_t* end) = 0; + + /** + * Called when the end of a compound item (MAP or ARRAY) is found. The item argument will be + * the same one passed to the item() call -- and may be empty if item() moved its value out. + * hdrBegin, valueBegin and end point to the beginning of the item header, the beginning of the + * first contained value, and one past the end of the last contained value, respectively. + * + * Note that the Item will have no content. + * + * As with item(), itemEnd() can change the ParseClient by returning a different one, or end the + * parsing by returning nullptr; + */ + virtual ParseClient* itemEnd(std::unique_ptr& item, const uint8_t* hdrBegin, + const uint8_t* valueBegin, const uint8_t* end) = 0; + + /** + * Called when parsing encounters an error. position is set to the first unparsed byte (one + * past the last successfully-parsed byte) and errorMessage contains an message explaining what + * sort of error occurred. + */ + virtual void error(const uint8_t* position, const std::string& errorMessage) = 0; +}; + +} // namespace cppbor diff --git a/src/database/CMakeLists.txt b/src/database/CMakeLists.txt index dffc3a26..26c14815 100644 --- a/src/database/CMakeLists.txt +++ b/src/database/CMakeLists.txt @@ -7,7 +7,7 @@ idf_component_register( "file_gatherer.cpp" "tag_parser.cpp" "index.cpp" INCLUDE_DIRS "include" REQUIRES "result" "span" "esp_psram" "fatfs" "libtags" "komihash" "cbor" - "tasks" "memory" "util" "tinyfsm" "events" "opusfile") + "tasks" "memory" "util" "tinyfsm" "events" "opusfile" "libcppbor") target_compile_options(${COMPONENT_LIB} PRIVATE ${EXTRA_WARNINGS}) diff --git a/src/database/database.cpp b/src/database/database.cpp index 9d29aea8..60aa69fc 100644 --- a/src/database/database.cpp +++ b/src/database/database.cpp @@ -126,9 +126,9 @@ auto Database::Update() -> std::future { ESP_LOGI(kTag, "dropping stale indexes"); { std::unique_ptr it{db_->NewIterator(read_options)}; - OwningSlice prefix = EncodeAllIndexesPrefix(); - it->Seek(prefix.slice); - while (it->Valid() && it->key().starts_with(prefix.slice)) { + std::string prefix = EncodeAllIndexesPrefix(); + it->Seek(prefix); + while (it->Valid() && it->key().starts_with(prefix)) { db_->Delete(leveldb::WriteOptions(), it->key()); it->Next(); } @@ -139,9 +139,9 @@ auto Database::Update() -> std::future { { uint64_t num_processed = 0; std::unique_ptr it{db_->NewIterator(read_options)}; - OwningSlice prefix = EncodeDataPrefix(); - it->Seek(prefix.slice); - while (it->Valid() && it->key().starts_with(prefix.slice)) { + std::string prefix = EncodeDataPrefix(); + it->Seek(prefix); + while (it->Valid() && it->key().starts_with(prefix)) { num_processed++; events::Ui().Dispatch(event::UpdateProgress{ .stage = event::UpdateProgress::Stage::kVerifyingExistingTracks, @@ -215,10 +215,10 @@ auto Database::Update() -> std::future { // Check for any existing record with the same hash. uint64_t hash = tags->Hash(); - OwningSlice key = EncodeHashKey(hash); + std::string key = EncodeHashKey(hash); std::optional existing_hash; std::string raw_entry; - if (db_->Get(leveldb::ReadOptions(), key.slice, &raw_entry).ok()) { + if (db_->Get(leveldb::ReadOptions(), key, &raw_entry).ok()) { existing_hash = ParseHashValue(raw_entry); } @@ -306,9 +306,9 @@ auto Database::GetBulkTracks(std::vector ids) std::unique_ptr it{ db_->NewIterator(leveldb::ReadOptions{})}; for (const TrackId& id : sorted_ids) { - OwningSlice key = EncodeDataKey(id); - it->Seek(key.slice); - if (!it->Valid() || it->key() != key.slice) { + std::string key = EncodeDataKey(id); + it->Seek(key); + if (!it->Valid() || it->key() != key) { // This id wasn't found at all. Skip it. continue; } @@ -354,9 +354,9 @@ auto Database::GetTracksByIndex(const IndexInfo& index, std::size_t page_size) .depth = 0, .components_hash = 0, }; - OwningSlice prefix = EncodeIndexPrefix(header); - Continuation c{.prefix = prefix.data, - .start_key = prefix.data, + std::string prefix = EncodeIndexPrefix(header); + Continuation c{.prefix = {prefix.data(), prefix.size()}, + .start_key = {prefix.data(), prefix.size()}, .forward = true, .was_prev_forward = true, .page_size = page_size}; @@ -366,8 +366,9 @@ auto Database::GetTracksByIndex(const IndexInfo& index, std::size_t page_size) auto Database::GetTracks(std::size_t page_size) -> std::future*> { return worker_task_->Dispatch*>([=, this]() -> Result* { - Continuation c{.prefix = EncodeDataPrefix().data, - .start_key = EncodeDataPrefix().data, + std::string prefix = EncodeDataPrefix(); + Continuation c{.prefix = {prefix.data(), prefix.size()}, + .start_key = {prefix.data(), prefix.size()}, .forward = true, .was_prev_forward = true, .page_size = page_size}; @@ -414,7 +415,7 @@ auto Database::dbMintNewTrackId() -> TrackId { } if (!db_->Put(leveldb::WriteOptions(), kTrackIdKey, - TrackIdToBytes(next_id + 1).slice) + TrackIdToBytes(next_id + 1)) .ok()) { ESP_LOGE(kTag, "failed to write next track id"); } @@ -423,25 +424,25 @@ auto Database::dbMintNewTrackId() -> TrackId { } auto Database::dbEntomb(TrackId id, uint64_t hash) -> void { - OwningSlice key = EncodeHashKey(hash); - OwningSlice val = EncodeHashValue(id); - if (!db_->Put(leveldb::WriteOptions(), key.slice, val.slice).ok()) { + std::string key = EncodeHashKey(hash); + std::string val = EncodeHashValue(id); + if (!db_->Put(leveldb::WriteOptions(), key, val).ok()) { ESP_LOGE(kTag, "failed to entomb #%llx (id #%lx)", hash, id); } } auto Database::dbPutTrackData(const TrackData& s) -> void { - OwningSlice key = EncodeDataKey(s.id()); - OwningSlice val = EncodeDataValue(s); - if (!db_->Put(leveldb::WriteOptions(), key.slice, val.slice).ok()) { + std::string key = EncodeDataKey(s.id()); + std::string val = EncodeDataValue(s); + if (!db_->Put(leveldb::WriteOptions(), key, val).ok()) { ESP_LOGE(kTag, "failed to write data for #%lx", s.id()); } } auto Database::dbGetTrackData(TrackId id) -> std::shared_ptr { - OwningSlice key = EncodeDataKey(id); + std::string key = EncodeDataKey(id); std::string raw_val; - if (!db_->Get(leveldb::ReadOptions(), key.slice, &raw_val).ok()) { + if (!db_->Get(leveldb::ReadOptions(), key, &raw_val).ok()) { ESP_LOGW(kTag, "no key found for #%lx", id); return {}; } @@ -449,17 +450,17 @@ auto Database::dbGetTrackData(TrackId id) -> std::shared_ptr { } auto Database::dbPutHash(const uint64_t& hash, TrackId i) -> void { - OwningSlice key = EncodeHashKey(hash); - OwningSlice val = EncodeHashValue(i); - if (!db_->Put(leveldb::WriteOptions(), key.slice, val.slice).ok()) { + std::string key = EncodeHashKey(hash); + std::string val = EncodeHashValue(i); + if (!db_->Put(leveldb::WriteOptions(), key, val).ok()) { ESP_LOGE(kTag, "failed to write hash for #%lx", i); } } auto Database::dbGetHash(const uint64_t& hash) -> std::optional { - OwningSlice key = EncodeHashKey(hash); + std::string key = EncodeHashKey(hash); std::string raw_val; - if (!db_->Get(leveldb::ReadOptions(), key.slice, &raw_val).ok()) { + if (!db_->Get(leveldb::ReadOptions(), key, &raw_val).ok()) { ESP_LOGW(kTag, "no key found for hash #%llx", hash); return {}; } @@ -672,10 +673,10 @@ auto IndexRecord::Expand(std::size_t page_size) const return {}; } IndexKey::Header new_header = ExpandHeader(key_.header, key_.item); - OwningSlice new_prefix = EncodeIndexPrefix(new_header); + std::string new_prefix = EncodeIndexPrefix(new_header); return Continuation{ - .prefix = new_prefix.data, - .start_key = new_prefix.data, + .prefix = {new_prefix.data(), new_prefix.size()}, + .start_key = {new_prefix.data(), new_prefix.size()}, .forward = true, .was_prev_forward = true, .page_size = page_size, diff --git a/src/database/include/records.hpp b/src/database/include/records.hpp index e7d7738c..e13c6568 100644 --- a/src/database/include/records.hpp +++ b/src/database/include/records.hpp @@ -21,33 +21,20 @@ namespace database { -/* - * Helper class for creating leveldb Slices bundled with the data they point to. - * Slices are otherwise non-owning, which can make handling them post-creation - * difficult. - */ -class OwningSlice { - public: - std::pmr::string data; - leveldb::Slice slice; - - explicit OwningSlice(std::pmr::string d); -}; - /* * Returns the prefix added to every TrackData key. This can be used to iterate * over every data record in the database. */ -auto EncodeDataPrefix() -> OwningSlice; +auto EncodeDataPrefix() -> std::string; /* Encodes a data key for a track with the specified id. */ -auto EncodeDataKey(const TrackId& id) -> OwningSlice; +auto EncodeDataKey(const TrackId& id) -> std::string; /* * Encodes a TrackData instance into bytes, in preparation for storing it within * the database. This encoding is consistent, and will remain stable over time. */ -auto EncodeDataValue(const TrackData& track) -> OwningSlice; +auto EncodeDataValue(const TrackData& track) -> std::string; /* * Parses bytes previously encoded via EncodeDataValue back into a TrackData. @@ -56,14 +43,14 @@ auto EncodeDataValue(const TrackData& track) -> OwningSlice; auto ParseDataValue(const leveldb::Slice& slice) -> std::shared_ptr; /* Encodes a hash key for the specified hash. */ -auto EncodeHashKey(const uint64_t& hash) -> OwningSlice; +auto EncodeHashKey(const uint64_t& hash) -> std::string; /* * Encodes a hash value (at this point just a track id) into bytes, in * preparation for storing within the database. This encoding is consistent, and * will remain stable over time. */ -auto EncodeHashValue(TrackId id) -> OwningSlice; +auto EncodeHashValue(TrackId id) -> std::string; /* * Parses bytes previously encoded via EncodeHashValue back into a track id. May @@ -72,17 +59,17 @@ auto EncodeHashValue(TrackId id) -> OwningSlice; auto ParseHashValue(const leveldb::Slice&) -> std::optional; /* Encodes a prefix that matches all index keys, of all ids and depths. */ -auto EncodeAllIndexesPrefix() -> OwningSlice; +auto EncodeAllIndexesPrefix() -> std::string; /* */ -auto EncodeIndexPrefix(const IndexKey::Header&) -> OwningSlice; +auto EncodeIndexPrefix(const IndexKey::Header&) -> std::string; -auto EncodeIndexKey(const IndexKey&) -> OwningSlice; +auto EncodeIndexKey(const IndexKey&) -> std::string; auto ParseIndexKey(const leveldb::Slice&) -> std::optional; /* Encodes a TrackId as bytes. */ -auto TrackIdToBytes(TrackId id) -> OwningSlice; +auto TrackIdToBytes(TrackId id) -> std::string; /* * Converts a track id encoded via TrackIdToBytes back into a TrackId. May diff --git a/src/database/include/track.hpp b/src/database/include/track.hpp index 3c7b20fa..44bfbc54 100644 --- a/src/database/include/track.hpp +++ b/src/database/include/track.hpp @@ -117,7 +117,6 @@ class TrackData { const TrackId id_; const std::pmr::string filepath_; const uint64_t tags_hash_; - const uint32_t play_count_; const bool is_tombstoned_; public: @@ -126,18 +125,15 @@ class TrackData { : id_(id), filepath_(path, &memory::kSpiRamResource), tags_hash_(hash), - play_count_(0), is_tombstoned_(false) {} TrackData(TrackId id, const std::pmr::string& path, uint64_t hash, - uint32_t play_count, bool is_tombstoned) : id_(id), filepath_(path, &memory::kSpiRamResource), tags_hash_(hash), - play_count_(play_count), is_tombstoned_(is_tombstoned) {} TrackData(TrackData&& other) = delete; @@ -147,7 +143,6 @@ class TrackData { auto id() const -> TrackId { return id_; } auto filepath() const -> std::pmr::string { return filepath_; } - auto play_count() const -> uint32_t { return play_count_; } auto tags_hash() const -> uint64_t { return tags_hash_; } auto is_tombstoned() const -> bool { return is_tombstoned_; } diff --git a/src/database/index.cpp b/src/database/index.cpp index 79939aaa..c7fd753a 100644 --- a/src/database/index.cpp +++ b/src/database/index.cpp @@ -94,7 +94,7 @@ auto Index(const IndexInfo& info, const Track& t, leveldb::WriteBatch* batch) } auto encoded = EncodeIndexKey(key); - batch->Put(encoded.slice, {value.data(), value.size()}); + batch->Put(encoded, {value.data(), value.size()}); // If there are more components after this, then we need to finish by // narrowing the header with the current title. diff --git a/src/database/records.cpp b/src/database/records.cpp index 103b3547..37ea429b 100644 --- a/src/database/records.cpp +++ b/src/database/records.cpp @@ -7,6 +7,7 @@ #include "records.hpp" #include +#include #include #include @@ -15,7 +16,8 @@ #include #include -#include "cbor.h" +#include "cppbor.h" +#include "cppbor_parse.h" #include "esp_log.h" #include "index.hpp" @@ -49,244 +51,76 @@ static const char kHashPrefix = 'H'; static const char kIndexPrefix = 'I'; static const char kFieldSeparator = '\0'; -using ostringstream = - std::basic_ostringstream, - std::pmr::polymorphic_allocator>; - -/* - * Helper function for allocating an appropriately-sized byte buffer, then - * encoding data into it. - * - * 'T' should be a callable that takes a CborEncoder* as - * an argument, and stores values within that encoder. 'T' will be called - * exactly twice: first to detemine the buffer size, and then second to do the - * encoding. - * - * 'out_buf' will be set to the location of newly allocated memory; it is up to - * the caller to free it. Returns the size of 'out_buf'. - */ -template -auto cbor_encode(uint8_t** out_buf, T fn) -> std::size_t { - // First pass: work out how many bytes we will encode into. - // FIXME: With benchmarking to help, we could consider preallocting a small - // buffer here to do the whole encoding in one pass. - CborEncoder size_encoder; - cbor_encoder_init(&size_encoder, NULL, 0, 0); - std::invoke(fn, &size_encoder); - std::size_t buf_size = cbor_encoder_get_extra_bytes_needed(&size_encoder); - - // Second pass: do the encoding. - CborEncoder encoder; - *out_buf = new uint8_t[buf_size]; - cbor_encoder_init(&encoder, *out_buf, buf_size, 0); - std::invoke(fn, &encoder); - - return buf_size; -} - -OwningSlice::OwningSlice(std::pmr::string d) - : data(d), slice(data.data(), data.size()) {} - /* 'D/' */ -auto EncodeDataPrefix() -> OwningSlice { - char data[2] = {kDataPrefix, kFieldSeparator}; - return OwningSlice({data, 2}); +auto EncodeDataPrefix() -> std::string { + return {kDataPrefix, kFieldSeparator}; } /* 'D/ 0xACAB' */ -auto EncodeDataKey(const TrackId& id) -> OwningSlice { - ostringstream output; - output.put(kDataPrefix).put(kFieldSeparator); - output << TrackIdToBytes(id).data; - return OwningSlice(output.str()); +auto EncodeDataKey(const TrackId& id) -> std::string { + return EncodeDataPrefix() + TrackIdToBytes(id); } -auto EncodeDataValue(const TrackData& track) -> OwningSlice { - uint8_t* buf; - std::size_t buf_len = cbor_encode(&buf, [&](CborEncoder* enc) { - CborEncoder array_encoder; - CborError err; - err = cbor_encoder_create_array(enc, &array_encoder, 5); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encode_int(&array_encoder, track.id()); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encode_text_string(&array_encoder, track.filepath().c_str(), - track.filepath().size()); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encode_uint(&array_encoder, track.tags_hash()); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encode_int(&array_encoder, track.play_count()); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encode_boolean(&array_encoder, track.is_tombstoned()); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encoder_close_container(enc, &array_encoder); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - }); - std::pmr::string as_str(reinterpret_cast(buf), buf_len); - delete buf; - return OwningSlice(as_str); +auto EncodeDataValue(const TrackData& track) -> std::string { + cppbor::Array val{ + cppbor::Uint{track.id()}, + cppbor::Tstr{track.filepath()}, + cppbor::Uint{track.tags_hash()}, + cppbor::Bool{track.is_tombstoned()}, + }; + return val.toString(); } auto ParseDataValue(const leveldb::Slice& slice) -> std::shared_ptr { - CborParser parser; - CborValue container; - CborError err; - err = cbor_parser_init(reinterpret_cast(slice.data()), - slice.size(), 0, &parser, &container); - if (err != CborNoError || !cbor_value_is_container(&container)) { - return {}; - } - - CborValue val; - err = cbor_value_enter_container(&container, &val); - if (err != CborNoError || !cbor_value_is_unsigned_integer(&val)) { - return {}; - } - - uint64_t raw_int; - err = cbor_value_get_uint64(&val, &raw_int); - if (err != CborNoError) { - return {}; - } - TrackId id = raw_int; - err = cbor_value_advance(&val); - if (err != CborNoError || !cbor_value_is_text_string(&val)) { + auto [item, unused, err] = cppbor::parseWithViews( + reinterpret_cast(slice.data()), slice.size()); + if (!item || item->type() != cppbor::ARRAY) { + return nullptr; + } + auto vals = item->asArray(); + if (vals->size() != 4 || vals->get(0)->type() != cppbor::UINT || + vals->get(1)->type() != cppbor::TSTR || + vals->get(2)->type() != cppbor::UINT || + vals->get(3)->type() != cppbor::SIMPLE) { return {}; } - - char* raw_path; - std::size_t len; - err = cbor_value_dup_text_string(&val, &raw_path, &len, &val); - if (err != CborNoError || !cbor_value_is_unsigned_integer(&val)) { - return {}; - } - std::pmr::string path(raw_path, len); - delete raw_path; - - err = cbor_value_get_uint64(&val, &raw_int); - if (err != CborNoError) { - return {}; - } - uint64_t hash = raw_int; - err = cbor_value_advance(&val); - if (err != CborNoError || !cbor_value_is_unsigned_integer(&val)) { - return {}; - } - - err = cbor_value_get_uint64(&val, &raw_int); - if (err != CborNoError) { - return {}; - } - uint32_t play_count = raw_int; - err = cbor_value_advance(&val); - if (err != CborNoError || !cbor_value_is_boolean(&val)) { - return {}; - } - - bool is_tombstoned; - err = cbor_value_get_boolean(&val, &is_tombstoned); - if (err != CborNoError) { - return {}; - } - - return std::make_shared(id, path, hash, play_count, is_tombstoned); + TrackId id = vals->get(0)->asUint()->unsignedValue(); + auto path = vals->get(1)->asViewTstr()->view(); + uint64_t hash = vals->get(2)->asUint()->unsignedValue(); + bool tombstoned = vals->get(3)->asBool()->value(); + return std::make_shared( + id, std::pmr::string{path.data(), path.size()}, hash, tombstoned); } /* 'H/ 0xBEEF' */ -auto EncodeHashKey(const uint64_t& hash) -> OwningSlice { - ostringstream output; - output.put(kHashPrefix).put(kFieldSeparator); - - uint8_t buf[16]; - CborEncoder enc; - cbor_encoder_init(&enc, buf, sizeof(buf), 0); - cbor_encode_uint(&enc, hash); - std::size_t len = cbor_encoder_get_buffer_size(&enc, buf); - output.write(reinterpret_cast(buf), len); - - return OwningSlice(output.str()); +auto EncodeHashKey(const uint64_t& hash) -> std::string { + return std::string{kHashPrefix, kFieldSeparator} + + cppbor::Uint{hash}.toString(); } auto ParseHashValue(const leveldb::Slice& slice) -> std::optional { return BytesToTrackId({slice.data(), slice.size()}); } -auto EncodeHashValue(TrackId id) -> OwningSlice { +auto EncodeHashValue(TrackId id) -> std::string { return TrackIdToBytes(id); } /* 'I/' */ -auto EncodeAllIndexesPrefix() -> OwningSlice { - char data[2] = {kIndexPrefix, kFieldSeparator}; - return OwningSlice({data, 2}); -} - -auto AppendIndexHeader(const IndexKey::Header& header, ostringstream* out) - -> void { - *out << kIndexPrefix << kFieldSeparator; - - // Construct the header. - uint8_t* buf; - std::size_t buf_len = cbor_encode(&buf, [&](CborEncoder* enc) { - CborEncoder array_encoder; - CborError err; - err = cbor_encoder_create_array(enc, &array_encoder, 3); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encode_uint(&array_encoder, header.id); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encode_uint(&array_encoder, header.depth); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encode_uint(&array_encoder, header.components_hash); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - err = cbor_encoder_close_container(enc, &array_encoder); - if (err != CborNoError && err != CborErrorOutOfMemory) { - ESP_LOGE(kTag, "encoding err %u", err); - return; - } - }); - std::pmr::string encoded{reinterpret_cast(buf), buf_len}; - delete buf; - *out << encoded << kFieldSeparator; +auto EncodeAllIndexesPrefix() -> std::string { + return {kIndexPrefix, kFieldSeparator}; } -auto EncodeIndexPrefix(const IndexKey::Header& header) -> OwningSlice { - ostringstream out; - AppendIndexHeader(header, &out); - return OwningSlice(out.str()); +auto EncodeIndexPrefix(const IndexKey::Header& header) -> std::string { + std::ostringstream out; + out.put(kIndexPrefix).put(kFieldSeparator); + cppbor::Array val{ + cppbor::Uint{header.id}, + cppbor::Uint{header.depth}, + cppbor::Uint{header.components_hash}, + }; + out << val.toString() << kFieldSeparator; + return out.str(); } /* @@ -303,93 +137,51 @@ auto EncodeIndexPrefix(const IndexKey::Header& header) -> OwningSlice { * components. We store disambiguation information in the trailer; just a track * id for now, but could reasonably be something like 'release year' as well. */ -auto EncodeIndexKey(const IndexKey& key) -> OwningSlice { - ostringstream out; +auto EncodeIndexKey(const IndexKey& key) -> std::string { + std::ostringstream out{}; - // Construct the header. - AppendIndexHeader(key.header, &out); + out << EncodeIndexPrefix(key.header); // The component should already be UTF-8 encoded, so just write it. if (key.item) { - out << *key.item; + out << *key.item << kFieldSeparator; } - // Construct the footer. - out << kFieldSeparator; if (key.track) { - auto encoded = TrackIdToBytes(*key.track); - out << encoded.data; + out << TrackIdToBytes(*key.track); } - return OwningSlice(out.str()); + + return out.str(); } auto ParseIndexKey(const leveldb::Slice& slice) -> std::optional { IndexKey result{}; auto prefix = EncodeAllIndexesPrefix(); - if (!slice.starts_with(prefix.slice)) { + if (!slice.starts_with(prefix)) { return {}; } - std::string key_data = slice.ToString().substr(prefix.data.size()); - std::size_t header_length = 0; - { - CborParser parser; - CborValue container; - CborError err; - err = cbor_parser_init(reinterpret_cast(key_data.data()), - key_data.size(), 0, &parser, &container); - if (err != CborNoError || !cbor_value_is_container(&container)) { - return {}; - } - - CborValue val; - err = cbor_value_enter_container(&container, &val); - if (err != CborNoError || !cbor_value_is_unsigned_integer(&val)) { - return {}; - } - - uint64_t raw_int; - err = cbor_value_get_uint64(&val, &raw_int); - if (err != CborNoError) { - return {}; - } - result.header.id = raw_int; - err = cbor_value_advance(&val); - if (err != CborNoError || !cbor_value_is_unsigned_integer(&val)) { - return {}; - } - - err = cbor_value_get_uint64(&val, &raw_int); - if (err != CborNoError) { - return {}; - } - result.header.depth = raw_int; - err = cbor_value_advance(&val); - if (err != CborNoError || !cbor_value_is_unsigned_integer(&val)) { - return {}; - } - - err = cbor_value_get_uint64(&val, &raw_int); - if (err != CborNoError) { - return {}; - } - result.header.components_hash = raw_int; - err = cbor_value_advance(&val); - if (err != CborNoError || !cbor_value_at_end(&val)) { - return {}; - } - - const uint8_t* next_byte = cbor_value_get_next_byte(&val); - header_length = - next_byte - reinterpret_cast(key_data.data()); + std::string key_data = slice.ToString().substr(prefix.size()); + auto [key, end_of_key, err] = cppbor::parseWithViews( + reinterpret_cast(key_data.data()), key_data.size()); + if (!key || key->type() != cppbor::ARRAY) { + return {}; } - - if (header_length == 0) { + auto as_array = key->asArray(); + if (as_array->size() != 3 || as_array->get(0)->type() != cppbor::UINT || + as_array->get(1)->type() != cppbor::UINT || + as_array->get(2)->type() != cppbor::UINT) { return {}; } + result.header.id = as_array->get(0)->asUint()->unsignedValue(); + result.header.depth = as_array->get(1)->asUint()->unsignedValue(); + result.header.components_hash = as_array->get(2)->asUint()->unsignedValue(); + + size_t header_length = + reinterpret_cast(end_of_key) - key_data.data(); - if (header_length >= key_data.size()) { + if (header_length == 0 || header_length >= key_data.size()) { return {}; } @@ -411,27 +203,17 @@ auto ParseIndexKey(const leveldb::Slice& slice) -> std::optional { return result; } -auto TrackIdToBytes(TrackId id) -> OwningSlice { - uint8_t buf[8]; - CborEncoder enc; - cbor_encoder_init(&enc, buf, sizeof(buf), 0); - cbor_encode_uint(&enc, id); - std::size_t len = cbor_encoder_get_buffer_size(&enc, buf); - std::pmr::string as_str(reinterpret_cast(buf), len); - return OwningSlice(as_str); +auto TrackIdToBytes(TrackId id) -> std::string { + return cppbor::Uint{id}.toString(); } auto BytesToTrackId(cpp::span bytes) -> std::optional { - CborParser parser; - CborValue val; - cbor_parser_init(reinterpret_cast(bytes.data()), bytes.size(), - 0, &parser, &val); - if (!cbor_value_is_unsigned_integer(&val)) { + auto [res, unused, err] = cppbor::parse( + reinterpret_cast(bytes.data()), bytes.size()); + if (!res || res->type() != cppbor::UINT) { return {}; } - uint64_t raw_id; - cbor_value_get_uint64(&val, &raw_id); - return raw_id; + return res->asUint()->unsignedValue(); } } // namespace database diff --git a/src/database/track.cpp b/src/database/track.cpp index f48bb8ed..ca180124 100644 --- a/src/database/track.cpp +++ b/src/database/track.cpp @@ -53,15 +53,15 @@ auto TrackTags::Hash() const -> uint64_t { } auto TrackData::UpdateHash(uint64_t new_hash) const -> TrackData { - return TrackData(id_, filepath_, new_hash, play_count_, is_tombstoned_); + return TrackData(id_, filepath_, new_hash, is_tombstoned_); } auto TrackData::Entomb() const -> TrackData { - return TrackData(id_, filepath_, tags_hash_, play_count_, true); + return TrackData(id_, filepath_, tags_hash_, true); } auto TrackData::Exhume(const std::pmr::string& new_path) const -> TrackData { - return TrackData(id_, new_path, tags_hash_, play_count_, false); + return TrackData(id_, new_path, tags_hash_, false); } auto Track::TitleOrFilename() const -> std::pmr::string { diff --git a/tools/cmake/common.cmake b/tools/cmake/common.cmake index 34bd1226..21ae2b8e 100644 --- a/tools/cmake/common.cmake +++ b/tools/cmake/common.cmake @@ -9,23 +9,24 @@ set(COMPONENTS "") # External dependencies +list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/bindey") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/catch2") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/cbor") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/fatfs") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/komihash") +list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/libcppbor") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/libfoxenflac") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/libmad") -list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/speexdsp") -list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/tremor") -list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/ogg") -list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/opusfile") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/libtags") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/lvgl") +list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/ogg") +list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/opusfile") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/result") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/shared_string") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/span") +list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/speexdsp") list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/tinyfsm") -list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/bindey") +list(APPEND EXTRA_COMPONENT_DIRS "$ENV{PROJ_PATH}/lib/tremor") include($ENV{IDF_PATH}/tools/cmake/project.cmake)