From acccd822f0147147dd8b16f059578df073c088c2 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Mon, 19 Jun 2023 15:36:43 +1000 Subject: [PATCH] back to back flac playback is working :) --- src/audio/audio_decoder.cpp | 120 ++++++++++++++---------- src/audio/audio_task.cpp | 7 +- src/audio/fatfs_audio_input.cpp | 19 ++-- src/audio/include/audio_decoder.hpp | 1 + src/audio/include/fatfs_audio_input.hpp | 1 + src/audio/include/stream_info.hpp | 19 ++-- src/audio/stream_info.cpp | 29 +++--- src/codecs/mad.cpp | 11 ++- src/system_fsm/running.cpp | 1 - 9 files changed, 123 insertions(+), 85 deletions(-) diff --git a/src/audio/audio_decoder.cpp b/src/audio/audio_decoder.cpp index b4af65fb..583f4b22 100644 --- a/src/audio/audio_decoder.cpp +++ b/src/audio/audio_decoder.cpp @@ -36,37 +36,27 @@ AudioDecoder::AudioDecoder() current_codec_(), current_input_format_(), current_output_format_(), + has_prepared_output_(false), has_samples_to_send_(false), has_input_remaining_(false) {} AudioDecoder::~AudioDecoder() {} auto AudioDecoder::ProcessStreamInfo(const StreamInfo& info) -> bool { + has_prepared_output_ = false; + current_codec_.reset(); + current_input_format_.reset(); + current_output_format_.reset(); + if (!std::holds_alternative(info.format)) { return false; } - ESP_LOGI(kTag, "got new stream"); - const auto& encoded = std::get(info.format); - - // Reuse the existing codec if we can. This will help with gapless playback, - // since we can potentially just continue to decode as we were before, - // without any setup overhead. - // TODO(jacqueline): Reconsider this. It makes a lot of things harder to smash - // streams together at this layer. - /* - if (current_codec_ != nullptr && current_input_format_) { - auto cur_encoding = std::get(*current_input_format_); - if (cur_encoding.type == encoded.type) { - ESP_LOGI(kTag, "reusing existing decoder"); - current_input_format_ = info.format; - return true; - } - } - */ + + const auto& new_format = std::get(info.format); current_input_format_ = info.format; ESP_LOGI(kTag, "creating new decoder"); - auto result = codecs::CreateCodecForType(encoded.type); + auto result = codecs::CreateCodecForType(new_format.type); if (result.has_value()) { current_codec_.reset(result.value()); } else { @@ -86,18 +76,31 @@ auto AudioDecoder::Process(const std::vector& inputs, auto input = inputs.begin(); const StreamInfo& info = input->info(); - // Check the input stream's format has changed (or, by extension, if this is - // the first stream). - if (!current_input_format_ || *current_input_format_ != info.format) { - has_samples_to_send_ = false; + // Is this a completely new stream? + if (!current_input_format_) { if (!ProcessStreamInfo(info)) { + // We couldn't handle the new stream. Signal to the producer that we don't + // have anything to do. + input->mark_consumer_finished(); return; } - ESP_LOGI(kTag, "beginning new stream"); + } + + // Have we determined what kind of samples this stream decodes to? + if (!current_output_format_) { auto res = current_codec_->BeginStream(input->data()); input->consume(res.first); + if (res.second.has_error()) { - // TODO(jacqueline): Handle errors. + auto err = res.second.error(); + if (err == codecs::ICodec::Error::kOutOfInput) { + // We didn't manage to clear whatever front matter is before this + // stream's header. We need to call BeginStream again with more data. + return; + } + // Somthing about the stream's header was malformed. Skip it. + ESP_LOGE(kTag, "error beginning stream"); + input->mark_consumer_finished(); return; } @@ -116,59 +119,82 @@ auto AudioDecoder::Process(const std::vector& inputs, } } - while (seek_to_sample_) { + if (seek_to_sample_) { ESP_LOGI(kTag, "seeking forwards..."); auto res = current_codec_->SeekStream(input->data(), *seek_to_sample_); input->consume(res.first); + if (res.second.has_error()) { auto err = res.second.error(); if (err == codecs::ICodec::Error::kOutOfInput) { return; } else { // TODO(jacqueline): Handle errors. - seek_to_sample_.reset(); } - } else { - seek_to_sample_.reset(); } + + seek_to_sample_.reset(); } has_input_remaining_ = true; while (true) { + // Make sure the output buffer is ready to receive samples in our format + // before starting to process data. // TODO(jacqueline): Pass through seek info here? - if (!output->prepare(*current_output_format_)) { + if (!has_prepared_output_ && !output->prepare(*current_output_format_)) { ESP_LOGI(kTag, "waiting for buffer to become free"); - break; + return; } + has_prepared_output_ = true; + // Parse frames and produce samples. auto res = current_codec_->ContinueStream(input->data(), output->data()); input->consume(res.first); + + // Handle any errors during processing. if (res.second.has_error()) { + // The codec ran out of input during processing. This is expected to + // happen throughout the stream. if (res.second.error() == codecs::ICodec::Error::kOutOfInput) { - ESP_LOGW(kTag, "out of input"); - ESP_LOGW(kTag, "(%u bytes left)", input->data().size_bytes()); + ESP_LOGI(kTag, "codec needs more data"); has_input_remaining_ = false; - // We can't be halfway through sending samples if the codec is asking - // for more input. has_samples_to_send_ = false; - input->mark_incomplete(); + if (input->is_producer_finished()) { + ESP_LOGI(kTag, "codec is all done."); + + // We're out of data, and so is the producer. Nothing left to be done + // with the input stream. + input->mark_consumer_finished(); + + // Upstream isn't going to give us any more data. Tell downstream + // that they shouldn't expact any more samples from this stream. + output->mark_producer_finished(); + break; + } } else { // TODO(jacqueline): Handle errors. - ESP_LOGE(kTag, "codec return fatal error"); + ESP_LOGE(kTag, "codec returned fatal error"); } + // Note that a codec that returns an error is not allowed to write + // samples. So it's safe to skip the latter part of the loop. return; - } - - codecs::ICodec::OutputInfo out_info = res.second.value(); - output->add(out_info.bytes_written); - has_samples_to_send_ = !out_info.is_finished_writing; - - if (has_samples_to_send_) { - // We weren't able to fit all the generated samples into the output - // buffer. Stop trying; we'll finish up during the next pass. - break; + } else { + // Some samples were written! Ensure the downstream element knows about + // them. + codecs::ICodec::OutputInfo out_info = res.second.value(); + output->add(out_info.bytes_written); + has_samples_to_send_ = !out_info.is_finished_writing; + + if (has_samples_to_send_) { + // The codec wasn't able to finish writing all of its samples into the + // output buffer. We need to return so that we can get a new buffer. + return; + } } } + + current_codec_.reset(); + current_input_format_.reset(); } } // namespace audio diff --git a/src/audio/audio_task.cpp b/src/audio/audio_task.cpp index 394a55b6..10bed656 100644 --- a/src/audio/audio_task.cpp +++ b/src/audio/audio_task.cpp @@ -125,8 +125,13 @@ void AudioTaskMain(std::unique_ptr pipeline, IAudioSink* sink) { InputStream sink_stream(&raw_sink_stream); if (sink_stream.info().bytes_in_stream == 0) { + if (sink_stream.is_producer_finished()) { + sink_stream.mark_consumer_finished(); + } else { + // The user is probably about to hear a skip :( + ESP_LOGW(kTag, "!! audio sink is underbuffered !!"); + } // No new bytes to sink, so skip sinking completely. - ESP_LOGW(kTag, "no bytes to sink"); continue; } diff --git a/src/audio/fatfs_audio_input.cpp b/src/audio/fatfs_audio_input.cpp index 77b104d3..d3bd9259 100644 --- a/src/audio/fatfs_audio_input.cpp +++ b/src/audio/fatfs_audio_input.cpp @@ -43,6 +43,7 @@ FatfsAudioInput::FatfsAudioInput() pending_path_(), current_file_(), is_file_open_(false), + has_prepared_output_(false), current_container_(), current_format_() {} @@ -57,10 +58,13 @@ auto FatfsAudioInput::OpenFile(const std::string& path) -> bool { if (is_file_open_) { f_close(¤t_file_); is_file_open_ = false; + has_prepared_output_ = false; } + if (pending_path_) { pending_path_ = {}; } + ESP_LOGI(kTag, "opening file %s", path.c_str()); database::TagParserImpl tag_parser; @@ -112,13 +116,11 @@ auto FatfsAudioInput::NeedsToProcess() const -> bool { auto FatfsAudioInput::Process(const std::vector& inputs, OutputStream* output) -> void { if (pending_path_) { - ESP_LOGI(kTag, "waiting for path"); if (!pending_path_->valid()) { pending_path_ = {}; } else { if (pending_path_->wait_for(std::chrono::seconds(0)) == std::future_status::ready) { - ESP_LOGI(kTag, "path ready!"); auto result = pending_path_->get(); if (result) { OpenFile(*result); @@ -131,9 +133,11 @@ auto FatfsAudioInput::Process(const std::vector& inputs, return; } - if (!output->prepare(*current_format_)) { + if (!has_prepared_output_ && !output->prepare(*current_format_)) { + ESP_LOGI(kTag, "waiting for buffer to free up"); return; } + has_prepared_output_ = true; std::size_t max_size = output->data().size_bytes(); if (max_size < output->data().size_bytes() / 2) { @@ -152,14 +156,11 @@ auto FatfsAudioInput::Process(const std::vector& inputs, output->add(size); if (size < max_size || f_eof(¤t_file_)) { + ESP_LOGI(kTag, "file finished. closing."); f_close(¤t_file_); is_file_open_ = false; - - // HACK: libmad requires an 8 byte padding at the end of each file. - if (current_container_ == database::Encoding::kMp3) { - std::fill_n(output->data().begin(), 8, std::byte(0)); - output->add(8); - } + has_prepared_output_ = false; + output->mark_producer_finished(); events::Dispatch({}); } diff --git a/src/audio/include/audio_decoder.hpp b/src/audio/include/audio_decoder.hpp index 4e7e127e..aa051685 100644 --- a/src/audio/include/audio_decoder.hpp +++ b/src/audio/include/audio_decoder.hpp @@ -43,6 +43,7 @@ class AudioDecoder : public IAudioElement { std::optional current_input_format_; std::optional current_output_format_; std::optional seek_to_sample_; + bool has_prepared_output_; bool has_samples_to_send_; bool has_input_remaining_; diff --git a/src/audio/include/fatfs_audio_input.hpp b/src/audio/include/fatfs_audio_input.hpp index ab392f54..77d3b96d 100644 --- a/src/audio/include/fatfs_audio_input.hpp +++ b/src/audio/include/fatfs_audio_input.hpp @@ -52,6 +52,7 @@ class FatfsAudioInput : public IAudioElement { std::optional>> pending_path_; FIL current_file_; bool is_file_open_; + bool has_prepared_output_; std::optional current_container_; std::optional current_format_; diff --git a/src/audio/include/stream_info.hpp b/src/audio/include/stream_info.hpp index 54b87003..4db3e5fd 100644 --- a/src/audio/include/stream_info.hpp +++ b/src/audio/include/stream_info.hpp @@ -26,10 +26,9 @@ struct StreamInfo { // stream's buffer. std::size_t bytes_in_stream{0}; - // The total length of this stream, in case its source is finite (e.g. a - // file on disk). May be absent for endless streams (internet streams, - // generated audio, etc.) - std::optional length_bytes{}; + bool is_producer_finished = true; + + bool is_consumer_finished = true; // std::optional seek_to_seconds{}; @@ -62,10 +61,8 @@ class RawStream { public: StreamInfo* info; cpp::span data; - bool is_incomplete; - RawStream(StreamInfo* i, cpp::span d) - : info(i), data(d), is_incomplete(false) {} + RawStream(StreamInfo* i, cpp::span d) : info(i), data(d) {} }; /* @@ -78,7 +75,9 @@ class InputStream { void consume(std::size_t bytes) const; - void mark_incomplete() const; + bool is_producer_finished() const; + + void mark_consumer_finished() const; const StreamInfo& info() const; @@ -100,7 +99,9 @@ class OutputStream { cpp::span data() const; - bool is_incomplete() const; + bool is_consumer_finished() const; + + void mark_producer_finished() const; private: RawStream* raw_; diff --git a/src/audio/stream_info.cpp b/src/audio/stream_info.cpp index 8fc31530..2337b3a3 100644 --- a/src/audio/stream_info.cpp +++ b/src/audio/stream_info.cpp @@ -28,8 +28,12 @@ void InputStream::consume(std::size_t bytes) const { raw_->info->bytes_in_stream = new_data.size_bytes(); } -void InputStream::mark_incomplete() const { - raw_->is_incomplete = true; +void InputStream::mark_consumer_finished() const { + raw_->info->is_consumer_finished = true; +} + +bool InputStream::is_producer_finished() const { + return raw_->info->is_producer_finished; } const StreamInfo& InputStream::info() const { @@ -46,17 +50,12 @@ void OutputStream::add(std::size_t bytes) const { } bool OutputStream::prepare(const StreamInfo::Format& new_format) { - if (std::holds_alternative(raw_->info->format)) { - raw_->info->format = new_format; - raw_->info->bytes_in_stream = 0; - return true; - } - if (new_format == raw_->info->format) { - return true; - } - if (raw_->is_incomplete) { + if (std::holds_alternative(raw_->info->format) || + raw_->info->is_consumer_finished) { raw_->info->format = new_format; raw_->info->bytes_in_stream = 0; + raw_->info->is_producer_finished = false; + raw_->info->is_consumer_finished = false; return true; } return false; @@ -70,8 +69,12 @@ cpp::span OutputStream::data() const { return raw_->data.subspan(raw_->info->bytes_in_stream); } -bool OutputStream::is_incomplete() const { - return raw_->is_incomplete; +void OutputStream::mark_producer_finished() const { + raw_->info->is_producer_finished = true; +} + +bool OutputStream::is_consumer_finished() const { + return raw_->info->is_consumer_finished; } } // namespace audio diff --git a/src/codecs/mad.cpp b/src/codecs/mad.cpp index 8b9897eb..f3f3cffe 100644 --- a/src/codecs/mad.cpp +++ b/src/codecs/mad.cpp @@ -44,6 +44,7 @@ MadMp3Decoder::~MadMp3Decoder() { } auto MadMp3Decoder::GetInputPosition() -> std::size_t { + assert(stream_.next_frame >= stream_.buffer); return stream_.next_frame - stream_.buffer; } @@ -51,7 +52,7 @@ auto MadMp3Decoder::BeginStream(const cpp::span input) -> Result { mad_stream_buffer(&stream_, reinterpret_cast(input.data()), - input.size()); + input.size_bytes()); // Whatever was last synthesized is now invalid, so ensure we don't try to // send it. current_sample_ = -1; @@ -65,11 +66,11 @@ auto MadMp3Decoder::BeginStream(const cpp::span input) // Recoverable errors are usually malformed parts of the stream. // We can recover from them by just retrying the decode. continue; - } else { - // Don't bother checking for other errors; if the first part of the stream - // doesn't even contain a header then something's gone wrong. - return {GetInputPosition(), cpp::fail(Error::kMalformedData)}; } + if (stream_.error == MAD_ERROR_BUFLEN) { + return {GetInputPosition(), cpp::fail(Error::kOutOfInput)}; + } + return {GetInputPosition(), cpp::fail(Error::kMalformedData)}; } uint8_t channels = MAD_NCHANNELS(&header); diff --git a/src/system_fsm/running.cpp b/src/system_fsm/running.cpp index 7e8a218d..87c25440 100644 --- a/src/system_fsm/running.cpp +++ b/src/system_fsm/running.cpp @@ -38,7 +38,6 @@ void Running::entry() { vTaskDelay(pdMS_TO_TICKS(250)); ESP_LOGI(kTag, "opening database"); - database::Database::Destroy(); auto database_res = database::Database::Open(); if (database_res.has_error()) { ESP_LOGW(kTag, "failed to open!");