From c635d5011c37c02246135fe0df404631ec111bd6 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Wed, 16 Aug 2023 11:40:49 +1000 Subject: [PATCH] Put more audio pipeline wiring in control of the audio fsm --- src/audio/audio_fsm.cpp | 29 ++++++++++++++------------ src/audio/audio_task.cpp | 15 ++++++------- src/audio/bt_audio_output.cpp | 2 +- src/audio/i2s_audio_output.cpp | 2 +- src/audio/include/audio_fsm.hpp | 7 ++++--- src/audio/include/audio_sink.hpp | 19 +++++++++++++---- src/audio/include/audio_task.hpp | 13 ++++++------ src/audio/include/bt_audio_output.hpp | 2 +- src/audio/include/i2s_audio_output.hpp | 2 +- src/audio/include/sink_mixer.hpp | 23 ++++++++++---------- src/audio/sink_mixer.cpp | 15 +++++++++---- 11 files changed, 75 insertions(+), 54 deletions(-) diff --git a/src/audio/audio_fsm.cpp b/src/audio/audio_fsm.cpp index 87fbbda2..1ea670af 100644 --- a/src/audio/audio_fsm.cpp +++ b/src/audio/audio_fsm.cpp @@ -10,6 +10,7 @@ #include #include +#include "audio_sink.hpp" #include "esp_log.h" #include "freertos/portmacro.h" #include "freertos/projdefs.h" @@ -23,6 +24,7 @@ #include "future_fetcher.hpp" #include "i2s_audio_output.hpp" #include "i2s_dac.hpp" +#include "sink_mixer.hpp" #include "system_events.hpp" #include "track.hpp" #include "track_queue.hpp" @@ -36,9 +38,10 @@ std::shared_ptr AudioState::sDac; std::weak_ptr AudioState::sDatabase; std::unique_ptr AudioState::sTask; -std::unique_ptr AudioState::sFileSource; -std::unique_ptr AudioState::sI2SOutput; -std::unique_ptr AudioState::sBtOutput; + +std::shared_ptr AudioState::sFileSource; +std::shared_ptr AudioState::sMixer; +std::shared_ptr AudioState::sOutput; TrackQueue* AudioState::sTrackQueue; std::optional AudioState::sCurrentTrack; @@ -59,11 +62,13 @@ auto AudioState::Init(drivers::IGpios* gpio_expander, sDatabase = database; sFileSource.reset(new FatfsAudioInput(tag_parser)); - sI2SOutput.reset(new I2SAudioOutput(sIGpios, sDac)); - sBtOutput.reset(new BluetoothAudioOutput(bluetooth)); + sOutput.reset(new I2SAudioOutput(sIGpios, sDac)); + // sOutput.reset(new BluetoothAudioOutput(bluetooth)); + + sMixer.reset(new SinkMixer()); + sMixer->SetOutput(sOutput); - AudioTask::Start(sFileSource.get(), sI2SOutput.get()); - // AudioTask::Start(sFileSource.get(), sBtOutput.get()); + AudioTask::Start(sFileSource, sMixer); return true; } @@ -73,14 +78,14 @@ void AudioState::react(const system_fsm::StorageMounted& ev) { } void AudioState::react(const system_fsm::KeyUpChanged& ev) { - if (ev.falling && sI2SOutput->AdjustVolumeUp()) { + if (ev.falling && sOutput->AdjustVolumeUp()) { ESP_LOGI(kTag, "volume up!"); events::Ui().Dispatch(VolumeChanged{}); } } void AudioState::react(const system_fsm::KeyDownChanged& ev) { - if (ev.falling && sI2SOutput->AdjustVolumeDown()) { + if (ev.falling && sOutput->AdjustVolumeDown()) { ESP_LOGI(kTag, "volume down!"); events::Ui().Dispatch(VolumeChanged{}); } @@ -131,8 +136,7 @@ void Standby::react(const QueueUpdate& ev) { void Playback::entry() { ESP_LOGI(kTag, "beginning playback"); - sI2SOutput->SetInUse(true); - // sBtOutput->SetInUse(true); + sOutput->SetInUse(true); } void Playback::exit() { @@ -140,8 +144,7 @@ void Playback::exit() { // TODO(jacqueline): Second case where it's useful to wait for the i2s buffer // to drain. vTaskDelay(pdMS_TO_TICKS(250)); - sI2SOutput->SetInUse(false); - // sBtOutput->SetInUse(false); + sOutput->SetInUse(false); } void Playback::react(const QueueUpdate& ev) { diff --git a/src/audio/audio_task.cpp b/src/audio/audio_task.cpp index 1ebd4e46..99b1c170 100644 --- a/src/audio/audio_task.cpp +++ b/src/audio/audio_task.cpp @@ -76,19 +76,16 @@ auto Timer::AddSamples(std::size_t samples) -> void { } } -auto AudioTask::Start(IAudioSource* source, IAudioSink* sink) -> AudioTask* { +auto AudioTask::Start(std::shared_ptr source, + std::shared_ptr sink) -> AudioTask* { AudioTask* task = new AudioTask(source, sink); tasks::StartPersistent([=]() { task->Main(); }); return task; } -AudioTask::AudioTask(IAudioSource* source, IAudioSink* sink) - : source_(source), - sink_(sink), - codec_(), - mixer_(new SinkMixer(sink)), - timer_(), - current_format_() { +AudioTask::AudioTask(std::shared_ptr source, + std::shared_ptr mixer) + : source_(source), mixer_(mixer), codec_(), timer_(), current_format_() { codec_buffer_ = { reinterpret_cast(heap_caps_calloc( kCodecBufferLength, sizeof(sample::Sample), MALLOC_CAP_SPIRAM)), @@ -133,7 +130,7 @@ auto AudioTask::BeginDecoding(std::shared_ptr stream) -> bool { timer_.reset(); } - current_sink_format_ = IAudioSink::Format{ + current_sink_format_ = IAudioOutput::Format{ .sample_rate = open_res->sample_rate_hz, .num_channels = open_res->num_channels, .bits_per_sample = 16, diff --git a/src/audio/bt_audio_output.cpp b/src/audio/bt_audio_output.cpp index 641bea21..2e54f69a 100644 --- a/src/audio/bt_audio_output.cpp +++ b/src/audio/bt_audio_output.cpp @@ -30,7 +30,7 @@ namespace audio { static constexpr size_t kDrainBufferSize = 48 * 1024; BluetoothAudioOutput::BluetoothAudioOutput(drivers::Bluetooth* bt) - : IAudioSink(kDrainBufferSize, MALLOC_CAP_SPIRAM), bluetooth_(bt) {} + : IAudioOutput(kDrainBufferSize, MALLOC_CAP_SPIRAM), bluetooth_(bt) {} BluetoothAudioOutput::~BluetoothAudioOutput() {} diff --git a/src/audio/i2s_audio_output.cpp b/src/audio/i2s_audio_output.cpp index 8c779e15..9e18597a 100644 --- a/src/audio/i2s_audio_output.cpp +++ b/src/audio/i2s_audio_output.cpp @@ -43,7 +43,7 @@ static constexpr size_t kDrainBufferSize = 8 * 1024; I2SAudioOutput::I2SAudioOutput(drivers::IGpios* expander, std::weak_ptr dac) - : IAudioSink(kDrainBufferSize, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), + : IAudioOutput(kDrainBufferSize, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), expander_(expander), dac_(dac.lock()), current_config_(), diff --git a/src/audio/include/audio_fsm.hpp b/src/audio/include/audio_fsm.hpp index cb307e68..6c785426 100644 --- a/src/audio/include/audio_fsm.hpp +++ b/src/audio/include/audio_fsm.hpp @@ -10,6 +10,7 @@ #include #include +#include "audio_sink.hpp" #include "tinyfsm.hpp" #include "audio_events.hpp" @@ -68,9 +69,9 @@ class AudioState : public tinyfsm::Fsm { static std::weak_ptr sDatabase; static std::unique_ptr sTask; - static std::unique_ptr sFileSource; - static std::unique_ptr sI2SOutput; - static std::unique_ptr sBtOutput; + static std::shared_ptr sFileSource; + static std::shared_ptr sMixer; + static std::shared_ptr sOutput; static TrackQueue* sTrackQueue; static std::optional sCurrentTrack; diff --git a/src/audio/include/audio_sink.hpp b/src/audio/include/audio_sink.hpp index cce14242..4baa4aa1 100644 --- a/src/audio/include/audio_sink.hpp +++ b/src/audio/include/audio_sink.hpp @@ -15,17 +15,28 @@ namespace audio { -class IAudioSink { +/* + * Interface for classes that use PCM samples to create noises for the user. + * + * These classes do not generally have any specific task for their work, and + * simply help to mediate working out the correct PCM format, and then sending + * those samples to the appropriate hardware driver. + */ +class IAudioOutput { private: StreamBufferHandle_t stream_; public: - IAudioSink(size_t buffer_size, uint32_t caps) + IAudioOutput(size_t buffer_size, uint32_t caps) : stream_(xStreamBufferCreateWithCaps(buffer_size, 1, caps)) {} - virtual ~IAudioSink() { vStreamBufferDeleteWithCaps(stream_); } + virtual ~IAudioOutput() { vStreamBufferDeleteWithCaps(stream_); } - virtual auto SetInUse(bool) -> void {} + /* + * Indicates whether this output is currently being sent samples. If this is + * false, the output should place itself into a low power state. + */ + virtual auto SetInUse(bool) -> void = 0; virtual auto SetVolumeImbalance(int_fast8_t balance) -> void = 0; virtual auto SetVolume(uint_fast8_t percent) -> void = 0; diff --git a/src/audio/include/audio_task.hpp b/src/audio/include/audio_task.hpp index d2c25add..08c5769c 100644 --- a/src/audio/include/audio_task.hpp +++ b/src/audio/include/audio_task.hpp @@ -34,7 +34,8 @@ class Timer { class AudioTask { public: - static auto Start(IAudioSource* source, IAudioSink* sink) -> AudioTask*; + static auto Start(std::shared_ptr source, + std::shared_ptr mixer) -> AudioTask*; auto Main() -> void; @@ -42,21 +43,21 @@ class AudioTask { AudioTask& operator=(const AudioTask&) = delete; private: - AudioTask(IAudioSource* source, IAudioSink* sink); + AudioTask(std::shared_ptr source, + std::shared_ptr mixer); auto BeginDecoding(std::shared_ptr) -> bool; auto ContinueDecoding() -> bool; - IAudioSource* source_; - IAudioSink* sink_; + std::shared_ptr source_; + std::shared_ptr mixer_; std::shared_ptr stream_; std::unique_ptr codec_; - std::unique_ptr mixer_; std::unique_ptr timer_; std::optional current_format_; - std::optional current_sink_format_; + std::optional current_sink_format_; cpp::span codec_buffer_; }; diff --git a/src/audio/include/bt_audio_output.hpp b/src/audio/include/bt_audio_output.hpp index d762bab6..a8e44f31 100644 --- a/src/audio/include/bt_audio_output.hpp +++ b/src/audio/include/bt_audio_output.hpp @@ -19,7 +19,7 @@ namespace audio { -class BluetoothAudioOutput : public IAudioSink { +class BluetoothAudioOutput : public IAudioOutput { public: BluetoothAudioOutput(drivers::Bluetooth* bt); ~BluetoothAudioOutput(); diff --git a/src/audio/include/i2s_audio_output.hpp b/src/audio/include/i2s_audio_output.hpp index 13cb183f..0068916e 100644 --- a/src/audio/include/i2s_audio_output.hpp +++ b/src/audio/include/i2s_audio_output.hpp @@ -17,7 +17,7 @@ namespace audio { -class I2SAudioOutput : public IAudioSink { +class I2SAudioOutput : public IAudioOutput { public: I2SAudioOutput(drivers::IGpios* expander, std::weak_ptr dac); ~I2SAudioOutput(); diff --git a/src/audio/include/sink_mixer.hpp b/src/audio/include/sink_mixer.hpp index 37143439..d046f835 100644 --- a/src/audio/include/sink_mixer.hpp +++ b/src/audio/include/sink_mixer.hpp @@ -9,36 +9,37 @@ #include #include -#include "resample.hpp" -#include "sample.hpp" - #include "audio_sink.hpp" #include "audio_source.hpp" #include "codec.hpp" +#include "resample.hpp" +#include "sample.hpp" namespace audio { /* * Handles the final downmix + resample + quantisation stage of audio, - * generation sending the result directly to an IAudioSink. + * generation sending the result directly to an IAudioOutput. */ class SinkMixer { public: - SinkMixer(IAudioSink* sink); + SinkMixer(); ~SinkMixer(); + auto SetOutput(std::shared_ptr) -> void; + auto MixAndSend(cpp::span, - const IAudioSink::Format& format, + const IAudioOutput::Format& format, bool is_eos) -> void; private: auto Main() -> void; - auto SetTargetFormat(const IAudioSink::Format& format) -> void; + auto SetTargetFormat(const IAudioOutput::Format& format) -> void; auto HandleSamples(cpp::span, bool) -> size_t; struct Args { - IAudioSink::Format format; + IAudioOutput::Format format; size_t samples_available; bool is_end_of_stream; }; @@ -52,9 +53,9 @@ class SinkMixer { cpp::span resampled_buffer_; - IAudioSink* sink_; - IAudioSink::Format source_format_; - IAudioSink::Format target_format_; + std::shared_ptr sink_; + IAudioOutput::Format source_format_; + IAudioOutput::Format target_format_; size_t leftover_bytes_; size_t leftover_offset_; }; diff --git a/src/audio/sink_mixer.cpp b/src/audio/sink_mixer.cpp index c6cb886f..ad7198dc 100644 --- a/src/audio/sink_mixer.cpp +++ b/src/audio/sink_mixer.cpp @@ -10,6 +10,7 @@ #include #include +#include "audio_sink.hpp" #include "esp_heap_caps.h" #include "esp_log.h" #include "freertos/portmacro.h" @@ -27,13 +28,12 @@ static constexpr std::size_t kSampleBufferLength = 240 * 2; namespace audio { -SinkMixer::SinkMixer(IAudioSink* sink) +SinkMixer::SinkMixer() : commands_(xQueueCreate(1, sizeof(Args))), resampler_(nullptr), source_(xStreamBufferCreateWithCaps(kSourceBufferLength, 1, - MALLOC_CAP_SPIRAM)), - sink_(sink) { + MALLOC_CAP_SPIRAM)) { input_buffer_ = { reinterpret_cast(heap_caps_calloc( kSampleBufferLength, sizeof(sample::Sample), MALLOC_CAP_SPIRAM)), @@ -54,8 +54,15 @@ SinkMixer::~SinkMixer() { vStreamBufferDelete(source_); } +auto SinkMixer::SetOutput(std::shared_ptr output) -> void { + // FIXME: We should add synchronisation here, but we should be careful about + // not impacting performance given that the output will change only very + // rarely (if ever). + sink_ = output; +} + auto SinkMixer::MixAndSend(cpp::span input, - const IAudioSink::Format& format, + const IAudioOutput::Format& format, bool is_eos) -> void { Args args{ .format = format,