Merge pull request 'Introduce basic TTS support using pre-recorded voice lines' (#103) from jqln/tts into main
Reviewed-on: https://codeberg.org/cool-tech-zone/tangara-fw/pulls/103custom
cooljqln
7 months ago
commit
c030a4510a
@ -0,0 +1,192 @@ |
||||
/*
|
||||
* Copyright 2024 jacqueline <me@jacqueline.id.au> |
||||
* |
||||
* SPDX-License-Identifier: GPL-3.0-only |
||||
*/ |
||||
|
||||
#include "tts/player.hpp" |
||||
#include <mutex> |
||||
|
||||
#include "audio/audio_events.hpp" |
||||
#include "audio/processor.hpp" |
||||
#include "audio/resample.hpp" |
||||
#include "codec.hpp" |
||||
#include "esp_log.h" |
||||
#include "events/event_queue.hpp" |
||||
#include "freertos/projdefs.h" |
||||
#include "portmacro.h" |
||||
#include "sample.hpp" |
||||
#include "types.hpp" |
||||
|
||||
namespace tts { |
||||
|
||||
[[maybe_unused]] static constexpr char kTag[] = "ttsplay"; |
||||
|
||||
Player::Player(tasks::WorkerPool& worker, |
||||
drivers::PcmBuffer& output, |
||||
audio::FatfsStreamFactory& factory) |
||||
: bg_(worker), |
||||
stream_factory_(factory), |
||||
output_(output), |
||||
stream_playing_(false), |
||||
stream_cancelled_(false) {} |
||||
|
||||
auto Player::playFile(const std::string& text, const std::string& file) |
||||
-> void { |
||||
bg_.Dispatch<void>([=, this]() { |
||||
{ |
||||
std::scoped_lock<std::mutex> lock{new_stream_mutex_}; |
||||
if (stream_playing_) { |
||||
stream_cancelled_ = true; |
||||
stream_playing_.wait(true); |
||||
} |
||||
stream_cancelled_ = false; |
||||
stream_playing_ = true; |
||||
} |
||||
|
||||
openAndDecode(text, file); |
||||
|
||||
if (!stream_cancelled_) { |
||||
events::Audio().Dispatch(audio::TtsPlaybackChanged{.is_playing = false}); |
||||
} |
||||
stream_playing_ = false; |
||||
stream_playing_.notify_all(); |
||||
}); |
||||
} |
||||
|
||||
auto Player::openAndDecode(const std::string& text, const std::string& path) |
||||
-> void { |
||||
auto stream = stream_factory_.create(path); |
||||
if (!stream) { |
||||
ESP_LOGW(kTag, "missing '%s' for '%s'", path.c_str(), text.c_str()); |
||||
return; |
||||
} |
||||
|
||||
// FIXME: Rather than hardcoding WAV support only, we should work out a
|
||||
// proper subset of 'low memory' decoders that can all be used for TTS
|
||||
// playback.
|
||||
if (stream->type() != codecs::StreamType::kWav) { |
||||
ESP_LOGE(kTag, "'%s' has unsupported encoding", path.c_str()); |
||||
return; |
||||
} |
||||
|
||||
auto decoder = codecs::CreateCodecForType(stream->type()); |
||||
if (!decoder) { |
||||
ESP_LOGE(kTag, "creating decoder failed"); |
||||
return; |
||||
} |
||||
|
||||
std::unique_ptr<codecs::ICodec> codec{*decoder}; |
||||
auto open_res = codec->OpenStream(stream, 0); |
||||
if (open_res.has_error()) { |
||||
ESP_LOGE(kTag, "opening stream failed"); |
||||
return; |
||||
} |
||||
|
||||
decodeToSink(*open_res, std::move(codec)); |
||||
} |
||||
|
||||
auto Player::decodeToSink(const codecs::ICodec::OutputFormat& format, |
||||
std::unique_ptr<codecs::ICodec> codec) -> void { |
||||
// Set up buffers to hold samples between the intermediary parts of
|
||||
// processing. We can just use the stack for these, since this method is
|
||||
// called only from background workers, which have enormous stacks.
|
||||
sample::Sample decode_storage[4096]; |
||||
audio::Buffer decode_buf(decode_storage); |
||||
|
||||
sample::Sample resample_storage[4096]; |
||||
audio::Buffer resample_buf(resample_storage); |
||||
|
||||
sample::Sample stereo_storage[4096]; |
||||
audio::Buffer stereo_buf(stereo_storage); |
||||
|
||||
// Work out what processing the codec's output needs.
|
||||
std::unique_ptr<audio::Resampler> resampler; |
||||
if (format.sample_rate_hz != 48000) { |
||||
resampler = std::make_unique<audio::Resampler>(format.sample_rate_hz, 48000, |
||||
format.num_channels); |
||||
} |
||||
bool double_samples = format.num_channels == 1; |
||||
|
||||
// Start our playback (wait for previous to end?)
|
||||
events::Audio().Dispatch(audio::TtsPlaybackChanged{.is_playing = true}); |
||||
|
||||
// FIXME: This decode-and-process loop is substantially the same as the audio
|
||||
// processor's filter loop. Ideally we should refactor both of these loops to
|
||||
// reuse code, however I'm holding off on doing this until we've implemented
|
||||
// more advanced audio processing features in the audio processor (EQ, tempo
|
||||
// shifting, etc.) as it's not clear to me yet how much the two codepaths will
|
||||
// be diverging later anyway.
|
||||
while ((codec || !decode_buf.isEmpty() || !resample_buf.isEmpty() || |
||||
!stereo_buf.isEmpty()) && |
||||
!stream_cancelled_) { |
||||
if (codec) { |
||||
auto decode_res = codec->DecodeTo(decode_buf.writeAcquire()); |
||||
if (decode_res.has_error()) { |
||||
ESP_LOGE(kTag, "decoding error"); |
||||
break; |
||||
} |
||||
decode_buf.writeCommit(decode_res->samples_written); |
||||
if (decode_res->is_stream_finished) { |
||||
codec.reset(); |
||||
} |
||||
} |
||||
|
||||
if (!decode_buf.isEmpty()) { |
||||
auto resample_input = decode_buf.readAcquire(); |
||||
auto resample_output = resample_buf.writeAcquire(); |
||||
|
||||
size_t read, wrote; |
||||
if (resampler) { |
||||
std::tie(read, wrote) = |
||||
resampler->Process(resample_input, resample_output, false); |
||||
} else { |
||||
read = wrote = std::min(resample_input.size(), resample_output.size()); |
||||
std::copy_n(resample_input.begin(), read, resample_output.begin()); |
||||
} |
||||
|
||||
decode_buf.readCommit(read); |
||||
resample_buf.writeCommit(wrote); |
||||
} |
||||
|
||||
if (!resample_buf.isEmpty()) { |
||||
auto channels_input = resample_buf.readAcquire(); |
||||
auto channels_output = stereo_buf.writeAcquire(); |
||||
size_t read, wrote; |
||||
if (double_samples) { |
||||
wrote = channels_output.size(); |
||||
read = wrote / 2; |
||||
if (read > channels_input.size()) { |
||||
read = channels_input.size(); |
||||
wrote = read * 2; |
||||
} |
||||
for (size_t i = 0; i < read; i++) { |
||||
channels_output[i * 2] = channels_input[i]; |
||||
channels_output[(i * 2) + 1] = channels_input[i]; |
||||
} |
||||
} else { |
||||
read = wrote = std::min(channels_input.size(), channels_output.size()); |
||||
std::copy_n(channels_input.begin(), read, channels_output.begin()); |
||||
} |
||||
resample_buf.readCommit(read); |
||||
stereo_buf.writeCommit(wrote); |
||||
} |
||||
|
||||
// The mixin PcmBuffer should almost always be draining, so we can force
|
||||
// samples into it more aggressively than with the main music PcmBuffer.
|
||||
while (!stereo_buf.isEmpty()) { |
||||
size_t sent = output_.send(stereo_buf.readAcquire()); |
||||
stereo_buf.readCommit(sent); |
||||
} |
||||
} |
||||
|
||||
while (!output_.isEmpty()) { |
||||
if (stream_cancelled_) { |
||||
output_.clear(); |
||||
} else { |
||||
vTaskDelay(pdMS_TO_TICKS(100)); |
||||
} |
||||
} |
||||
} |
||||
|
||||
} // namespace tts
|
||||
@ -0,0 +1,47 @@ |
||||
/*
|
||||
* Copyright 2024 jacqueline <me@jacqueline.id.au> |
||||
* |
||||
* SPDX-License-Identifier: GPL-3.0-only |
||||
*/ |
||||
|
||||
#pragma once |
||||
|
||||
#include <string> |
||||
|
||||
#include "audio/fatfs_stream_factory.hpp" |
||||
#include "codec.hpp" |
||||
#include "drivers/pcm_buffer.hpp" |
||||
#include "tasks.hpp" |
||||
|
||||
namespace tts { |
||||
|
||||
/*
|
||||
* A TTS Player is the output stage of the TTS pipeline. It receives a stream |
||||
* of filenames that should be played, and handles decoding these files and |
||||
* sending them to the output buffer. |
||||
*/ |
||||
class Player { |
||||
public: |
||||
Player(tasks::WorkerPool&, drivers::PcmBuffer&, audio::FatfsStreamFactory&); |
||||
|
||||
auto playFile(const std::string& text, const std::string& path) -> void; |
||||
|
||||
// Not copyable or movable.
|
||||
Player(const Player&) = delete; |
||||
Player& operator=(const Player&) = delete; |
||||
|
||||
private: |
||||
tasks::WorkerPool& bg_; |
||||
audio::FatfsStreamFactory& stream_factory_; |
||||
drivers::PcmBuffer& output_; |
||||
|
||||
std::mutex new_stream_mutex_; |
||||
std::atomic<bool> stream_playing_; |
||||
std::atomic<bool> stream_cancelled_; |
||||
|
||||
auto openAndDecode(const std::string& text, const std::string& path) -> void; |
||||
auto decodeToSink(const codecs::ICodec::OutputFormat&, |
||||
std::unique_ptr<codecs::ICodec>) -> void; |
||||
}; |
||||
|
||||
} // namespace tts
|
||||
Loading…
Reference in new issue