Start on TTS support by logging the data that will become TTS lines

Includes some misc cleanup of haptic double-triggering (or
non-triggering), since those cases all end up being TTS event
double-reporting, which to me crosses the threshold from "annoying" to
"usability issue"

lib/luavgl/src/group.c

@@ -248,6 +248,14 @@ static int luavgl_group_get_focused(lua_State *L)
   return 1;
 }
 
+static int luavgl_group_clear_focus(lua_State *L)
+{
+  luavgl_group_t *g = luavgl_check_group(L, 1);
+  g->group->obj_focus = NULL;
+
+  return 0;
+}
+
 static int luavgl_group_remove_obj(lua_State *L)
 {
   lv_obj_t *obj = luavgl_to_obj(L, 1);
@@ -319,6 +327,7 @@ static const luaL_Reg group_methods[] = {
     {"get_wrap",      luavgl_group_get_wrap     },
     {"get_obj_count", luavgl_group_get_obj_count},
     {"get_focused",   luavgl_group_get_focused  },
+    {"clear_focus",   luavgl_group_clear_focus  },
 
     {NULL,            NULL                      },
 };

lua/playing.lua

@@ -126,6 +126,7 @@ return screen:new {
       range = { min = 0, max = 100 },
       value = 0,
     }
+    local scrubber_desc = widgets.Description(scrubber, "Scrubber")
 
     scrubber:onevent(lvgl.EVENT.RELEASED, function()
       local track = playback.track:get()
@@ -165,6 +166,7 @@ return screen:new {
     end)
     local repeat_img = repeat_btn:Image { src = img.repeat_src }
     theme.set_style(repeat_img, icon_enabled_class)
+    local repeat_desc = widgets.Description(repeat_btn)
 
 
     local prev_btn = controls:Button {}
@@ -177,6 +179,7 @@ return screen:new {
     end)
     local prev_img = prev_btn:Image { src = img.prev }
     theme.set_style(prev_img, icon_enabled_class)
+    local prev_desc = widgets.Description(prev_btn, "Previous track")
 
     local play_pause_btn = controls:Button {}
     play_pause_btn:onClicked(function()
@@ -185,11 +188,13 @@ return screen:new {
     play_pause_btn:focus()
     local play_pause_img = play_pause_btn:Image { src = img.pause }
     theme.set_style(play_pause_img, icon_enabled_class)
+    local play_pause_desc = widgets.Description(play_pause_btn, "Play")
 
     local next_btn = controls:Button {}
     next_btn:onClicked(queue.next)
     local next_img = next_btn:Image { src = img.next }
     theme.set_style(next_img, icon_disabled_class)
+    local next_desc = widgets.Description(next_btn, "Next track")
 
     local shuffle_btn = controls:Button {}
     shuffle_btn:onClicked(function()
@@ -197,6 +202,7 @@ return screen:new {
     end)
     local shuffle_img = shuffle_btn:Image { src = img.shuffle }
     theme.set_style(shuffle_img, icon_enabled_class)
+    local shuffle_desc = widgets.Description(shuffle_btn)
 
     controls:Object({ flex_grow = 1, h = 1 }) -- spacer
 
@@ -205,8 +211,10 @@ return screen:new {
       playback.playing:bind(function(playing)
         if playing then
           play_pause_img:set_src(img.pause)
+          play_pause_desc:set { text = "Pause" }
         else
           play_pause_img:set_src(img.play)
+          play_pause_desc:set { text = "Play" }
         end
       end),
       playback.position:bind(function(pos)
@@ -241,9 +249,19 @@ return screen:new {
       end),
       queue.random:bind(function(shuffling)
         theme.set_style(shuffle_img, shuffling and icon_enabled_class or icon_disabled_class)
+        if shuffling then
+          shuffle_desc:set { text = "Disable shuffle" }
+        else
+          shuffle_desc:set { text = "Enable shuffle" }
+        end
       end),
       queue.repeat_track:bind(function(en)
         theme.set_style(repeat_img, en and icon_enabled_class or icon_disabled_class)
+        if en then
+          repeat_desc:set { text = "Disable track repeat" }
+        else
+          repeat_desc:set { text = "Enable track repeat" }
+        end
       end),
       queue.size:bind(function(num)
         if not num then return end

lua/widgets.lua

@@ -25,6 +25,15 @@ local img = {
 
 local widgets = {}
 
+function widgets.Description(obj, text)
+  local label = obj:Label {}
+  if text then
+    label:set { text = text }
+  end
+  label:add_flag(lvgl.FLAG.HIDDEN)
+  return label
+end
+
 widgets.MenuScreen = screen:new {
   show_back = false,
   title = "",
@@ -50,7 +59,7 @@ widgets.MenuScreen = screen:new {
 }
 
 function widgets.Row(parent, left, right)
-    local container = parent:Object{
+  local container = parent:Object {
     flex = {
       flex_direction = "row",
       justify_content = "flex-start",
@@ -61,11 +70,11 @@ function widgets.Row(parent, left, right)
     h = lvgl.SIZE_CONTENT
   }
   container:add_style(styles.list_item)
-    container:Label{
+  container:Label {
     text = left,
     flex_grow = 1
   }
-    container:Label{
+  container:Label {
     text = right
   }
 end
@@ -102,7 +111,8 @@ function widgets.StatusBar(parent, opts)
       w = lvgl.SIZE_CONTENT,
       h = 12,
     }
-    back:Label({ text = "<", align = lvgl.ALIGN.CENTER })
+    local label = back:Label({ text = "<", align = lvgl.ALIGN.CENTER })
+    widgets.Description(label, "Back")
     back:onClicked(opts.back_cb)
   end
 
@@ -191,7 +201,7 @@ function widgets.StatusBar(parent, opts)
 end
 
 function widgets.IconBtn(parent, icon, text)
-    local btn = parent:Button{
+  local btn = parent:Button {
     flex = {
       flex_direction = "row",
       justify_content = "flex-start",
@@ -205,10 +215,10 @@ function widgets.IconBtn(parent, icon, text)
     pad_left = 1,
     pad_column = 1
   }
-    btn:Image{
+  btn:Image {
     src = icon
   }
-    btn:Label{
+  btn:Label {
     text = text,
     text_font = font.fusion_10
   }
@@ -231,11 +241,14 @@ function widgets.InfiniteList(parent, iterator, opts)
     local group = lvgl.group.get_default()
     local focused_obj = group:get_focused()
     local num_children = infinite_list.root:get_child_cnt()
+    if (focused_obj) then
+      group:clear_focus()
+    end
     -- remove all children from the group and re-add them
-        for i = 0, num_children-1 do
+    for i = 0, num_children - 1 do
       lvgl.group.remove_obj(infinite_list.root:get_child(i))
     end
-        for i = 0, num_children-1 do
+    for i = 0, num_children - 1 do
       group:add_obj(infinite_list.root:get_child(i))
     end
     if (focused_obj) then
@@ -291,7 +304,7 @@ function widgets.InfiniteList(parent, iterator, opts)
         local to_add = fwd_iterator:next()
         if to_add then
           remove_top()
-                add_item(to_add, last_index+1)
+          add_item(to_add, last_index + 1)
         end
       end
       if this_item < last_selected then
@@ -300,7 +313,7 @@ function widgets.InfiniteList(parent, iterator, opts)
           local to_add = bck_iterator:prev();
           if to_add then
             remove_last()
-                  add_item(to_add, first_index-1)
+            add_item(to_add, first_index - 1)
             refresh_group()
           end
         end

src/tangara/CMakeLists.txt

@@ -4,7 +4,7 @@
 
 idf_component_register(
   SRC_DIRS "app_console" "audio" "battery" "database" "dev_console" "events"
-  "input" "lua" "system_fsm" "ui"
+  "input" "lua" "system_fsm" "ui" "tts"
   INCLUDE_DIRS "."
   REQUIRES "codecs" "drivers" "locale" "memory" "tasks" "util" "graphics"
   "tinyfsm" "lvgl" "esp_timer" "luavgl" "esp_app_format" "libcppbor" "libtags"

src/tangara/input/device_factory.cpp

@@ -9,6 +9,7 @@
 #include <memory>
 
 #include "input/feedback_haptics.hpp"
+#include "input/feedback_tts.hpp"
 #include "input/input_device.hpp"
 #include "input/input_nav_buttons.hpp"
 #include "input/input_touch_dpad.hpp"
@@ -52,7 +53,10 @@ auto DeviceFactory::createInputs(drivers::NvsStorage::InputModes mode)
 
 auto DeviceFactory::createFeedbacks()
     -> std::vector<std::shared_ptr<IFeedbackDevice>> {
-  return {std::make_shared<Haptics>(services_->haptics())};
+  return {
+      std::make_shared<Haptics>(services_->haptics()),
+      std::make_shared<TextToSpeech>(services_->tts()),
+  };
 }
 
 }  // namespace input

src/tangara/input/feedback_device.hpp

@@ -7,6 +7,7 @@
 #pragma once
 
 #include <cstdint>
+#include "core/lv_group.h"
 
 namespace input {
 
@@ -23,7 +24,7 @@ class IFeedbackDevice {
  public:
   virtual ~IFeedbackDevice() {}
 
-  virtual auto feedback(uint8_t event_type) -> void = 0;
+  virtual auto feedback(lv_group_t* group, uint8_t event_type) -> void = 0;
 
   // TODO: Add configuration; likely the same shape of interface that
   // IInputDevice uses.

src/tangara/input/feedback_haptics.cpp

@@ -8,6 +8,7 @@
 
 #include <cstdint>
 
+#include "core/lv_group.h"
 #include "lvgl/lvgl.h"
 
 #include "core/lv_event.h"
@@ -21,7 +22,13 @@ using Effect = drivers::Haptics::Effect;
 
 Haptics::Haptics(drivers::Haptics& haptics_) : haptics_(haptics_) {}
 
-auto Haptics::feedback(uint8_t event_type) -> void {
+auto Haptics::feedback(lv_group_t* group, uint8_t event_type) -> void {
+  lv_obj_t* obj = lv_group_get_focused(group);
+  if (obj == last_selection_) {
+    return;
+  }
+  last_selection_ = obj;
+
   switch (event_type) {
     case LV_EVENT_FOCUSED:
       haptics_.PlayWaveformEffect(Effect::kMediumClick1_100Pct);

src/tangara/input/feedback_haptics.hpp

@@ -8,6 +8,8 @@
 
 #include <cstdint>
 
+#include "core/lv_obj.h"
+
 #include "drivers/haptics.hpp"
 #include "input/feedback_device.hpp"
 
@@ -17,10 +19,11 @@ class Haptics : public IFeedbackDevice {
  public:
   Haptics(drivers::Haptics& haptics_);
 
-  auto feedback(uint8_t event_type) -> void override;
+  auto feedback(lv_group_t*, uint8_t event_type) -> void override;
 
  private:
   drivers::Haptics& haptics_;
+  lv_obj_t* last_selection_;
 };
 
 }  // namespace input

src/tangara/input/feedback_tts.cpp

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2024 jacqueline <me@jacqueline.id.au>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#include "input/feedback_tts.hpp"
+
+#include <cstdint>
+#include <variant>
+
+#include "lvgl/lvgl.h"
+
+#include "core/lv_event.h"
+#include "core/lv_group.h"
+#include "core/lv_obj.h"
+#include "core/lv_obj_class.h"
+#include "core/lv_obj_tree.h"
+#include "extra/widgets/list/lv_list.h"
+#include "tts/events.hpp"
+#include "widgets/lv_label.h"
+
+#include "tts/events.hpp"
+#include "tts/provider.hpp"
+
+namespace input {
+
+TextToSpeech::TextToSpeech(tts::Provider& tts)
+    : tts_(tts), last_obj_(nullptr) {}
+
+auto TextToSpeech::feedback(lv_group_t* group, uint8_t event_type) -> void {
+  if (group != last_group_) {
+    last_group_ = group;
+    last_obj_ = nullptr;
+    if (group) {
+      tts_.feed(tts::SimpleEvent::kContextChanged);
+    }
+  }
+
+  if (group) {
+    lv_obj_t* focused = lv_group_get_focused(group);
+    if (focused == last_obj_) {
+      return;
+    }
+
+    last_obj_ = focused;
+    if (focused != nullptr) {
+      describe(*focused);
+    }
+  }
+}
+
+auto TextToSpeech::describe(lv_obj_t& obj) -> void {
+  if (lv_obj_check_type(&obj, &lv_btn_class) ||
+      lv_obj_check_type(&obj, &lv_list_btn_class)) {
+    auto desc = findDescription(obj);
+    tts_.feed(tts::SelectionChanged{
+        .new_selection =
+            tts::SelectionChanged::Selection{
+                .description = desc,
+                .is_interactive = true,
+            },
+    });
+  } else {
+    auto desc = findDescription(obj);
+    tts_.feed(tts::SelectionChanged{
+        .new_selection =
+            tts::SelectionChanged::Selection{
+                .description = desc,
+                .is_interactive = false,
+            },
+    });
+  }
+}
+
+auto TextToSpeech::findDescription(lv_obj_t& obj)
+    -> std::optional<std::string> {
+  if (lv_obj_get_child_cnt(&obj) > 0) {
+    for (size_t i = 0; i < lv_obj_get_child_cnt(&obj); i++) {
+      auto res = findDescription(*lv_obj_get_child(&obj, i));
+      if (res) {
+        return res;
+      }
+    }
+  }
+
+  if (lv_obj_check_type(&obj, &lv_label_class)) {
+    std::string text = lv_label_get_text(&obj);
+    if (!text.empty()) {
+      return text;
+    }
+  }
+
+  return {};
+}
+
+}  // namespace input

src/tangara/input/feedback_tts.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2024 jacqueline <me@jacqueline.id.au>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#pragma once
+
+#include <cstdint>
+
+#include "core/lv_obj.h"
+#include "drivers/haptics.hpp"
+
+#include "input/feedback_device.hpp"
+#include "tts/events.hpp"
+#include "tts/provider.hpp"
+
+namespace input {
+
+class TextToSpeech : public IFeedbackDevice {
+ public:
+  TextToSpeech(tts::Provider&);
+
+  auto feedback(lv_group_t*, uint8_t event_type) -> void override;
+
+ private:
+  tts::Provider& tts_;
+
+  auto describe(lv_obj_t&) -> void;
+  auto findDescription(lv_obj_t&) -> std::optional<std::string>;
+
+  lv_group_t* last_group_;
+  lv_obj_t* last_obj_;
+};
+
+}  // namespace input

src/tangara/input/lvgl_input_driver.cpp

@@ -10,6 +10,8 @@
 #include <memory>
 #include <variant>
 
+#include "core/lv_event.h"
+#include "core/lv_indev.h"
 #include "lua.hpp"
 #include "lvgl.h"
 
@@ -91,6 +93,16 @@ LvglInputDriver::LvglInputDriver(drivers::NvsStorage& nvs,
   registration_ = lv_indev_drv_register(&driver_);
 }
 
+auto LvglInputDriver::setGroup(lv_group_t* g) -> void {
+  if (!g) {
+    return;
+  }
+  lv_indev_set_group(registration_, g);
+  // Emit a synthetic 'focus' event for the current selection, since otherwise
+  // our feedback devices won't know that the selection changed.
+  feedback(LV_EVENT_FOCUSED);
+}
+
 auto LvglInputDriver::read(lv_indev_data_t* data) -> void {
   // TODO: we should pass lock state on to the individual devices, since they
   // may wish to either ignore the lock state, or power down until unlock.
@@ -107,7 +119,7 @@ auto LvglInputDriver::feedback(uint8_t event) -> void {
     return;
   }
   for (auto&& device : feedbacks_) {
-    device->feedback(event);
+    device->feedback(registration_->group, event);
   }
 }
 

src/tangara/input/lvgl_input_driver.hpp

@@ -17,12 +17,12 @@
 #include "input/device_factory.hpp"
 #include "input/feedback_device.hpp"
 
+#include "drivers/nvs.hpp"
+#include "drivers/touchwheel.hpp"
 #include "input/input_device.hpp"
 #include "input/input_hook.hpp"
 #include "lua/lua_thread.hpp"
 #include "lua/property.hpp"
-#include "drivers/nvs.hpp"
-#include "drivers/touchwheel.hpp"
 
 namespace input {
 
@@ -37,10 +37,10 @@ class LvglInputDriver {
 
   auto mode() -> lua::Property& { return mode_; }
 
+  auto setGroup(lv_group_t*) -> void;
   auto read(lv_indev_data_t* data) -> void;
   auto feedback(uint8_t) -> void;
 
-  auto registration() -> lv_indev_t* { return registration_; }
   auto lock(bool l) -> void { is_locked_ = l; }
 
   auto pushHooks(lua_State* L) -> int;

src/tangara/system_fsm/booting.cpp

@@ -38,6 +38,7 @@
 #include "system_fsm/service_locator.hpp"
 #include "system_fsm/system_events.hpp"
 #include "tasks.hpp"
+#include "tts/provider.hpp"
 #include "ui/ui_fsm.hpp"
 
 namespace system_fsm {
@@ -99,6 +100,7 @@ auto Booting::entry() -> void {
       std::make_unique<audio::TrackQueue>(sServices->bg_worker()));
   sServices->tag_parser(std::make_unique<database::TagParserImpl>());
   sServices->collator(locale::CreateCollator());
+  sServices->tts(std::make_unique<tts::Provider>());
 
   ESP_LOGI(kTag, "init bluetooth");
   sServices->bluetooth(std::make_unique<drivers::Bluetooth>(

src/tangara/system_fsm/service_locator.hpp

@@ -10,17 +10,18 @@
 
 #include "audio/track_queue.hpp"
 #include "battery/battery.hpp"
-#include "drivers/bluetooth.hpp"
 #include "collation.hpp"
 #include "database/database.hpp"
 #include "database/tag_parser.hpp"
+#include "drivers/bluetooth.hpp"
 #include "drivers/gpios.hpp"
 #include "drivers/haptics.hpp"
 #include "drivers/nvs.hpp"
 #include "drivers/samd.hpp"
 #include "drivers/storage.hpp"
-#include "tasks.hpp"
 #include "drivers/touchwheel.hpp"
+#include "tasks.hpp"
+#include "tts/provider.hpp"
 
 namespace system_fsm {
 
@@ -69,6 +70,13 @@ class ServiceLocator {
 
   auto battery(std::unique_ptr<battery::Battery> i) { battery_ = std::move(i); }
 
+  auto tts() -> tts::Provider& {
+    assert(tts_ != nullptr);
+    return *tts_;
+  }
+
+  auto tts(std::unique_ptr<tts::Provider> i) { tts_ = std::move(i); }
+
   auto touchwheel() -> std::optional<drivers::TouchWheel*> {
     if (!touchwheel_) {
       return {};
@@ -140,6 +148,7 @@ class ServiceLocator {
 
   std::unique_ptr<audio::TrackQueue> queue_;
   std::unique_ptr<battery::Battery> battery_;
+  std::unique_ptr<tts::Provider> tts_;
 
   std::shared_ptr<database::Database> database_;
   std::unique_ptr<database::ITagParser> tag_parser_;

src/tangara/tts/events.hpp

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2024 jacqueline <me@jacqueline.id.au>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#pragma once
+
+#include <optional>
+#include <string>
+#include <variant>
+
+namespace tts {
+
+/*
+ * 'Simple' TTS events are events that do not have any extra event-specific
+ * details.
+ */
+enum class SimpleEvent {
+  /*
+   * Indicates that the screen's content has substantially changed. e.g. a new
+   * screen has been pushed.
+   */
+  kContextChanged,
+};
+
+/*
+ * Event indicating that the currently selected LVGL object has changed.
+ */
+struct SelectionChanged {
+  struct Selection {
+    std::optional<std::string> description;
+    bool is_interactive;
+  };
+
+  std::optional<Selection> new_selection;
+};
+
+using Event = std::variant<SimpleEvent, SelectionChanged>;
+
+}  // namespace tts

src/tangara/tts/provider.cpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 jacqueline <me@jacqueline.id.au>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#include "tts/provider.hpp"
+
+#include <optional>
+#include <string>
+#include <variant>
+
+#include "esp_log.h"
+
+#include "tts/events.hpp"
+
+namespace tts {
+
+[[maybe_unused]] static constexpr char kTag[] = "tts";
+
+Provider::Provider() {}
+
+auto Provider::feed(const Event& e) -> void {
+  if (std::holds_alternative<SimpleEvent>(e)) {
+    // ESP_LOGI(kTag, "context changed");
+  } else if (std::holds_alternative<SelectionChanged>(e)) {
+    auto ev = std::get<SelectionChanged>(e);
+    if (!ev.new_selection) {
+      // ESP_LOGI(kTag, "no selection");
+    } else {
+      // ESP_LOGI(kTag, "new selection: '%s', interactive? %i",
+      // ev.new_selection->description.value_or("").c_str(),
+      // ev.new_selection->is_interactive);
+    }
+  }
+}
+
+}  // namespace tts

src/tangara/tts/provider.hpp

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2024 jacqueline <me@jacqueline.id.au>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#pragma once
+
+#include <optional>
+#include <string>
+#include <variant>
+
+#include "tts/events.hpp"
+
+namespace tts {
+
+class Provider {
+ public:
+  Provider();
+  auto feed(const Event&) -> void;
+};
+
+}  // namespace tts

src/tangara/ui/lvgl_task.cpp

@@ -68,14 +68,14 @@ auto UiTask::Main() -> void {
     if (screen != current_screen_ && screen != nullptr) {
       lv_scr_load(screen->root());
       if (input_) {
-        lv_indev_set_group(input_->registration(), screen->group());
+        input_->setGroup(screen->group());
       }
       current_screen_ = screen;
     }
 
     if (input_ && current_screen_->group() != current_group) {
       current_group = current_screen_->group();
-      lv_indev_set_group(input_->registration(), current_group);
+      input_->setGroup(current_group);
     }
 
     TickType_t delay = lv_timer_handler();