refactor(hands): 移除空壳 Hand — Whiteboard/Slideshow/Speech (Phase 5)
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled

删除 3 个仅含 UI 占位的 Hand,清理 Rust 实现与前端引用:
- Rust: whiteboard.rs(422行) + slideshow.rs(797行) + speech.rs(442行)
- 前端: WhiteboardCanvas + SlideshowRenderer + speech-synth + 相关类型/常量
- 配置: 3 个 HAND.toml
- 净减 ~5400 行,Hands 9→6(启用) + Quiz/Browser/Researcher/Collector/Clip/Twitter/Reminder
This commit is contained in:
iven
2026-04-17 19:55:59 +08:00
parent 14fa7e150a
commit cb9e48f11d
21 changed files with 11 additions and 3014 deletions

View File

@@ -1,9 +1,6 @@
//! Educational Hands - Teaching and presentation capabilities
//!
//! This module provides hands for interactive classroom experiences:
//! - Whiteboard: Drawing and annotation
//! - Slideshow: Presentation control
//! - Speech: Text-to-speech synthesis
//! This module provides hands for interactive experiences:
//! - Quiz: Assessment and evaluation
//! - Browser: Web automation
//! - Researcher: Deep research and analysis
@@ -11,9 +8,6 @@
//! - Clip: Video processing
//! - Twitter: Social media automation
mod whiteboard;
mod slideshow;
mod speech;
pub mod quiz;
mod browser;
mod researcher;
@@ -22,9 +16,6 @@ mod clip;
mod twitter;
pub mod reminder;
pub use whiteboard::*;
pub use slideshow::*;
pub use speech::*;
pub use quiz::*;
pub use browser::*;
pub use researcher::*;

View File

@@ -1,797 +0,0 @@
//! Slideshow Hand - Presentation control capabilities
//!
//! Provides slideshow control for teaching:
//! - next_slide/prev_slide: Navigation
//! - goto_slide: Jump to specific slide
//! - spotlight: Highlight elements
//! - laser: Show laser pointer
//! - highlight: Highlight areas
//! - play_animation: Trigger animations
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::sync::Arc;
use tokio::sync::RwLock;
use zclaw_types::Result;
use crate::{Hand, HandConfig, HandContext, HandResult, HandStatus};
/// Slideshow action types
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "action", rename_all = "snake_case")]
pub enum SlideshowAction {
/// Go to next slide
NextSlide,
/// Go to previous slide
PrevSlide,
/// Go to specific slide
GotoSlide {
slide_number: usize,
},
/// Spotlight/highlight an element
Spotlight {
element_id: String,
#[serde(default = "default_spotlight_duration")]
duration_ms: u64,
},
/// Show laser pointer at position
Laser {
x: f64,
y: f64,
#[serde(default = "default_laser_duration")]
duration_ms: u64,
},
/// Highlight a rectangular area
Highlight {
x: f64,
y: f64,
width: f64,
height: f64,
#[serde(default)]
color: Option<String>,
#[serde(default = "default_highlight_duration")]
duration_ms: u64,
},
/// Play animation
PlayAnimation {
animation_id: String,
},
/// Pause auto-play
Pause,
/// Resume auto-play
Resume,
/// Start auto-play
AutoPlay {
#[serde(default = "default_interval")]
interval_ms: u64,
},
/// Stop auto-play
StopAutoPlay,
/// Get current state
GetState,
/// Set slide content (for dynamic slides)
SetContent {
slide_number: usize,
content: SlideContent,
},
}
fn default_spotlight_duration() -> u64 { 2000 }
fn default_laser_duration() -> u64 { 3000 }
fn default_highlight_duration() -> u64 { 2000 }
fn default_interval() -> u64 { 5000 }
/// Slide content structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SlideContent {
pub title: String,
#[serde(default)]
pub subtitle: Option<String>,
#[serde(default)]
pub content: Vec<ContentBlock>,
#[serde(default)]
pub notes: Option<String>,
#[serde(default)]
pub background: Option<String>,
}
/// Presentation/slideshow rendering content block. Domain-specific for slide content.
/// Distinct from zclaw_types::ContentBlock (LLM messages) and zclaw_protocols::ContentBlock (MCP).
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContentBlock {
Text { text: String, style: Option<TextStyle> },
Image { url: String, alt: Option<String> },
List { items: Vec<String>, ordered: bool },
Code { code: String, language: Option<String> },
Math { latex: String },
Table { headers: Vec<String>, rows: Vec<Vec<String>> },
Chart { chart_type: String, data: serde_json::Value },
}
/// Text style options
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct TextStyle {
#[serde(default)]
pub bold: bool,
#[serde(default)]
pub italic: bool,
#[serde(default)]
pub size: Option<u32>,
#[serde(default)]
pub color: Option<String>,
}
/// Slideshow state
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SlideshowState {
pub current_slide: usize,
pub total_slides: usize,
pub is_playing: bool,
pub auto_play_interval_ms: u64,
pub slides: Vec<SlideContent>,
}
impl Default for SlideshowState {
fn default() -> Self {
Self {
current_slide: 0,
total_slides: 0,
is_playing: false,
auto_play_interval_ms: 5000,
slides: Vec::new(),
}
}
}
/// Slideshow Hand implementation
pub struct SlideshowHand {
config: HandConfig,
state: Arc<RwLock<SlideshowState>>,
}
impl SlideshowHand {
/// Create a new slideshow hand
pub fn new() -> Self {
Self {
config: HandConfig {
id: "slideshow".to_string(),
name: "幻灯片".to_string(),
description: "控制演示文稿的播放、导航和标注".to_string(),
needs_approval: false,
dependencies: vec![],
input_schema: Some(serde_json::json!({
"type": "object",
"properties": {
"action": { "type": "string" },
"slide_number": { "type": "integer" },
"element_id": { "type": "string" },
}
})),
tags: vec!["presentation".to_string(), "education".to_string()],
enabled: true,
max_concurrent: 0,
timeout_secs: 0,
},
state: Arc::new(RwLock::new(SlideshowState::default())),
}
}
/// Create with slides (async version)
pub async fn with_slides_async(slides: Vec<SlideContent>) -> Self {
let hand = Self::new();
let mut state = hand.state.write().await;
state.total_slides = slides.len();
state.slides = slides;
drop(state);
hand
}
/// Execute a slideshow action
pub async fn execute_action(&self, action: SlideshowAction) -> Result<HandResult> {
let mut state = self.state.write().await;
match action {
SlideshowAction::NextSlide => {
if state.current_slide < state.total_slides.saturating_sub(1) {
state.current_slide += 1;
}
Ok(HandResult::success(serde_json::json!({
"status": "next",
"current_slide": state.current_slide,
"total_slides": state.total_slides,
})))
}
SlideshowAction::PrevSlide => {
if state.current_slide > 0 {
state.current_slide -= 1;
}
Ok(HandResult::success(serde_json::json!({
"status": "prev",
"current_slide": state.current_slide,
"total_slides": state.total_slides,
})))
}
SlideshowAction::GotoSlide { slide_number } => {
if slide_number < state.total_slides {
state.current_slide = slide_number;
Ok(HandResult::success(serde_json::json!({
"status": "goto",
"current_slide": state.current_slide,
"slide_content": state.slides.get(slide_number),
})))
} else {
Ok(HandResult::error(format!("Slide {} out of range", slide_number)))
}
}
SlideshowAction::Spotlight { element_id, duration_ms } => {
Ok(HandResult::success(serde_json::json!({
"status": "spotlight",
"element_id": element_id,
"duration_ms": duration_ms,
})))
}
SlideshowAction::Laser { x, y, duration_ms } => {
Ok(HandResult::success(serde_json::json!({
"status": "laser",
"x": x,
"y": y,
"duration_ms": duration_ms,
})))
}
SlideshowAction::Highlight { x, y, width, height, color, duration_ms } => {
Ok(HandResult::success(serde_json::json!({
"status": "highlight",
"x": x, "y": y,
"width": width, "height": height,
"color": color.unwrap_or_else(|| "#ffcc00".to_string()),
"duration_ms": duration_ms,
})))
}
SlideshowAction::PlayAnimation { animation_id } => {
Ok(HandResult::success(serde_json::json!({
"status": "animation",
"animation_id": animation_id,
})))
}
SlideshowAction::Pause => {
state.is_playing = false;
Ok(HandResult::success(serde_json::json!({
"status": "paused",
})))
}
SlideshowAction::Resume => {
state.is_playing = true;
Ok(HandResult::success(serde_json::json!({
"status": "resumed",
})))
}
SlideshowAction::AutoPlay { interval_ms } => {
state.is_playing = true;
state.auto_play_interval_ms = interval_ms;
Ok(HandResult::success(serde_json::json!({
"status": "autoplay",
"interval_ms": interval_ms,
})))
}
SlideshowAction::StopAutoPlay => {
state.is_playing = false;
Ok(HandResult::success(serde_json::json!({
"status": "stopped",
})))
}
SlideshowAction::GetState => {
Ok(HandResult::success(serde_json::to_value(&*state).unwrap_or(Value::Null)))
}
SlideshowAction::SetContent { slide_number, content } => {
if slide_number < state.slides.len() {
state.slides[slide_number] = content.clone();
Ok(HandResult::success(serde_json::json!({
"status": "content_set",
"slide_number": slide_number,
})))
} else if slide_number == state.slides.len() {
state.slides.push(content);
state.total_slides = state.slides.len();
Ok(HandResult::success(serde_json::json!({
"status": "slide_added",
"slide_number": slide_number,
})))
} else {
Ok(HandResult::error(format!("Invalid slide number: {}", slide_number)))
}
}
}
}
/// Get current state
pub async fn get_state(&self) -> SlideshowState {
self.state.read().await.clone()
}
/// Add a slide
pub async fn add_slide(&self, content: SlideContent) {
let mut state = self.state.write().await;
state.slides.push(content);
state.total_slides = state.slides.len();
}
}
impl Default for SlideshowHand {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl Hand for SlideshowHand {
fn config(&self) -> &HandConfig {
&self.config
}
async fn execute(&self, _context: &HandContext, input: Value) -> Result<HandResult> {
let action: SlideshowAction = match serde_json::from_value(input) {
Ok(a) => a,
Err(e) => {
return Ok(HandResult::error(format!("Invalid slideshow action: {}", e)));
}
};
self.execute_action(action).await
}
fn status(&self) -> HandStatus {
HandStatus::Idle
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
// === Config & Defaults ===
#[tokio::test]
async fn test_slideshow_creation() {
let hand = SlideshowHand::new();
assert_eq!(hand.config().id, "slideshow");
assert_eq!(hand.config().name, "幻灯片");
assert!(!hand.config().needs_approval);
assert!(hand.config().enabled);
assert!(hand.config().tags.contains(&"presentation".to_string()));
}
#[test]
fn test_default_impl() {
let hand = SlideshowHand::default();
assert_eq!(hand.config().id, "slideshow");
}
#[test]
fn test_needs_approval() {
let hand = SlideshowHand::new();
assert!(!hand.needs_approval());
}
#[test]
fn test_status() {
let hand = SlideshowHand::new();
assert_eq!(hand.status(), HandStatus::Idle);
}
#[test]
fn test_default_state() {
let state = SlideshowState::default();
assert_eq!(state.current_slide, 0);
assert_eq!(state.total_slides, 0);
assert!(!state.is_playing);
assert_eq!(state.auto_play_interval_ms, 5000);
assert!(state.slides.is_empty());
}
// === Navigation ===
#[tokio::test]
async fn test_navigation() {
let hand = SlideshowHand::with_slides_async(vec![
SlideContent { title: "Slide 1".to_string(), subtitle: None, content: vec![], notes: None, background: None },
SlideContent { title: "Slide 2".to_string(), subtitle: None, content: vec![], notes: None, background: None },
SlideContent { title: "Slide 3".to_string(), subtitle: None, content: vec![], notes: None, background: None },
]).await;
// Next
hand.execute_action(SlideshowAction::NextSlide).await.unwrap();
assert_eq!(hand.get_state().await.current_slide, 1);
// Goto
hand.execute_action(SlideshowAction::GotoSlide { slide_number: 2 }).await.unwrap();
assert_eq!(hand.get_state().await.current_slide, 2);
// Prev
hand.execute_action(SlideshowAction::PrevSlide).await.unwrap();
assert_eq!(hand.get_state().await.current_slide, 1);
}
#[tokio::test]
async fn test_next_slide_at_end() {
let hand = SlideshowHand::with_slides_async(vec![
SlideContent { title: "Only Slide".to_string(), subtitle: None, content: vec![], notes: None, background: None },
]).await;
// At slide 0, should not advance past last slide
hand.execute_action(SlideshowAction::NextSlide).await.unwrap();
assert_eq!(hand.get_state().await.current_slide, 0);
}
#[tokio::test]
async fn test_prev_slide_at_beginning() {
let hand = SlideshowHand::with_slides_async(vec![
SlideContent { title: "Slide 1".to_string(), subtitle: None, content: vec![], notes: None, background: None },
SlideContent { title: "Slide 2".to_string(), subtitle: None, content: vec![], notes: None, background: None },
]).await;
// At slide 0, should not go below 0
hand.execute_action(SlideshowAction::PrevSlide).await.unwrap();
assert_eq!(hand.get_state().await.current_slide, 0);
}
#[tokio::test]
async fn test_goto_slide_out_of_range() {
let hand = SlideshowHand::with_slides_async(vec![
SlideContent { title: "Slide 1".to_string(), subtitle: None, content: vec![], notes: None, background: None },
]).await;
let result = hand.execute_action(SlideshowAction::GotoSlide { slide_number: 5 }).await.unwrap();
assert!(!result.success);
}
#[tokio::test]
async fn test_goto_slide_returns_content() {
let hand = SlideshowHand::with_slides_async(vec![
SlideContent { title: "First".to_string(), subtitle: None, content: vec![], notes: None, background: None },
SlideContent { title: "Second".to_string(), subtitle: None, content: vec![], notes: None, background: None },
]).await;
let result = hand.execute_action(SlideshowAction::GotoSlide { slide_number: 1 }).await.unwrap();
assert!(result.success);
assert_eq!(result.output["slide_content"]["title"], "Second");
}
// === Spotlight & Laser & Highlight ===
#[tokio::test]
async fn test_spotlight() {
let hand = SlideshowHand::new();
let action = SlideshowAction::Spotlight {
element_id: "title".to_string(),
duration_ms: 2000,
};
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
assert_eq!(result.output["element_id"], "title");
assert_eq!(result.output["duration_ms"], 2000);
}
#[tokio::test]
async fn test_spotlight_default_duration() {
let hand = SlideshowHand::new();
let action = SlideshowAction::Spotlight {
element_id: "elem".to_string(),
duration_ms: default_spotlight_duration(),
};
let result = hand.execute_action(action).await.unwrap();
assert_eq!(result.output["duration_ms"], 2000);
}
#[tokio::test]
async fn test_laser() {
let hand = SlideshowHand::new();
let action = SlideshowAction::Laser {
x: 100.0,
y: 200.0,
duration_ms: 3000,
};
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
assert_eq!(result.output["x"], 100.0);
assert_eq!(result.output["y"], 200.0);
}
#[tokio::test]
async fn test_highlight_default_color() {
let hand = SlideshowHand::new();
let action = SlideshowAction::Highlight {
x: 10.0, y: 20.0, width: 100.0, height: 50.0,
color: None, duration_ms: 2000,
};
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
assert_eq!(result.output["color"], "#ffcc00");
}
#[tokio::test]
async fn test_highlight_custom_color() {
let hand = SlideshowHand::new();
let action = SlideshowAction::Highlight {
x: 0.0, y: 0.0, width: 50.0, height: 50.0,
color: Some("#ff0000".to_string()), duration_ms: 1000,
};
let result = hand.execute_action(action).await.unwrap();
assert_eq!(result.output["color"], "#ff0000");
}
// === AutoPlay / Pause / Resume ===
#[tokio::test]
async fn test_autoplay_pause_resume() {
let hand = SlideshowHand::new();
// AutoPlay
let result = hand.execute_action(SlideshowAction::AutoPlay { interval_ms: 3000 }).await.unwrap();
assert!(result.success);
assert!(hand.get_state().await.is_playing);
assert_eq!(hand.get_state().await.auto_play_interval_ms, 3000);
// Pause
hand.execute_action(SlideshowAction::Pause).await.unwrap();
assert!(!hand.get_state().await.is_playing);
// Resume
hand.execute_action(SlideshowAction::Resume).await.unwrap();
assert!(hand.get_state().await.is_playing);
// Stop
hand.execute_action(SlideshowAction::StopAutoPlay).await.unwrap();
assert!(!hand.get_state().await.is_playing);
}
#[tokio::test]
async fn test_autoplay_default_interval() {
let hand = SlideshowHand::new();
hand.execute_action(SlideshowAction::AutoPlay { interval_ms: default_interval() }).await.unwrap();
assert_eq!(hand.get_state().await.auto_play_interval_ms, 5000);
}
// === PlayAnimation ===
#[tokio::test]
async fn test_play_animation() {
let hand = SlideshowHand::new();
let result = hand.execute_action(SlideshowAction::PlayAnimation {
animation_id: "fade_in".to_string(),
}).await.unwrap();
assert!(result.success);
assert_eq!(result.output["animation_id"], "fade_in");
}
// === GetState ===
#[tokio::test]
async fn test_get_state() {
let hand = SlideshowHand::with_slides_async(vec![
SlideContent { title: "A".to_string(), subtitle: None, content: vec![], notes: None, background: None },
]).await;
let result = hand.execute_action(SlideshowAction::GetState).await.unwrap();
assert!(result.success);
assert_eq!(result.output["total_slides"], 1);
assert_eq!(result.output["current_slide"], 0);
}
// === SetContent ===
#[tokio::test]
async fn test_set_content() {
let hand = SlideshowHand::new();
let content = SlideContent {
title: "Test Slide".to_string(),
subtitle: Some("Subtitle".to_string()),
content: vec![ContentBlock::Text {
text: "Hello".to_string(),
style: None,
}],
notes: Some("Speaker notes".to_string()),
background: None,
};
let result = hand.execute_action(SlideshowAction::SetContent {
slide_number: 0,
content,
}).await.unwrap();
assert!(result.success);
assert_eq!(hand.get_state().await.total_slides, 1);
assert_eq!(hand.get_state().await.slides[0].title, "Test Slide");
}
#[tokio::test]
async fn test_set_content_append() {
let hand = SlideshowHand::with_slides_async(vec![
SlideContent { title: "First".to_string(), subtitle: None, content: vec![], notes: None, background: None },
]).await;
let content = SlideContent {
title: "Appended".to_string(), subtitle: None, content: vec![], notes: None, background: None,
};
let result = hand.execute_action(SlideshowAction::SetContent {
slide_number: 1,
content,
}).await.unwrap();
assert!(result.success);
assert_eq!(result.output["status"], "slide_added");
assert_eq!(hand.get_state().await.total_slides, 2);
}
#[tokio::test]
async fn test_set_content_invalid_index() {
let hand = SlideshowHand::new();
let content = SlideContent {
title: "Gap".to_string(), subtitle: None, content: vec![], notes: None, background: None,
};
let result = hand.execute_action(SlideshowAction::SetContent {
slide_number: 5,
content,
}).await.unwrap();
assert!(!result.success);
}
// === Action Deserialization ===
#[test]
fn test_deserialize_next_slide() {
let action: SlideshowAction = serde_json::from_value(json!({"action": "next_slide"})).unwrap();
assert!(matches!(action, SlideshowAction::NextSlide));
}
#[test]
fn test_deserialize_goto_slide() {
let action: SlideshowAction = serde_json::from_value(json!({"action": "goto_slide", "slide_number": 3})).unwrap();
match action {
SlideshowAction::GotoSlide { slide_number } => assert_eq!(slide_number, 3),
_ => panic!("Expected GotoSlide"),
}
}
#[test]
fn test_deserialize_laser() {
let action: SlideshowAction = serde_json::from_value(json!({
"action": "laser", "x": 50.0, "y": 75.0
})).unwrap();
match action {
SlideshowAction::Laser { x, y, .. } => {
assert_eq!(x, 50.0);
assert_eq!(y, 75.0);
}
_ => panic!("Expected Laser"),
}
}
#[test]
fn test_deserialize_autoplay() {
let action: SlideshowAction = serde_json::from_value(json!({"action": "auto_play"})).unwrap();
match action {
SlideshowAction::AutoPlay { interval_ms } => assert_eq!(interval_ms, 5000),
_ => panic!("Expected AutoPlay"),
}
}
#[test]
fn test_deserialize_invalid_action() {
let result = serde_json::from_value::<SlideshowAction>(json!({"action": "nonexistent"}));
assert!(result.is_err());
}
// === ContentBlock Deserialization ===
#[test]
fn test_content_block_text() {
let block: ContentBlock = serde_json::from_value(json!({
"type": "text", "text": "Hello"
})).unwrap();
match block {
ContentBlock::Text { text, style } => {
assert_eq!(text, "Hello");
assert!(style.is_none());
}
_ => panic!("Expected Text"),
}
}
#[test]
fn test_content_block_list() {
let block: ContentBlock = serde_json::from_value(json!({
"type": "list", "items": ["A", "B"], "ordered": true
})).unwrap();
match block {
ContentBlock::List { items, ordered } => {
assert_eq!(items, vec!["A", "B"]);
assert!(ordered);
}
_ => panic!("Expected List"),
}
}
#[test]
fn test_content_block_code() {
let block: ContentBlock = serde_json::from_value(json!({
"type": "code", "code": "fn main() {}", "language": "rust"
})).unwrap();
match block {
ContentBlock::Code { code, language } => {
assert_eq!(code, "fn main() {}");
assert_eq!(language, Some("rust".to_string()));
}
_ => panic!("Expected Code"),
}
}
#[test]
fn test_content_block_table() {
let block: ContentBlock = serde_json::from_value(json!({
"type": "table",
"headers": ["Name", "Age"],
"rows": [["Alice", "30"]]
})).unwrap();
match block {
ContentBlock::Table { headers, rows } => {
assert_eq!(headers, vec!["Name", "Age"]);
assert_eq!(rows, vec![vec!["Alice", "30"]]);
}
_ => panic!("Expected Table"),
}
}
// === Hand trait via execute ===
#[tokio::test]
async fn test_hand_execute_dispatch() {
let hand = SlideshowHand::with_slides_async(vec![
SlideContent { title: "S1".to_string(), subtitle: None, content: vec![], notes: None, background: None },
SlideContent { title: "S2".to_string(), subtitle: None, content: vec![], notes: None, background: None },
]).await;
let ctx = HandContext::default();
let result = hand.execute(&ctx, json!({"action": "next_slide"})).await.unwrap();
assert!(result.success);
assert_eq!(result.output["current_slide"], 1);
}
#[tokio::test]
async fn test_hand_execute_invalid_action() {
let hand = SlideshowHand::new();
let ctx = HandContext::default();
let result = hand.execute(&ctx, json!({"action": "invalid"})).await.unwrap();
assert!(!result.success);
}
// === add_slide helper ===
#[tokio::test]
async fn test_add_slide() {
let hand = SlideshowHand::new();
hand.add_slide(SlideContent {
title: "Dynamic".to_string(), subtitle: None, content: vec![], notes: None, background: None,
}).await;
hand.add_slide(SlideContent {
title: "Dynamic 2".to_string(), subtitle: None, content: vec![], notes: None, background: None,
}).await;
let state = hand.get_state().await;
assert_eq!(state.total_slides, 2);
assert_eq!(state.slides.len(), 2);
}
}

View File

@@ -1,442 +0,0 @@
//! Speech Hand - Text-to-Speech synthesis capabilities
//!
//! Provides speech synthesis for teaching:
//! - speak: Convert text to speech
//! - speak_ssml: Advanced speech with SSML markup
//! - pause/resume/stop: Playback control
//! - list_voices: Get available voices
//! - set_voice: Configure voice settings
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::sync::Arc;
use tokio::sync::RwLock;
use zclaw_types::Result;
use crate::{Hand, HandConfig, HandContext, HandResult, HandStatus};
/// TTS Provider types
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "lowercase")]
pub enum TtsProvider {
#[default]
Browser,
Azure,
OpenAI,
ElevenLabs,
Local,
}
/// Speech action types
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "action", rename_all = "snake_case")]
pub enum SpeechAction {
/// Speak text
Speak {
text: String,
#[serde(default)]
voice: Option<String>,
#[serde(default = "default_rate")]
rate: f32,
#[serde(default = "default_pitch")]
pitch: f32,
#[serde(default = "default_volume")]
volume: f32,
#[serde(default)]
language: Option<String>,
},
/// Speak with SSML markup
SpeakSsml {
ssml: String,
#[serde(default)]
voice: Option<String>,
},
/// Pause playback
Pause,
/// Resume playback
Resume,
/// Stop playback
Stop,
/// List available voices
ListVoices {
#[serde(default)]
language: Option<String>,
},
/// Set default voice
SetVoice {
voice: String,
#[serde(default)]
language: Option<String>,
},
/// Set provider
SetProvider {
provider: TtsProvider,
#[serde(default)]
api_key: Option<String>,
#[serde(default)]
region: Option<String>,
},
}
fn default_rate() -> f32 { 1.0 }
fn default_pitch() -> f32 { 1.0 }
fn default_volume() -> f32 { 1.0 }
/// Voice information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VoiceInfo {
pub id: String,
pub name: String,
pub language: String,
pub gender: String,
#[serde(default)]
pub preview_url: Option<String>,
}
/// Playback state
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub enum PlaybackState {
#[default]
Idle,
Playing,
Paused,
}
/// Speech configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SpeechConfig {
pub provider: TtsProvider,
pub default_voice: Option<String>,
pub default_language: String,
pub default_rate: f32,
pub default_pitch: f32,
pub default_volume: f32,
}
impl Default for SpeechConfig {
fn default() -> Self {
Self {
provider: TtsProvider::Browser,
default_voice: None,
default_language: "zh-CN".to_string(),
default_rate: 1.0,
default_pitch: 1.0,
default_volume: 1.0,
}
}
}
/// Speech state
#[derive(Debug, Clone, Default)]
pub struct SpeechState {
pub config: SpeechConfig,
pub playback: PlaybackState,
pub current_text: Option<String>,
pub position_ms: u64,
pub available_voices: Vec<VoiceInfo>,
}
/// Speech Hand implementation
pub struct SpeechHand {
config: HandConfig,
state: Arc<RwLock<SpeechState>>,
}
impl SpeechHand {
/// Create a new speech hand
pub fn new() -> Self {
Self {
config: HandConfig {
id: "speech".to_string(),
name: "语音合成".to_string(),
description: "文本转语音合成输出".to_string(),
needs_approval: false,
dependencies: vec![],
input_schema: Some(serde_json::json!({
"type": "object",
"properties": {
"action": { "type": "string" },
"text": { "type": "string" },
"voice": { "type": "string" },
"rate": { "type": "number" },
}
})),
tags: vec!["audio".to_string(), "tts".to_string(), "education".to_string(), "demo".to_string()],
enabled: true,
max_concurrent: 0,
timeout_secs: 0,
},
state: Arc::new(RwLock::new(SpeechState {
config: SpeechConfig::default(),
playback: PlaybackState::Idle,
available_voices: Self::get_default_voices(),
..Default::default()
})),
}
}
/// Create with custom provider
pub fn with_provider(provider: TtsProvider) -> Self {
let hand = Self::new();
let mut state = hand.state.blocking_write();
state.config.provider = provider;
drop(state);
hand
}
/// Get default voices
fn get_default_voices() -> Vec<VoiceInfo> {
vec![
VoiceInfo {
id: "zh-CN-XiaoxiaoNeural".to_string(),
name: "Xiaoxiao".to_string(),
language: "zh-CN".to_string(),
gender: "female".to_string(),
preview_url: None,
},
VoiceInfo {
id: "zh-CN-YunxiNeural".to_string(),
name: "Yunxi".to_string(),
language: "zh-CN".to_string(),
gender: "male".to_string(),
preview_url: None,
},
VoiceInfo {
id: "en-US-JennyNeural".to_string(),
name: "Jenny".to_string(),
language: "en-US".to_string(),
gender: "female".to_string(),
preview_url: None,
},
VoiceInfo {
id: "en-US-GuyNeural".to_string(),
name: "Guy".to_string(),
language: "en-US".to_string(),
gender: "male".to_string(),
preview_url: None,
},
]
}
/// Execute a speech action
pub async fn execute_action(&self, action: SpeechAction) -> Result<HandResult> {
let mut state = self.state.write().await;
match action {
SpeechAction::Speak { text, voice, rate, pitch, volume, language } => {
let voice_id = voice.or(state.config.default_voice.clone())
.unwrap_or_else(|| "default".to_string());
let lang = language.unwrap_or_else(|| state.config.default_language.clone());
let actual_rate = if rate == 1.0 { state.config.default_rate } else { rate };
let actual_pitch = if pitch == 1.0 { state.config.default_pitch } else { pitch };
let actual_volume = if volume == 1.0 { state.config.default_volume } else { volume };
state.playback = PlaybackState::Playing;
state.current_text = Some(text.clone());
// Determine TTS method based on provider:
// - Browser: frontend uses Web Speech API (zero deps, works offline)
// - OpenAI: frontend calls speech_tts command (high-quality, needs API key)
// - Others: future support
let tts_method = match state.config.provider {
TtsProvider::Browser => "browser",
TtsProvider::OpenAI => "openai_api",
TtsProvider::Azure => "azure_api",
TtsProvider::ElevenLabs => "elevenlabs_api",
TtsProvider::Local => "local_engine",
};
let estimated_duration_ms = (text.chars().count() as f64 / 5.0 * 1000.0) as u64;
Ok(HandResult::success(serde_json::json!({
"status": "speaking",
"tts_method": tts_method,
"text": text,
"voice": voice_id,
"language": lang,
"rate": actual_rate,
"pitch": actual_pitch,
"volume": actual_volume,
"provider": format!("{:?}", state.config.provider).to_lowercase(),
"duration_ms": estimated_duration_ms,
"instruction": "Frontend should play this via TTS engine"
})))
}
SpeechAction::SpeakSsml { ssml, voice } => {
let voice_id = voice.or(state.config.default_voice.clone())
.unwrap_or_else(|| "default".to_string());
state.playback = PlaybackState::Playing;
state.current_text = Some(ssml.clone());
Ok(HandResult::success(serde_json::json!({
"status": "speaking_ssml",
"ssml": ssml,
"voice": voice_id,
"provider": state.config.provider,
})))
}
SpeechAction::Pause => {
state.playback = PlaybackState::Paused;
Ok(HandResult::success(serde_json::json!({
"status": "paused",
"position_ms": state.position_ms,
})))
}
SpeechAction::Resume => {
state.playback = PlaybackState::Playing;
Ok(HandResult::success(serde_json::json!({
"status": "resumed",
"position_ms": state.position_ms,
})))
}
SpeechAction::Stop => {
state.playback = PlaybackState::Idle;
state.current_text = None;
state.position_ms = 0;
Ok(HandResult::success(serde_json::json!({
"status": "stopped",
})))
}
SpeechAction::ListVoices { language } => {
let voices: Vec<_> = state.available_voices.iter()
.filter(|v| {
language.as_ref()
.map(|l| v.language.starts_with(l))
.unwrap_or(true)
})
.cloned()
.collect();
Ok(HandResult::success(serde_json::json!({
"voices": voices,
"count": voices.len(),
})))
}
SpeechAction::SetVoice { voice, language } => {
state.config.default_voice = Some(voice.clone());
if let Some(lang) = language {
state.config.default_language = lang;
}
Ok(HandResult::success(serde_json::json!({
"status": "voice_set",
"voice": voice,
"language": state.config.default_language,
})))
}
SpeechAction::SetProvider { provider, api_key, region: _ } => {
state.config.provider = provider.clone();
// In real implementation, would configure provider
Ok(HandResult::success(serde_json::json!({
"status": "provider_set",
"provider": provider,
"configured": api_key.is_some(),
})))
}
}
}
/// Get current state
pub async fn get_state(&self) -> SpeechState {
self.state.read().await.clone()
}
}
impl Default for SpeechHand {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl Hand for SpeechHand {
fn config(&self) -> &HandConfig {
&self.config
}
async fn execute(&self, _context: &HandContext, input: Value) -> Result<HandResult> {
let action: SpeechAction = match serde_json::from_value(input) {
Ok(a) => a,
Err(e) => {
return Ok(HandResult::error(format!("Invalid speech action: {}", e)));
}
};
self.execute_action(action).await
}
fn status(&self) -> HandStatus {
HandStatus::Idle
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_speech_creation() {
let hand = SpeechHand::new();
assert_eq!(hand.config().id, "speech");
}
#[tokio::test]
async fn test_speak() {
let hand = SpeechHand::new();
let action = SpeechAction::Speak {
text: "Hello, world!".to_string(),
voice: None,
rate: 1.0,
pitch: 1.0,
volume: 1.0,
language: None,
};
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
}
#[tokio::test]
async fn test_pause_resume() {
let hand = SpeechHand::new();
// Speak first
hand.execute_action(SpeechAction::Speak {
text: "Test".to_string(),
voice: None, rate: 1.0, pitch: 1.0, volume: 1.0, language: None,
}).await.unwrap();
// Pause
let result = hand.execute_action(SpeechAction::Pause).await.unwrap();
assert!(result.success);
// Resume
let result = hand.execute_action(SpeechAction::Resume).await.unwrap();
assert!(result.success);
}
#[tokio::test]
async fn test_list_voices() {
let hand = SpeechHand::new();
let action = SpeechAction::ListVoices { language: Some("zh".to_string()) };
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
}
#[tokio::test]
async fn test_set_voice() {
let hand = SpeechHand::new();
let action = SpeechAction::SetVoice {
voice: "zh-CN-XiaoxiaoNeural".to_string(),
language: Some("zh-CN".to_string()),
};
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
let state = hand.get_state().await;
assert_eq!(state.config.default_voice, Some("zh-CN-XiaoxiaoNeural".to_string()));
}
}

View File

@@ -1,422 +0,0 @@
//! Whiteboard Hand - Drawing and annotation capabilities
//!
//! Provides whiteboard drawing actions for teaching:
//! - draw_text: Draw text on the whiteboard
//! - draw_shape: Draw shapes (rectangle, circle, arrow, etc.)
//! - draw_line: Draw lines and curves
//! - draw_chart: Draw charts (bar, line, pie)
//! - draw_latex: Render LaTeX formulas
//! - draw_table: Draw data tables
//! - clear: Clear the whiteboard
//! - export: Export as image
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use zclaw_types::Result;
use crate::{Hand, HandConfig, HandContext, HandResult, HandStatus};
/// Whiteboard action types
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "action", rename_all = "snake_case")]
pub enum WhiteboardAction {
/// Draw text
DrawText {
x: f64,
y: f64,
text: String,
#[serde(default = "default_font_size")]
font_size: u32,
#[serde(default)]
color: Option<String>,
#[serde(default)]
font_family: Option<String>,
},
/// Draw a shape
DrawShape {
shape: ShapeType,
x: f64,
y: f64,
width: f64,
height: f64,
#[serde(default)]
fill: Option<String>,
#[serde(default)]
stroke: Option<String>,
#[serde(default = "default_stroke_width")]
stroke_width: u32,
},
/// Draw a line
DrawLine {
points: Vec<Point>,
#[serde(default)]
color: Option<String>,
#[serde(default = "default_stroke_width")]
stroke_width: u32,
},
/// Draw a chart
DrawChart {
chart_type: ChartType,
data: ChartData,
x: f64,
y: f64,
width: f64,
height: f64,
#[serde(default)]
title: Option<String>,
},
/// Draw LaTeX formula
DrawLatex {
latex: String,
x: f64,
y: f64,
#[serde(default = "default_font_size")]
font_size: u32,
#[serde(default)]
color: Option<String>,
},
/// Draw a table
DrawTable {
headers: Vec<String>,
rows: Vec<Vec<String>>,
x: f64,
y: f64,
#[serde(default)]
column_widths: Option<Vec<f64>>,
},
/// Erase area
Erase {
x: f64,
y: f64,
width: f64,
height: f64,
},
/// Clear whiteboard
Clear,
/// Undo last action
Undo,
/// Redo last undone action
Redo,
/// Export as image
Export {
#[serde(default = "default_export_format")]
format: String,
},
}
fn default_font_size() -> u32 { 16 }
fn default_stroke_width() -> u32 { 2 }
fn default_export_format() -> String { "png".to_string() }
/// Shape types
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ShapeType {
Rectangle,
RoundedRectangle,
Circle,
Ellipse,
Triangle,
Arrow,
Star,
Checkmark,
Cross,
}
/// Point for line drawing
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Point {
pub x: f64,
pub y: f64,
}
/// Chart types
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ChartType {
Bar,
Line,
Pie,
Scatter,
Area,
Radar,
}
/// Chart data
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChartData {
pub labels: Vec<String>,
pub datasets: Vec<Dataset>,
}
/// Dataset for charts
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Dataset {
pub label: String,
pub values: Vec<f64>,
#[serde(default)]
pub color: Option<String>,
}
/// Whiteboard state (for undo/redo)
#[derive(Debug, Clone, Default)]
pub struct WhiteboardState {
pub actions: Vec<WhiteboardAction>,
pub undone: Vec<WhiteboardAction>,
pub canvas_width: f64,
pub canvas_height: f64,
}
/// Whiteboard Hand implementation
pub struct WhiteboardHand {
config: HandConfig,
state: std::sync::Arc<tokio::sync::RwLock<WhiteboardState>>,
}
impl WhiteboardHand {
/// Create a new whiteboard hand
pub fn new() -> Self {
Self {
config: HandConfig {
id: "whiteboard".to_string(),
name: "白板".to_string(),
description: "在虚拟白板上绘制和标注".to_string(),
needs_approval: false,
dependencies: vec![],
input_schema: Some(serde_json::json!({
"type": "object",
"properties": {
"action": { "type": "string" },
"x": { "type": "number" },
"y": { "type": "number" },
"text": { "type": "string" },
}
})),
tags: vec!["presentation".to_string(), "education".to_string()],
enabled: true,
max_concurrent: 0,
timeout_secs: 0,
},
state: std::sync::Arc::new(tokio::sync::RwLock::new(WhiteboardState {
canvas_width: 1920.0,
canvas_height: 1080.0,
..Default::default()
})),
}
}
/// Create with custom canvas size
pub fn with_size(width: f64, height: f64) -> Self {
let hand = Self::new();
let mut state = hand.state.blocking_write();
state.canvas_width = width;
state.canvas_height = height;
drop(state);
hand
}
/// Execute a whiteboard action
pub async fn execute_action(&self, action: WhiteboardAction) -> Result<HandResult> {
let mut state = self.state.write().await;
match &action {
WhiteboardAction::Clear => {
state.actions.clear();
state.undone.clear();
return Ok(HandResult::success(serde_json::json!({
"status": "cleared",
"action_count": 0
})));
}
WhiteboardAction::Undo => {
if let Some(last) = state.actions.pop() {
state.undone.push(last);
return Ok(HandResult::success(serde_json::json!({
"status": "undone",
"remaining_actions": state.actions.len()
})));
}
return Ok(HandResult::success(serde_json::json!({
"status": "no_action_to_undo"
})));
}
WhiteboardAction::Redo => {
if let Some(redone) = state.undone.pop() {
state.actions.push(redone);
return Ok(HandResult::success(serde_json::json!({
"status": "redone",
"total_actions": state.actions.len()
})));
}
return Ok(HandResult::success(serde_json::json!({
"status": "no_action_to_redo"
})));
}
WhiteboardAction::Export { format } => {
// In real implementation, would render to image
return Ok(HandResult::success(serde_json::json!({
"status": "exported",
"format": format,
"data_url": format!("data:image/{};base64,<rendered_data>", format)
})));
}
_ => {
// Regular drawing action
state.actions.push(action.clone());
return Ok(HandResult::success(serde_json::json!({
"status": "drawn",
"action": action,
"total_actions": state.actions.len()
})));
}
}
}
/// Get current state
pub async fn get_state(&self) -> WhiteboardState {
self.state.read().await.clone()
}
/// Get all actions
pub async fn get_actions(&self) -> Vec<WhiteboardAction> {
self.state.read().await.actions.clone()
}
}
impl Default for WhiteboardHand {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl Hand for WhiteboardHand {
fn config(&self) -> &HandConfig {
&self.config
}
async fn execute(&self, _context: &HandContext, input: Value) -> Result<HandResult> {
// Parse action from input
let action: WhiteboardAction = match serde_json::from_value(input.clone()) {
Ok(a) => a,
Err(e) => {
return Ok(HandResult::error(format!("Invalid whiteboard action: {}", e)));
}
};
self.execute_action(action).await
}
fn status(&self) -> HandStatus {
// Check if there are any actions
HandStatus::Idle
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_whiteboard_creation() {
let hand = WhiteboardHand::new();
assert_eq!(hand.config().id, "whiteboard");
}
#[tokio::test]
async fn test_draw_text() {
let hand = WhiteboardHand::new();
let action = WhiteboardAction::DrawText {
x: 100.0,
y: 100.0,
text: "Hello World".to_string(),
font_size: 24,
color: Some("#333333".to_string()),
font_family: None,
};
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
let state = hand.get_state().await;
assert_eq!(state.actions.len(), 1);
}
#[tokio::test]
async fn test_draw_shape() {
let hand = WhiteboardHand::new();
let action = WhiteboardAction::DrawShape {
shape: ShapeType::Rectangle,
x: 50.0,
y: 50.0,
width: 200.0,
height: 100.0,
fill: Some("#4CAF50".to_string()),
stroke: None,
stroke_width: 2,
};
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
}
#[tokio::test]
async fn test_undo_redo() {
let hand = WhiteboardHand::new();
// Draw something
hand.execute_action(WhiteboardAction::DrawText {
x: 0.0, y: 0.0, text: "Test".to_string(), font_size: 16, color: None, font_family: None,
}).await.unwrap();
// Undo
let result = hand.execute_action(WhiteboardAction::Undo).await.unwrap();
assert!(result.success);
assert_eq!(hand.get_state().await.actions.len(), 0);
// Redo
let result = hand.execute_action(WhiteboardAction::Redo).await.unwrap();
assert!(result.success);
assert_eq!(hand.get_state().await.actions.len(), 1);
}
#[tokio::test]
async fn test_clear() {
let hand = WhiteboardHand::new();
// Draw something
hand.execute_action(WhiteboardAction::DrawText {
x: 0.0, y: 0.0, text: "Test".to_string(), font_size: 16, color: None, font_family: None,
}).await.unwrap();
// Clear
let result = hand.execute_action(WhiteboardAction::Clear).await.unwrap();
assert!(result.success);
assert_eq!(hand.get_state().await.actions.len(), 0);
}
#[tokio::test]
async fn test_chart() {
let hand = WhiteboardHand::new();
let action = WhiteboardAction::DrawChart {
chart_type: ChartType::Bar,
data: ChartData {
labels: vec!["A".to_string(), "B".to_string(), "C".to_string()],
datasets: vec![Dataset {
label: "Values".to_string(),
values: vec![10.0, 20.0, 15.0],
color: Some("#2196F3".to_string()),
}],
},
x: 100.0,
y: 100.0,
width: 400.0,
height: 300.0,
title: Some("Test Chart".to_string()),
};
let result = hand.execute_action(action).await.unwrap();
assert!(result.success);
}
}

View File

@@ -27,7 +27,7 @@ use crate::config::KernelConfig;
use zclaw_memory::MemoryStore;
use zclaw_runtime::{LlmDriver, ToolRegistry, tool::SkillExecutor};
use zclaw_skills::SkillRegistry;
use zclaw_hands::{HandRegistry, hands::{BrowserHand, SlideshowHand, SpeechHand, QuizHand, WhiteboardHand, ResearcherHand, CollectorHand, ClipHand, TwitterHand, ReminderHand, quiz::LlmQuizGenerator}};
use zclaw_hands::{HandRegistry, hands::{BrowserHand, QuizHand, ResearcherHand, CollectorHand, ClipHand, TwitterHand, ReminderHand, quiz::LlmQuizGenerator}};
pub use adapters::KernelSkillExecutor;
pub use messaging::ChatModeConfig;
@@ -93,10 +93,7 @@ impl Kernel {
let quiz_model = config.model().to_string();
let quiz_generator = Arc::new(LlmQuizGenerator::new(driver.clone(), quiz_model));
hands.register(Arc::new(BrowserHand::new())).await;
hands.register(Arc::new(SlideshowHand::new())).await;
hands.register(Arc::new(SpeechHand::new())).await;
hands.register(Arc::new(QuizHand::with_generator(quiz_generator))).await;
hands.register(Arc::new(WhiteboardHand::new())).await;
hands.register(Arc::new(ResearcherHand::new())).await;
hands.register(Arc::new(CollectorHand::new())).await;
hands.register(Arc::new(ClipHand::new())).await;

View File

@@ -2,12 +2,11 @@
* SceneRenderer — Renders a single classroom scene.
*
* Supports scene types: slide, quiz, discussion, interactive, text, pbl, media.
* Executes scene actions (speech, whiteboard, quiz, discussion).
* Executes scene actions (quiz, discussion).
*/
import { useState, useEffect, useCallback } from 'react';
import { useState, useEffect } from 'react';
import type { GeneratedScene, SceneContent, SceneAction, AgentProfile } from '../../types/classroom';
import { WhiteboardCanvas } from './WhiteboardCanvas';
interface SceneRendererProps {
scene: GeneratedScene;
@@ -15,14 +14,10 @@ interface SceneRendererProps {
autoPlay?: boolean;
}
export function SceneRenderer({ scene, agents, autoPlay = true }: SceneRendererProps) {
export function SceneRenderer({ scene, autoPlay = true }: SceneRendererProps) {
const { content } = scene;
const [actionIndex, setActionIndex] = useState(0);
const [isPlaying, setIsPlaying] = useState(autoPlay);
const [whiteboardItems, setWhiteboardItems] = useState<Array<{
type: string;
data: SceneAction;
}>>([]);
const actions = content.actions ?? [];
const currentAction = actions[actionIndex] ?? null;
@@ -37,27 +32,12 @@ export function SceneRenderer({ scene, agents, autoPlay = true }: SceneRendererP
const delay = getActionDelay(actions[actionIndex]);
const timer = setTimeout(() => {
processAction(actions[actionIndex]);
setActionIndex((i) => i + 1);
}, delay);
return () => clearTimeout(timer);
}, [actionIndex, isPlaying, actions]);
const processAction = useCallback((action: SceneAction) => {
switch (action.type) {
case 'whiteboard_draw_text':
case 'whiteboard_draw_shape':
case 'whiteboard_draw_chart':
case 'whiteboard_draw_latex':
setWhiteboardItems((prev) => [...prev, { type: action.type, data: action }]);
break;
case 'whiteboard_clear':
setWhiteboardItems([]);
break;
}
}, []);
// Render scene based on type
return (
<div className="flex flex-col h-full">
@@ -72,31 +52,21 @@ export function SceneRenderer({ scene, agents, autoPlay = true }: SceneRendererP
</div>
{/* Main content area */}
<div className="flex-1 flex gap-4 overflow-hidden">
{/* Content panel */}
<div className="flex-1 overflow-auto">
{renderContent(content)}
</div>
{/* Whiteboard area */}
{whiteboardItems.length > 0 && (
<div className="w-96 shrink-0">
<WhiteboardCanvas items={whiteboardItems} />
</div>
)}
<div className="flex-1 overflow-auto">
{renderContent(content)}
</div>
{/* Current action indicator */}
{currentAction && (
<div className="mt-4 p-3 rounded-lg bg-indigo-50 dark:bg-indigo-900/20 border border-indigo-100 dark:border-indigo-800">
{renderCurrentAction(currentAction, agents)}
{renderCurrentAction(currentAction)}
</div>
)}
{/* Playback controls */}
<div className="flex items-center justify-center gap-2 mt-4">
<button
onClick={() => { setActionIndex(0); setWhiteboardItems([]); }}
onClick={() => { setActionIndex(0); }}
className="px-2 py-1 text-xs rounded bg-gray-100 dark:bg-gray-700"
>
Restart
@@ -121,12 +91,6 @@ export function SceneRenderer({ scene, agents, autoPlay = true }: SceneRendererP
function getActionDelay(action: SceneAction): number {
switch (action.type) {
case 'speech': return 2000;
case 'whiteboard_draw_text': return 800;
case 'whiteboard_draw_shape': return 600;
case 'whiteboard_draw_chart': return 1000;
case 'whiteboard_draw_latex': return 1000;
case 'whiteboard_clear': return 300;
case 'quiz_show': return 5000;
case 'discussion': return 10000;
default: return 1000;
@@ -167,26 +131,12 @@ function renderContent(content: SceneContent) {
);
}
function renderCurrentAction(action: SceneAction, agents: AgentProfile[]) {
function renderCurrentAction(action: SceneAction) {
switch (action.type) {
case 'speech': {
const agent = agents.find(a => a.role === action.agentRole);
return (
<div className="flex items-start gap-2">
<span className="text-lg">{agent?.avatar ?? '💬'}</span>
<div>
<span className="text-xs font-medium text-gray-600">{agent?.name ?? action.agentRole}</span>
<p className="text-sm text-gray-700 dark:text-gray-300">{action.text}</p>
</div>
</div>
);
}
case 'quiz_show':
return <div className="text-sm text-amber-600">Quiz: {action.quizId}</div>;
case 'discussion':
return <div className="text-sm text-green-600">Discussion: {action.topic}</div>;
default:
return <div className="text-xs text-gray-400">{action.type}</div>;
}
}

View File

@@ -1,295 +0,0 @@
/**
* WhiteboardCanvas — SVG-based whiteboard for classroom scene rendering.
*
* Supports incremental drawing operations:
* - Text (positioned labels)
* - Shapes (rectangles, circles, arrows)
* - Charts (bar/line/pie via simple SVG)
* - LaTeX (rendered as styled text blocks)
*/
import { useCallback } from 'react';
import type { SceneAction } from '../../types/classroom';
interface WhiteboardCanvasProps {
items: WhiteboardItem[];
width?: number;
height?: number;
}
export interface WhiteboardItem {
type: string;
data: SceneAction;
}
export function WhiteboardCanvas({
items,
width = 800,
height = 600,
}: WhiteboardCanvasProps) {
const renderItem = useCallback((item: WhiteboardItem, index: number) => {
switch (item.type) {
case 'whiteboard_draw_text':
return <TextItem key={index} data={item.data as TextDrawData} />;
case 'whiteboard_draw_shape':
return <ShapeItem key={index} data={item.data as ShapeDrawData} />;
case 'whiteboard_draw_chart':
return <ChartItem key={index} data={item.data as ChartDrawData} />;
case 'whiteboard_draw_latex':
return <LatexItem key={index} data={item.data as LatexDrawData} />;
default:
return null;
}
}, []);
return (
<div className="w-full h-full border border-gray-200 dark:border-gray-700 rounded-lg bg-white dark:bg-gray-900 overflow-auto">
<svg
viewBox={`0 0 ${width} ${height}`}
className="w-full h-full"
xmlns="http://www.w3.org/2000/svg"
>
{/* Grid background */}
<defs>
<pattern id="grid" width="40" height="40" patternUnits="userSpaceOnUse">
<path d="M 40 0 L 0 0 0 40" fill="none" stroke="#f0f0f0" strokeWidth="0.5" />
</pattern>
</defs>
<rect width={width} height={height} fill="url(#grid)" />
{/* Rendered items */}
{items.map((item, i) => renderItem(item, i))}
</svg>
</div>
);
}
// ---------------------------------------------------------------------------
// Sub-components
// ---------------------------------------------------------------------------
interface TextDrawData {
type: 'whiteboard_draw_text';
x: number;
y: number;
text: string;
fontSize?: number;
color?: string;
}
function TextItem({ data }: { data: TextDrawData }) {
return (
<text
x={data.x}
y={data.y}
fontSize={data.fontSize ?? 16}
fill={data.color ?? '#333333'}
fontFamily="system-ui, sans-serif"
>
{data.text}
</text>
);
}
interface ShapeDrawData {
type: 'whiteboard_draw_shape';
shape: string;
x: number;
y: number;
width: number;
height: number;
fill?: string;
}
function ShapeItem({ data }: { data: ShapeDrawData }) {
switch (data.shape) {
case 'circle':
return (
<ellipse
cx={data.x + data.width / 2}
cy={data.y + data.height / 2}
rx={data.width / 2}
ry={data.height / 2}
fill={data.fill ?? '#e5e7eb'}
stroke="#9ca3af"
strokeWidth={1}
/>
);
case 'arrow':
return (
<g>
<line
x1={data.x}
y1={data.y + data.height / 2}
x2={data.x + data.width}
y2={data.y + data.height / 2}
stroke={data.fill ?? '#6b7280'}
strokeWidth={2}
markerEnd="url(#arrowhead)"
/>
<defs>
<marker id="arrowhead" markerWidth="10" markerHeight="7" refX="10" refY="3.5" orient="auto">
<polygon points="0 0, 10 3.5, 0 7" fill={data.fill ?? '#6b7280'} />
</marker>
</defs>
</g>
);
default: // rectangle
return (
<rect
x={data.x}
y={data.y}
width={data.width}
height={data.height}
fill={data.fill ?? '#e5e7eb'}
stroke="#9ca3af"
strokeWidth={1}
rx={4}
/>
);
}
}
interface ChartDrawData {
type: 'whiteboard_draw_chart';
chartType: string;
data: Record<string, unknown>;
x: number;
y: number;
width: number;
height: number;
}
function ChartItem({ data }: { data: ChartDrawData }) {
const chartData = data.data;
const labels = (chartData?.labels as string[]) ?? [];
const values = (chartData?.values as number[]) ?? [];
if (labels.length === 0 || values.length === 0) return null;
switch (data.chartType) {
case 'bar':
return <BarChart data={data} labels={labels} values={values} />;
case 'line':
return <LineChart data={data} labels={labels} values={values} />;
default:
return <BarChart data={data} labels={labels} values={values} />;
}
}
function BarChart({ data, labels, values }: {
data: ChartDrawData;
labels: string[];
values: number[];
}) {
const maxVal = Math.max(...values, 1);
const barWidth = data.width / (labels.length * 2);
const chartHeight = data.height - 30;
return (
<g transform={`translate(${data.x}, ${data.y})`}>
{values.map((val, i) => {
const barHeight = (val / maxVal) * chartHeight;
return (
<g key={i}>
<rect
x={i * (barWidth * 2) + barWidth / 2}
y={chartHeight - barHeight}
width={barWidth}
height={barHeight}
fill="#6366f1"
rx={2}
/>
<text
x={i * (barWidth * 2) + barWidth}
y={data.height - 5}
textAnchor="middle"
fontSize={10}
fill="#666"
>
{labels[i]}
</text>
</g>
);
})}
</g>
);
}
function LineChart({ data, labels, values }: {
data: ChartDrawData;
labels: string[];
values: number[];
}) {
const maxVal = Math.max(...values, 1);
const chartHeight = data.height - 30;
const stepX = data.width / Math.max(labels.length - 1, 1);
const points = values.map((val, i) => {
const x = i * stepX;
const y = chartHeight - (val / maxVal) * chartHeight;
return `${x},${y}`;
}).join(' ');
return (
<g transform={`translate(${data.x}, ${data.y})`}>
<polyline
points={points}
fill="none"
stroke="#6366f1"
strokeWidth={2}
/>
{values.map((val, i) => {
const x = i * stepX;
const y = chartHeight - (val / maxVal) * chartHeight;
return (
<g key={i}>
<circle cx={x} cy={y} r={3} fill="#6366f1" />
<text
x={x}
y={data.height - 5}
textAnchor="middle"
fontSize={10}
fill="#666"
>
{labels[i]}
</text>
</g>
);
})}
</g>
);
}
interface LatexDrawData {
type: 'whiteboard_draw_latex';
latex: string;
x: number;
y: number;
}
function LatexItem({ data }: { data: LatexDrawData }) {
return (
<g transform={`translate(${data.x}, ${data.y})`}>
<rect
x={-4}
y={-20}
width={data.latex.length * 10 + 8}
height={28}
fill="#fef3c7"
stroke="#f59e0b"
strokeWidth={1}
rx={4}
/>
<text
x={0}
y={0}
fontSize={14}
fill="#92400e"
fontFamily="'Courier New', monospace"
>
{data.latex}
</text>
</g>
);
}

View File

@@ -8,5 +8,4 @@ export { ClassroomPlayer } from './ClassroomPlayer';
export { SceneRenderer } from './SceneRenderer';
export { AgentChat } from './AgentChat';
export { NotesSidebar } from './NotesSidebar';
export { WhiteboardCanvas } from './WhiteboardCanvas';
export { TtsPlayer } from './TtsPlayer';

View File

@@ -16,7 +16,6 @@ import type { PresentationType, PresentationAnalysis } from './types';
import { TypeSwitcher } from './TypeSwitcher';
import { QuizRenderer } from './renderers/QuizRenderer';
const SlideshowRenderer = React.lazy(() => import('./renderers/SlideshowRenderer').then(m => ({ default: m.SlideshowRenderer })));
const DocumentRenderer = React.lazy(() => import('./renderers/DocumentRenderer').then(m => ({ default: m.DocumentRenderer })));
const ChartRenderer = React.lazy(() => import('./renderers/ChartRenderer').then(m => ({ default: m.ChartRenderer })));
@@ -79,7 +78,7 @@ export function PresentationContainer({
if (supportedTypes && supportedTypes.length > 0) {
return supportedTypes.filter((t): t is PresentationType => t !== 'auto');
}
return (['quiz', 'slideshow', 'document', 'chart', 'whiteboard'] as PresentationType[]);
return (['quiz', 'document', 'chart'] as PresentationType[]);
}, [supportedTypes]);
const renderContent = () => {
@@ -96,13 +95,6 @@ export function PresentationContainer({
case 'quiz':
return <QuizRenderer data={data as Parameters<typeof QuizRenderer>[0]['data']} />;
case 'slideshow':
return (
<React.Suspense fallback={<div className="h-64 animate-pulse bg-gray-100" />}>
<SlideshowRenderer data={data as Parameters<typeof SlideshowRenderer>[0]['data']} />
</React.Suspense>
);
case 'document':
return (
<React.Suspense fallback={<div className="h-64 animate-pulse bg-gray-100" />}>
@@ -110,16 +102,6 @@ export function PresentationContainer({
</React.Suspense>
);
case 'whiteboard':
return (
<div className="flex flex-col items-center justify-center h-64 bg-gray-50 gap-3">
<span className="inline-flex items-center px-3 py-1 rounded-full text-xs font-medium bg-amber-100 text-amber-700">
</span>
<p className="text-gray-500"></p>
</div>
);
case 'chart':
return (
<React.Suspense fallback={<div className="h-64 animate-pulse bg-gray-100" />}>

View File

@@ -7,9 +7,7 @@
import {
BarChart3,
FileText,
Presentation,
CheckCircle,
PenTool,
} from 'lucide-react';
import type { PresentationType, PresentationAnalysis } from './types';
@@ -34,11 +32,6 @@ const typeConfig: Record<PresentationType, { icon: React.ReactNode; label: strin
label: '图表',
description: '数据可视化',
},
slideshow: {
icon: <Presentation className="w-4 h-4" />,
label: '幻灯片',
description: '演示文稿风格',
},
quiz: {
icon: <CheckCircle className="w-4 h-4" />,
label: '测验',
@@ -49,11 +42,6 @@ const typeConfig: Record<PresentationType, { icon: React.ReactNode; label: strin
label: '文档',
description: 'Markdown 文档',
},
whiteboard: {
icon: <PenTool className="w-4 h-4" />,
label: '白板',
description: '交互式画布',
},
auto: {
icon: <CheckCircle className="w-4 h-4" />,
label: '自动',

View File

@@ -19,7 +19,6 @@ export { PresentationContainer } from './PresentationContainer';
export { TypeSwitcher } from './TypeSwitcher';
export { QuizRenderer } from './renderers/QuizRenderer';
export { DocumentRenderer } from './renderers/DocumentRenderer';
export { SlideshowRenderer } from './renderers/SlideshowRenderer';
export type {
PresentationType,
PresentationAnalysis,
@@ -27,7 +26,5 @@ export type {
QuizData,
QuizQuestion,
QuestionType,
SlideshowData,
DocumentData,
WhiteboardData,
} from './types';

View File

@@ -1,283 +0,0 @@
/**
* Slideshow Renderer
*
* Renders presentation as a slideshow with slide navigation.
* Supports: title, content, image, code, twoColumn slide types.
*/
import { useState, useEffect, useCallback } from 'react';
import Markdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import {
ChevronLeft,
ChevronRight,
Maximize2,
Minimize2,
Play,
Pause,
} from 'lucide-react';
import type { SlideshowData, Slide } from '../types';
interface SlideshowRendererProps {
data: SlideshowData;
/** Auto-play interval in seconds (0 = disabled) */
autoPlayInterval?: number;
/** Show progress indicator */
showProgress?: boolean;
/** Show speaker notes */
showNotes?: boolean;
/** Custom className */
className?: string;
}
export function SlideshowRenderer({
data,
autoPlayInterval = 0,
showProgress = true,
showNotes = true,
className = '',
}: SlideshowRendererProps) {
const [currentIndex, setCurrentIndex] = useState(0);
const [isPlaying, setIsPlaying] = useState(false);
const [isFullscreen, setIsFullscreen] = useState(false);
const slides = data.slides || [];
const totalSlides = slides.length;
const handleNext = useCallback(() => {
setCurrentIndex((prev) => (prev + 1) % totalSlides);
}, [totalSlides]);
const handlePrev = useCallback(() => {
setCurrentIndex((prev) => (prev - 1 + totalSlides) % totalSlides);
}, [totalSlides]);
const toggleFullscreen = useCallback(() => {
setIsFullscreen((prev) => !prev);
}, []);
// Handle keyboard navigation
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
if (e.key === 'ArrowRight' || e.key === ' ') {
e.preventDefault();
handleNext();
} else if (e.key === 'ArrowLeft') {
e.preventDefault();
handlePrev();
} else if (e.key === 'f') {
toggleFullscreen();
}
};
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [handleNext, handlePrev, toggleFullscreen]);
// Auto-play
useEffect(() => {
if (isPlaying && autoPlayInterval > 0) {
const timer = setInterval(handleNext, autoPlayInterval * 1000);
return () => clearInterval(timer);
}
}, [isPlaying, autoPlayInterval, handleNext]);
const currentSlide = slides[currentIndex];
if (!currentSlide) {
return (
<div className={`flex items-center justify-center h-64 bg-gray-50 ${className}`}>
<p className="text-gray-500"></p>
</div>
);
}
return (
<div
className={`flex flex-col h-full ${
isFullscreen ? 'fixed inset-0 z-50 bg-white' : ''
} ${className}`}
>
{/* Slide Content */}
<div className="flex-1 flex items-center justify-center p-8">
<div className="max-w-4xl w-full">
<SlideContent slide={currentSlide} />
</div>
</div>
{/* Controls */}
<div className="flex items-center justify-between p-4 bg-gray-50 border-t">
<div className="flex items-center gap-2">
<button
onClick={handlePrev}
disabled={totalSlides <= 1}
className="p-2 hover:bg-gray-200 rounded disabled:opacity-50"
>
<ChevronLeft className="w-5 h-5" />
</button>
<button
onClick={() => setIsPlaying(!isPlaying)}
disabled={autoPlayInterval === 0}
className="p-2 hover:bg-gray-200 rounded disabled:opacity-50"
>
{isPlaying ? (
<Pause className="w-5 h-5" />
) : (
<Play className="w-5 h-5" />
)}
</button>
<button
onClick={handleNext}
disabled={totalSlides <= 1}
className="p-2 hover:bg-gray-200 rounded disabled:opacity-50"
>
<ChevronRight className="w-5 h-5" />
</button>
</div>
{/* Progress */}
{showProgress && (
<div className="text-sm text-gray-500">
{currentIndex + 1} / {totalSlides}
</div>
)}
{/* Fullscreen */}
<button
onClick={toggleFullscreen}
className="p-2 hover:bg-gray-200 rounded"
>
{isFullscreen ? (
<Minimize2 className="w-5 h-5" />
) : (
<Maximize2 className="w-5 h-5" />
)}
</button>
</div>
{/* Speaker Notes */}
{showNotes && currentSlide.notes && (
<div className="p-4 bg-yellow-50 border-t text-sm text-gray-600">
{currentSlide.notes}
</div>
)}
</div>
);
}
/** Renders a single slide based on its type */
function SlideContent({ slide }: { slide: Slide }) {
switch (slide.type) {
case 'title':
return (
<div className="text-center py-12">
{slide.title && (
<h1 className="text-4xl font-bold mb-4">{slide.title}</h1>
)}
{slide.content && (
<p className="text-xl text-gray-600 max-w-2xl mx-auto">
{slide.content}
</p>
)}
</div>
);
case 'content':
return (
<div>
{slide.title && (
<h2 className="text-3xl font-bold text-center mb-6">{slide.title}</h2>
)}
{slide.content && (
<div className="prose prose-gray max-w-none">
<Markdown remarkPlugins={[remarkGfm]}>{slide.content}</Markdown>
</div>
)}
</div>
);
case 'image':
return (
<div className="text-center">
{slide.title && (
<h2 className="text-2xl font-bold mb-4">{slide.title}</h2>
)}
{slide.image && (
<img
src={slide.image}
alt={slide.title || '幻灯片图片'}
className="max-w-full max-h-[60vh] mx-auto rounded-lg shadow-md"
/>
)}
{slide.content && (
<p className="mt-4 text-gray-600">{slide.content}</p>
)}
</div>
);
case 'code':
return (
<div>
{slide.title && (
<h2 className="text-2xl font-bold mb-4">{slide.title}</h2>
)}
{slide.code && (
<pre className="bg-gray-900 text-gray-100 p-6 rounded-lg overflow-x-auto text-sm">
{slide.language && (
<div className="text-xs text-gray-400 mb-3 uppercase tracking-wider">
{slide.language}
</div>
)}
<code>{slide.code}</code>
</pre>
)}
{slide.content && (
<p className="mt-4 text-gray-600">{slide.content}</p>
)}
</div>
);
case 'twoColumn':
return (
<div>
{slide.title && (
<h2 className="text-2xl font-bold mb-4">{slide.title}</h2>
)}
<div className="grid grid-cols-2 gap-8">
<div className="prose prose-sm max-w-none">
{slide.leftContent && (
<Markdown remarkPlugins={[remarkGfm]}>
{slide.leftContent}
</Markdown>
)}
</div>
<div className="prose prose-sm max-w-none">
{slide.rightContent && (
<Markdown remarkPlugins={[remarkGfm]}>
{slide.rightContent}
</Markdown>
)}
</div>
</div>
</div>
);
default:
return (
<div>
{slide.title && (
<h2 className="text-2xl font-bold mb-4">{slide.title}</h2>
)}
{slide.content && (
<div className="prose prose-gray max-w-none">
<Markdown remarkPlugins={[remarkGfm]}>{slide.content}</Markdown>
</div>
)}
</div>
);
}
}
export default SlideshowRenderer;

View File

@@ -8,9 +8,7 @@
export type PresentationType =
| 'chart'
| 'quiz'
| 'slideshow'
| 'document'
| 'whiteboard'
| 'auto';
export interface PresentationAnalysis {
@@ -84,34 +82,6 @@ export interface QuizOption {
isCorrect?: boolean;
}
export interface SlideshowData {
title?: string;
slides: Slide[];
theme?: SlideshowTheme;
autoPlay?: boolean;
interval?: number;
}
export interface Slide {
id: string;
type: 'title' | 'content' | 'image' | 'code' | 'twoColumn';
title?: string;
content?: string;
image?: string;
code?: string;
language?: string;
leftContent?: string;
rightContent?: string;
notes?: string;
}
export interface SlideshowTheme {
backgroundColor?: string;
textColor?: string;
accentColor?: string;
fontFamily?: string;
}
export interface DocumentData {
title?: string;
content?: string;
@@ -121,25 +91,3 @@ export interface DocumentData {
url?: string;
}
export interface WhiteboardData {
title?: string;
elements: WhiteboardElement[];
background?: string;
gridSize?: number;
}
export interface WhiteboardElement {
id: string;
type: 'rect' | 'circle' | 'line' | 'text' | 'image' | 'path';
x: number;
y: number;
width?: number;
height?: number;
fill?: string;
stroke?: string;
strokeWidth?: number;
text?: string;
fontSize?: number;
src?: string;
points?: number[];
}

View File

@@ -16,11 +16,7 @@ export const HAND_IDS = {
TRADER: 'trader',
CLIP: 'clip',
TWITTER: 'twitter',
// Additional hands from backend
SLIDESHOW: 'slideshow',
SPEECH: 'speech',
QUIZ: 'quiz',
WHITEBOARD: 'whiteboard',
} as const;
export type HandIdType = typeof HAND_IDS[keyof typeof HAND_IDS];
@@ -49,10 +45,7 @@ export const HAND_CATEGORY_MAP: Record<string, HandCategoryType> = {
[HAND_IDS.LEAD]: HAND_CATEGORIES.COMMUNICATION,
[HAND_IDS.TWITTER]: HAND_CATEGORIES.COMMUNICATION,
[HAND_IDS.CLIP]: HAND_CATEGORIES.CONTENT,
[HAND_IDS.SLIDESHOW]: HAND_CATEGORIES.CONTENT,
[HAND_IDS.SPEECH]: HAND_CATEGORIES.CONTENT,
[HAND_IDS.QUIZ]: HAND_CATEGORIES.PRODUCTIVITY,
[HAND_IDS.WHITEBOARD]: HAND_CATEGORIES.PRODUCTIVITY,
};
// === Helper Functions ===

View File

@@ -12,7 +12,6 @@ import { useHandStore } from '../store/handStore';
import { useWorkflowStore } from '../store/workflowStore';
import { useChatStore } from '../store/chatStore';
import type { GatewayClient } from '../lib/gateway-client';
import { speechSynth } from '../lib/speech-synth';
import { createLogger } from '../lib/logger';
const log = createLogger('useAutomationEvents');
@@ -166,22 +165,6 @@ export function useAutomationEvents(
runId: eventData.run_id,
});
// Trigger browser TTS for SpeechHand results
if (eventData.hand_name === 'speech' && eventData.hand_result && typeof eventData.hand_result === 'object') {
const res = eventData.hand_result as Record<string, unknown>;
if (res.tts_method === 'browser' && typeof res.text === 'string' && res.text) {
speechSynth.speak({
text: res.text,
voice: typeof res.voice === 'string' ? res.voice : undefined,
language: typeof res.language === 'string' ? res.language : undefined,
rate: typeof res.rate === 'number' ? res.rate : undefined,
pitch: typeof res.pitch === 'number' ? res.pitch : undefined,
volume: typeof res.volume === 'number' ? res.volume : undefined,
}).catch((err: unknown) => {
log.warn('Browser TTS failed:', err);
});
}
}
}
// Handle error status

View File

@@ -1,195 +0,0 @@
/**
* Speech Synthesis Service — Browser TTS via Web Speech API
*
* Provides text-to-speech playback using the browser's native SpeechSynthesis API.
* Zero external dependencies, works offline, supports Chinese and English voices.
*
* Architecture:
* - SpeechHand (Rust) returns tts_method + text + voice config
* - This service handles Browser TTS playback in the webview
* - OpenAI/Azure TTS is handled via backend API calls
*/
export interface SpeechSynthOptions {
text: string;
voice?: string;
language?: string;
rate?: number;
pitch?: number;
volume?: number;
}
export interface SpeechSynthState {
playing: boolean;
paused: boolean;
currentText: string | null;
voices: SpeechSynthesisVoice[];
}
type SpeechEventCallback = (state: SpeechSynthState) => void;
class SpeechSynthService {
private synth: SpeechSynthesis | null = null;
private currentUtterance: SpeechSynthesisUtterance | null = null;
private listeners: Set<SpeechEventCallback> = new Set();
private cachedVoices: SpeechSynthesisVoice[] = [];
constructor() {
if (typeof window !== 'undefined' && window.speechSynthesis) {
this.synth = window.speechSynthesis;
this.loadVoices();
// Voices may load asynchronously
this.synth.onvoiceschanged = () => this.loadVoices();
}
}
private loadVoices() {
if (!this.synth) return;
this.cachedVoices = this.synth.getVoices();
this.notify();
}
private notify() {
const state = this.getState();
this.listeners.forEach(cb => cb(state));
}
/** Subscribe to state changes */
subscribe(callback: SpeechEventCallback): () => void {
this.listeners.add(callback);
return () => this.listeners.delete(callback);
}
/** Get current state */
getState(): SpeechSynthState {
return {
playing: this.synth?.speaking ?? false,
paused: this.synth?.paused ?? false,
currentText: this.currentUtterance?.text ?? null,
voices: this.cachedVoices,
};
}
/** Check if TTS is available */
isAvailable(): boolean {
return this.synth != null;
}
/** Get available voices, optionally filtered by language */
getVoices(language?: string): SpeechSynthesisVoice[] {
if (!language) return this.cachedVoices;
const langPrefix = language.split('-')[0].toLowerCase();
return this.cachedVoices.filter(v =>
v.lang.toLowerCase().startsWith(langPrefix)
);
}
/** Speak text with given options */
speak(options: SpeechSynthOptions): Promise<void> {
return new Promise((resolve, reject) => {
if (!this.synth) {
reject(new Error('Speech synthesis not available'));
return;
}
// Cancel any ongoing speech
this.stop();
const utterance = new SpeechSynthesisUtterance(options.text);
this.currentUtterance = utterance;
// Set language
utterance.lang = options.language ?? 'zh-CN';
// Set voice if specified
if (options.voice && options.voice !== 'default') {
const voice = this.cachedVoices.find(v =>
v.name === options.voice || v.voiceURI === options.voice
);
if (voice) utterance.voice = voice;
} else {
// Auto-select best voice for the language
this.selectBestVoice(utterance, options.language ?? 'zh-CN');
}
// Set parameters
utterance.rate = options.rate ?? 1.0;
utterance.pitch = options.pitch ?? 1.0;
utterance.volume = options.volume ?? 1.0;
utterance.onstart = () => {
this.notify();
};
utterance.onend = () => {
this.currentUtterance = null;
this.notify();
resolve();
};
utterance.onerror = (event) => {
this.currentUtterance = null;
this.notify();
// "canceled" is not a real error (happens on stop())
if (event.error !== 'canceled') {
reject(new Error(`Speech error: ${event.error}`));
} else {
resolve();
}
};
this.synth.speak(utterance);
});
}
/** Pause current speech */
pause() {
this.synth?.pause();
this.notify();
}
/** Resume paused speech */
resume() {
this.synth?.resume();
this.notify();
}
/** Stop current speech */
stop() {
this.synth?.cancel();
this.currentUtterance = null;
this.notify();
}
/** Auto-select the best voice for a language */
private selectBestVoice(utterance: SpeechSynthesisUtterance, language: string) {
const langPrefix = language.split('-')[0].toLowerCase();
const candidates = this.cachedVoices.filter(v =>
v.lang.toLowerCase().startsWith(langPrefix)
);
if (candidates.length === 0) return;
// Prefer voices with "Neural" or "Enhanced" in name (higher quality)
const neural = candidates.find(v =>
v.name.includes('Neural') || v.name.includes('Enhanced') || v.name.includes('Premium')
);
if (neural) {
utterance.voice = neural;
return;
}
// Prefer local voices (work offline)
const local = candidates.find(v => v.localService);
if (local) {
utterance.voice = local;
return;
}
// Fall back to first matching voice
utterance.voice = candidates[0];
}
}
// Singleton instance
export const speechSynth = new SpeechSynthService();

View File

@@ -24,7 +24,6 @@ import { getSkillDiscovery } from '../../lib/skill-discovery';
import { useOfflineStore, isOffline } from '../../store/offlineStore';
import { useConnectionStore } from '../../store/connectionStore';
import { createLogger } from '../../lib/logger';
import { speechSynth } from '../../lib/speech-synth';
import { generateRandomString } from '../../lib/crypto-utils';
import type { ChatModeType, ChatModeConfig, Subtask } from '../../components/ai';
import type { ToolCallStep } from '../../components/ai';
@@ -440,22 +439,6 @@ export const useStreamStore = create<StreamState>()(
};
_chat?.updateMessages(msgs => [...msgs, handMsg]);
if (name === 'speech' && status === 'completed' && result && typeof result === 'object') {
const res = result as Record<string, unknown>;
if (res.tts_method === 'browser' && typeof res.text === 'string' && res.text) {
speechSynth.speak({
text: res.text as string,
voice: (res.voice as string) || undefined,
language: (res.language as string) || undefined,
rate: typeof res.rate === 'number' ? res.rate : undefined,
pitch: typeof res.pitch === 'number' ? res.pitch : undefined,
volume: typeof res.volume === 'number' ? res.volume : undefined,
}).catch((err: unknown) => {
const logger = createLogger('speech-synth');
logger.warn('Browser TTS failed', { error: String(err) });
});
}
}
},
onSubtaskStatus: (taskId: string, description: string, status: string, detail?: string) => {
// Map backend status to frontend Subtask status

View File

@@ -46,14 +46,6 @@ export enum GenerationStage {
// --- Scene Actions ---
export type SceneAction =
| { type: 'speech'; text: string; agentRole: string }
| { type: 'whiteboard_draw_text'; x: number; y: number; text: string; fontSize?: number; color?: string }
| { type: 'whiteboard_draw_shape'; shape: string; x: number; y: number; width: number; height: number; fill?: string }
| { type: 'whiteboard_draw_chart'; chartType: string; data: unknown; x: number; y: number; width: number; height: number }
| { type: 'whiteboard_draw_latex'; latex: string; x: number; y: number }
| { type: 'whiteboard_clear' }
| { type: 'slideshow_spotlight'; elementId: string }
| { type: 'slideshow_next' }
| { type: 'quiz_show'; quizId: string }
| { type: 'discussion'; topic: string; durationSeconds?: number };

View File

@@ -1,119 +0,0 @@
# Slideshow Hand - 幻灯片控制能力包
#
# ZCLAW Hand 配置
# 提供幻灯片演示控制能力,支持翻页、聚焦、激光笔等
[hand]
name = "slideshow"
version = "1.0.0"
description = "幻灯片控制能力包 - 控制演示文稿的播放、导航和标注"
author = "ZCLAW Team"
type = "presentation"
requires_approval = false
timeout = 30
max_concurrent = 1
tags = ["slideshow", "presentation", "slides", "education", "teaching"]
[hand.config]
# 支持的幻灯片格式
supported_formats = ["pptx", "pdf", "html", "markdown"]
# 自动翻页间隔0 表示禁用
auto_advance_interval = 0
# 是否显示进度条
show_progress = true
# 是否显示页码
show_page_number = true
# 激光笔颜色
laser_color = "#ff0000"
# 聚焦框颜色
spotlight_color = "#ffcc00"
[hand.triggers]
manual = true
schedule = false
webhook = false
[[hand.triggers.events]]
type = "chat.intent"
pattern = "幻灯片|演示|翻页|下一页|上一页|slide|presentation|next|prev"
priority = 5
[hand.permissions]
requires = [
"slideshow.navigate",
"slideshow.annotate",
"slideshow.control"
]
roles = ["operator.read"]
[hand.rate_limit]
max_requests = 200
window_seconds = 3600
[hand.audit]
log_inputs = true
log_outputs = false
retention_days = 7
# 幻灯片动作定义
[[hand.actions]]
id = "next_slide"
name = "下一页"
description = "切换到下一张幻灯片"
params = {}
[[hand.actions]]
id = "prev_slide"
name = "上一页"
description = "切换到上一张幻灯片"
params = {}
[[hand.actions]]
id = "goto_slide"
name = "跳转到指定页"
description = "跳转到指定编号的幻灯片"
params = { slide_number = "number" }
[[hand.actions]]
id = "spotlight"
name = "聚焦元素"
description = "用高亮框聚焦指定元素"
params = { element_id = "string", duration = "number?" }
[[hand.actions]]
id = "laser"
name = "激光笔"
description = "在幻灯片上显示激光笔指示"
params = { x = "number", y = "number", duration = "number?" }
[[hand.actions]]
id = "highlight"
name = "高亮区域"
description = "高亮显示幻灯片上的区域"
params = { x = "number", y = "number", width = "number", height = "number", color = "string?" }
[[hand.actions]]
id = "play_animation"
name = "播放动画"
description = "触发幻灯片上的动画效果"
params = { animation_id = "string" }
[[hand.actions]]
id = "pause"
name = "暂停"
description = "暂停自动播放"
params = {}
[[hand.actions]]
id = "resume"
name = "继续"
description = "继续自动播放"
params = {}

View File

@@ -1,127 +0,0 @@
# Speech Hand - 语音合成能力包
#
# ZCLAW Hand 配置
# 提供文本转语音 (TTS) 能力,支持多种语音和语言
[hand]
name = "speech"
version = "1.0.0"
description = "语音合成能力包 - 将文本转换为自然语音输出"
author = "ZCLAW Team"
type = "media"
requires_approval = false
timeout = 120
max_concurrent = 3
tags = ["speech", "tts", "voice", "audio", "education", "accessibility", "demo"]
[hand.config]
# TTS 提供商: browser, azure, openai, elevenlabs, local
provider = "browser"
# 默认语音
default_voice = "default"
# 默认语速 (0.5 - 2.0)
default_rate = 1.0
# 默认音调 (0.5 - 2.0)
default_pitch = 1.0
# 默认音量 (0 - 1.0)
default_volume = 1.0
# 语言代码
default_language = "zh-CN"
# 是否缓存音频
cache_audio = true
# Azure TTS 配置 (如果 provider = "azure")
[hand.config.azure]
# voice_name = "zh-CN-XiaoxiaoNeural"
# region = "eastasia"
# OpenAI TTS 配置 (如果 provider = "openai")
[hand.config.openai]
# model = "tts-1"
# voice = "alloy"
# 浏览器 TTS 配置 (如果 provider = "browser")
[hand.config.browser]
# 使用系统默认语音
use_system_voice = true
# 语音名称映射
voice_mapping = { "zh-CN" = "Microsoft Huihui", "en-US" = "Microsoft David" }
[hand.triggers]
manual = true
schedule = false
webhook = false
[[hand.triggers.events]]
type = "chat.intent"
pattern = "朗读|念|说|播放语音|speak|read|say|tts"
priority = 5
[hand.permissions]
requires = [
"speech.synthesize",
"speech.play",
"speech.stop"
]
roles = ["operator.read"]
[hand.rate_limit]
max_requests = 100
window_seconds = 3600
[hand.audit]
log_inputs = true
log_outputs = false # 音频不记录
retention_days = 3
# 语音动作定义
[[hand.actions]]
id = "speak"
name = "朗读文本"
description = "将文本转换为语音并播放"
params = { text = "string", voice = "string?", rate = "number?", pitch = "number?" }
[[hand.actions]]
id = "speak_ssml"
name = "朗读 SSML"
description = "使用 SSML 标记朗读文本(支持更精细控制)"
params = { ssml = "string", voice = "string?" }
[[hand.actions]]
id = "pause"
name = "暂停播放"
description = "暂停当前语音播放"
params = {}
[[hand.actions]]
id = "resume"
name = "继续播放"
description = "继续暂停的语音播放"
params = {}
[[hand.actions]]
id = "stop"
name = "停止播放"
description = "停止当前语音播放"
params = {}
[[hand.actions]]
id = "list_voices"
name = "列出可用语音"
description = "获取可用的语音列表"
params = { language = "string?" }
[[hand.actions]]
id = "set_voice"
name = "设置默认语音"
description = "更改默认语音设置"
params = { voice = "string", language = "string?" }

View File

@@ -1,126 +0,0 @@
# Whiteboard Hand - 白板绘制能力包
#
# ZCLAW Hand 配置
# 提供交互式白板绘制能力,支持文本、图形、公式、图表等
[hand]
name = "whiteboard"
version = "1.0.0"
description = "白板绘制能力包 - 绘制文本、图形、公式、图表等教学内容"
author = "ZCLAW Team"
type = "presentation"
requires_approval = false
timeout = 60
max_concurrent = 1
tags = ["whiteboard", "drawing", "presentation", "education", "teaching"]
[hand.config]
# 画布尺寸
canvas_width = 1920
canvas_height = 1080
# 默认画笔颜色
default_color = "#333333"
# 默认线宽
default_line_width = 2
# 支持的绘制动作
supported_actions = [
"draw_text",
"draw_shape",
"draw_line",
"draw_chart",
"draw_latex",
"draw_table",
"erase",
"clear",
"undo",
"redo"
]
# 字体配置
[hand.config.fonts]
text_font = "system-ui"
math_font = "KaTeX_Main"
code_font = "JetBrains Mono"
[hand.triggers]
manual = true
schedule = false
webhook = false
[[hand.triggers.events]]
type = "chat.intent"
pattern = "画|绘制|白板|展示|draw|whiteboard|sketch"
priority = 5
[hand.permissions]
requires = [
"whiteboard.draw",
"whiteboard.clear",
"whiteboard.export"
]
roles = ["operator.read"]
[hand.rate_limit]
max_requests = 100
window_seconds = 3600
[hand.audit]
log_inputs = true
log_outputs = false # 绘制内容不记录
retention_days = 7
# 绘制动作定义
[[hand.actions]]
id = "draw_text"
name = "绘制文本"
description = "在白板上绘制文本"
params = { x = "number", y = "number", text = "string", font_size = "number?", color = "string?" }
[[hand.actions]]
id = "draw_shape"
name = "绘制图形"
description = "绘制矩形、圆形、箭头等基本图形"
params = { shape = "string", x = "number", y = "number", width = "number", height = "number", fill = "string?" }
[[hand.actions]]
id = "draw_line"
name = "绘制线条"
description = "绘制直线或曲线"
params = { points = "array", color = "string?", line_width = "number?" }
[[hand.actions]]
id = "draw_chart"
name = "绘制图表"
description = "绘制柱状图、折线图、饼图等"
params = { chart_type = "string", data = "object", x = "number", y = "number", width = "number", height = "number" }
[[hand.actions]]
id = "draw_latex"
name = "绘制公式"
description = "渲染 LaTeX 数学公式"
params = { latex = "string", x = "number", y = "number", font_size = "number?" }
[[hand.actions]]
id = "draw_table"
name = "绘制表格"
description = "绘制数据表格"
params = { headers = "array", rows = "array", x = "number", y = "number" }
[[hand.actions]]
id = "clear"
name = "清空画布"
description = "清空白板所有内容"
params = {}
[[hand.actions]]
id = "export"
name = "导出图片"
description = "将白板内容导出为图片(⚠️ 导出功能开发中,当前返回占位数据)"
demo = true
params = { format = "string?" }