feat: Batch 5-9 — GrowthIntegration桥接、验证补全、死代码清理、Pipeline模板、Speech/Twitter真实实现
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled

Batch 5 (P0): GrowthIntegration 接入 Tauri
- Kernel 新增 set_viking()/set_extraction_driver() 桥接 SqliteStorage
- 中间件链共享存储,MemoryExtractor 接入 LLM 驱动

Batch 6 (P1): 输入验证 + Heartbeat
- Relay 验证补全(stream 兼容检查、API key 格式校验)
- UUID 类型校验、SessionId 错误返回
- Heartbeat 默认开启 + 首次聊天自动初始化

Batch 7 (P2): 死代码清理
- zclaw-channels 整体移除(317 行)
- multi-agent 特性门控、admin 方法标注

Batch 8 (P2): Pipeline 模板
- PipelineMetadata 新增 annotations 字段
- pipeline_templates 命令 + 2 个示例模板
- fallback driver base_url 修复(doubao/qwen/deepseek 端点)

Batch 9 (P1): SpeechHand/TwitterHand 真实实现
- SpeechHand: tts_method 字段 + Browser TTS 前端集成 (Web Speech API)
- TwitterHand: 12 个 action 全部替换为 Twitter API v2 真实 HTTP 调用
- chatStore/useAutomationEvents 双路径 TTS 触发
This commit is contained in:
iven
2026-03-30 09:24:50 +08:00
parent 5595083b96
commit 13c0b18bbc
39 changed files with 1155 additions and 507 deletions

View File

@@ -68,6 +68,7 @@
"@types/react-window": "^2.0.0",
"@types/uuid": "^10.0.0",
"@vitejs/plugin-react": "^4.7.0",
"@vitejs/plugin-react-oxc": "^0.4.3",
"@vitest/coverage-v8": "2.1.9",
"autoprefixer": "^10.4.27",
"eslint": "^10.1.0",

19
desktop/pnpm-lock.yaml generated
View File

@@ -99,6 +99,9 @@ importers:
'@vitejs/plugin-react':
specifier: ^4.7.0
version: 4.7.0(vite@8.0.3(esbuild@0.27.4)(jiti@2.6.1))
'@vitejs/plugin-react-oxc':
specifier: ^0.4.3
version: 0.4.3(vite@8.0.3(esbuild@0.27.4)(jiti@2.6.1))
'@vitest/coverage-v8':
specifier: 2.1.9
version: 2.1.9(vitest@2.1.9(jsdom@25.0.1)(lightningcss@1.32.0))
@@ -787,6 +790,9 @@ packages:
'@rolldown/pluginutils@1.0.0-beta.27':
resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==}
'@rolldown/pluginutils@1.0.0-beta.47':
resolution: {integrity: sha512-8QagwMH3kNCuzD8EWL8R2YPW5e4OrHNSAHRFDdmFqEwEaD/KcNKjVoumo+gP2vW5eKB2UPbM6vTYiGZX0ixLnw==}
'@rolldown/pluginutils@1.0.0-rc.12':
resolution: {integrity: sha512-HHMwmarRKvoFsJorqYlFeFRzXZqCt2ETQlEDOb9aqssrnVBB1/+xgTGtuTrIk5vzLNX1MjMtTf7W9z3tsSbrxw==}
@@ -1303,6 +1309,12 @@ packages:
'@ungap/structured-clone@1.3.0':
resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==}
'@vitejs/plugin-react-oxc@0.4.3':
resolution: {integrity: sha512-eJv6hHOIOVXzA4b2lZwccu/7VNmk9372fGOqsx5tNxiJHLtFBokyCTQUhlgjjXxl7guLPauHp0TqGTVyn1HvQA==}
engines: {node: ^20.19.0 || >=22.12.0}
peerDependencies:
vite: ^6.3.0 || ^7.0.0
'@vitejs/plugin-react@4.7.0':
resolution: {integrity: sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==}
engines: {node: ^14.18.0 || >=16.0.0}
@@ -3880,6 +3892,8 @@ snapshots:
'@rolldown/pluginutils@1.0.0-beta.27': {}
'@rolldown/pluginutils@1.0.0-beta.47': {}
'@rolldown/pluginutils@1.0.0-rc.12': {}
'@rollup/rollup-android-arm-eabi@4.60.0':
@@ -4322,6 +4336,11 @@ snapshots:
'@ungap/structured-clone@1.3.0': {}
'@vitejs/plugin-react-oxc@0.4.3(vite@8.0.3(esbuild@0.27.4)(jiti@2.6.1))':
dependencies:
'@rolldown/pluginutils': 1.0.0-beta.47
vite: 8.0.3(esbuild@0.27.4)(jiti@2.6.1)
'@vitejs/plugin-react@4.7.0(vite@8.0.3(esbuild@0.27.4)(jiti@2.6.1))':
dependencies:
'@babel/core': 7.29.0

View File

@@ -17,6 +17,9 @@ tauri-build = { version = "2", features = [] }
[features]
default = []
# Multi-agent orchestration (A2A protocol, Director, agent delegation)
# Disabled by default — enable when multi-agent UI is ready.
multi-agent = ["zclaw-kernel/multi-agent"]
dev-server = ["dep:axum", "dep:tower-http"]
[dependencies]
@@ -24,7 +27,7 @@ dev-server = ["dep:axum", "dep:tower-http"]
zclaw-types = { workspace = true }
zclaw-memory = { workspace = true }
zclaw-runtime = { workspace = true }
zclaw-kernel = { workspace = true, features = ["multi-agent"] }
zclaw-kernel = { workspace = true }
zclaw-skills = { workspace = true }
zclaw-hands = { workspace = true }
zclaw-pipeline = { workspace = true }

View File

@@ -246,6 +246,7 @@ pub fn is_extraction_driver_configured() -> bool {
/// Get the global extraction driver.
///
/// Returns `None` if not yet configured via `configure_extraction_driver`.
#[allow(dead_code)]
pub fn get_extraction_driver() -> Option<Arc<TauriExtractionDriver>> {
EXTRACTION_DRIVER.get().cloned()
}

View File

@@ -100,12 +100,12 @@ pub type HeartbeatCheckFn = Box<dyn Fn(String) -> std::pin::Pin<Box<dyn std::fut
impl Default for HeartbeatConfig {
fn default() -> Self {
Self {
enabled: false,
enabled: true,
interval_minutes: 30,
quiet_hours_start: Some("22:00".to_string()),
quiet_hours_end: Some("08:00".to_string()),
notify_channel: NotifyChannel::Ui,
proactivity_level: ProactivityLevel::Light,
proactivity_level: ProactivityLevel::Standard,
max_alerts_per_tick: 5,
}
}

View File

@@ -57,6 +57,52 @@ impl fmt::Display for ValidationError {
impl std::error::Error for ValidationError {}
/// Validate a UUID string (for agent_id, session_id, etc.)
///
/// Provides a clear error message when the UUID format is invalid,
/// instead of a generic "invalid characters" error from `validate_identifier`.
pub fn validate_uuid(value: &str, field_name: &str) -> Result<(), ValidationError> {
let len = value.len();
if len == 0 {
return Err(ValidationError::RequiredFieldEmpty {
field: field_name.to_string(),
});
}
// UUID format: 8-4-4-4-12 hex digits with hyphens (36 chars total)
if len != 36 {
return Err(ValidationError::InvalidCharacters {
field: field_name.to_string(),
invalid_chars: format!("expected UUID format (36 chars), got {} chars", len),
});
}
// Quick structure check: positions 8,13,18,23 should be '-'
let bytes = value.as_bytes();
if bytes[8] != b'-' || bytes[13] != b'-' || bytes[18] != b'-' || bytes[23] != b'-' {
return Err(ValidationError::InvalidCharacters {
field: field_name.to_string(),
invalid_chars: "not a valid UUID (expected format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)".into(),
});
}
// Check all non-hyphen positions are hex digits
for (i, &b) in bytes.iter().enumerate() {
if i == 8 || i == 13 || i == 18 || i == 23 {
continue;
}
if !b.is_ascii_hexdigit() {
return Err(ValidationError::InvalidCharacters {
field: field_name.to_string(),
invalid_chars: format!("'{}' at position {} is not a hex digit", b as char, i),
});
}
}
Ok(())
}
/// Validate an identifier (agent_id, pipeline_id, skill_id, etc.)
///
/// # Rules

View File

@@ -25,6 +25,11 @@ pub type SessionStreamGuard = Arc<dashmap::DashMap<String, Arc<Mutex<()>>>>;
fn validate_agent_id(agent_id: &str) -> Result<String, String> {
validate_identifier(agent_id, "agent_id")
.map_err(|e| format!("Invalid agent_id: {}", e))?;
// AgentId is a UUID wrapper — validate UUID format for better error messages
if agent_id.contains('-') {
crate::intelligence::validation::validate_uuid(agent_id, "agent_id")
.map_err(|e| format!("Invalid agent_id: {}", e))?;
}
Ok(agent_id.to_string())
}
@@ -209,7 +214,7 @@ pub async fn kernel_init(
let model = config.llm.model.clone();
// Boot kernel
let kernel = Kernel::boot(config.clone())
let mut kernel = Kernel::boot(config.clone())
.await
.map_err(|e| format!("Failed to initialize kernel: {}", e))?;
@@ -222,6 +227,33 @@ pub async fn kernel_init(
model.clone(),
);
// Bridge SqliteStorage to Kernel's GrowthIntegration
// This connects the middleware chain (MemoryMiddleware, CompactionMiddleware)
// to the same persistent SqliteStorage used by viking_commands and intelligence_hooks.
{
match crate::viking_commands::get_storage().await {
Ok(sqlite_storage) => {
// Wrap SqliteStorage in VikingAdapter (SqliteStorage implements VikingStorage)
let viking = std::sync::Arc::new(zclaw_runtime::VikingAdapter::new(sqlite_storage));
kernel.set_viking(viking);
tracing::info!("[kernel_init] Bridged persistent SqliteStorage to Kernel GrowthIntegration");
}
Err(e) => {
tracing::warn!(
"[kernel_init] Failed to get SqliteStorage, GrowthIntegration will use in-memory storage: {}",
e
);
}
}
// Set the LLM extraction driver on the kernel for memory extraction via middleware
let extraction_driver = crate::intelligence::extraction_adapter::TauriExtractionDriver::new(
driver.clone(),
model.clone(),
);
kernel.set_extraction_driver(std::sync::Arc::new(extraction_driver));
}
// Configure summary driver so the Growth system can generate L0/L1 summaries
if let Some(api_key) = config_request.as_ref().and_then(|r| r.api_key.clone()) {
crate::summarizer_adapter::configure_summary_driver(
@@ -530,6 +562,21 @@ pub async fn agent_chat_stream(
format!("Session {} already has an active stream", session_id)
})?;
// AUTO-INIT HEARTBEAT: Ensure heartbeat engine exists for this agent.
// Uses default config (enabled: true, 30min interval) so heartbeat runs
// automatically from the first conversation without manual setup.
{
let mut engines = heartbeat_state.lock().await;
if !engines.contains_key(&request.agent_id) {
let engine = crate::intelligence::heartbeat::HeartbeatEngine::new(
request.agent_id.clone(),
None, // Use default config (enabled: true)
);
engines.insert(request.agent_id.clone(), engine);
tracing::info!("[agent_chat_stream] Auto-initialized heartbeat for agent: {}", request.agent_id);
}
}
// PRE-CONVERSATION: Build intelligence-enhanced system prompt
let enhanced_prompt = crate::intelligence_hooks::pre_conversation_hook(
&request.agent_id,
@@ -550,15 +597,22 @@ pub async fn agent_chat_stream(
// Use intelligence-enhanced system prompt if available
let prompt_arg = if enhanced_prompt.is_empty() { None } else { Some(enhanced_prompt) };
// Parse session_id for session reuse (carry conversation history across turns)
let session_id_parsed = std::str::FromStr::from_str(&session_id)
.ok()
.map(|uuid| zclaw_types::SessionId::from_uuid(uuid));
if session_id_parsed.is_none() {
tracing::warn!(
"session_id '{}' is not a valid UUID, will create a new session (context will be lost)",
session_id
);
}
// Empty session_id means first message in a new conversation — that's valid.
// Non-empty session_id MUST be a valid UUID; if not, return error instead of
// silently losing context by creating a new session.
let session_id_parsed = if session_id.is_empty() {
None
} else {
match uuid::Uuid::parse_str(&session_id) {
Ok(uuid) => Some(zclaw_types::SessionId::from_uuid(uuid)),
Err(e) => {
return Err(format!(
"Invalid session_id '{}': {}. Cannot reuse conversation context.",
session_id, e
));
}
}
};
let rx = kernel.send_message_stream_with_prompt(&id, message.clone(), prompt_arg, session_id_parsed)
.await
.map_err(|e| format!("Failed to start streaming: {}", e))?;
@@ -1775,9 +1829,10 @@ pub async fn scheduled_task_list(
}
// ============================================================
// A2A (Agent-to-Agent) Commands
// A2A (Agent-to-Agent) Commands — gated behind multi-agent feature
// ============================================================
#[cfg(feature = "multi-agent")]
/// Send a direct A2A message from one agent to another
#[tauri::command]
pub async fn agent_a2a_send(
@@ -1810,6 +1865,7 @@ pub async fn agent_a2a_send(
}
/// Broadcast a message from one agent to all other agents
#[cfg(feature = "multi-agent")]
#[tauri::command]
pub async fn agent_a2a_broadcast(
state: State<'_, KernelState>,
@@ -1830,6 +1886,7 @@ pub async fn agent_a2a_broadcast(
}
/// Discover agents with a specific capability
#[cfg(feature = "multi-agent")]
#[tauri::command]
pub async fn agent_a2a_discover(
state: State<'_, KernelState>,
@@ -1850,6 +1907,7 @@ pub async fn agent_a2a_discover(
}
/// Delegate a task to another agent and wait for response
#[cfg(feature = "multi-agent")]
#[tauri::command]
pub async fn agent_a2a_delegate_task(
state: State<'_, KernelState>,

View File

@@ -1352,15 +1352,19 @@ pub fn run() {
kernel_commands::scheduled_task_create,
kernel_commands::scheduled_task_list,
// A2A commands (Agent-to-Agent messaging)
// A2A commands gated behind multi-agent feature
#[cfg(feature = "multi-agent")]
kernel_commands::agent_a2a_send,
#[cfg(feature = "multi-agent")]
kernel_commands::agent_a2a_broadcast,
#[cfg(feature = "multi-agent")]
kernel_commands::agent_a2a_discover,
#[cfg(feature = "multi-agent")]
kernel_commands::agent_a2a_delegate_task,
// Pipeline commands (DSL-based workflows)
pipeline_commands::pipeline_list,
pipeline_commands::pipeline_get,
pipeline_commands::pipeline_templates, pipeline_commands::pipeline_get,
pipeline_commands::pipeline_run,
pipeline_commands::pipeline_progress,
pipeline_commands::pipeline_cancel,

View File

@@ -681,9 +681,8 @@ fn scan_pipelines_with_paths(
tracing::debug!("[scan] File content length: {} bytes", content.len());
match parse_pipeline_yaml(&content) {
Ok(pipeline) => {
// Debug: log parsed pipeline metadata
println!(
"[DEBUG scan] Parsed YAML: {} -> category: {:?}, industry: {:?}",
tracing::debug!(
"[scan] Parsed YAML: {} -> category: {:?}, industry: {:?}",
pipeline.metadata.name,
pipeline.metadata.category,
pipeline.metadata.industry
@@ -744,8 +743,8 @@ fn scan_pipelines_full_sync(
fn pipeline_to_info(pipeline: &Pipeline) -> PipelineInfo {
let industry = pipeline.metadata.industry.clone().unwrap_or_default();
println!(
"[DEBUG pipeline_to_info] Pipeline: {}, category: {:?}, industry: {:?}",
tracing::debug!(
"[pipeline_to_info] Pipeline: {}, category: {:?}, industry: {:?}",
pipeline.metadata.name,
pipeline.metadata.category,
pipeline.metadata.industry
@@ -1040,16 +1039,30 @@ fn create_llm_driver_from_config() -> Option<Arc<dyn LlmActionDriver>> {
// Convert api_key to SecretString
let secret_key = SecretString::new(api_key);
// Create the runtime driver
// Create the runtime driver — use with_base_url when a custom endpoint is configured
// (essential for Chinese providers like doubao, qwen, deepseek, kimi)
let runtime_driver: Arc<dyn zclaw_runtime::LlmDriver> = match provider.as_str() {
"anthropic" => {
Arc::new(zclaw_runtime::AnthropicDriver::new(secret_key))
if let Some(url) = base_url {
Arc::new(zclaw_runtime::AnthropicDriver::with_base_url(secret_key, url))
} else {
Arc::new(zclaw_runtime::AnthropicDriver::new(secret_key))
}
}
"openai" | "doubao" | "qwen" | "deepseek" | "kimi" => {
Arc::new(zclaw_runtime::OpenAiDriver::new(secret_key))
"openai" | "doubao" | "qwen" | "deepseek" | "kimi" | "zhipu" => {
// Chinese providers typically need a custom base_url
if let Some(url) = base_url {
Arc::new(zclaw_runtime::OpenAiDriver::with_base_url(secret_key, url))
} else {
Arc::new(zclaw_runtime::OpenAiDriver::new(secret_key))
}
}
"gemini" => {
Arc::new(zclaw_runtime::GeminiDriver::new(secret_key))
if let Some(url) = base_url {
Arc::new(zclaw_runtime::GeminiDriver::with_base_url(secret_key, url))
} else {
Arc::new(zclaw_runtime::GeminiDriver::new(secret_key))
}
}
"local" | "ollama" => {
let url = base_url.unwrap_or_else(|| "http://localhost:11434".to_string());
@@ -1077,3 +1090,83 @@ pub async fn analyze_presentation(
// Convert analysis to JSON
serde_json::to_value(&analysis).map_err(|e| e.to_string())
}
/// Pipeline template metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PipelineTemplateInfo {
pub id: String,
pub display_name: String,
pub description: String,
pub category: String,
pub industry: String,
pub tags: Vec<String>,
pub icon: String,
pub version: String,
pub author: String,
pub inputs: Vec<PipelineInputInfo>,
}
/// List available pipeline templates from the `_templates/` directory.
///
/// Templates are pipeline YAML files that users can browse and instantiate.
/// They live in `pipelines/_templates/` and are not directly runnable
/// (they serve as blueprints).
#[tauri::command]
pub async fn pipeline_templates(
state: State<'_, Arc<PipelineState>>,
) -> Result<Vec<PipelineTemplateInfo>, String> {
let pipelines = state.pipelines.read().await;
// Filter pipelines that have `is_template: true` in metadata
// or are in the _templates directory
let templates: Vec<PipelineTemplateInfo> = pipelines.iter()
.filter_map(|(id, pipeline)| {
// Check if this pipeline has template metadata
let is_template = pipeline.metadata.annotations
.as_ref()
.and_then(|a| a.get("is_template"))
.and_then(|v| v.as_bool())
.unwrap_or(false);
if !is_template {
return None;
}
Some(PipelineTemplateInfo {
id: pipeline.metadata.name.clone(),
display_name: pipeline.metadata.display_name.clone()
.unwrap_or_else(|| pipeline.metadata.name.clone()),
description: pipeline.metadata.description.clone().unwrap_or_default(),
category: pipeline.metadata.category.clone().unwrap_or_default(),
industry: pipeline.metadata.industry.clone().unwrap_or_default(),
tags: pipeline.metadata.tags.clone(),
icon: pipeline.metadata.icon.clone().unwrap_or_else(|| "📦".to_string()),
version: pipeline.metadata.version.clone(),
author: pipeline.metadata.author.clone().unwrap_or_default(),
inputs: pipeline.spec.inputs.iter().map(|input| {
PipelineInputInfo {
name: input.name.clone(),
input_type: match input.input_type {
zclaw_pipeline::InputType::String => "string".to_string(),
zclaw_pipeline::InputType::Number => "number".to_string(),
zclaw_pipeline::InputType::Boolean => "boolean".to_string(),
zclaw_pipeline::InputType::Select => "select".to_string(),
zclaw_pipeline::InputType::MultiSelect => "multi-select".to_string(),
zclaw_pipeline::InputType::File => "file".to_string(),
zclaw_pipeline::InputType::Text => "text".to_string(),
},
required: input.required,
label: input.label.clone().unwrap_or_else(|| input.name.clone()),
placeholder: input.placeholder.clone(),
default: input.default.clone(),
options: input.options.clone(),
}
}).collect(),
})
})
.collect();
tracing::debug!("[pipeline_templates] Found {} templates", templates.len());
Ok(templates)
}

View File

@@ -35,7 +35,7 @@ import {
// === Default Config ===
const DEFAULT_HEARTBEAT_CONFIG: HeartbeatConfigType = {
enabled: false,
enabled: true,
interval_minutes: 30,
quiet_hours_start: null,
quiet_hours_end: null,

View File

@@ -12,6 +12,7 @@ import { useHandStore } from '../store/handStore';
import { useWorkflowStore } from '../store/workflowStore';
import { useChatStore } from '../store/chatStore';
import type { GatewayClient } from '../lib/gateway-client';
import { speechSynth } from '../lib/speech-synth';
// === Event Types ===
@@ -161,6 +162,23 @@ export function useAutomationEvents(
handResult: eventData.hand_result,
runId: eventData.run_id,
});
// Trigger browser TTS for SpeechHand results
if (eventData.hand_name === 'speech' && eventData.hand_result && typeof eventData.hand_result === 'object') {
const res = eventData.hand_result as Record<string, unknown>;
if (res.tts_method === 'browser' && typeof res.text === 'string' && res.text) {
speechSynth.speak({
text: res.text,
voice: typeof res.voice === 'string' ? res.voice : undefined,
language: typeof res.language === 'string' ? res.language : undefined,
rate: typeof res.rate === 'number' ? res.rate : undefined,
pitch: typeof res.pitch === 'number' ? res.pitch : undefined,
volume: typeof res.volume === 'number' ? res.volume : undefined,
}).catch((err: unknown) => {
console.warn('[useAutomationEvents] Browser TTS failed:', err);
});
}
}
}
// Handle error status

View File

@@ -920,6 +920,12 @@ export class SaaSClient {
return this.request('GET', '/api/v1/config/pull' + qs);
}
// ==========================================================================
// Admin Panel API — Reserved for future admin UI (Next.js admin dashboard)
// These methods are not called by the desktop app but are kept as thin API
// wrappers for when the admin panel is built.
// ==========================================================================
// --- Provider Management (Admin) ---
/** List all providers */

View File

@@ -0,0 +1,195 @@
/**
* Speech Synthesis Service — Browser TTS via Web Speech API
*
* Provides text-to-speech playback using the browser's native SpeechSynthesis API.
* Zero external dependencies, works offline, supports Chinese and English voices.
*
* Architecture:
* - SpeechHand (Rust) returns tts_method + text + voice config
* - This service handles Browser TTS playback in the webview
* - OpenAI/Azure TTS is handled via backend API calls
*/
export interface SpeechSynthOptions {
text: string;
voice?: string;
language?: string;
rate?: number;
pitch?: number;
volume?: number;
}
export interface SpeechSynthState {
playing: boolean;
paused: boolean;
currentText: string | null;
voices: SpeechSynthesisVoice[];
}
type SpeechEventCallback = (state: SpeechSynthState) => void;
class SpeechSynthService {
private synth: SpeechSynthesis | null = null;
private currentUtterance: SpeechSynthesisUtterance | null = null;
private listeners: Set<SpeechEventCallback> = new Set();
private cachedVoices: SpeechSynthesisVoice[] = [];
constructor() {
if (typeof window !== 'undefined' && window.speechSynthesis) {
this.synth = window.speechSynthesis;
this.loadVoices();
// Voices may load asynchronously
this.synth.onvoiceschanged = () => this.loadVoices();
}
}
private loadVoices() {
if (!this.synth) return;
this.cachedVoices = this.synth.getVoices();
this.notify();
}
private notify() {
const state = this.getState();
this.listeners.forEach(cb => cb(state));
}
/** Subscribe to state changes */
subscribe(callback: SpeechEventCallback): () => void {
this.listeners.add(callback);
return () => this.listeners.delete(callback);
}
/** Get current state */
getState(): SpeechSynthState {
return {
playing: this.synth?.speaking ?? false,
paused: this.synth?.paused ?? false,
currentText: this.currentUtterance?.text ?? null,
voices: this.cachedVoices,
};
}
/** Check if TTS is available */
isAvailable(): boolean {
return this.synth != null;
}
/** Get available voices, optionally filtered by language */
getVoices(language?: string): SpeechSynthesisVoice[] {
if (!language) return this.cachedVoices;
const langPrefix = language.split('-')[0].toLowerCase();
return this.cachedVoices.filter(v =>
v.lang.toLowerCase().startsWith(langPrefix)
);
}
/** Speak text with given options */
speak(options: SpeechSynthOptions): Promise<void> {
return new Promise((resolve, reject) => {
if (!this.synth) {
reject(new Error('Speech synthesis not available'));
return;
}
// Cancel any ongoing speech
this.stop();
const utterance = new SpeechSynthesisUtterance(options.text);
this.currentUtterance = utterance;
// Set language
utterance.lang = options.language ?? 'zh-CN';
// Set voice if specified
if (options.voice && options.voice !== 'default') {
const voice = this.cachedVoices.find(v =>
v.name === options.voice || v.voiceURI === options.voice
);
if (voice) utterance.voice = voice;
} else {
// Auto-select best voice for the language
this.selectBestVoice(utterance, options.language ?? 'zh-CN');
}
// Set parameters
utterance.rate = options.rate ?? 1.0;
utterance.pitch = options.pitch ?? 1.0;
utterance.volume = options.volume ?? 1.0;
utterance.onstart = () => {
this.notify();
};
utterance.onend = () => {
this.currentUtterance = null;
this.notify();
resolve();
};
utterance.onerror = (event) => {
this.currentUtterance = null;
this.notify();
// "canceled" is not a real error (happens on stop())
if (event.error !== 'canceled') {
reject(new Error(`Speech error: ${event.error}`));
} else {
resolve();
}
};
this.synth.speak(utterance);
});
}
/** Pause current speech */
pause() {
this.synth?.pause();
this.notify();
}
/** Resume paused speech */
resume() {
this.synth?.resume();
this.notify();
}
/** Stop current speech */
stop() {
this.synth?.cancel();
this.currentUtterance = null;
this.notify();
}
/** Auto-select the best voice for a language */
private selectBestVoice(utterance: SpeechSynthesisUtterance, language: string) {
const langPrefix = language.split('-')[0].toLowerCase();
const candidates = this.cachedVoices.filter(v =>
v.lang.toLowerCase().startsWith(langPrefix)
);
if (candidates.length === 0) return;
// Prefer voices with "Neural" or "Enhanced" in name (higher quality)
const neural = candidates.find(v =>
v.name.includes('Neural') || v.name.includes('Enhanced') || v.name.includes('Premium')
);
if (neural) {
utterance.voice = neural;
return;
}
// Prefer local voices (work offline)
const local = candidates.find(v => v.localService);
if (local) {
utterance.voice = local;
return;
}
// Fall back to first matching voice
utterance.voice = candidates[0];
}
}
// Singleton instance
export const speechSynth = new SpeechSynthService();

View File

@@ -8,6 +8,7 @@ import { getSkillDiscovery } from '../lib/skill-discovery';
import { useOfflineStore, isOffline } from './offlineStore';
import { useConnectionStore } from './connectionStore';
import { createLogger } from '../lib/logger';
import { speechSynth } from '../lib/speech-synth';
import { generateRandomString } from '../lib/crypto-utils';
const log = createLogger('ChatStore');
@@ -461,6 +462,24 @@ export const useChatStore = create<ChatState>()(
handResult: result,
};
set((state) => ({ messages: [...state.messages, handMsg] }));
// Trigger browser TTS when SpeechHand completes with browser method
if (name === 'speech' && status === 'completed' && result && typeof result === 'object') {
const res = result as Record<string, unknown>;
if (res.tts_method === 'browser' && typeof res.text === 'string' && res.text) {
speechSynth.speak({
text: res.text as string,
voice: (res.voice as string) || undefined,
language: (res.language as string) || undefined,
rate: typeof res.rate === 'number' ? res.rate : undefined,
pitch: typeof res.pitch === 'number' ? res.pitch : undefined,
volume: typeof res.volume === 'number' ? res.volume : undefined,
}).catch((err: unknown) => {
const logger = createLogger('speech-synth');
logger.warn('Browser TTS failed', { error: String(err) });
});
}
}
},
onComplete: (inputTokens?: number, outputTokens?: number) => {
const state = get();

View File

@@ -1,5 +1,5 @@
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
import react from "@vitejs/plugin-react-oxc";
import tailwindcss from "@tailwindcss/vite";
const host = process.env.TAURI_DEV_HOST;
@@ -36,6 +36,15 @@ export default defineConfig(async () => ({
changeOrigin: true,
secure: false,
ws: true, // Enable WebSocket proxy for streaming
configure: (proxy) => {
// Suppress ECONNREFUSED errors during startup while Kernel is still compiling
proxy.on('error', (err) => {
if ('code' in err && (err as NodeJS.ErrnoException).code === 'ECONNREFUSED') {
return; // Silently ignore — Kernel not ready yet
}
console.error('[proxy error]', err);
});
},
},
},
},