refactor(hands): 移除空壳 Hand — Whiteboard/Slideshow/Speech (Phase 5)
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled

删除 3 个仅含 UI 占位的 Hand,清理 Rust 实现与前端引用:
- Rust: whiteboard.rs(422行) + slideshow.rs(797行) + speech.rs(442行)
- 前端: WhiteboardCanvas + SlideshowRenderer + speech-synth + 相关类型/常量
- 配置: 3 个 HAND.toml
- 净减 ~5400 行,Hands 9→6(启用) + Quiz/Browser/Researcher/Collector/Clip/Twitter/Reminder
This commit is contained in:
iven
2026-04-17 19:55:59 +08:00
parent 14fa7e150a
commit cb9e48f11d
21 changed files with 11 additions and 3014 deletions

View File

@@ -2,12 +2,11 @@
* SceneRenderer — Renders a single classroom scene.
*
* Supports scene types: slide, quiz, discussion, interactive, text, pbl, media.
* Executes scene actions (speech, whiteboard, quiz, discussion).
* Executes scene actions (quiz, discussion).
*/
import { useState, useEffect, useCallback } from 'react';
import { useState, useEffect } from 'react';
import type { GeneratedScene, SceneContent, SceneAction, AgentProfile } from '../../types/classroom';
import { WhiteboardCanvas } from './WhiteboardCanvas';
interface SceneRendererProps {
scene: GeneratedScene;
@@ -15,14 +14,10 @@ interface SceneRendererProps {
autoPlay?: boolean;
}
export function SceneRenderer({ scene, agents, autoPlay = true }: SceneRendererProps) {
export function SceneRenderer({ scene, autoPlay = true }: SceneRendererProps) {
const { content } = scene;
const [actionIndex, setActionIndex] = useState(0);
const [isPlaying, setIsPlaying] = useState(autoPlay);
const [whiteboardItems, setWhiteboardItems] = useState<Array<{
type: string;
data: SceneAction;
}>>([]);
const actions = content.actions ?? [];
const currentAction = actions[actionIndex] ?? null;
@@ -37,27 +32,12 @@ export function SceneRenderer({ scene, agents, autoPlay = true }: SceneRendererP
const delay = getActionDelay(actions[actionIndex]);
const timer = setTimeout(() => {
processAction(actions[actionIndex]);
setActionIndex((i) => i + 1);
}, delay);
return () => clearTimeout(timer);
}, [actionIndex, isPlaying, actions]);
const processAction = useCallback((action: SceneAction) => {
switch (action.type) {
case 'whiteboard_draw_text':
case 'whiteboard_draw_shape':
case 'whiteboard_draw_chart':
case 'whiteboard_draw_latex':
setWhiteboardItems((prev) => [...prev, { type: action.type, data: action }]);
break;
case 'whiteboard_clear':
setWhiteboardItems([]);
break;
}
}, []);
// Render scene based on type
return (
<div className="flex flex-col h-full">
@@ -72,31 +52,21 @@ export function SceneRenderer({ scene, agents, autoPlay = true }: SceneRendererP
</div>
{/* Main content area */}
<div className="flex-1 flex gap-4 overflow-hidden">
{/* Content panel */}
<div className="flex-1 overflow-auto">
{renderContent(content)}
</div>
{/* Whiteboard area */}
{whiteboardItems.length > 0 && (
<div className="w-96 shrink-0">
<WhiteboardCanvas items={whiteboardItems} />
</div>
)}
<div className="flex-1 overflow-auto">
{renderContent(content)}
</div>
{/* Current action indicator */}
{currentAction && (
<div className="mt-4 p-3 rounded-lg bg-indigo-50 dark:bg-indigo-900/20 border border-indigo-100 dark:border-indigo-800">
{renderCurrentAction(currentAction, agents)}
{renderCurrentAction(currentAction)}
</div>
)}
{/* Playback controls */}
<div className="flex items-center justify-center gap-2 mt-4">
<button
onClick={() => { setActionIndex(0); setWhiteboardItems([]); }}
onClick={() => { setActionIndex(0); }}
className="px-2 py-1 text-xs rounded bg-gray-100 dark:bg-gray-700"
>
Restart
@@ -121,12 +91,6 @@ export function SceneRenderer({ scene, agents, autoPlay = true }: SceneRendererP
function getActionDelay(action: SceneAction): number {
switch (action.type) {
case 'speech': return 2000;
case 'whiteboard_draw_text': return 800;
case 'whiteboard_draw_shape': return 600;
case 'whiteboard_draw_chart': return 1000;
case 'whiteboard_draw_latex': return 1000;
case 'whiteboard_clear': return 300;
case 'quiz_show': return 5000;
case 'discussion': return 10000;
default: return 1000;
@@ -167,26 +131,12 @@ function renderContent(content: SceneContent) {
);
}
function renderCurrentAction(action: SceneAction, agents: AgentProfile[]) {
function renderCurrentAction(action: SceneAction) {
switch (action.type) {
case 'speech': {
const agent = agents.find(a => a.role === action.agentRole);
return (
<div className="flex items-start gap-2">
<span className="text-lg">{agent?.avatar ?? '💬'}</span>
<div>
<span className="text-xs font-medium text-gray-600">{agent?.name ?? action.agentRole}</span>
<p className="text-sm text-gray-700 dark:text-gray-300">{action.text}</p>
</div>
</div>
);
}
case 'quiz_show':
return <div className="text-sm text-amber-600">Quiz: {action.quizId}</div>;
case 'discussion':
return <div className="text-sm text-green-600">Discussion: {action.topic}</div>;
default:
return <div className="text-xs text-gray-400">{action.type}</div>;
}
}

View File

@@ -1,295 +0,0 @@
/**
* WhiteboardCanvas — SVG-based whiteboard for classroom scene rendering.
*
* Supports incremental drawing operations:
* - Text (positioned labels)
* - Shapes (rectangles, circles, arrows)
* - Charts (bar/line/pie via simple SVG)
* - LaTeX (rendered as styled text blocks)
*/
import { useCallback } from 'react';
import type { SceneAction } from '../../types/classroom';
interface WhiteboardCanvasProps {
items: WhiteboardItem[];
width?: number;
height?: number;
}
export interface WhiteboardItem {
type: string;
data: SceneAction;
}
export function WhiteboardCanvas({
items,
width = 800,
height = 600,
}: WhiteboardCanvasProps) {
const renderItem = useCallback((item: WhiteboardItem, index: number) => {
switch (item.type) {
case 'whiteboard_draw_text':
return <TextItem key={index} data={item.data as TextDrawData} />;
case 'whiteboard_draw_shape':
return <ShapeItem key={index} data={item.data as ShapeDrawData} />;
case 'whiteboard_draw_chart':
return <ChartItem key={index} data={item.data as ChartDrawData} />;
case 'whiteboard_draw_latex':
return <LatexItem key={index} data={item.data as LatexDrawData} />;
default:
return null;
}
}, []);
return (
<div className="w-full h-full border border-gray-200 dark:border-gray-700 rounded-lg bg-white dark:bg-gray-900 overflow-auto">
<svg
viewBox={`0 0 ${width} ${height}`}
className="w-full h-full"
xmlns="http://www.w3.org/2000/svg"
>
{/* Grid background */}
<defs>
<pattern id="grid" width="40" height="40" patternUnits="userSpaceOnUse">
<path d="M 40 0 L 0 0 0 40" fill="none" stroke="#f0f0f0" strokeWidth="0.5" />
</pattern>
</defs>
<rect width={width} height={height} fill="url(#grid)" />
{/* Rendered items */}
{items.map((item, i) => renderItem(item, i))}
</svg>
</div>
);
}
// ---------------------------------------------------------------------------
// Sub-components
// ---------------------------------------------------------------------------
interface TextDrawData {
type: 'whiteboard_draw_text';
x: number;
y: number;
text: string;
fontSize?: number;
color?: string;
}
function TextItem({ data }: { data: TextDrawData }) {
return (
<text
x={data.x}
y={data.y}
fontSize={data.fontSize ?? 16}
fill={data.color ?? '#333333'}
fontFamily="system-ui, sans-serif"
>
{data.text}
</text>
);
}
interface ShapeDrawData {
type: 'whiteboard_draw_shape';
shape: string;
x: number;
y: number;
width: number;
height: number;
fill?: string;
}
function ShapeItem({ data }: { data: ShapeDrawData }) {
switch (data.shape) {
case 'circle':
return (
<ellipse
cx={data.x + data.width / 2}
cy={data.y + data.height / 2}
rx={data.width / 2}
ry={data.height / 2}
fill={data.fill ?? '#e5e7eb'}
stroke="#9ca3af"
strokeWidth={1}
/>
);
case 'arrow':
return (
<g>
<line
x1={data.x}
y1={data.y + data.height / 2}
x2={data.x + data.width}
y2={data.y + data.height / 2}
stroke={data.fill ?? '#6b7280'}
strokeWidth={2}
markerEnd="url(#arrowhead)"
/>
<defs>
<marker id="arrowhead" markerWidth="10" markerHeight="7" refX="10" refY="3.5" orient="auto">
<polygon points="0 0, 10 3.5, 0 7" fill={data.fill ?? '#6b7280'} />
</marker>
</defs>
</g>
);
default: // rectangle
return (
<rect
x={data.x}
y={data.y}
width={data.width}
height={data.height}
fill={data.fill ?? '#e5e7eb'}
stroke="#9ca3af"
strokeWidth={1}
rx={4}
/>
);
}
}
interface ChartDrawData {
type: 'whiteboard_draw_chart';
chartType: string;
data: Record<string, unknown>;
x: number;
y: number;
width: number;
height: number;
}
function ChartItem({ data }: { data: ChartDrawData }) {
const chartData = data.data;
const labels = (chartData?.labels as string[]) ?? [];
const values = (chartData?.values as number[]) ?? [];
if (labels.length === 0 || values.length === 0) return null;
switch (data.chartType) {
case 'bar':
return <BarChart data={data} labels={labels} values={values} />;
case 'line':
return <LineChart data={data} labels={labels} values={values} />;
default:
return <BarChart data={data} labels={labels} values={values} />;
}
}
function BarChart({ data, labels, values }: {
data: ChartDrawData;
labels: string[];
values: number[];
}) {
const maxVal = Math.max(...values, 1);
const barWidth = data.width / (labels.length * 2);
const chartHeight = data.height - 30;
return (
<g transform={`translate(${data.x}, ${data.y})`}>
{values.map((val, i) => {
const barHeight = (val / maxVal) * chartHeight;
return (
<g key={i}>
<rect
x={i * (barWidth * 2) + barWidth / 2}
y={chartHeight - barHeight}
width={barWidth}
height={barHeight}
fill="#6366f1"
rx={2}
/>
<text
x={i * (barWidth * 2) + barWidth}
y={data.height - 5}
textAnchor="middle"
fontSize={10}
fill="#666"
>
{labels[i]}
</text>
</g>
);
})}
</g>
);
}
function LineChart({ data, labels, values }: {
data: ChartDrawData;
labels: string[];
values: number[];
}) {
const maxVal = Math.max(...values, 1);
const chartHeight = data.height - 30;
const stepX = data.width / Math.max(labels.length - 1, 1);
const points = values.map((val, i) => {
const x = i * stepX;
const y = chartHeight - (val / maxVal) * chartHeight;
return `${x},${y}`;
}).join(' ');
return (
<g transform={`translate(${data.x}, ${data.y})`}>
<polyline
points={points}
fill="none"
stroke="#6366f1"
strokeWidth={2}
/>
{values.map((val, i) => {
const x = i * stepX;
const y = chartHeight - (val / maxVal) * chartHeight;
return (
<g key={i}>
<circle cx={x} cy={y} r={3} fill="#6366f1" />
<text
x={x}
y={data.height - 5}
textAnchor="middle"
fontSize={10}
fill="#666"
>
{labels[i]}
</text>
</g>
);
})}
</g>
);
}
interface LatexDrawData {
type: 'whiteboard_draw_latex';
latex: string;
x: number;
y: number;
}
function LatexItem({ data }: { data: LatexDrawData }) {
return (
<g transform={`translate(${data.x}, ${data.y})`}>
<rect
x={-4}
y={-20}
width={data.latex.length * 10 + 8}
height={28}
fill="#fef3c7"
stroke="#f59e0b"
strokeWidth={1}
rx={4}
/>
<text
x={0}
y={0}
fontSize={14}
fill="#92400e"
fontFamily="'Courier New', monospace"
>
{data.latex}
</text>
</g>
);
}

View File

@@ -8,5 +8,4 @@ export { ClassroomPlayer } from './ClassroomPlayer';
export { SceneRenderer } from './SceneRenderer';
export { AgentChat } from './AgentChat';
export { NotesSidebar } from './NotesSidebar';
export { WhiteboardCanvas } from './WhiteboardCanvas';
export { TtsPlayer } from './TtsPlayer';

View File

@@ -16,7 +16,6 @@ import type { PresentationType, PresentationAnalysis } from './types';
import { TypeSwitcher } from './TypeSwitcher';
import { QuizRenderer } from './renderers/QuizRenderer';
const SlideshowRenderer = React.lazy(() => import('./renderers/SlideshowRenderer').then(m => ({ default: m.SlideshowRenderer })));
const DocumentRenderer = React.lazy(() => import('./renderers/DocumentRenderer').then(m => ({ default: m.DocumentRenderer })));
const ChartRenderer = React.lazy(() => import('./renderers/ChartRenderer').then(m => ({ default: m.ChartRenderer })));
@@ -79,7 +78,7 @@ export function PresentationContainer({
if (supportedTypes && supportedTypes.length > 0) {
return supportedTypes.filter((t): t is PresentationType => t !== 'auto');
}
return (['quiz', 'slideshow', 'document', 'chart', 'whiteboard'] as PresentationType[]);
return (['quiz', 'document', 'chart'] as PresentationType[]);
}, [supportedTypes]);
const renderContent = () => {
@@ -96,13 +95,6 @@ export function PresentationContainer({
case 'quiz':
return <QuizRenderer data={data as Parameters<typeof QuizRenderer>[0]['data']} />;
case 'slideshow':
return (
<React.Suspense fallback={<div className="h-64 animate-pulse bg-gray-100" />}>
<SlideshowRenderer data={data as Parameters<typeof SlideshowRenderer>[0]['data']} />
</React.Suspense>
);
case 'document':
return (
<React.Suspense fallback={<div className="h-64 animate-pulse bg-gray-100" />}>
@@ -110,16 +102,6 @@ export function PresentationContainer({
</React.Suspense>
);
case 'whiteboard':
return (
<div className="flex flex-col items-center justify-center h-64 bg-gray-50 gap-3">
<span className="inline-flex items-center px-3 py-1 rounded-full text-xs font-medium bg-amber-100 text-amber-700">
</span>
<p className="text-gray-500"></p>
</div>
);
case 'chart':
return (
<React.Suspense fallback={<div className="h-64 animate-pulse bg-gray-100" />}>

View File

@@ -7,9 +7,7 @@
import {
BarChart3,
FileText,
Presentation,
CheckCircle,
PenTool,
} from 'lucide-react';
import type { PresentationType, PresentationAnalysis } from './types';
@@ -34,11 +32,6 @@ const typeConfig: Record<PresentationType, { icon: React.ReactNode; label: strin
label: '图表',
description: '数据可视化',
},
slideshow: {
icon: <Presentation className="w-4 h-4" />,
label: '幻灯片',
description: '演示文稿风格',
},
quiz: {
icon: <CheckCircle className="w-4 h-4" />,
label: '测验',
@@ -49,11 +42,6 @@ const typeConfig: Record<PresentationType, { icon: React.ReactNode; label: strin
label: '文档',
description: 'Markdown 文档',
},
whiteboard: {
icon: <PenTool className="w-4 h-4" />,
label: '白板',
description: '交互式画布',
},
auto: {
icon: <CheckCircle className="w-4 h-4" />,
label: '自动',

View File

@@ -19,7 +19,6 @@ export { PresentationContainer } from './PresentationContainer';
export { TypeSwitcher } from './TypeSwitcher';
export { QuizRenderer } from './renderers/QuizRenderer';
export { DocumentRenderer } from './renderers/DocumentRenderer';
export { SlideshowRenderer } from './renderers/SlideshowRenderer';
export type {
PresentationType,
PresentationAnalysis,
@@ -27,7 +26,5 @@ export type {
QuizData,
QuizQuestion,
QuestionType,
SlideshowData,
DocumentData,
WhiteboardData,
} from './types';

View File

@@ -1,283 +0,0 @@
/**
* Slideshow Renderer
*
* Renders presentation as a slideshow with slide navigation.
* Supports: title, content, image, code, twoColumn slide types.
*/
import { useState, useEffect, useCallback } from 'react';
import Markdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import {
ChevronLeft,
ChevronRight,
Maximize2,
Minimize2,
Play,
Pause,
} from 'lucide-react';
import type { SlideshowData, Slide } from '../types';
interface SlideshowRendererProps {
data: SlideshowData;
/** Auto-play interval in seconds (0 = disabled) */
autoPlayInterval?: number;
/** Show progress indicator */
showProgress?: boolean;
/** Show speaker notes */
showNotes?: boolean;
/** Custom className */
className?: string;
}
export function SlideshowRenderer({
data,
autoPlayInterval = 0,
showProgress = true,
showNotes = true,
className = '',
}: SlideshowRendererProps) {
const [currentIndex, setCurrentIndex] = useState(0);
const [isPlaying, setIsPlaying] = useState(false);
const [isFullscreen, setIsFullscreen] = useState(false);
const slides = data.slides || [];
const totalSlides = slides.length;
const handleNext = useCallback(() => {
setCurrentIndex((prev) => (prev + 1) % totalSlides);
}, [totalSlides]);
const handlePrev = useCallback(() => {
setCurrentIndex((prev) => (prev - 1 + totalSlides) % totalSlides);
}, [totalSlides]);
const toggleFullscreen = useCallback(() => {
setIsFullscreen((prev) => !prev);
}, []);
// Handle keyboard navigation
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
if (e.key === 'ArrowRight' || e.key === ' ') {
e.preventDefault();
handleNext();
} else if (e.key === 'ArrowLeft') {
e.preventDefault();
handlePrev();
} else if (e.key === 'f') {
toggleFullscreen();
}
};
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [handleNext, handlePrev, toggleFullscreen]);
// Auto-play
useEffect(() => {
if (isPlaying && autoPlayInterval > 0) {
const timer = setInterval(handleNext, autoPlayInterval * 1000);
return () => clearInterval(timer);
}
}, [isPlaying, autoPlayInterval, handleNext]);
const currentSlide = slides[currentIndex];
if (!currentSlide) {
return (
<div className={`flex items-center justify-center h-64 bg-gray-50 ${className}`}>
<p className="text-gray-500"></p>
</div>
);
}
return (
<div
className={`flex flex-col h-full ${
isFullscreen ? 'fixed inset-0 z-50 bg-white' : ''
} ${className}`}
>
{/* Slide Content */}
<div className="flex-1 flex items-center justify-center p-8">
<div className="max-w-4xl w-full">
<SlideContent slide={currentSlide} />
</div>
</div>
{/* Controls */}
<div className="flex items-center justify-between p-4 bg-gray-50 border-t">
<div className="flex items-center gap-2">
<button
onClick={handlePrev}
disabled={totalSlides <= 1}
className="p-2 hover:bg-gray-200 rounded disabled:opacity-50"
>
<ChevronLeft className="w-5 h-5" />
</button>
<button
onClick={() => setIsPlaying(!isPlaying)}
disabled={autoPlayInterval === 0}
className="p-2 hover:bg-gray-200 rounded disabled:opacity-50"
>
{isPlaying ? (
<Pause className="w-5 h-5" />
) : (
<Play className="w-5 h-5" />
)}
</button>
<button
onClick={handleNext}
disabled={totalSlides <= 1}
className="p-2 hover:bg-gray-200 rounded disabled:opacity-50"
>
<ChevronRight className="w-5 h-5" />
</button>
</div>
{/* Progress */}
{showProgress && (
<div className="text-sm text-gray-500">
{currentIndex + 1} / {totalSlides}
</div>
)}
{/* Fullscreen */}
<button
onClick={toggleFullscreen}
className="p-2 hover:bg-gray-200 rounded"
>
{isFullscreen ? (
<Minimize2 className="w-5 h-5" />
) : (
<Maximize2 className="w-5 h-5" />
)}
</button>
</div>
{/* Speaker Notes */}
{showNotes && currentSlide.notes && (
<div className="p-4 bg-yellow-50 border-t text-sm text-gray-600">
{currentSlide.notes}
</div>
)}
</div>
);
}
/** Renders a single slide based on its type */
function SlideContent({ slide }: { slide: Slide }) {
switch (slide.type) {
case 'title':
return (
<div className="text-center py-12">
{slide.title && (
<h1 className="text-4xl font-bold mb-4">{slide.title}</h1>
)}
{slide.content && (
<p className="text-xl text-gray-600 max-w-2xl mx-auto">
{slide.content}
</p>
)}
</div>
);
case 'content':
return (
<div>
{slide.title && (
<h2 className="text-3xl font-bold text-center mb-6">{slide.title}</h2>
)}
{slide.content && (
<div className="prose prose-gray max-w-none">
<Markdown remarkPlugins={[remarkGfm]}>{slide.content}</Markdown>
</div>
)}
</div>
);
case 'image':
return (
<div className="text-center">
{slide.title && (
<h2 className="text-2xl font-bold mb-4">{slide.title}</h2>
)}
{slide.image && (
<img
src={slide.image}
alt={slide.title || '幻灯片图片'}
className="max-w-full max-h-[60vh] mx-auto rounded-lg shadow-md"
/>
)}
{slide.content && (
<p className="mt-4 text-gray-600">{slide.content}</p>
)}
</div>
);
case 'code':
return (
<div>
{slide.title && (
<h2 className="text-2xl font-bold mb-4">{slide.title}</h2>
)}
{slide.code && (
<pre className="bg-gray-900 text-gray-100 p-6 rounded-lg overflow-x-auto text-sm">
{slide.language && (
<div className="text-xs text-gray-400 mb-3 uppercase tracking-wider">
{slide.language}
</div>
)}
<code>{slide.code}</code>
</pre>
)}
{slide.content && (
<p className="mt-4 text-gray-600">{slide.content}</p>
)}
</div>
);
case 'twoColumn':
return (
<div>
{slide.title && (
<h2 className="text-2xl font-bold mb-4">{slide.title}</h2>
)}
<div className="grid grid-cols-2 gap-8">
<div className="prose prose-sm max-w-none">
{slide.leftContent && (
<Markdown remarkPlugins={[remarkGfm]}>
{slide.leftContent}
</Markdown>
)}
</div>
<div className="prose prose-sm max-w-none">
{slide.rightContent && (
<Markdown remarkPlugins={[remarkGfm]}>
{slide.rightContent}
</Markdown>
)}
</div>
</div>
</div>
);
default:
return (
<div>
{slide.title && (
<h2 className="text-2xl font-bold mb-4">{slide.title}</h2>
)}
{slide.content && (
<div className="prose prose-gray max-w-none">
<Markdown remarkPlugins={[remarkGfm]}>{slide.content}</Markdown>
</div>
)}
</div>
);
}
}
export default SlideshowRenderer;

View File

@@ -8,9 +8,7 @@
export type PresentationType =
| 'chart'
| 'quiz'
| 'slideshow'
| 'document'
| 'whiteboard'
| 'auto';
export interface PresentationAnalysis {
@@ -84,34 +82,6 @@ export interface QuizOption {
isCorrect?: boolean;
}
export interface SlideshowData {
title?: string;
slides: Slide[];
theme?: SlideshowTheme;
autoPlay?: boolean;
interval?: number;
}
export interface Slide {
id: string;
type: 'title' | 'content' | 'image' | 'code' | 'twoColumn';
title?: string;
content?: string;
image?: string;
code?: string;
language?: string;
leftContent?: string;
rightContent?: string;
notes?: string;
}
export interface SlideshowTheme {
backgroundColor?: string;
textColor?: string;
accentColor?: string;
fontFamily?: string;
}
export interface DocumentData {
title?: string;
content?: string;
@@ -121,25 +91,3 @@ export interface DocumentData {
url?: string;
}
export interface WhiteboardData {
title?: string;
elements: WhiteboardElement[];
background?: string;
gridSize?: number;
}
export interface WhiteboardElement {
id: string;
type: 'rect' | 'circle' | 'line' | 'text' | 'image' | 'path';
x: number;
y: number;
width?: number;
height?: number;
fill?: string;
stroke?: string;
strokeWidth?: number;
text?: string;
fontSize?: number;
src?: string;
points?: number[];
}

View File

@@ -16,11 +16,7 @@ export const HAND_IDS = {
TRADER: 'trader',
CLIP: 'clip',
TWITTER: 'twitter',
// Additional hands from backend
SLIDESHOW: 'slideshow',
SPEECH: 'speech',
QUIZ: 'quiz',
WHITEBOARD: 'whiteboard',
} as const;
export type HandIdType = typeof HAND_IDS[keyof typeof HAND_IDS];
@@ -49,10 +45,7 @@ export const HAND_CATEGORY_MAP: Record<string, HandCategoryType> = {
[HAND_IDS.LEAD]: HAND_CATEGORIES.COMMUNICATION,
[HAND_IDS.TWITTER]: HAND_CATEGORIES.COMMUNICATION,
[HAND_IDS.CLIP]: HAND_CATEGORIES.CONTENT,
[HAND_IDS.SLIDESHOW]: HAND_CATEGORIES.CONTENT,
[HAND_IDS.SPEECH]: HAND_CATEGORIES.CONTENT,
[HAND_IDS.QUIZ]: HAND_CATEGORIES.PRODUCTIVITY,
[HAND_IDS.WHITEBOARD]: HAND_CATEGORIES.PRODUCTIVITY,
};
// === Helper Functions ===

View File

@@ -12,7 +12,6 @@ import { useHandStore } from '../store/handStore';
import { useWorkflowStore } from '../store/workflowStore';
import { useChatStore } from '../store/chatStore';
import type { GatewayClient } from '../lib/gateway-client';
import { speechSynth } from '../lib/speech-synth';
import { createLogger } from '../lib/logger';
const log = createLogger('useAutomationEvents');
@@ -166,22 +165,6 @@ export function useAutomationEvents(
runId: eventData.run_id,
});
// Trigger browser TTS for SpeechHand results
if (eventData.hand_name === 'speech' && eventData.hand_result && typeof eventData.hand_result === 'object') {
const res = eventData.hand_result as Record<string, unknown>;
if (res.tts_method === 'browser' && typeof res.text === 'string' && res.text) {
speechSynth.speak({
text: res.text,
voice: typeof res.voice === 'string' ? res.voice : undefined,
language: typeof res.language === 'string' ? res.language : undefined,
rate: typeof res.rate === 'number' ? res.rate : undefined,
pitch: typeof res.pitch === 'number' ? res.pitch : undefined,
volume: typeof res.volume === 'number' ? res.volume : undefined,
}).catch((err: unknown) => {
log.warn('Browser TTS failed:', err);
});
}
}
}
// Handle error status

View File

@@ -1,195 +0,0 @@
/**
* Speech Synthesis Service — Browser TTS via Web Speech API
*
* Provides text-to-speech playback using the browser's native SpeechSynthesis API.
* Zero external dependencies, works offline, supports Chinese and English voices.
*
* Architecture:
* - SpeechHand (Rust) returns tts_method + text + voice config
* - This service handles Browser TTS playback in the webview
* - OpenAI/Azure TTS is handled via backend API calls
*/
export interface SpeechSynthOptions {
text: string;
voice?: string;
language?: string;
rate?: number;
pitch?: number;
volume?: number;
}
export interface SpeechSynthState {
playing: boolean;
paused: boolean;
currentText: string | null;
voices: SpeechSynthesisVoice[];
}
type SpeechEventCallback = (state: SpeechSynthState) => void;
class SpeechSynthService {
private synth: SpeechSynthesis | null = null;
private currentUtterance: SpeechSynthesisUtterance | null = null;
private listeners: Set<SpeechEventCallback> = new Set();
private cachedVoices: SpeechSynthesisVoice[] = [];
constructor() {
if (typeof window !== 'undefined' && window.speechSynthesis) {
this.synth = window.speechSynthesis;
this.loadVoices();
// Voices may load asynchronously
this.synth.onvoiceschanged = () => this.loadVoices();
}
}
private loadVoices() {
if (!this.synth) return;
this.cachedVoices = this.synth.getVoices();
this.notify();
}
private notify() {
const state = this.getState();
this.listeners.forEach(cb => cb(state));
}
/** Subscribe to state changes */
subscribe(callback: SpeechEventCallback): () => void {
this.listeners.add(callback);
return () => this.listeners.delete(callback);
}
/** Get current state */
getState(): SpeechSynthState {
return {
playing: this.synth?.speaking ?? false,
paused: this.synth?.paused ?? false,
currentText: this.currentUtterance?.text ?? null,
voices: this.cachedVoices,
};
}
/** Check if TTS is available */
isAvailable(): boolean {
return this.synth != null;
}
/** Get available voices, optionally filtered by language */
getVoices(language?: string): SpeechSynthesisVoice[] {
if (!language) return this.cachedVoices;
const langPrefix = language.split('-')[0].toLowerCase();
return this.cachedVoices.filter(v =>
v.lang.toLowerCase().startsWith(langPrefix)
);
}
/** Speak text with given options */
speak(options: SpeechSynthOptions): Promise<void> {
return new Promise((resolve, reject) => {
if (!this.synth) {
reject(new Error('Speech synthesis not available'));
return;
}
// Cancel any ongoing speech
this.stop();
const utterance = new SpeechSynthesisUtterance(options.text);
this.currentUtterance = utterance;
// Set language
utterance.lang = options.language ?? 'zh-CN';
// Set voice if specified
if (options.voice && options.voice !== 'default') {
const voice = this.cachedVoices.find(v =>
v.name === options.voice || v.voiceURI === options.voice
);
if (voice) utterance.voice = voice;
} else {
// Auto-select best voice for the language
this.selectBestVoice(utterance, options.language ?? 'zh-CN');
}
// Set parameters
utterance.rate = options.rate ?? 1.0;
utterance.pitch = options.pitch ?? 1.0;
utterance.volume = options.volume ?? 1.0;
utterance.onstart = () => {
this.notify();
};
utterance.onend = () => {
this.currentUtterance = null;
this.notify();
resolve();
};
utterance.onerror = (event) => {
this.currentUtterance = null;
this.notify();
// "canceled" is not a real error (happens on stop())
if (event.error !== 'canceled') {
reject(new Error(`Speech error: ${event.error}`));
} else {
resolve();
}
};
this.synth.speak(utterance);
});
}
/** Pause current speech */
pause() {
this.synth?.pause();
this.notify();
}
/** Resume paused speech */
resume() {
this.synth?.resume();
this.notify();
}
/** Stop current speech */
stop() {
this.synth?.cancel();
this.currentUtterance = null;
this.notify();
}
/** Auto-select the best voice for a language */
private selectBestVoice(utterance: SpeechSynthesisUtterance, language: string) {
const langPrefix = language.split('-')[0].toLowerCase();
const candidates = this.cachedVoices.filter(v =>
v.lang.toLowerCase().startsWith(langPrefix)
);
if (candidates.length === 0) return;
// Prefer voices with "Neural" or "Enhanced" in name (higher quality)
const neural = candidates.find(v =>
v.name.includes('Neural') || v.name.includes('Enhanced') || v.name.includes('Premium')
);
if (neural) {
utterance.voice = neural;
return;
}
// Prefer local voices (work offline)
const local = candidates.find(v => v.localService);
if (local) {
utterance.voice = local;
return;
}
// Fall back to first matching voice
utterance.voice = candidates[0];
}
}
// Singleton instance
export const speechSynth = new SpeechSynthService();

View File

@@ -24,7 +24,6 @@ import { getSkillDiscovery } from '../../lib/skill-discovery';
import { useOfflineStore, isOffline } from '../../store/offlineStore';
import { useConnectionStore } from '../../store/connectionStore';
import { createLogger } from '../../lib/logger';
import { speechSynth } from '../../lib/speech-synth';
import { generateRandomString } from '../../lib/crypto-utils';
import type { ChatModeType, ChatModeConfig, Subtask } from '../../components/ai';
import type { ToolCallStep } from '../../components/ai';
@@ -440,22 +439,6 @@ export const useStreamStore = create<StreamState>()(
};
_chat?.updateMessages(msgs => [...msgs, handMsg]);
if (name === 'speech' && status === 'completed' && result && typeof result === 'object') {
const res = result as Record<string, unknown>;
if (res.tts_method === 'browser' && typeof res.text === 'string' && res.text) {
speechSynth.speak({
text: res.text as string,
voice: (res.voice as string) || undefined,
language: (res.language as string) || undefined,
rate: typeof res.rate === 'number' ? res.rate : undefined,
pitch: typeof res.pitch === 'number' ? res.pitch : undefined,
volume: typeof res.volume === 'number' ? res.volume : undefined,
}).catch((err: unknown) => {
const logger = createLogger('speech-synth');
logger.warn('Browser TTS failed', { error: String(err) });
});
}
}
},
onSubtaskStatus: (taskId: string, description: string, status: string, detail?: string) => {
// Map backend status to frontend Subtask status

View File

@@ -46,14 +46,6 @@ export enum GenerationStage {
// --- Scene Actions ---
export type SceneAction =
| { type: 'speech'; text: string; agentRole: string }
| { type: 'whiteboard_draw_text'; x: number; y: number; text: string; fontSize?: number; color?: string }
| { type: 'whiteboard_draw_shape'; shape: string; x: number; y: number; width: number; height: number; fill?: string }
| { type: 'whiteboard_draw_chart'; chartType: string; data: unknown; x: number; y: number; width: number; height: number }
| { type: 'whiteboard_draw_latex'; latex: string; x: number; y: number }
| { type: 'whiteboard_clear' }
| { type: 'slideshow_spotlight'; elementId: string }
| { type: 'slideshow_next' }
| { type: 'quiz_show'; quizId: string }
| { type: 'discussion'; topic: string; durationSeconds?: number };