From 6bd9b841aa9e21c68e33a8ebd00222498a594235 Mon Sep 17 00:00:00 2001 From: iven Date: Tue, 17 Mar 2026 08:56:02 +0800 Subject: [PATCH] feat(browser-hand): implement Browser Hand UI components Add complete Browser Hand UI system for browser automation: Components: - BrowserHandCard: Main card with status display and screenshot preview - TaskTemplateModal: Template selection and parameter configuration - ScreenshotPreview: Screenshot display with fullscreen capability Templates: - Basic operations: navigate, screenshot, form fill, click, execute JS - Scraping: text, list, images, links, tables - Automation: login+action, multi-page, monitoring, pagination Features: - 15 built-in task templates across 3 categories - Real-time execution status with progress bar - Screenshot preview with zoom and fullscreen - Integration with HandsPanel for seamless UX - Zustand store for state management - Comprehensive test coverage (16 tests) Co-Authored-By: Claude Opus 4.6 --- .../BrowserHand/BrowserHandCard.tsx | 234 +++++++ .../BrowserHand/ScreenshotPreview.tsx | 151 ++++ .../BrowserHand/TaskTemplateModal.tsx | 417 +++++++++++ desktop/src/components/BrowserHand/index.ts | 42 ++ .../BrowserHand/templates/automation.ts | 654 ++++++++++++++++++ .../components/BrowserHand/templates/basic.ts | 411 +++++++++++ .../components/BrowserHand/templates/index.ts | 240 +++++++ .../BrowserHand/templates/scraping.ts | 535 ++++++++++++++ .../components/BrowserHand/templates/types.ts | 240 +++++++ desktop/src/components/HandsPanel.tsx | 29 +- desktop/src/store/browserHandStore.ts | 496 +++++++++++++ desktop/src/store/index.ts | 16 + tests/desktop/browserHandStore.test.ts | 273 ++++++++ 13 files changed, 3729 insertions(+), 9 deletions(-) create mode 100644 desktop/src/components/BrowserHand/BrowserHandCard.tsx create mode 100644 desktop/src/components/BrowserHand/ScreenshotPreview.tsx create mode 100644 desktop/src/components/BrowserHand/TaskTemplateModal.tsx create mode 100644 desktop/src/components/BrowserHand/index.ts create mode 100644 desktop/src/components/BrowserHand/templates/automation.ts create mode 100644 desktop/src/components/BrowserHand/templates/basic.ts create mode 100644 desktop/src/components/BrowserHand/templates/index.ts create mode 100644 desktop/src/components/BrowserHand/templates/scraping.ts create mode 100644 desktop/src/components/BrowserHand/templates/types.ts create mode 100644 desktop/src/store/browserHandStore.ts create mode 100644 tests/desktop/browserHandStore.test.ts diff --git a/desktop/src/components/BrowserHand/BrowserHandCard.tsx b/desktop/src/components/BrowserHand/BrowserHandCard.tsx new file mode 100644 index 0000000..0de7228 --- /dev/null +++ b/desktop/src/components/BrowserHand/BrowserHandCard.tsx @@ -0,0 +1,234 @@ +/** + * BrowserHandCard Component + * + * Main card for Browser Hand with real-time status and screenshot preview. + */ + +import React from 'react'; +import { + Globe, + Camera, + RefreshCw, + Settings, + Play, + Loader2, + CheckCircle, + XCircle, + AlertCircle, + ExternalLink, +} from 'lucide-react'; +import { cn } from '../../lib/utils'; +import { useBrowserHandStore } from '../../store/browserHandStore'; +import { ScreenshotPreview } from './ScreenshotPreview'; +import { TaskTemplateModal } from './TaskTemplateModal'; +import type { Hand } from '../../types/hands'; + +interface BrowserHandCardProps { + hand: Hand; + onOpenSettings?: () => void; +} + +export function BrowserHandCard({ onOpenSettings }: BrowserHandCardProps) { + const { + execution, + sessions, + activeSessionId, + isTemplateModalOpen, + isLoading, + error, + openTemplateModal, + closeTemplateModal, + takeScreenshot, + createSession, + closeSession, + clearError, + } = useBrowserHandStore(); + + const [isStarting, setIsStarting] = React.useState(false); + + // Auto-start session if needed + React.useEffect(() => { + if (sessions.length === 0 && !activeSessionId) { + setIsStarting(true); + createSession({ headless: true }) + .then(() => setIsStarting(false)) + .catch(() => setIsStarting(false)); + } + }, [sessions.length, activeSessionId, createSession]); + + // Get status display + const getStatusDisplay = () => { + if (isStarting || isLoading) { + return { text: '连接中...', color: 'text-yellow-500', icon: Loader2 }; + } + if (error) { + return { text: '错误', color: 'text-red-500', icon: XCircle }; + } + if (execution.isRunning) { + return { text: '运行中', color: 'text-blue-500', icon: Play }; + } + if (activeSessionId) { + return { text: '就绪', color: 'text-green-500', icon: CheckCircle }; + } + return { text: '未连接', color: 'text-gray-500', icon: AlertCircle }; + }; + + const status = getStatusDisplay(); + const StatusIcon = status.icon; + + return ( +
+ {/* Header */} +
+
+
+ +
+
+

+ Browser Hand +

+

+ 浏览器自动化能力 +

+
+
+
+ + + {status.text} + +
+
+ + {/* Screenshot Preview */} +
+ { + if (activeSessionId) { + takeScreenshot(); + } + }} + altText={execution.isRunning ? '执行中...' : '等待截图'} + /> +
+ + {/* Status Bar */} + {(execution.isRunning || execution.currentUrl) && ( +
+ {execution.currentUrl && ( +
+ + {execution.currentUrl} +
+ )} + {execution.isRunning && ( + <> +
+ + {execution.currentAction || '处理中...'} +
+
+
+
+ + )} +
+ )} + + {/* Error Display */} + {error && ( +
+
+
+ + {error} +
+ +
+
+ )} + + {/* Actions */} +
+
+ + + + + + + {onOpenSettings && ( + + )} +
+
+ + {/* Template Modal */} + { + const { executeTemplate } = useBrowserHandStore.getState(); + executeTemplate(template.id, params); + }} + /> +
+ ); +} diff --git a/desktop/src/components/BrowserHand/ScreenshotPreview.tsx b/desktop/src/components/BrowserHand/ScreenshotPreview.tsx new file mode 100644 index 0000000..0f5606b --- /dev/null +++ b/desktop/src/components/BrowserHand/ScreenshotPreview.tsx @@ -0,0 +1,151 @@ +/** + * ScreenshotPreview Component + * + * Displays browser screenshots with zoom and fullscreen capabilities. + */ + +import React from 'react'; +import { Expand, RefreshCw, Loader2, Camera, X } from 'lucide-react'; + +import { cn } from '../../lib/utils'; + +interface ScreenshotPreviewProps { + /** Base64 encoded screenshot data */ + base64: string | null; + /** Loading state */ + isLoading?: boolean; + /** Callback when refresh is requested */ + onRefresh?: () => void; + /** Callback when clicked (for fullscreen) */ + onClick?: () => void; + /** Alt text when no screenshot */ + altText?: string; + /** Container class name */ + className?: string; +} + +export function ScreenshotPreview({ + base64, + isLoading = false, + onRefresh, + onClick, + altText = '等待截图', + className = '', +}: ScreenshotPreviewProps) { + const [isFullscreen, setIsFullscreen] = React.useState(false); + + // Handle keyboard shortcut for fullscreen toggle + React.useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Escape' && isFullscreen) { + setIsFullscreen(false); + } + }; + + if (isFullscreen) { + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + } + }, [isFullscreen]); + + if (!base64 && !isLoading) { + return ( +
+ +

{altText}

+
+ ); + } + + const handleClick = () => { + if (onClick) { + onClick(); + } + setIsFullscreen(true); + }; + + return ( +
+ {/* Loading overlay */} + {isLoading && ( +
+ +
+ )} + + {/* Toolbar */} +
+ {onRefresh && ( + + )} + +
+ + {/* Screenshot image */} +
+ Browser screenshot +
+ + {/* Fullscreen modal */} + {isFullscreen && ( +
setIsFullscreen(false)} + > + Browser screenshot fullscreen e.stopPropagation()} + /> + +
+ )} +
+ ); +} diff --git a/desktop/src/components/BrowserHand/TaskTemplateModal.tsx b/desktop/src/components/BrowserHand/TaskTemplateModal.tsx new file mode 100644 index 0000000..352f241 --- /dev/null +++ b/desktop/src/components/BrowserHand/TaskTemplateModal.tsx @@ -0,0 +1,417 @@ +/** + * TaskTemplateModal Component + * + * Modal for selecting task templates and configuring parameters. + */ + +import React, { useState, useEffect } from 'react'; +import { + X, + Play, + AlertCircle, + Camera, + FileText, + List, + MousePointerClick, + Code, + Image, + Link, + Table, + LogIn, + Layers, + Activity, + ClipboardList, + ChevronsRight, + Info, +} from 'lucide-react'; +import { cn } from '../../lib/utils'; +import { + validateTemplateParams, + mergeParamsWithDefaults, + getTemplatesByCategory, + type TaskTemplate, + type TemplateCategory, + type TaskTemplateParam, +} from './templates'; + +interface TaskTemplateModalProps { + isOpen: boolean; + onClose: () => void; + onSelect: (template: TaskTemplate, params: Record) => void; +} + +const categoryIcons: Record> = { + basic: Camera, + scraping: FileText, + automation: Layers, +}; + +const categoryColors: Record = { + basic: 'bg-blue-500', + scraping: 'bg-green-500', + automation: 'bg-purple-500', +}; + +export function TaskTemplateModal({ + isOpen, + onClose, + onSelect, +}: TaskTemplateModalProps) { + const [selectedCategory, setSelectedCategory] = useState('basic'); + const [selectedTemplate, setSelectedTemplate] = useState(null); + const [params, setParams] = useState>({}); + const [validationErrors, setValidationErrors] = useState([]); + + const basicTemplates = getTemplatesByCategory('basic'); + const scrapingTemplates = getTemplatesByCategory('scraping'); + const automationTemplates = getTemplatesByCategory('automation'); + + // Reset when modal closes + useEffect(() => { + if (!isOpen) { + setSelectedTemplate(null); + setParams({}); + setValidationErrors([]); + } + }, [isOpen]); + + // Handle template selection + const handleTemplateSelect = (template: TaskTemplate) => { + setSelectedTemplate(template); + setParams({}); + setValidationErrors([]); + }; + + // Handle param change + const handleParamChange = (key: string, value: unknown) => { + setParams((prev) => ({ ...prev, [key]: value })); + setValidationErrors((prev) => prev.filter((e) => e !== key)); + }; + + // Handle form submission + const handleSubmit = () => { + if (!selectedTemplate) return; + + // Validate params + const validation = validateTemplateParams(selectedTemplate.params, params); + if (!validation.valid) { + setValidationErrors(validation.errors.map((e) => e.message)); + return; + } + + // Merge with defaults and execute + const mergedParams = mergeParamsWithDefaults(selectedTemplate.params, params); + onSelect(selectedTemplate, mergedParams); + onClose(); + }; + + // Get icon component + const getIconComponent = (iconName: string) => { + const icons: Record> = { + Camera, + FileText, + List, + MousePointerClick, + Code, + Image, + Link, + Table, + LogIn, + Layers, + Activity, + ClipboardList, + ChevronsRight, + Info, + }; + const IconComponent = icons[iconName] || Info; + return ; + }; + + if (!isOpen) return null; + + return ( +
+ {/* Backdrop */} +
+ + {/* Modal */} +
e.stopPropagation()} + > + {/* Header */} +
+

+ 选择任务模板 +

+ +
+ + {/* Category Tabs */} +
+ {(['basic', 'scraping', 'automation'] as TemplateCategory[]).map((category) => { + const CategoryIcon = categoryIcons[category]; + return ( + + ); + })} +
+ + {/* Content */} +
+ {/* Template List */} +
+
+ {(selectedCategory === 'basic' ? basicTemplates : + selectedCategory === 'scraping' ? scrapingTemplates : + automationTemplates + ).map((template) => ( + + ))} +
+
+ + {/* Parameter Form */} +
+ {selectedTemplate ? ( +
+
+
+ {getIconComponent(selectedTemplate.icon)} +
+
+

+ {selectedTemplate.name} +

+

+ {selectedTemplate.description} +

+
+
+ + {selectedTemplate.params.map((param) => ( +
+ + {param.description && ( +

+ {param.description} +

+ )} + + {renderParamInput(param, params[param.key], handleParamChange)} + + {validationErrors.includes(`${param.label} 是必填项`) && !params[param.key] && ( +

{param.label} 是必填项

+ )} +
+ ))} + + {validationErrors.length > 0 && ( +
+
+ + 请检查以下错误： +
+
    + {validationErrors.map((error, i) => ( +
  • {error}
    • + ))} +
+
+ )} +
+ ) : ( +
+ +

选择左侧的模板以配置参数

+
+ )} +
+
+ + {/* Footer */} +
+ + +
+
+
+ ); +} + +// Helper function to render param input +function renderParamInput( + param: TaskTemplateParam, + value: unknown, + onChange: (key: string, value: unknown) => void +) { + const inputId = `param-${param.key}`; + + const baseInputClasses = 'w-full px-3 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:ring-2 focus:ring-blue-500 focus:border-transparent'; + + switch (param.type) { + case 'text': + case 'url': + return ( + onChange(param.key, e.target.value)} + placeholder={param.placeholder} + className={baseInputClasses} + /> + ); + + case 'number': + return ( + onChange(param.key, e.target.value ? Number(e.target.value) : undefined)} + placeholder={param.placeholder} + min={param.min} + max={param.max} + className={baseInputClasses} + /> + ); + + case 'boolean': + return ( + + ); + + case 'select': + return ( + + ); + + case 'textarea': + case 'json': + return ( +