feat(hands): implement 4 new Hands and fix BrowserHand registration
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled

- Add ResearcherHand: DuckDuckGo search, web fetch, report generation
- Add CollectorHand: data collection, aggregation, multiple output formats
- Add ClipHand: video processing (trim, convert, thumbnail, concat)
- Add TwitterHand: Twitter/X automation (tweet, retweet, like, search)
- Fix BrowserHand not registered in Kernel (critical bug)
- Add HandError variant to ZclawError enum
- Update documentation: 9/11 Hands implemented (82%)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
iven
2026-03-24 13:22:44 +08:00
parent 3ff08faa56
commit 1441f98c5e
15 changed files with 2376 additions and 36 deletions

View File

@@ -0,0 +1,642 @@
//! Clip Hand - Video processing and editing capabilities
//!
//! This hand provides video processing features:
//! - Trim: Cut video segments
//! - Convert: Format conversion
//! - Resize: Resolution changes
//! - Thumbnail: Generate thumbnails
//! - Concat: Join videos
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::process::Command;
use std::sync::Arc;
use tokio::sync::RwLock;
use zclaw_types::Result;
use crate::{Hand, HandConfig, HandContext, HandResult};
/// Video format options
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum VideoFormat {
Mp4,
Webm,
Mov,
Avi,
Gif,
}
impl Default for VideoFormat {
fn default() -> Self {
Self::Mp4
}
}
/// Video resolution presets
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Resolution {
Original,
P480,
P720,
P1080,
P4k,
Custom { width: u32, height: u32 },
}
impl Default for Resolution {
fn default() -> Self {
Self::Original
}
}
/// Trim configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TrimConfig {
/// Input video path
pub input_path: String,
/// Output video path
pub output_path: String,
/// Start time in seconds
#[serde(default)]
pub start_time: Option<f64>,
/// End time in seconds
#[serde(default)]
pub end_time: Option<f64>,
/// Duration in seconds (alternative to end_time)
#[serde(default)]
pub duration: Option<f64>,
}
/// Convert configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ConvertConfig {
/// Input video path
pub input_path: String,
/// Output video path
pub output_path: String,
/// Output format
#[serde(default)]
pub format: VideoFormat,
/// Resolution
#[serde(default)]
pub resolution: Resolution,
/// Video bitrate (e.g., "2M")
#[serde(default)]
pub video_bitrate: Option<String>,
/// Audio bitrate (e.g., "128k")
#[serde(default)]
pub audio_bitrate: Option<String>,
}
/// Thumbnail configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ThumbnailConfig {
/// Input video path
pub input_path: String,
/// Output image path
pub output_path: String,
/// Time position in seconds
#[serde(default)]
pub time: f64,
/// Output width
#[serde(default)]
pub width: Option<u32>,
/// Output height
#[serde(default)]
pub height: Option<u32>,
}
/// Concat configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ConcatConfig {
/// Input video paths
pub input_paths: Vec<String>,
/// Output video path
pub output_path: String,
}
/// Video info result
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VideoInfo {
pub path: String,
pub duration_secs: f64,
pub width: u32,
pub height: u32,
pub fps: f64,
pub format: String,
pub video_codec: String,
pub audio_codec: Option<String>,
pub bitrate_kbps: Option<u32>,
pub file_size_bytes: u64,
}
/// Clip action types
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "action")]
pub enum ClipAction {
#[serde(rename = "trim")]
Trim { config: TrimConfig },
#[serde(rename = "convert")]
Convert { config: ConvertConfig },
#[serde(rename = "resize")]
Resize { input_path: String, output_path: String, resolution: Resolution },
#[serde(rename = "thumbnail")]
Thumbnail { config: ThumbnailConfig },
#[serde(rename = "concat")]
Concat { config: ConcatConfig },
#[serde(rename = "info")]
Info { path: String },
#[serde(rename = "check_ffmpeg")]
CheckFfmpeg,
}
/// Clip Hand implementation
pub struct ClipHand {
config: HandConfig,
ffmpeg_path: Arc<RwLock<Option<String>>>,
}
impl ClipHand {
/// Create a new clip hand
pub fn new() -> Self {
Self {
config: HandConfig {
id: "clip".to_string(),
name: "Clip".to_string(),
description: "Video processing and editing capabilities using FFmpeg".to_string(),
needs_approval: false,
dependencies: vec!["ffmpeg".to_string()],
input_schema: Some(serde_json::json!({
"type": "object",
"oneOf": [
{
"properties": {
"action": { "const": "trim" },
"config": {
"type": "object",
"properties": {
"inputPath": { "type": "string" },
"outputPath": { "type": "string" },
"startTime": { "type": "number" },
"endTime": { "type": "number" },
"duration": { "type": "number" }
},
"required": ["inputPath", "outputPath"]
}
},
"required": ["action", "config"]
},
{
"properties": {
"action": { "const": "convert" },
"config": {
"type": "object",
"properties": {
"inputPath": { "type": "string" },
"outputPath": { "type": "string" },
"format": { "type": "string", "enum": ["mp4", "webm", "mov", "avi", "gif"] },
"resolution": { "type": "string" }
},
"required": ["inputPath", "outputPath"]
}
},
"required": ["action", "config"]
},
{
"properties": {
"action": { "const": "thumbnail" },
"config": {
"type": "object",
"properties": {
"inputPath": { "type": "string" },
"outputPath": { "type": "string" },
"time": { "type": "number" }
},
"required": ["inputPath", "outputPath"]
}
},
"required": ["action", "config"]
},
{
"properties": {
"action": { "const": "concat" },
"config": {
"type": "object",
"properties": {
"inputPaths": { "type": "array", "items": { "type": "string" } },
"outputPath": { "type": "string" }
},
"required": ["inputPaths", "outputPath"]
}
},
"required": ["action", "config"]
},
{
"properties": {
"action": { "const": "info" },
"path": { "type": "string" }
},
"required": ["action", "path"]
},
{
"properties": {
"action": { "const": "check_ffmpeg" }
},
"required": ["action"]
}
]
})),
tags: vec!["video".to_string(), "media".to_string(), "editing".to_string()],
enabled: true,
},
ffmpeg_path: Arc::new(RwLock::new(None)),
}
}
/// Find FFmpeg executable
async fn find_ffmpeg(&self) -> Option<String> {
// Check cached path
{
let cached = self.ffmpeg_path.read().await;
if cached.is_some() {
return cached.clone();
}
}
// Try common locations
let candidates = if cfg!(windows) {
vec!["ffmpeg.exe", "C:\\ffmpeg\\bin\\ffmpeg.exe", "C:\\Program Files\\ffmpeg\\bin\\ffmpeg.exe"]
} else {
vec!["ffmpeg", "/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg"]
};
for candidate in candidates {
if Command::new(candidate).arg("-version").output().is_ok() {
let mut cached = self.ffmpeg_path.write().await;
*cached = Some(candidate.to_string());
return Some(candidate.to_string());
}
}
None
}
/// Execute trim operation
async fn execute_trim(&self, config: &TrimConfig) -> Result<Value> {
let ffmpeg = self.find_ffmpeg().await
.ok_or_else(|| zclaw_types::ZclawError::HandError("FFmpeg not found. Please install FFmpeg.".to_string()))?;
let mut args: Vec<String> = vec!["-i".to_string(), config.input_path.clone()];
// Add start time
if let Some(start) = config.start_time {
args.push("-ss".to_string());
args.push(start.to_string());
}
// Add duration or end time
if let Some(duration) = config.duration {
args.push("-t".to_string());
args.push(duration.to_string());
} else if let Some(end) = config.end_time {
if let Some(start) = config.start_time {
args.push("-t".to_string());
args.push((end - start).to_string());
} else {
args.push("-to".to_string());
args.push(end.to_string());
}
}
args.extend_from_slice(&["-c".to_string(), "copy".to_string(), config.output_path.clone()]);
let output = Command::new(&ffmpeg)
.args(&args)
.output()
.map_err(|e| zclaw_types::ZclawError::HandError(format!("FFmpeg execution failed: {}", e)))?;
if output.status.success() {
Ok(json!({
"success": true,
"output_path": config.output_path,
"message": "Video trimmed successfully"
}))
} else {
let stderr = String::from_utf8_lossy(&output.stderr);
Ok(json!({
"success": false,
"error": stderr,
"message": "Failed to trim video"
}))
}
}
/// Execute convert operation
async fn execute_convert(&self, config: &ConvertConfig) -> Result<Value> {
let ffmpeg = self.find_ffmpeg().await
.ok_or_else(|| zclaw_types::ZclawError::HandError("FFmpeg not found".to_string()))?;
let mut args: Vec<String> = vec!["-i".to_string(), config.input_path.clone()];
// Add resolution
if let Resolution::Custom { width, height } = config.resolution {
args.push("-vf".to_string());
args.push(format!("scale={}:{}", width, height));
} else {
let scale = match &config.resolution {
Resolution::P480 => "scale=854:480",
Resolution::P720 => "scale=1280:720",
Resolution::P1080 => "scale=1920:1080",
Resolution::P4k => "scale=3840:2160",
_ => "",
};
if !scale.is_empty() {
args.push("-vf".to_string());
args.push(scale.to_string());
}
}
// Add bitrates
if let Some(ref vbr) = config.video_bitrate {
args.push("-b:v".to_string());
args.push(vbr.clone());
}
if let Some(ref abr) = config.audio_bitrate {
args.push("-b:a".to_string());
args.push(abr.clone());
}
args.push(config.output_path.clone());
let output = Command::new(&ffmpeg)
.args(&args)
.output()
.map_err(|e| zclaw_types::ZclawError::HandError(format!("FFmpeg execution failed: {}", e)))?;
if output.status.success() {
Ok(json!({
"success": true,
"output_path": config.output_path,
"format": format!("{:?}", config.format),
"message": "Video converted successfully"
}))
} else {
let stderr = String::from_utf8_lossy(&output.stderr);
Ok(json!({
"success": false,
"error": stderr,
"message": "Failed to convert video"
}))
}
}
/// Execute thumbnail extraction
async fn execute_thumbnail(&self, config: &ThumbnailConfig) -> Result<Value> {
let ffmpeg = self.find_ffmpeg().await
.ok_or_else(|| zclaw_types::ZclawError::HandError("FFmpeg not found".to_string()))?;
let mut args: Vec<String> = vec![
"-i".to_string(), config.input_path.clone(),
"-ss".to_string(), config.time.to_string(),
"-vframes".to_string(), "1".to_string(),
];
// Add scale if dimensions specified
if let (Some(w), Some(h)) = (config.width, config.height) {
args.push("-vf".to_string());
args.push(format!("scale={}:{}", w, h));
}
args.push(config.output_path.clone());
let output = Command::new(&ffmpeg)
.args(&args)
.output()
.map_err(|e| zclaw_types::ZclawError::HandError(format!("FFmpeg execution failed: {}", e)))?;
if output.status.success() {
Ok(json!({
"success": true,
"output_path": config.output_path,
"time": config.time,
"message": "Thumbnail extracted successfully"
}))
} else {
let stderr = String::from_utf8_lossy(&output.stderr);
Ok(json!({
"success": false,
"error": stderr,
"message": "Failed to extract thumbnail"
}))
}
}
/// Execute video concatenation
async fn execute_concat(&self, config: &ConcatConfig) -> Result<Value> {
let ffmpeg = self.find_ffmpeg().await
.ok_or_else(|| zclaw_types::ZclawError::HandError("FFmpeg not found".to_string()))?;
// Create concat file
let concat_content: String = config.input_paths.iter()
.map(|p| format!("file '{}'", p))
.collect::<Vec<_>>()
.join("\n");
let temp_file = std::env::temp_dir().join("zclaw_concat.txt");
std::fs::write(&temp_file, &concat_content)
.map_err(|e| zclaw_types::ZclawError::HandError(format!("Failed to create concat file: {}", e)))?;
let args = vec![
"-f", "concat",
"-safe", "0",
"-i", temp_file.to_str().unwrap(),
"-c", "copy",
&config.output_path,
];
let output = Command::new(&ffmpeg)
.args(&args)
.output()
.map_err(|e| zclaw_types::ZclawError::HandError(format!("FFmpeg execution failed: {}", e)))?;
// Cleanup temp file
let _ = std::fs::remove_file(&temp_file);
if output.status.success() {
Ok(json!({
"success": true,
"output_path": config.output_path,
"videos_concatenated": config.input_paths.len(),
"message": "Videos concatenated successfully"
}))
} else {
let stderr = String::from_utf8_lossy(&output.stderr);
Ok(json!({
"success": false,
"error": stderr,
"message": "Failed to concatenate videos"
}))
}
}
/// Get video information
async fn execute_info(&self, path: &str) -> Result<Value> {
let ffprobe = {
let ffmpeg = self.find_ffmpeg().await
.ok_or_else(|| zclaw_types::ZclawError::HandError("FFmpeg not found".to_string()))?;
ffmpeg.replace("ffmpeg", "ffprobe")
};
let args = vec![
"-v", "quiet",
"-print_format", "json",
"-show_format",
"-show_streams",
path,
];
let output = Command::new(&ffprobe)
.args(&args)
.output()
.map_err(|e| zclaw_types::ZclawError::HandError(format!("FFprobe execution failed: {}", e)))?;
if output.status.success() {
let stdout = String::from_utf8_lossy(&output.stdout);
let info: Value = serde_json::from_str(&stdout)
.unwrap_or_else(|_| json!({"raw": stdout.to_string()}));
Ok(json!({
"success": true,
"path": path,
"info": info
}))
} else {
let stderr = String::from_utf8_lossy(&output.stderr);
Ok(json!({
"success": false,
"error": stderr,
"message": "Failed to get video info"
}))
}
}
/// Check FFmpeg availability
async fn check_ffmpeg(&self) -> Result<Value> {
match self.find_ffmpeg().await {
Some(path) => {
// Get version info
let output = Command::new(&path)
.arg("-version")
.output()
.ok();
let version = output.and_then(|o| {
let stdout = String::from_utf8_lossy(&o.stdout);
stdout.lines().next().map(|s| s.to_string())
}).unwrap_or_else(|| "Unknown version".to_string());
Ok(json!({
"available": true,
"path": path,
"version": version
}))
}
None => Ok(json!({
"available": false,
"message": "FFmpeg not found. Please install FFmpeg to use video processing features.",
"install_url": if cfg!(windows) {
"https://ffmpeg.org/download.html#build-windows"
} else if cfg!(target_os = "macos") {
"brew install ffmpeg"
} else {
"sudo apt install ffmpeg"
}
}))
}
}
}
impl Default for ClipHand {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl Hand for ClipHand {
fn config(&self) -> &HandConfig {
&self.config
}
async fn execute(&self, _context: &HandContext, input: Value) -> Result<HandResult> {
let action: ClipAction = serde_json::from_value(input.clone())
.map_err(|e| zclaw_types::ZclawError::HandError(format!("Invalid action: {}", e)))?;
let start = std::time::Instant::now();
let result = match action {
ClipAction::Trim { config } => self.execute_trim(&config).await?,
ClipAction::Convert { config } => self.execute_convert(&config).await?,
ClipAction::Resize { input_path, output_path, resolution } => {
let convert_config = ConvertConfig {
input_path,
output_path,
format: VideoFormat::Mp4,
resolution,
video_bitrate: None,
audio_bitrate: None,
};
self.execute_convert(&convert_config).await?
}
ClipAction::Thumbnail { config } => self.execute_thumbnail(&config).await?,
ClipAction::Concat { config } => self.execute_concat(&config).await?,
ClipAction::Info { path } => self.execute_info(&path).await?,
ClipAction::CheckFfmpeg => self.check_ffmpeg().await?,
};
let duration_ms = start.elapsed().as_millis() as u64;
Ok(HandResult {
success: result["success"].as_bool().unwrap_or(false),
output: result,
error: None,
duration_ms: Some(duration_ms),
status: "completed".to_string(),
})
}
fn needs_approval(&self) -> bool {
false
}
fn check_dependencies(&self) -> Result<Vec<String>> {
let mut missing = Vec::new();
// Check FFmpeg
if Command::new("ffmpeg").arg("-version").output().is_err() {
if Command::new("C:\\ffmpeg\\bin\\ffmpeg.exe").arg("-version").output().is_err() {
missing.push("FFmpeg not found. Install from https://ffmpeg.org/".to_string());
}
}
Ok(missing)
}
fn status(&self) -> crate::HandStatus {
// Check if FFmpeg is available
if Command::new("ffmpeg").arg("-version").output().is_ok() {
crate::HandStatus::Idle
} else if Command::new("C:\\ffmpeg\\bin\\ffmpeg.exe").arg("-version").output().is_ok() {
crate::HandStatus::Idle
} else {
crate::HandStatus::Failed
}
}
}