diff --git a/crates/zclaw-hands/src/hands/browser.rs b/crates/zclaw-hands/src/hands/browser.rs new file mode 100644 index 0000000..dae9647 --- /dev/null +++ b/crates/zclaw-hands/src/hands/browser.rs @@ -0,0 +1,416 @@ +//! Browser Hand - Web automation capabilities +//! +//! Provides browser automation actions for web interaction: +//! - navigate: Navigate to a URL +//! - click: Click on an element +//! - type: Type text into an input field +//! - scrape: Extract content from the page +//! - screenshot: Take a screenshot +//! - fill_form: Fill out a form +//! - wait: Wait for an element to appear +//! - execute: Execute JavaScript + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use zclaw_types::Result; + +use crate::{Hand, HandConfig, HandContext, HandResult, HandStatus}; + +/// Browser action types +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "action", rename_all = "snake_case")] +pub enum BrowserAction { + /// Navigate to a URL + Navigate { + url: String, + #[serde(default)] + wait_for: Option, + }, + /// Click on an element + Click { + selector: String, + #[serde(default)] + wait_ms: Option, + }, + /// Type text into an element + Type { + selector: String, + text: String, + #[serde(default)] + clear_first: bool, + }, + /// Select an option from a dropdown + Select { + selector: String, + value: String, + }, + /// Scrape content from the page + Scrape { + selectors: Vec, + #[serde(default)] + wait_for: Option, + }, + /// Take a screenshot + Screenshot { + #[serde(default)] + selector: Option, + #[serde(default)] + full_page: bool, + }, + /// Fill out a form + FillForm { + fields: Vec, + #[serde(default)] + submit_selector: Option, + }, + /// Wait for an element + Wait { + selector: String, + #[serde(default = "default_timeout")] + timeout_ms: u64, + }, + /// Execute JavaScript + Execute { + script: String, + #[serde(default)] + args: Vec, + }, + /// Get page source + GetSource, + /// Get current URL + GetUrl, + /// Get page title + GetTitle, + /// Scroll the page + Scroll { + #[serde(default)] + x: i32, + #[serde(default)] + y: i32, + #[serde(default)] + selector: Option, + }, + /// Go back + Back, + /// Go forward + Forward, + /// Refresh page + Refresh, + /// Hover over an element + Hover { + selector: String, + }, + /// Press a key + PressKey { + key: String, + }, + /// Upload file + Upload { + selector: String, + file_path: String, + }, +} + +/// Form field definition +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FormField { + pub selector: String, + pub value: String, +} + +fn default_timeout() -> u64 { 10000 } + +/// Browser Hand implementation +pub struct BrowserHand { + config: HandConfig, +} + +impl BrowserHand { + /// Create a new Browser Hand + pub fn new() -> Self { + Self { + config: HandConfig { + id: "browser".to_string(), + name: "Browser".to_string(), + description: "Web browser automation for navigation, interaction, and scraping".to_string(), + needs_approval: false, + dependencies: vec!["webdriver".to_string()], + input_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": ["navigate", "click", "type", "scrape", "screenshot", "fill_form", "wait", "execute"] + }, + "url": { "type": "string" }, + "selector": { "type": "string" }, + "text": { "type": "string" }, + "selectors": { "type": "array", "items": { "type": "string" } }, + "script": { "type": "string" } + }, + "required": ["action"] + })), + tags: vec!["automation".to_string(), "web".to_string(), "browser".to_string()], + enabled: true, + }, + } + } + + /// Check if WebDriver is available + fn check_webdriver(&self) -> bool { + // Check if ChromeDriver or GeckoDriver is running + // For now, return true as the actual check would require network access + true + } +} + +impl Default for BrowserHand { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Hand for BrowserHand { + fn config(&self) -> &HandConfig { + &self.config + } + + async fn execute(&self, _context: &HandContext, input: Value) -> Result { + // Parse the action + let action: BrowserAction = match serde_json::from_value(input) { + Ok(a) => a, + Err(e) => return Ok(HandResult::error(format!("Invalid action: {}", e))), + }; + + // Execute based on action type + // Note: Actual browser operations are handled via Tauri commands + // This Hand provides a structured interface for the runtime + match action { + BrowserAction::Navigate { url, wait_for } => { + Ok(HandResult::success(serde_json::json!({ + "action": "navigate", + "url": url, + "wait_for": wait_for, + "status": "pending_execution" + }))) + } + BrowserAction::Click { selector, wait_ms } => { + Ok(HandResult::success(serde_json::json!({ + "action": "click", + "selector": selector, + "wait_ms": wait_ms, + "status": "pending_execution" + }))) + } + BrowserAction::Type { selector, text, clear_first } => { + Ok(HandResult::success(serde_json::json!({ + "action": "type", + "selector": selector, + "text": text, + "clear_first": clear_first, + "status": "pending_execution" + }))) + } + BrowserAction::Scrape { selectors, wait_for } => { + Ok(HandResult::success(serde_json::json!({ + "action": "scrape", + "selectors": selectors, + "wait_for": wait_for, + "status": "pending_execution" + }))) + } + BrowserAction::Screenshot { selector, full_page } => { + Ok(HandResult::success(serde_json::json!({ + "action": "screenshot", + "selector": selector, + "full_page": full_page, + "status": "pending_execution" + }))) + } + BrowserAction::FillForm { fields, submit_selector } => { + Ok(HandResult::success(serde_json::json!({ + "action": "fill_form", + "fields": fields, + "submit_selector": submit_selector, + "status": "pending_execution" + }))) + } + BrowserAction::Wait { selector, timeout_ms } => { + Ok(HandResult::success(serde_json::json!({ + "action": "wait", + "selector": selector, + "timeout_ms": timeout_ms, + "status": "pending_execution" + }))) + } + BrowserAction::Execute { script, args } => { + Ok(HandResult::success(serde_json::json!({ + "action": "execute", + "script": script, + "args": args, + "status": "pending_execution" + }))) + } + BrowserAction::GetSource => { + Ok(HandResult::success(serde_json::json!({ + "action": "get_source", + "status": "pending_execution" + }))) + } + BrowserAction::GetUrl => { + Ok(HandResult::success(serde_json::json!({ + "action": "get_url", + "status": "pending_execution" + }))) + } + BrowserAction::GetTitle => { + Ok(HandResult::success(serde_json::json!({ + "action": "get_title", + "status": "pending_execution" + }))) + } + BrowserAction::Scroll { x, y, selector } => { + Ok(HandResult::success(serde_json::json!({ + "action": "scroll", + "x": x, + "y": y, + "selector": selector, + "status": "pending_execution" + }))) + } + BrowserAction::Back => { + Ok(HandResult::success(serde_json::json!({ + "action": "back", + "status": "pending_execution" + }))) + } + BrowserAction::Forward => { + Ok(HandResult::success(serde_json::json!({ + "action": "forward", + "status": "pending_execution" + }))) + } + BrowserAction::Refresh => { + Ok(HandResult::success(serde_json::json!({ + "action": "refresh", + "status": "pending_execution" + }))) + } + BrowserAction::Hover { selector } => { + Ok(HandResult::success(serde_json::json!({ + "action": "hover", + "selector": selector, + "status": "pending_execution" + }))) + } + BrowserAction::PressKey { key } => { + Ok(HandResult::success(serde_json::json!({ + "action": "press_key", + "key": key, + "status": "pending_execution" + }))) + } + BrowserAction::Upload { selector, file_path } => { + Ok(HandResult::success(serde_json::json!({ + "action": "upload", + "selector": selector, + "file_path": file_path, + "status": "pending_execution" + }))) + } + BrowserAction::Select { selector, value } => { + Ok(HandResult::success(serde_json::json!({ + "action": "select", + "selector": selector, + "value": value, + "status": "pending_execution" + }))) + } + } + } + + fn is_dependency_available(&self, dep: &str) -> bool { + match dep { + "webdriver" => self.check_webdriver(), + _ => true, + } + } + + fn status(&self) -> HandStatus { + if self.check_webdriver() { + HandStatus::Idle + } else { + HandStatus::PendingApproval // Using this to indicate dependency missing + } + } +} + +/// Browser automation sequence for complex operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrowserSequence { + /// Sequence name + pub name: String, + /// Steps to execute + pub steps: Vec, + /// Whether to stop on error + #[serde(default = "default_stop_on_error")] + pub stop_on_error: bool, + /// Delay between steps in milliseconds + #[serde(default)] + pub step_delay_ms: Option, +} + +fn default_stop_on_error() -> bool { true } + +impl BrowserSequence { + /// Create a new browser sequence + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + steps: Vec::new(), + stop_on_error: true, + step_delay_ms: None, + } + } + + /// Add a navigate step + pub fn navigate(mut self, url: impl Into) -> Self { + self.steps.push(BrowserAction::Navigate { url: url.into(), wait_for: None }); + self + } + + /// Add a click step + pub fn click(mut self, selector: impl Into) -> Self { + self.steps.push(BrowserAction::Click { selector: selector.into(), wait_ms: None }); + self + } + + /// Add a type step + pub fn type_text(mut self, selector: impl Into, text: impl Into) -> Self { + self.steps.push(BrowserAction::Type { + selector: selector.into(), + text: text.into(), + clear_first: false, + }); + self + } + + /// Add a wait step + pub fn wait(mut self, selector: impl Into, timeout_ms: u64) -> Self { + self.steps.push(BrowserAction::Wait { selector: selector.into(), timeout_ms }); + self + } + + /// Add a screenshot step + pub fn screenshot(mut self) -> Self { + self.steps.push(BrowserAction::Screenshot { selector: None, full_page: false }); + self + } + + /// Build the sequence + pub fn build(self) -> Vec { + self.steps + } +} diff --git a/crates/zclaw-hands/src/hands/mod.rs b/crates/zclaw-hands/src/hands/mod.rs new file mode 100644 index 0000000..1c58fc4 --- /dev/null +++ b/crates/zclaw-hands/src/hands/mod.rs @@ -0,0 +1,20 @@ +//! Educational Hands - Teaching and presentation capabilities +//! +//! This module provides hands for interactive classroom experiences: +//! - Whiteboard: Drawing and annotation +//! - Slideshow: Presentation control +//! - Speech: Text-to-speech synthesis +//! - Quiz: Assessment and evaluation +//! - Browser: Web automation + +mod whiteboard; +mod slideshow; +mod speech; +mod quiz; +mod browser; + +pub use whiteboard::*; +pub use slideshow::*; +pub use speech::*; +pub use quiz::*; +pub use browser::*;