//! Browser Hand - Web automation capabilities //! //! Provides browser automation actions for web interaction: //! - navigate: Navigate to a URL //! - click: Click on an element //! - type: Type text into an input field //! - scrape: Extract content from the page //! - screenshot: Take a screenshot //! - fill_form: Fill out a form //! - wait: Wait for an element to appear //! - execute: Execute JavaScript use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::Value; use zclaw_types::Result; use crate::{Hand, HandConfig, HandContext, HandResult, HandStatus}; /// Browser action types #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "action", rename_all = "snake_case")] pub enum BrowserAction { /// Navigate to a URL Navigate { url: String, #[serde(default)] wait_for: Option, }, /// Click on an element Click { selector: String, #[serde(default)] wait_ms: Option, }, /// Type text into an element Type { selector: String, text: String, #[serde(default)] clear_first: bool, }, /// Select an option from a dropdown Select { selector: String, value: String, }, /// Scrape content from the page Scrape { selectors: Vec, #[serde(default)] wait_for: Option, }, /// Take a screenshot Screenshot { #[serde(default)] selector: Option, #[serde(default)] full_page: bool, }, /// Fill out a form FillForm { fields: Vec, #[serde(default)] submit_selector: Option, }, /// Wait for an element Wait { selector: String, #[serde(default = "default_timeout")] timeout_ms: u64, }, /// Execute JavaScript Execute { script: String, #[serde(default)] args: Vec, }, /// Get page source GetSource, /// Get current URL GetUrl, /// Get page title GetTitle, /// Scroll the page Scroll { #[serde(default)] x: i32, #[serde(default)] y: i32, #[serde(default)] selector: Option, }, /// Go back Back, /// Go forward Forward, /// Refresh page Refresh, /// Hover over an element Hover { selector: String, }, /// Press a key PressKey { key: String, }, /// Upload file Upload { selector: String, file_path: String, }, } /// Form field definition #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FormField { pub selector: String, pub value: String, } fn default_timeout() -> u64 { 10000 } /// Browser Hand implementation pub struct BrowserHand { config: HandConfig, } impl BrowserHand { /// Create a new Browser Hand pub fn new() -> Self { Self { config: HandConfig { id: "browser".to_string(), name: "浏览器".to_string(), description: "网页浏览器自动化,支持导航、交互和数据采集".to_string(), needs_approval: false, dependencies: vec!["webdriver".to_string()], input_schema: Some(serde_json::json!({ "type": "object", "properties": { "action": { "type": "string", "enum": ["navigate", "click", "type", "scrape", "screenshot", "fill_form", "wait", "execute"] }, "url": { "type": "string" }, "selector": { "type": "string" }, "text": { "type": "string" }, "selectors": { "type": "array", "items": { "type": "string" } }, "script": { "type": "string" } }, "required": ["action"] })), tags: vec!["automation".to_string(), "web".to_string(), "browser".to_string()], enabled: true, }, } } /// Check if WebDriver is available fn check_webdriver(&self) -> bool { // Check if ChromeDriver or GeckoDriver is running // For now, return true as the actual check would require network access true } } impl Default for BrowserHand { fn default() -> Self { Self::new() } } #[async_trait] impl Hand for BrowserHand { fn config(&self) -> &HandConfig { &self.config } async fn execute(&self, _context: &HandContext, input: Value) -> Result { // Parse the action let action: BrowserAction = match serde_json::from_value(input) { Ok(a) => a, Err(e) => return Ok(HandResult::error(format!("Invalid action: {}", e))), }; // Execute based on action type // Note: Actual browser operations are handled via Tauri commands // This Hand provides a structured interface for the runtime match action { BrowserAction::Navigate { url, wait_for } => { Ok(HandResult::success(serde_json::json!({ "action": "navigate", "url": url, "wait_for": wait_for, "status": "pending_execution" }))) } BrowserAction::Click { selector, wait_ms } => { Ok(HandResult::success(serde_json::json!({ "action": "click", "selector": selector, "wait_ms": wait_ms, "status": "pending_execution" }))) } BrowserAction::Type { selector, text, clear_first } => { Ok(HandResult::success(serde_json::json!({ "action": "type", "selector": selector, "text": text, "clear_first": clear_first, "status": "pending_execution" }))) } BrowserAction::Scrape { selectors, wait_for } => { Ok(HandResult::success(serde_json::json!({ "action": "scrape", "selectors": selectors, "wait_for": wait_for, "status": "pending_execution" }))) } BrowserAction::Screenshot { selector, full_page } => { Ok(HandResult::success(serde_json::json!({ "action": "screenshot", "selector": selector, "full_page": full_page, "status": "pending_execution" }))) } BrowserAction::FillForm { fields, submit_selector } => { Ok(HandResult::success(serde_json::json!({ "action": "fill_form", "fields": fields, "submit_selector": submit_selector, "status": "pending_execution" }))) } BrowserAction::Wait { selector, timeout_ms } => { Ok(HandResult::success(serde_json::json!({ "action": "wait", "selector": selector, "timeout_ms": timeout_ms, "status": "pending_execution" }))) } BrowserAction::Execute { script, args } => { Ok(HandResult::success(serde_json::json!({ "action": "execute", "script": script, "args": args, "status": "pending_execution" }))) } BrowserAction::GetSource => { Ok(HandResult::success(serde_json::json!({ "action": "get_source", "status": "pending_execution" }))) } BrowserAction::GetUrl => { Ok(HandResult::success(serde_json::json!({ "action": "get_url", "status": "pending_execution" }))) } BrowserAction::GetTitle => { Ok(HandResult::success(serde_json::json!({ "action": "get_title", "status": "pending_execution" }))) } BrowserAction::Scroll { x, y, selector } => { Ok(HandResult::success(serde_json::json!({ "action": "scroll", "x": x, "y": y, "selector": selector, "status": "pending_execution" }))) } BrowserAction::Back => { Ok(HandResult::success(serde_json::json!({ "action": "back", "status": "pending_execution" }))) } BrowserAction::Forward => { Ok(HandResult::success(serde_json::json!({ "action": "forward", "status": "pending_execution" }))) } BrowserAction::Refresh => { Ok(HandResult::success(serde_json::json!({ "action": "refresh", "status": "pending_execution" }))) } BrowserAction::Hover { selector } => { Ok(HandResult::success(serde_json::json!({ "action": "hover", "selector": selector, "status": "pending_execution" }))) } BrowserAction::PressKey { key } => { Ok(HandResult::success(serde_json::json!({ "action": "press_key", "key": key, "status": "pending_execution" }))) } BrowserAction::Upload { selector, file_path } => { Ok(HandResult::success(serde_json::json!({ "action": "upload", "selector": selector, "file_path": file_path, "status": "pending_execution" }))) } BrowserAction::Select { selector, value } => { Ok(HandResult::success(serde_json::json!({ "action": "select", "selector": selector, "value": value, "status": "pending_execution" }))) } } } fn is_dependency_available(&self, dep: &str) -> bool { match dep { "webdriver" => self.check_webdriver(), _ => true, } } fn status(&self) -> HandStatus { if self.check_webdriver() { HandStatus::Idle } else { HandStatus::PendingApproval // Using this to indicate dependency missing } } } /// Browser automation sequence for complex operations #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BrowserSequence { /// Sequence name pub name: String, /// Steps to execute pub steps: Vec, /// Whether to stop on error #[serde(default = "default_stop_on_error")] pub stop_on_error: bool, /// Delay between steps in milliseconds #[serde(default)] pub step_delay_ms: Option, } fn default_stop_on_error() -> bool { true } impl BrowserSequence { /// Create a new browser sequence pub fn new(name: impl Into) -> Self { Self { name: name.into(), steps: Vec::new(), stop_on_error: true, step_delay_ms: None, } } /// Add a navigate step pub fn navigate(mut self, url: impl Into) -> Self { self.steps.push(BrowserAction::Navigate { url: url.into(), wait_for: None }); self } /// Add a click step pub fn click(mut self, selector: impl Into) -> Self { self.steps.push(BrowserAction::Click { selector: selector.into(), wait_ms: None }); self } /// Add a type step pub fn type_text(mut self, selector: impl Into, text: impl Into) -> Self { self.steps.push(BrowserAction::Type { selector: selector.into(), text: text.into(), clear_first: false, }); self } /// Add a wait step pub fn wait(mut self, selector: impl Into, timeout_ms: u64) -> Self { self.steps.push(BrowserAction::Wait { selector: selector.into(), timeout_ms }); self } /// Add a screenshot step pub fn screenshot(mut self) -> Self { self.steps.push(BrowserAction::Screenshot { selector: None, full_page: false }); self } /// Build the sequence pub fn build(self) -> Vec { self.steps } }