813b49a986
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
P0 KernelClient 功能断裂修复: - Skill CUD: registry.rs create/update/delete + serialize_skill_md + kernel proxy - Workflow CUD: pipeline_commands.rs create/update/delete + serde_yaml依赖 - Agent更新: registry update方法 + AgentConfigUpdated事件 + agent_update命令 - Hand流式事件: HandStart/HandEnd变体替换ToolStart/ToolEnd - 后端验证: hand_get/hand_run_status/hand_run_list确认实现完整 - Approval闭环: respond_to_approval后台spawn+5分钟超时轮询 P2/P3 质量改进: - Browser WebDriver: TCP探测ChromeDriver/GeckoDriver/Edge端口替换硬编码true - api-fallbacks: 移除假技能和16个捏造安全层，替换为真实能力映射 - dead_code清理: 移除5个模块级#![allow(dead_code)]，删除3个真正死方法， 删除未注册的compactor_compact_llm命令，warnings从8降到3 - 所有变更通过cargo check + tsc --noEmit验证
428 lines
13 KiB
Rust
428 lines
13 KiB
Rust
//! Browser Hand - Web automation capabilities
|
|
//!
|
|
//! Provides browser automation actions for web interaction:
|
|
//! - navigate: Navigate to a URL
|
|
//! - click: Click on an element
|
|
//! - type: Type text into an input field
|
|
//! - scrape: Extract content from the page
|
|
//! - screenshot: Take a screenshot
|
|
//! - fill_form: Fill out a form
|
|
//! - wait: Wait for an element to appear
|
|
//! - execute: Execute JavaScript
|
|
|
|
use async_trait::async_trait;
|
|
use serde::{Deserialize, Serialize};
|
|
use serde_json::Value;
|
|
use zclaw_types::Result;
|
|
|
|
use crate::{Hand, HandConfig, HandContext, HandResult, HandStatus};
|
|
|
|
/// Browser action types
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(tag = "action", rename_all = "snake_case")]
|
|
pub enum BrowserAction {
|
|
/// Navigate to a URL
|
|
Navigate {
|
|
url: String,
|
|
#[serde(default)]
|
|
wait_for: Option<String>,
|
|
},
|
|
/// Click on an element
|
|
Click {
|
|
selector: String,
|
|
#[serde(default)]
|
|
wait_ms: Option<u64>,
|
|
},
|
|
/// Type text into an element
|
|
Type {
|
|
selector: String,
|
|
text: String,
|
|
#[serde(default)]
|
|
clear_first: bool,
|
|
},
|
|
/// Select an option from a dropdown
|
|
Select {
|
|
selector: String,
|
|
value: String,
|
|
},
|
|
/// Scrape content from the page
|
|
Scrape {
|
|
selectors: Vec<String>,
|
|
#[serde(default)]
|
|
wait_for: Option<String>,
|
|
},
|
|
/// Take a screenshot
|
|
Screenshot {
|
|
#[serde(default)]
|
|
selector: Option<String>,
|
|
#[serde(default)]
|
|
full_page: bool,
|
|
},
|
|
/// Fill out a form
|
|
FillForm {
|
|
fields: Vec<FormField>,
|
|
#[serde(default)]
|
|
submit_selector: Option<String>,
|
|
},
|
|
/// Wait for an element
|
|
Wait {
|
|
selector: String,
|
|
#[serde(default = "default_timeout")]
|
|
timeout_ms: u64,
|
|
},
|
|
/// Execute JavaScript
|
|
Execute {
|
|
script: String,
|
|
#[serde(default)]
|
|
args: Vec<Value>,
|
|
},
|
|
/// Get page source
|
|
GetSource,
|
|
/// Get current URL
|
|
GetUrl,
|
|
/// Get page title
|
|
GetTitle,
|
|
/// Scroll the page
|
|
Scroll {
|
|
#[serde(default)]
|
|
x: i32,
|
|
#[serde(default)]
|
|
y: i32,
|
|
#[serde(default)]
|
|
selector: Option<String>,
|
|
},
|
|
/// Go back
|
|
Back,
|
|
/// Go forward
|
|
Forward,
|
|
/// Refresh page
|
|
Refresh,
|
|
/// Hover over an element
|
|
Hover {
|
|
selector: String,
|
|
},
|
|
/// Press a key
|
|
PressKey {
|
|
key: String,
|
|
},
|
|
/// Upload file
|
|
Upload {
|
|
selector: String,
|
|
file_path: String,
|
|
},
|
|
}
|
|
|
|
/// Form field definition
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct FormField {
|
|
pub selector: String,
|
|
pub value: String,
|
|
}
|
|
|
|
fn default_timeout() -> u64 { 10000 }
|
|
|
|
/// Browser Hand implementation
|
|
pub struct BrowserHand {
|
|
config: HandConfig,
|
|
}
|
|
|
|
impl BrowserHand {
|
|
/// Create a new Browser Hand
|
|
pub fn new() -> Self {
|
|
Self {
|
|
config: HandConfig {
|
|
id: "browser".to_string(),
|
|
name: "浏览器".to_string(),
|
|
description: "网页浏览器自动化，支持导航、交互和数据采集".to_string(),
|
|
needs_approval: false,
|
|
dependencies: vec!["webdriver".to_string()],
|
|
input_schema: Some(serde_json::json!({
|
|
"type": "object",
|
|
"properties": {
|
|
"action": {
|
|
"type": "string",
|
|
"enum": ["navigate", "click", "type", "scrape", "screenshot", "fill_form", "wait", "execute"]
|
|
},
|
|
"url": { "type": "string" },
|
|
"selector": { "type": "string" },
|
|
"text": { "type": "string" },
|
|
"selectors": { "type": "array", "items": { "type": "string" } },
|
|
"script": { "type": "string" }
|
|
},
|
|
"required": ["action"]
|
|
})),
|
|
tags: vec!["automation".to_string(), "web".to_string(), "browser".to_string()],
|
|
enabled: true,
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Check if WebDriver is available by probing common ports
|
|
fn check_webdriver(&self) -> bool {
|
|
use std::net::TcpStream;
|
|
use std::time::Duration;
|
|
|
|
// Probe default WebDriver ports: ChromeDriver (9515), GeckoDriver (4444), Edge (17556)
|
|
let ports = [9515, 4444, 17556];
|
|
for port in ports {
|
|
let addr = format!("127.0.0.1:{}", port);
|
|
if let Ok(addr) = addr.parse() {
|
|
if TcpStream::connect_timeout(&addr, Duration::from_millis(500)).is_ok() {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
false
|
|
}
|
|
}
|
|
|
|
impl Default for BrowserHand {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl Hand for BrowserHand {
|
|
fn config(&self) -> &HandConfig {
|
|
&self.config
|
|
}
|
|
|
|
async fn execute(&self, _context: &HandContext, input: Value) -> Result<HandResult> {
|
|
// Parse the action
|
|
let action: BrowserAction = match serde_json::from_value(input) {
|
|
Ok(a) => a,
|
|
Err(e) => return Ok(HandResult::error(format!("Invalid action: {}", e))),
|
|
};
|
|
|
|
// Execute based on action type
|
|
// Note: Actual browser operations are handled via Tauri commands
|
|
// This Hand provides a structured interface for the runtime
|
|
match action {
|
|
BrowserAction::Navigate { url, wait_for } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "navigate",
|
|
"url": url,
|
|
"wait_for": wait_for,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Click { selector, wait_ms } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "click",
|
|
"selector": selector,
|
|
"wait_ms": wait_ms,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Type { selector, text, clear_first } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "type",
|
|
"selector": selector,
|
|
"text": text,
|
|
"clear_first": clear_first,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Scrape { selectors, wait_for } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "scrape",
|
|
"selectors": selectors,
|
|
"wait_for": wait_for,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Screenshot { selector, full_page } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "screenshot",
|
|
"selector": selector,
|
|
"full_page": full_page,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::FillForm { fields, submit_selector } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "fill_form",
|
|
"fields": fields,
|
|
"submit_selector": submit_selector,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Wait { selector, timeout_ms } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "wait",
|
|
"selector": selector,
|
|
"timeout_ms": timeout_ms,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Execute { script, args } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "execute",
|
|
"script": script,
|
|
"args": args,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::GetSource => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "get_source",
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::GetUrl => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "get_url",
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::GetTitle => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "get_title",
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Scroll { x, y, selector } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "scroll",
|
|
"x": x,
|
|
"y": y,
|
|
"selector": selector,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Back => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "back",
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Forward => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "forward",
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Refresh => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "refresh",
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Hover { selector } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "hover",
|
|
"selector": selector,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::PressKey { key } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "press_key",
|
|
"key": key,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Upload { selector, file_path } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "upload",
|
|
"selector": selector,
|
|
"file_path": file_path,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
BrowserAction::Select { selector, value } => {
|
|
Ok(HandResult::success(serde_json::json!({
|
|
"action": "select",
|
|
"selector": selector,
|
|
"value": value,
|
|
"status": "pending_execution"
|
|
})))
|
|
}
|
|
}
|
|
}
|
|
|
|
fn is_dependency_available(&self, dep: &str) -> bool {
|
|
match dep {
|
|
"webdriver" => self.check_webdriver(),
|
|
_ => true,
|
|
}
|
|
}
|
|
|
|
fn status(&self) -> HandStatus {
|
|
if self.check_webdriver() {
|
|
HandStatus::Idle
|
|
} else {
|
|
HandStatus::PendingApproval // Using this to indicate dependency missing
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Browser automation sequence for complex operations
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BrowserSequence {
|
|
/// Sequence name
|
|
pub name: String,
|
|
/// Steps to execute
|
|
pub steps: Vec<BrowserAction>,
|
|
/// Whether to stop on error
|
|
#[serde(default = "default_stop_on_error")]
|
|
pub stop_on_error: bool,
|
|
/// Delay between steps in milliseconds
|
|
#[serde(default)]
|
|
pub step_delay_ms: Option<u64>,
|
|
}
|
|
|
|
fn default_stop_on_error() -> bool { true }
|
|
|
|
impl BrowserSequence {
|
|
/// Create a new browser sequence
|
|
pub fn new(name: impl Into<String>) -> Self {
|
|
Self {
|
|
name: name.into(),
|
|
steps: Vec::new(),
|
|
stop_on_error: true,
|
|
step_delay_ms: None,
|
|
}
|
|
}
|
|
|
|
/// Add a navigate step
|
|
pub fn navigate(mut self, url: impl Into<String>) -> Self {
|
|
self.steps.push(BrowserAction::Navigate { url: url.into(), wait_for: None });
|
|
self
|
|
}
|
|
|
|
/// Add a click step
|
|
pub fn click(mut self, selector: impl Into<String>) -> Self {
|
|
self.steps.push(BrowserAction::Click { selector: selector.into(), wait_ms: None });
|
|
self
|
|
}
|
|
|
|
/// Add a type step
|
|
pub fn type_text(mut self, selector: impl Into<String>, text: impl Into<String>) -> Self {
|
|
self.steps.push(BrowserAction::Type {
|
|
selector: selector.into(),
|
|
text: text.into(),
|
|
clear_first: false,
|
|
});
|
|
self
|
|
}
|
|
|
|
/// Add a wait step
|
|
pub fn wait(mut self, selector: impl Into<String>, timeout_ms: u64) -> Self {
|
|
self.steps.push(BrowserAction::Wait { selector: selector.into(), timeout_ms });
|
|
self
|
|
}
|
|
|
|
/// Add a screenshot step
|
|
pub fn screenshot(mut self) -> Self {
|
|
self.steps.push(BrowserAction::Screenshot { selector: None, full_page: false });
|
|
self
|
|
}
|
|
|
|
/// Build the sequence
|
|
pub fn build(self) -> Vec<BrowserAction> {
|
|
self.steps
|
|
}
|
|
}
|