fix(browser): stability enhancements + MCP frontend client
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
S7 Browser Hand: - Remove dead code: browser/actions.rs (314 lines of unused BrowserAction/ActionResult types) - Fix browser_scrape_page: log failed selector matches instead of silently swallowing errors - Fix element_to_info: document known limitation for always-None location/size fields - Fix browserHandStore: reuse activeSessionId in executeScript/takeScreenshot/executeTemplate instead of creating orphan Browser sessions - Add Browser.connect(sessionId) method for session reuse MCP Frontend: - Add desktop/src/lib/mcp-client.ts (77 lines) — typed client for MCP Tauri commands (startMcpService, stopMcpService, listMcpServices, callMcpTool) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -1329,6 +1329,7 @@ dependencies = [
|
|||||||
"zclaw-kernel",
|
"zclaw-kernel",
|
||||||
"zclaw-memory",
|
"zclaw-memory",
|
||||||
"zclaw-pipeline",
|
"zclaw-pipeline",
|
||||||
|
"zclaw-protocols",
|
||||||
"zclaw-runtime",
|
"zclaw-runtime",
|
||||||
"zclaw-skills",
|
"zclaw-skills",
|
||||||
"zclaw-types",
|
"zclaw-types",
|
||||||
|
|||||||
@@ -1,313 +1,5 @@
|
|||||||
// Browser action definitions for Hands system
|
// Removed dead code (was 314 lines of unused BrowserAction/ActionResult/ElementInfo
|
||||||
// Note: These types are reserved for future Browser Hand automation features
|
// types guarded by #![allow(dead_code)]). The types defined here duplicated structs
|
||||||
|
// already present in client.rs and were never consumed by any other module.
|
||||||
#![allow(dead_code)]
|
// If browser action types are needed in the future, use the types from client.rs
|
||||||
|
// and session.rs directly.
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
/// Browser action types
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
#[serde(tag = "type", rename_all = "snake_case")]
|
|
||||||
pub enum BrowserAction {
|
|
||||||
/// Create a new browser session
|
|
||||||
CreateSession {
|
|
||||||
webdriver_url: Option<String>,
|
|
||||||
headless: Option<bool>,
|
|
||||||
browser_type: Option<String>,
|
|
||||||
window_size: Option<(u32, u32)>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Close browser session
|
|
||||||
CloseSession {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Navigate to URL
|
|
||||||
Navigate {
|
|
||||||
session_id: String,
|
|
||||||
url: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Go back
|
|
||||||
Back {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Go forward
|
|
||||||
Forward {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Refresh page
|
|
||||||
Refresh {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Click element
|
|
||||||
Click {
|
|
||||||
session_id: String,
|
|
||||||
selector: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Type text
|
|
||||||
Type {
|
|
||||||
session_id: String,
|
|
||||||
selector: String,
|
|
||||||
text: String,
|
|
||||||
clear_first: Option<bool>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Get element text
|
|
||||||
GetText {
|
|
||||||
session_id: String,
|
|
||||||
selector: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Get element attribute
|
|
||||||
GetAttribute {
|
|
||||||
session_id: String,
|
|
||||||
selector: String,
|
|
||||||
attribute: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Find element
|
|
||||||
FindElement {
|
|
||||||
session_id: String,
|
|
||||||
selector: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Find multiple elements
|
|
||||||
FindElements {
|
|
||||||
session_id: String,
|
|
||||||
selector: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Execute JavaScript
|
|
||||||
ExecuteScript {
|
|
||||||
session_id: String,
|
|
||||||
script: String,
|
|
||||||
args: Option<Vec<serde_json::Value>>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Take screenshot
|
|
||||||
Screenshot {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Take element screenshot
|
|
||||||
ElementScreenshot {
|
|
||||||
session_id: String,
|
|
||||||
selector: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Wait for element
|
|
||||||
WaitForElement {
|
|
||||||
session_id: String,
|
|
||||||
selector: String,
|
|
||||||
timeout_ms: Option<u64>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Get page source
|
|
||||||
GetSource {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Get current URL
|
|
||||||
GetCurrentUrl {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Get page title
|
|
||||||
GetTitle {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// List all sessions
|
|
||||||
ListSessions,
|
|
||||||
|
|
||||||
/// Get session info
|
|
||||||
GetSession {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Action execution result
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
#[serde(tag = "type", rename_all = "snake_case")]
|
|
||||||
pub enum ActionResult {
|
|
||||||
/// Session created
|
|
||||||
SessionCreated {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Session closed
|
|
||||||
SessionClosed {
|
|
||||||
session_id: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Navigation result
|
|
||||||
Navigated {
|
|
||||||
url: Option<String>,
|
|
||||||
title: Option<String>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Element clicked
|
|
||||||
Clicked {
|
|
||||||
selector: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Text typed
|
|
||||||
Typed {
|
|
||||||
selector: String,
|
|
||||||
text: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Text retrieved
|
|
||||||
TextRetrieved {
|
|
||||||
selector: String,
|
|
||||||
text: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Attribute retrieved
|
|
||||||
AttributeRetrieved {
|
|
||||||
selector: String,
|
|
||||||
attribute: String,
|
|
||||||
value: Option<String>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Element found
|
|
||||||
ElementFound {
|
|
||||||
element: ElementInfo,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Elements found
|
|
||||||
ElementsFound {
|
|
||||||
elements: Vec<ElementInfo>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Script executed
|
|
||||||
ScriptExecuted {
|
|
||||||
result: serde_json::Value,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Screenshot taken
|
|
||||||
ScreenshotTaken {
|
|
||||||
base64: String,
|
|
||||||
format: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Page source retrieved
|
|
||||||
SourceRetrieved {
|
|
||||||
source: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// URL retrieved
|
|
||||||
UrlRetrieved {
|
|
||||||
url: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Title retrieved
|
|
||||||
TitleRetrieved {
|
|
||||||
title: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Sessions listed
|
|
||||||
SessionsListed {
|
|
||||||
sessions: Vec<SessionInfo>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Session info retrieved
|
|
||||||
SessionInfo {
|
|
||||||
session: SessionInfo,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Operation completed (no specific data)
|
|
||||||
Completed,
|
|
||||||
|
|
||||||
/// Error occurred
|
|
||||||
Error {
|
|
||||||
message: String,
|
|
||||||
code: String,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct ElementInfo {
|
|
||||||
pub selector: String,
|
|
||||||
pub tag_name: Option<String>,
|
|
||||||
pub text: Option<String>,
|
|
||||||
pub is_displayed: bool,
|
|
||||||
pub is_enabled: bool,
|
|
||||||
pub is_selected: bool,
|
|
||||||
pub location: Option<ElementLocation>,
|
|
||||||
pub size: Option<ElementSize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct ElementLocation {
|
|
||||||
pub x: i32,
|
|
||||||
pub y: i32,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct ElementSize {
|
|
||||||
pub width: u64,
|
|
||||||
pub height: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct SessionInfo {
|
|
||||||
pub id: String,
|
|
||||||
pub name: String,
|
|
||||||
pub current_url: Option<String>,
|
|
||||||
pub title: Option<String>,
|
|
||||||
pub status: String,
|
|
||||||
pub created_at: String,
|
|
||||||
pub last_activity: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// High-level browser task (for Hand integration)
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
#[serde(tag = "task", rename_all = "snake_case")]
|
|
||||||
pub enum BrowserTask {
|
|
||||||
/// Scrape page content
|
|
||||||
ScrapePage {
|
|
||||||
url: String,
|
|
||||||
selectors: Vec<String>,
|
|
||||||
wait_for: Option<String>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Fill form
|
|
||||||
FillForm {
|
|
||||||
url: String,
|
|
||||||
fields: Vec<FormField>,
|
|
||||||
submit_selector: Option<String>,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Take page snapshot
|
|
||||||
PageSnapshot {
|
|
||||||
url: String,
|
|
||||||
include_screenshot: bool,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Navigate and extract
|
|
||||||
NavigateAndExtract {
|
|
||||||
url: String,
|
|
||||||
extraction_script: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Multi-page scraping
|
|
||||||
MultiPageScrape {
|
|
||||||
start_url: String,
|
|
||||||
next_page_selector: String,
|
|
||||||
item_selector: String,
|
|
||||||
max_pages: Option<u32>,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct FormField {
|
|
||||||
pub selector: String,
|
|
||||||
pub value: String,
|
|
||||||
pub field_type: Option<String>,
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -407,8 +407,15 @@ impl BrowserClient {
|
|||||||
let is_displayed = element.is_displayed().await.unwrap_or(false);
|
let is_displayed = element.is_displayed().await.unwrap_or(false);
|
||||||
let is_enabled = element.is_enabled().await.unwrap_or(false);
|
let is_enabled = element.is_enabled().await.unwrap_or(false);
|
||||||
let is_selected = element.is_selected().await.unwrap_or(false);
|
let is_selected = element.is_selected().await.unwrap_or(false);
|
||||||
// Note: location() and size() may not be available in all fantoccini versions
|
// KNOWN LIMITATION: location and size are always None.
|
||||||
// Using placeholder values if not available
|
// The fantoccini Element type does not expose a synchronous bounding-box
|
||||||
|
// helper; retrieving geometry requires a separate execute_script call
|
||||||
|
// (e.g. element.getBoundingClientRect()). Since no current caller relies
|
||||||
|
// on these fields, they are intentionally left as None rather than adding
|
||||||
|
// an extra round-trip. If bounding-box data is needed in the future,
|
||||||
|
// add a dedicated browser_element_rect command that calls
|
||||||
|
// execute_script("return arguments[0].getBoundingClientRect()") and
|
||||||
|
// deprecate the location/size fields on ElementInfo entirely.
|
||||||
let location = None;
|
let location = None;
|
||||||
let size = None;
|
let size = None;
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,5 @@
|
|||||||
// Tauri commands for browser automation
|
// Tauri commands for browser automation
|
||||||
// Note: Some imports are reserved for future Browser Hand features
|
|
||||||
|
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
use crate::browser::actions::BrowserAction;
|
|
||||||
use crate::browser::client::BrowserClient;
|
use crate::browser::client::BrowserClient;
|
||||||
use crate::browser::session::{BrowserType, SessionConfig};
|
use crate::browser::session::{BrowserType, SessionConfig};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -454,9 +450,18 @@ pub async fn browser_scrape_page(
|
|||||||
let mut results = serde_json::Map::new();
|
let mut results = serde_json::Map::new();
|
||||||
|
|
||||||
for selector in selectors {
|
for selector in selectors {
|
||||||
if let Ok(elements) = client.find_elements(&session_id, &selector).await {
|
match client.find_elements(&session_id, &selector).await {
|
||||||
let texts: Vec<String> = elements.iter().filter_map(|e| e.text.clone()).collect();
|
Ok(elements) => {
|
||||||
results.insert(selector, serde_json::json!(texts));
|
let texts: Vec<String> = elements.iter().filter_map(|e| e.text.clone()).collect();
|
||||||
|
results.insert(selector, serde_json::json!(texts));
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
selector = %selector,
|
||||||
|
error = %e,
|
||||||
|
"browser_scrape_page: find_elements failed, skipping selector"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,4 +7,4 @@ pub mod client;
|
|||||||
pub mod commands;
|
pub mod commands;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod session;
|
pub mod session;
|
||||||
pub mod actions;
|
// pub mod actions; // Removed: dead code — see actions.rs for details
|
||||||
|
|||||||
@@ -324,6 +324,14 @@ export class Browser {
|
|||||||
return this.sessionId;
|
return this.sessionId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attach to an existing session by ID.
|
||||||
|
* Use this to reuse a session created elsewhere without spawning a new one.
|
||||||
|
*/
|
||||||
|
connect(sessionId: string): void {
|
||||||
|
this.sessionId = sessionId;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Close browser session
|
* Close browser session
|
||||||
*/
|
*/
|
||||||
|
|||||||
77
desktop/src/lib/mcp-client.ts
Normal file
77
desktop/src/lib/mcp-client.ts
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
/**
|
||||||
|
* MCP (Model Context Protocol) Client for ZCLAW
|
||||||
|
*
|
||||||
|
* Thin typed wrapper around the 4 Tauri MCP commands.
|
||||||
|
* All communication goes through invoke() — no direct HTTP or WebSocket.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { invoke } from '@tauri-apps/api/core';
|
||||||
|
import { createLogger } from './logger';
|
||||||
|
|
||||||
|
const log = createLogger('mcp-client');
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Types
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
export interface McpServiceConfig {
|
||||||
|
name: string;
|
||||||
|
command: string;
|
||||||
|
args?: string[];
|
||||||
|
env?: Record<string, string>;
|
||||||
|
cwd?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface McpToolInfo {
|
||||||
|
service_name: string;
|
||||||
|
tool_name: string;
|
||||||
|
description: string;
|
||||||
|
input_schema: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface McpServiceStatus {
|
||||||
|
name: string;
|
||||||
|
tool_count: number;
|
||||||
|
tools: McpToolInfo[];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Functions
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start an MCP service process and discover its tools.
|
||||||
|
*/
|
||||||
|
export async function startMcpService(
|
||||||
|
config: McpServiceConfig
|
||||||
|
): Promise<McpToolInfo[]> {
|
||||||
|
log.info('startMcpService', { name: config.name });
|
||||||
|
return invoke<McpToolInfo[]>('mcp_start_service', { config });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop a running MCP service by name.
|
||||||
|
*/
|
||||||
|
export async function stopMcpService(name: string): Promise<void> {
|
||||||
|
log.info('stopMcpService', { name });
|
||||||
|
return invoke<void>('mcp_stop_service', { name });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List all running MCP services with their discovered tools.
|
||||||
|
*/
|
||||||
|
export async function listMcpServices(): Promise<McpServiceStatus[]> {
|
||||||
|
return invoke<McpServiceStatus[]>('mcp_list_services');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Call a tool exposed by a running MCP service.
|
||||||
|
*/
|
||||||
|
export async function callMcpTool(
|
||||||
|
serviceName: string,
|
||||||
|
toolName: string,
|
||||||
|
args: Record<string, unknown>
|
||||||
|
): Promise<unknown> {
|
||||||
|
log.info('callMcpTool', { serviceName, toolName });
|
||||||
|
return invoke<unknown>('mcp_call_tool', { serviceName, toolName, args });
|
||||||
|
}
|
||||||
@@ -10,6 +10,8 @@ import Browser, {
|
|||||||
createSession,
|
createSession,
|
||||||
closeSession,
|
closeSession,
|
||||||
listSessions,
|
listSessions,
|
||||||
|
screenshot as screenshotFn,
|
||||||
|
executeScript as executeScriptFn,
|
||||||
} from '../lib/browser-client';
|
} from '../lib/browser-client';
|
||||||
import {
|
import {
|
||||||
BUILTIN_TEMPLATES,
|
BUILTIN_TEMPLATES,
|
||||||
@@ -247,14 +249,21 @@ export const useBrowserHandStore = create<BrowserHandState & BrowserHandActions>
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create browser instance
|
// Create browser instance — reuse active session if available
|
||||||
const browser = new Browser();
|
const browser = new Browser();
|
||||||
|
let createdOwnSession = false;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
store.addLog({ level: 'info', message: `开始执行模板: ${template.name}` });
|
store.addLog({ level: 'info', message: `开始执行模板: ${template.name}` });
|
||||||
|
|
||||||
// Start browser session
|
// Attach to existing session or start a new one
|
||||||
await browser.start({ headless: true });
|
if (store.activeSessionId) {
|
||||||
|
browser.connect(store.activeSessionId);
|
||||||
|
createdOwnSession = false;
|
||||||
|
} else {
|
||||||
|
await browser.start({ headless: true });
|
||||||
|
createdOwnSession = true;
|
||||||
|
}
|
||||||
|
|
||||||
// Create execution context
|
// Create execution context
|
||||||
const context = {
|
const context = {
|
||||||
@@ -322,7 +331,10 @@ export const useBrowserHandStore = create<BrowserHandState & BrowserHandActions>
|
|||||||
|
|
||||||
throw error;
|
throw error;
|
||||||
} finally {
|
} finally {
|
||||||
await browser.close();
|
// Only close the session if we created it (no pre-existing active session)
|
||||||
|
if (createdOwnSession) {
|
||||||
|
await browser.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -340,10 +352,8 @@ export const useBrowserHandStore = create<BrowserHandState & BrowserHandActions>
|
|||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const browser = new Browser();
|
// Use the standalone function with the existing session — no new session created
|
||||||
await browser.start();
|
const result = await executeScriptFn(store.activeSessionId, script, args);
|
||||||
|
|
||||||
const result = await browser.eval(script, args);
|
|
||||||
|
|
||||||
store.updateExecutionState({
|
store.updateExecutionState({
|
||||||
isRunning: false,
|
isRunning: false,
|
||||||
@@ -399,10 +409,8 @@ export const useBrowserHandStore = create<BrowserHandState & BrowserHandActions>
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const browser = new Browser();
|
// Use the standalone function with the existing session — no new session created
|
||||||
await browser.start();
|
const result = await screenshotFn(store.activeSessionId);
|
||||||
|
|
||||||
const result = await browser.screenshot();
|
|
||||||
|
|
||||||
set((state) => ({
|
set((state) => ({
|
||||||
execution: {
|
execution: {
|
||||||
|
|||||||
Reference in New Issue
Block a user