diff --git a/desktop/package.json b/desktop/package.json index 05a8afc..b010342 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -23,6 +23,7 @@ "@tauri-apps/api": "^2", "@tauri-apps/plugin-opener": "^2", "clsx": "^2.1.1", + "date-fns": "^4.1.0", "framer-motion": "^12.36.0", "lucide-react": "^0.577.0", "react": "^19.1.0", @@ -31,6 +32,7 @@ "smol-toml": "^1.6.0", "tailwind-merge": "^3.5.0", "tweetnacl": "^1.0.3", + "uuid": "^11.0.0", "zustand": "^5.0.11" }, "devDependencies": { @@ -39,6 +41,7 @@ "@types/react": "^19.1.8", "@types/react-dom": "^19.1.6", "@types/react-window": "^2.0.0", + "@types/uuid": "^10.0.0", "@vitejs/plugin-react": "^4.6.0", "autoprefixer": "^10.4.27", "postcss": "^8.5.8", diff --git a/desktop/pnpm-lock.yaml b/desktop/pnpm-lock.yaml index abe3c34..5158939 100644 --- a/desktop/pnpm-lock.yaml +++ b/desktop/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: clsx: specifier: ^2.1.1 version: 2.1.1 + date-fns: + specifier: ^4.1.0 + version: 4.1.0 framer-motion: specifier: ^12.36.0 version: 12.36.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -41,6 +44,9 @@ importers: tweetnacl: specifier: ^1.0.3 version: 1.0.3 + uuid: + specifier: ^11.0.0 + version: 11.1.0 zustand: specifier: ^5.0.11 version: 5.0.11(@types/react@19.2.14)(react@19.2.4) @@ -60,6 +66,9 @@ importers: '@types/react-window': specifier: ^2.0.0 version: 2.0.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@types/uuid': + specifier: ^10.0.0 + version: 10.0.0 '@vitejs/plugin-react': specifier: ^4.6.0 version: 4.7.0(vite@7.3.1(jiti@2.6.1)(lightningcss@1.31.1)) @@ -680,6 +689,9 @@ packages: '@types/react@19.2.14': resolution: {integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==} + '@types/uuid@10.0.0': + resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} + '@vitejs/plugin-react@4.7.0': resolution: {integrity: sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==} engines: {node: ^14.18.0 || >=16.0.0} @@ -716,6 +728,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + date-fns@4.1.0: + resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==} + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -986,6 +1001,10 @@ packages: peerDependencies: browserslist: '>= 4.21.0' + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + vite@7.3.1: resolution: {integrity: sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1494,6 +1513,8 @@ snapshots: dependencies: csstype: 3.2.3 + '@types/uuid@10.0.0': {} + '@vitejs/plugin-react@4.7.0(vite@7.3.1(jiti@2.6.1)(lightningcss@1.31.1))': dependencies: '@babel/core': 7.29.0 @@ -1533,6 +1554,8 @@ snapshots: csstype@3.2.3: {} + date-fns@4.1.0: {} + debug@4.4.3: dependencies: ms: 2.1.3 @@ -1768,6 +1791,8 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 + uuid@11.1.0: {} + vite@7.3.1(jiti@2.6.1)(lightningcss@1.31.1): dependencies: esbuild: 0.27.3 diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 73ee0e5..843f694 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -483,12 +483,23 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "cookie" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" +dependencies = [ + "time", + "version_check", +] + [[package]] name = "cookie" version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" dependencies = [ + "percent-encoding", "time", "version_check", ] @@ -719,8 +730,11 @@ dependencies = [ name = "desktop" version = "0.1.0" dependencies = [ + "base64 0.22.1", "chrono", "dirs 5.0.1", + "fantoccini", + "futures", "regex", "reqwest 0.11.27", "serde", @@ -728,7 +742,9 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-opener", + "thiserror 2.0.18", "tokio", + "uuid", ] [[package]] @@ -984,6 +1000,30 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fantoccini" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a6a7a9a454c24453f9807c7f12b37e31ae43f3eb41888ae1f79a9a3e3be3f5" +dependencies = [ + "base64 0.22.1", + "cookie 0.18.1", + "futures-util", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-tls 0.6.0", + "hyper-util", + "mime", + "openssl", + "serde", + "serde_json", + "time", + "tokio", + "url", + "webdriver", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1104,6 +1144,21 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -1111,6 +1166,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1178,6 +1234,7 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1712,6 +1769,22 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -3188,7 +3261,7 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.32", - "hyper-tls", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -3962,7 +4035,7 @@ checksum = "da77cc00fb9028caf5b5d4650f75e31f1ef3693459dfca7f7e506d1ecef0ba2d" dependencies = [ "anyhow", "bytes", - "cookie", + "cookie 0.18.1", "dirs 6.0.0", "dunce", "embed_plist", @@ -4113,7 +4186,7 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2826d79a3297ed08cd6ea7f412644ef58e32969504bc4fbd8d7dbeabc4445ea2" dependencies = [ - "cookie", + "cookie 0.18.1", "dpi", "gtk", "http 1.4.0", @@ -4918,6 +4991,26 @@ dependencies = [ "string_cache_codegen 0.6.1", ] +[[package]] +name = "webdriver" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144ab979b12d36d65065635e646549925de229954de2eb3b47459b432a42db71" +dependencies = [ + "base64 0.21.7", + "bytes", + "cookie 0.16.2", + "http 0.2.12", + "log", + "serde", + "serde_derive", + "serde_json", + "thiserror 1.0.69", + "time", + "unicode-segmentation", + "url", +] + [[package]] name = "webkit2gtk" version = "2.0.2" @@ -5638,7 +5731,7 @@ checksum = "a24eda84b5d488f99344e54b807138896cee8df0b2d16c793f1f6b80e6d8df1f" dependencies = [ "base64 0.22.1", "block2", - "cookie", + "cookie 0.18.1", "crossbeam-channel", "dirs 6.0.0", "dom_query", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 52657a8..3551ace 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -24,7 +24,14 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" tokio = { version = "1", features = ["full"] } reqwest = { version = "0.11", features = ["json", "blocking"] } -chrono = "0.4" +chrono = { version = "0.4", features = ["serde"] } regex = "1" dirs = "5" +# Browser automation +fantoccini = "0.21" +futures = "0.3" +base64 = "0.22" +thiserror = "2" +uuid = { version = "1", features = ["v4", "serde"] } + diff --git a/desktop/src-tauri/src/browser/actions.rs b/desktop/src-tauri/src/browser/actions.rs new file mode 100644 index 0000000..ba4dc9a --- /dev/null +++ b/desktop/src-tauri/src/browser/actions.rs @@ -0,0 +1,310 @@ +// Browser action definitions for Hands system + +use serde::{Deserialize, Serialize}; + +/// Browser action types +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BrowserAction { + /// Create a new browser session + CreateSession { + webdriver_url: Option, + headless: Option, + browser_type: Option, + window_size: Option<(u32, u32)>, + }, + + /// Close browser session + CloseSession { + session_id: String, + }, + + /// Navigate to URL + Navigate { + session_id: String, + url: String, + }, + + /// Go back + Back { + session_id: String, + }, + + /// Go forward + Forward { + session_id: String, + }, + + /// Refresh page + Refresh { + session_id: String, + }, + + /// Click element + Click { + session_id: String, + selector: String, + }, + + /// Type text + Type { + session_id: String, + selector: String, + text: String, + clear_first: Option, + }, + + /// Get element text + GetText { + session_id: String, + selector: String, + }, + + /// Get element attribute + GetAttribute { + session_id: String, + selector: String, + attribute: String, + }, + + /// Find element + FindElement { + session_id: String, + selector: String, + }, + + /// Find multiple elements + FindElements { + session_id: String, + selector: String, + }, + + /// Execute JavaScript + ExecuteScript { + session_id: String, + script: String, + args: Option>, + }, + + /// Take screenshot + Screenshot { + session_id: String, + }, + + /// Take element screenshot + ElementScreenshot { + session_id: String, + selector: String, + }, + + /// Wait for element + WaitForElement { + session_id: String, + selector: String, + timeout_ms: Option, + }, + + /// Get page source + GetSource { + session_id: String, + }, + + /// Get current URL + GetCurrentUrl { + session_id: String, + }, + + /// Get page title + GetTitle { + session_id: String, + }, + + /// List all sessions + ListSessions, + + /// Get session info + GetSession { + session_id: String, + }, +} + +/// Action execution result +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ActionResult { + /// Session created + SessionCreated { + session_id: String, + }, + + /// Session closed + SessionClosed { + session_id: String, + }, + + /// Navigation result + Navigated { + url: Option, + title: Option, + }, + + /// Element clicked + Clicked { + selector: String, + }, + + /// Text typed + Typed { + selector: String, + text: String, + }, + + /// Text retrieved + TextRetrieved { + selector: String, + text: String, + }, + + /// Attribute retrieved + AttributeRetrieved { + selector: String, + attribute: String, + value: Option, + }, + + /// Element found + ElementFound { + element: ElementInfo, + }, + + /// Elements found + ElementsFound { + elements: Vec, + }, + + /// Script executed + ScriptExecuted { + result: serde_json::Value, + }, + + /// Screenshot taken + ScreenshotTaken { + base64: String, + format: String, + }, + + /// Page source retrieved + SourceRetrieved { + source: String, + }, + + /// URL retrieved + UrlRetrieved { + url: String, + }, + + /// Title retrieved + TitleRetrieved { + title: String, + }, + + /// Sessions listed + SessionsListed { + sessions: Vec, + }, + + /// Session info retrieved + SessionInfo { + session: SessionInfo, + }, + + /// Operation completed (no specific data) + Completed, + + /// Error occurred + Error { + message: String, + code: String, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ElementInfo { + pub selector: String, + pub tag_name: Option, + pub text: Option, + pub is_displayed: bool, + pub is_enabled: bool, + pub is_selected: bool, + pub location: Option, + pub size: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ElementLocation { + pub x: i32, + pub y: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ElementSize { + pub width: u64, + pub height: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SessionInfo { + pub id: String, + pub name: String, + pub current_url: Option, + pub title: Option, + pub status: String, + pub created_at: String, + pub last_activity: String, +} + +/// High-level browser task (for Hand integration) +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "task", rename_all = "snake_case")] +pub enum BrowserTask { + /// Scrape page content + ScrapePage { + url: String, + selectors: Vec, + wait_for: Option, + }, + + /// Fill form + FillForm { + url: String, + fields: Vec, + submit_selector: Option, + }, + + /// Take page snapshot + PageSnapshot { + url: String, + include_screenshot: bool, + }, + + /// Navigate and extract + NavigateAndExtract { + url: String, + extraction_script: String, + }, + + /// Multi-page scraping + MultiPageScrape { + start_url: String, + next_page_selector: String, + item_selector: String, + max_pages: Option, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FormField { + pub selector: String, + pub value: String, + pub field_type: Option, +} diff --git a/desktop/src-tauri/src/browser/client.rs b/desktop/src-tauri/src/browser/client.rs new file mode 100644 index 0000000..6291210 --- /dev/null +++ b/desktop/src-tauri/src/browser/client.rs @@ -0,0 +1,493 @@ +// Browser client using Fantoccini WebDriver + +use crate::browser::error::{BrowserError, Result}; +use crate::browser::session::{BrowserSession, BrowserType, SessionConfig, SessionManager}; +use base64::{engine::general_purpose::STANDARD, Engine}; +use fantoccini::elements::Element; +use fantoccini::{Client, ClientBuilder, Locator}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::RwLock; +use uuid::Uuid; + +/// Main browser automation client +pub struct BrowserClient { + /// Active WebDriver connections + connections: Arc>>, + /// Session manager + session_manager: SessionManager, +} + +impl BrowserClient { + pub fn new() -> Self { + Self { + connections: Arc::new(RwLock::new(HashMap::new())), + session_manager: SessionManager::new(), + } + } + + /// Create a new browser session + pub async fn create_session(&self, config: SessionConfig) -> Result { + let session_id = Uuid::new_v4().to_string(); + + // Build WebDriver capabilities as Map + let capabilities = self.build_capabilities(&config)?; + + // Connect to WebDriver + let client = ClientBuilder::native() + .capabilities(capabilities) + .connect(&config.webdriver_url) + .await + .map_err(|e| BrowserError::ConnectionFailed(e.to_string()))?; + + // Store connection + { + let mut connections = self.connections.write().await; + connections.insert(session_id.clone(), client); + } + + // Create session record + let session = BrowserSession::new(session_id.clone(), config); + self.session_manager.add_session(session).await; + + Ok(session_id) + } + + /// Close a browser session + pub async fn close_session(&self, session_id: &str) -> Result<()> { + let client = self.get_client(session_id).await?; + + // Close the browser + client + .close() + .await + .map_err(|e| BrowserError::ConnectionFailed(e.to_string()))?; + + // Remove from connections + { + let mut connections = self.connections.write().await; + connections.remove(session_id); + } + + // Remove session record + self.session_manager.remove_session(session_id).await; + + Ok(()) + } + + /// Get session information + pub async fn get_session(&self, session_id: &str) -> Result { + self.session_manager + .get_session(session_id) + .await + .ok_or_else(|| BrowserError::SessionNotFound(session_id.to_string())) + } + + /// List all sessions + pub async fn list_sessions(&self) -> Vec { + self.session_manager.list_sessions().await + } + + /// Navigate to URL + pub async fn navigate(&self, session_id: &str, url: &str) -> Result { + let client = self.get_client(session_id).await?; + + client + .goto(url) + .await + .map_err(|_| BrowserError::NavigationFailed { + url: url.to_string(), + })?; + + // Get current URL and title + let current_url = client.current_url().await.ok().map(|u| u.to_string()); + let title = client.title().await.ok(); + + // Update session + self.session_manager + .update_session(session_id, |s| { + s.update_location(current_url.clone(), title.clone()); + }) + .await; + + Ok(NavigationResult { + url: current_url, + title, + }) + } + + /// Go back + pub async fn back(&self, session_id: &str) -> Result<()> { + let client = self.get_client(session_id).await?; + client.back().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?; + self.update_session_location(session_id).await; + Ok(()) + } + + /// Go forward + pub async fn forward(&self, session_id: &str) -> Result<()> { + let client = self.get_client(session_id).await?; + client.forward().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?; + self.update_session_location(session_id).await; + Ok(()) + } + + /// Refresh page + pub async fn refresh(&self, session_id: &str) -> Result<()> { + let client = self.get_client(session_id).await?; + client.refresh().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?; + Ok(()) + } + + /// Find element by CSS selector + pub async fn find_element(&self, session_id: &str, selector: &str) -> Result { + let client = self.get_client(session_id).await?; + + let element = client + .find(Locator::Css(selector)) + .await + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + self.element_to_info(&element, selector).await + } + + /// Find multiple elements + pub async fn find_elements(&self, session_id: &str, selector: &str) -> Result> { + let client = self.get_client(session_id).await?; + + let elements = client + .find_all(Locator::Css(selector)) + .await + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + let mut infos = Vec::new(); + for element in elements { + if let Ok(info) = self.element_to_info(&element, selector).await { + infos.push(info); + } + } + + Ok(infos) + } + + /// Click element + pub async fn click(&self, session_id: &str, selector: &str) -> Result<()> { + let client = self.get_client(session_id).await?; + + let element = client + .find(Locator::Css(selector)) + .await + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + element.click().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?; + self.update_session_location(session_id).await; + Ok(()) + } + + /// Type text into element + pub async fn type_text(&self, session_id: &str, selector: &str, text: &str) -> Result<()> { + let client = self.get_client(session_id).await?; + + let element = client + .find(Locator::Css(selector)) + .await + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + element.send_keys(text).await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?; + Ok(()) + } + + /// Clear and type text + pub async fn clear_and_type(&self, session_id: &str, selector: &str, text: &str) -> Result<()> { + let client = self.get_client(session_id).await?; + + let element = client + .find(Locator::Css(selector)) + .await + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + element.clear().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?; + element.send_keys(text).await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?; + Ok(()) + } + + /// Get element text + pub async fn get_text(&self, session_id: &str, selector: &str) -> Result { + let client = self.get_client(session_id).await?; + + let element = client + .find(Locator::Css(selector)) + .await + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + element.text().await.map_err(|e| BrowserError::CommandFailed(e.to_string())) + } + + /// Get element attribute + pub async fn get_attribute( + &self, + session_id: &str, + selector: &str, + attribute: &str, + ) -> Result> { + let client = self.get_client(session_id).await?; + + let element = client + .find(Locator::Css(selector)) + .await + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + element.attr(attribute).await.map_err(|e| BrowserError::CommandFailed(e.to_string())) + } + + /// Execute JavaScript + pub async fn execute_script( + &self, + session_id: &str, + script: &str, + args: Vec, + ) -> Result { + let client = self.get_client(session_id).await?; + client.execute(script, args).await.map_err(|e| BrowserError::ScriptError { + message: e.to_string(), + }) + } + + /// Take screenshot + pub async fn screenshot(&self, session_id: &str) -> Result { + let client = self.get_client(session_id).await?; + + let screenshot = client.screenshot().await.map_err(|e| BrowserError::ScreenshotFailed { + reason: e.to_string(), + })?; + + let base64_data = STANDARD.encode(&screenshot); + + Ok(ScreenshotResult { + data: screenshot, + base64: base64_data, + format: "png".to_string(), + }) + } + + /// Take element screenshot + pub async fn element_screenshot( + &self, + session_id: &str, + selector: &str, + ) -> Result { + let client = self.get_client(session_id).await?; + + let element = client + .find(Locator::Css(selector)) + .await + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + let screenshot = element.screenshot().await.map_err(|e| BrowserError::ScreenshotFailed { + reason: e.to_string(), + })?; + + let base64_data = STANDARD.encode(&screenshot); + + Ok(ScreenshotResult { + data: screenshot, + base64: base64_data, + format: "png".to_string(), + }) + } + + /// Wait for element with custom timeout + pub async fn wait_for_element( + &self, + session_id: &str, + selector: &str, + timeout_ms: u64, + ) -> Result { + let client = self.get_client(session_id).await?; + let locator = Locator::Css(selector); + + // Use wait_for_find with proper API + let element = tokio::time::timeout( + Duration::from_millis(timeout_ms), + client.wait_for_find(locator) + ) + .await + .map_err(|_| BrowserError::Timeout { + selector: selector.to_string(), + })? + .map_err(|_| BrowserError::ElementNotFound { + selector: selector.to_string(), + })?; + + self.element_to_info(&element, selector).await + } + + /// Get page source + pub async fn get_source(&self, session_id: &str) -> Result { + let client = self.get_client(session_id).await?; + client.source().await.map_err(|e| BrowserError::CommandFailed(e.to_string())) + } + + /// Get current URL + pub async fn get_current_url(&self, session_id: &str) -> Result { + let client = self.get_client(session_id).await?; + let url = client.current_url().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?; + Ok(url.to_string()) + } + + /// Get page title + pub async fn get_title(&self, session_id: &str) -> Result { + let client = self.get_client(session_id).await?; + client.title().await.map_err(|e| BrowserError::CommandFailed(e.to_string())) + } + + // Private helper methods + + async fn get_client(&self, session_id: &str) -> Result { + let connections = self.connections.read().await; + connections + .get(session_id) + .cloned() + .ok_or_else(|| BrowserError::SessionNotFound(session_id.to_string())) + } + + fn build_capabilities(&self, config: &SessionConfig) -> Result> { + let browser_name = match config.browser_type { + BrowserType::Chrome => "chrome", + BrowserType::Firefox => "firefox", + BrowserType::Edge => "MicrosoftEdge", + BrowserType::Safari => "safari", + }; + + let mut args = vec![]; + + if config.headless { + args.push("--headless".to_string()); + } + + if let Some((width, height)) = config.window_size { + args.push(format!("--window-size={},{}", width, height)); + } + + args.extend(config.browser_args.clone()); + + let mut caps = serde_json::Map::new(); + caps.insert("browserName".to_string(), serde_json::json!(browser_name)); + + let mut chrome_options = serde_json::Map::new(); + chrome_options.insert("args".to_string(), serde_json::json!(args)); + chrome_options.insert("w3c".to_string(), serde_json::json!(true)); + + caps.insert("goog:chromeOptions".to_string(), serde_json::Value::Object(chrome_options)); + + Ok(caps) + } + + async fn element_to_info(&self, element: &Element, selector: &str) -> Result { + let tag_name = element.tag_name().await.ok(); + let text = element.text().await.ok(); + let is_displayed = element.is_displayed().await.unwrap_or(false); + let is_enabled = element.is_enabled().await.unwrap_or(false); + let is_selected = element.is_selected().await.unwrap_or(false); + // Note: location() and size() may not be available in all fantoccini versions + // Using placeholder values if not available + let location = None; + let size = None; + + Ok(ElementInfo { + selector: selector.to_string(), + tag_name, + text, + is_displayed, + is_enabled, + is_selected, + location, + size, + }) + } + + async fn update_session_location(&self, session_id: &str) { + if let Ok(client) = self.get_client(session_id).await { + let current_url = client.current_url().await.ok().map(|u| u.to_string()); + let title = client.title().await.ok(); + + self.session_manager + .update_session(session_id, |s| { + s.update_location(current_url, title); + }) + .await; + } + } +} + +impl Default for BrowserClient { + fn default() -> Self { + Self::new() + } +} + +impl Clone for BrowserClient { + fn clone(&self) -> Self { + Self { + connections: Arc::clone(&self.connections), + session_manager: self.session_manager.clone(), + } + } +} + +// Result types + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NavigationResult { + pub url: Option, + pub title: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ElementInfo { + pub selector: String, + pub tag_name: Option, + pub text: Option, + pub is_displayed: bool, + pub is_enabled: bool, + pub is_selected: bool, + pub location: Option, + pub size: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ElementLocation { + pub x: i32, + pub y: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ElementSize { + pub width: u64, + pub height: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScreenshotResult { + pub data: Vec, + pub base64: String, + pub format: String, +} diff --git a/desktop/src-tauri/src/browser/commands.rs b/desktop/src-tauri/src/browser/commands.rs new file mode 100644 index 0000000..97c7313 --- /dev/null +++ b/desktop/src-tauri/src/browser/commands.rs @@ -0,0 +1,531 @@ +// Tauri commands for browser automation + +use crate::browser::actions::{ActionResult, BrowserAction, BrowserTask, FormField}; +use crate::browser::client::BrowserClient; +use crate::browser::error::BrowserError; +use crate::browser::session::{BrowserType, SessionConfig}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::sync::RwLock; +use tauri::State; + +/// Global browser client state +pub struct BrowserState { + client: Arc>, +} + +impl BrowserState { + pub fn new() -> Self { + Self { + client: Arc::new(RwLock::new(BrowserClient::new())), + } + } +} + +impl Default for BrowserState { + fn default() -> Self { + Self::new() + } +} + +impl Clone for BrowserState { + fn clone(&self) -> Self { + Self { + client: Arc::clone(&self.client), + } + } +} + +// ============================================================================ +// Session Management Commands +// ============================================================================ + +/// Create a new browser session +#[tauri::command] +pub async fn browser_create_session( + state: State<'_, BrowserState>, + webdriver_url: Option, + headless: Option, + browser_type: Option, + window_width: Option, + window_height: Option, +) -> Result { + let browser_type = match browser_type.as_deref() { + Some("firefox") => BrowserType::Firefox, + Some("edge") => BrowserType::Edge, + Some("safari") => BrowserType::Safari, + _ => BrowserType::Chrome, + }; + + let config = SessionConfig { + webdriver_url: webdriver_url.unwrap_or_else(|| "http://localhost:4444".to_string()), + browser_type, + headless: headless.unwrap_or(true), + window_size: window_width.zip(window_height), + ..Default::default() + }; + + let client = state.client.read().await; + let session_id = client.create_session(config).await.map_err(|e| e.to_string())?; + + Ok(BrowserSessionResult { session_id }) +} + +/// Close a browser session +#[tauri::command] +pub async fn browser_close_session( + state: State<'_, BrowserState>, + session_id: String, +) -> Result<(), String> { + let client = state.client.read().await; + client.close_session(&session_id).await.map_err(|e| e.to_string()) +} + +/// List all browser sessions +#[tauri::command] +pub async fn browser_list_sessions( + state: State<'_, BrowserState>, +) -> Result, String> { + let client = state.client.read().await; + let sessions = client.list_sessions().await; + + Ok(sessions + .into_iter() + .map(|s| BrowserSessionInfo { + id: s.id, + name: s.name, + current_url: s.current_url, + title: s.title, + status: format!("{:?}", s.status).to_lowercase(), + created_at: s.created_at.to_rfc3339(), + last_activity: s.last_activity.to_rfc3339(), + }) + .collect()) +} + +/// Get session info +#[tauri::command] +pub async fn browser_get_session( + state: State<'_, BrowserState>, + session_id: String, +) -> Result { + let client = state.client.read().await; + let session = client.get_session(&session_id).await.map_err(|e| e.to_string())?; + + Ok(BrowserSessionInfo { + id: session.id, + name: session.name, + current_url: session.current_url, + title: session.title, + status: format!("{:?}", session.status).to_lowercase(), + created_at: session.created_at.to_rfc3339(), + last_activity: session.last_activity.to_rfc3339(), + }) +} + +// ============================================================================ +// Navigation Commands +// ============================================================================ + +/// Navigate to URL +#[tauri::command] +pub async fn browser_navigate( + state: State<'_, BrowserState>, + session_id: String, + url: String, +) -> Result { + let client = state.client.read().await; + let result = client.navigate(&session_id, &url).await.map_err(|e| e.to_string())?; + + Ok(BrowserNavigationResult { + url: result.url, + title: result.title, + }) +} + +/// Go back +#[tauri::command] +pub async fn browser_back( + state: State<'_, BrowserState>, + session_id: String, +) -> Result<(), String> { + let client = state.client.read().await; + client.back(&session_id).await.map_err(|e| e.to_string()) +} + +/// Go forward +#[tauri::command] +pub async fn browser_forward( + state: State<'_, BrowserState>, + session_id: String, +) -> Result<(), String> { + let client = state.client.read().await; + client.forward(&session_id).await.map_err(|e| e.to_string()) +} + +/// Refresh page +#[tauri::command] +pub async fn browser_refresh( + state: State<'_, BrowserState>, + session_id: String, +) -> Result<(), String> { + let client = state.client.read().await; + client.refresh(&session_id).await.map_err(|e| e.to_string()) +} + +/// Get current URL +#[tauri::command] +pub async fn browser_get_url( + state: State<'_, BrowserState>, + session_id: String, +) -> Result { + let client = state.client.read().await; + client.get_current_url(&session_id).await.map_err(|e| e.to_string()) +} + +/// Get page title +#[tauri::command] +pub async fn browser_get_title( + state: State<'_, BrowserState>, + session_id: String, +) -> Result { + let client = state.client.read().await; + client.get_title(&session_id).await.map_err(|e| e.to_string()) +} + +// ============================================================================ +// Element Interaction Commands +// ============================================================================ + +/// Find element +#[tauri::command] +pub async fn browser_find_element( + state: State<'_, BrowserState>, + session_id: String, + selector: String, +) -> Result { + let client = state.client.read().await; + let element = client.find_element(&session_id, &selector).await.map_err(|e| e.to_string())?; + + Ok(BrowserElementInfo { + selector: element.selector, + tag_name: element.tag_name, + text: element.text, + is_displayed: element.is_displayed, + is_enabled: element.is_enabled, + is_selected: element.is_selected, + location: element.location.map(|l| BrowserElementLocation { x: l.x, y: l.y }), + size: element.size.map(|s| BrowserElementSize { + width: s.width, + height: s.height, + }), + }) +} + +/// Find multiple elements +#[tauri::command] +pub async fn browser_find_elements( + state: State<'_, BrowserState>, + session_id: String, + selector: String, +) -> Result, String> { + let client = state.client.read().await; + let elements = client.find_elements(&session_id, &selector).await.map_err(|e| e.to_string())?; + + Ok(elements + .into_iter() + .map(|e| BrowserElementInfo { + selector: e.selector, + tag_name: e.tag_name, + text: e.text, + is_displayed: e.is_displayed, + is_enabled: e.is_enabled, + is_selected: e.is_selected, + location: e.location.map(|l| BrowserElementLocation { x: l.x, y: l.y }), + size: e.size.map(|s| BrowserElementSize { + width: s.width, + height: s.height, + }), + }) + .collect()) +} + +/// Click element +#[tauri::command] +pub async fn browser_click( + state: State<'_, BrowserState>, + session_id: String, + selector: String, +) -> Result<(), String> { + let client = state.client.read().await; + client.click(&session_id, &selector).await.map_err(|e| e.to_string()) +} + +/// Type text into element +#[tauri::command] +pub async fn browser_type( + state: State<'_, BrowserState>, + session_id: String, + selector: String, + text: String, + clear_first: Option, +) -> Result<(), String> { + let client = state.client.read().await; + + if clear_first.unwrap_or(false) { + client + .clear_and_type(&session_id, &selector, &text) + .await + .map_err(|e| e.to_string()) + } else { + client + .type_text(&session_id, &selector, &text) + .await + .map_err(|e| e.to_string()) + } +} + +/// Get element text +#[tauri::command] +pub async fn browser_get_text( + state: State<'_, BrowserState>, + session_id: String, + selector: String, +) -> Result { + let client = state.client.read().await; + client.get_text(&session_id, &selector).await.map_err(|e| e.to_string()) +} + +/// Get element attribute +#[tauri::command] +pub async fn browser_get_attribute( + state: State<'_, BrowserState>, + session_id: String, + selector: String, + attribute: String, +) -> Result, String> { + let client = state.client.read().await; + client + .get_attribute(&session_id, &selector, &attribute) + .await + .map_err(|e| e.to_string()) +} + +/// Wait for element +#[tauri::command] +pub async fn browser_wait_for_element( + state: State<'_, BrowserState>, + session_id: String, + selector: String, + timeout_ms: Option, +) -> Result { + let client = state.client.read().await; + let element = client + .wait_for_element(&session_id, &selector, timeout_ms.unwrap_or(10000)) + .await + .map_err(|e| e.to_string())?; + + Ok(BrowserElementInfo { + selector: element.selector, + tag_name: element.tag_name, + text: element.text, + is_displayed: element.is_displayed, + is_enabled: element.is_enabled, + is_selected: element.is_selected, + location: element.location.map(|l| BrowserElementLocation { x: l.x, y: l.y }), + size: element.size.map(|s| BrowserElementSize { + width: s.width, + height: s.height, + }), + }) +} + +// ============================================================================ +// Advanced Commands +// ============================================================================ + +/// Execute JavaScript +#[tauri::command] +pub async fn browser_execute_script( + state: State<'_, BrowserState>, + session_id: String, + script: String, + args: Option>, +) -> Result { + let client = state.client.read().await; + client + .execute_script(&session_id, &script, args.unwrap_or_default()) + .await + .map_err(|e| e.to_string()) +} + +/// Take screenshot +#[tauri::command] +pub async fn browser_screenshot( + state: State<'_, BrowserState>, + session_id: String, +) -> Result { + let client = state.client.read().await; + let result = client.screenshot(&session_id).await.map_err(|e| e.to_string())?; + + Ok(BrowserScreenshotResult { + base64: result.base64, + format: result.format, + }) +} + +/// Take element screenshot +#[tauri::command] +pub async fn browser_element_screenshot( + state: State<'_, BrowserState>, + session_id: String, + selector: String, +) -> Result { + let client = state.client.read().await; + let result = client + .element_screenshot(&session_id, &selector) + .await + .map_err(|e| e.to_string())?; + + Ok(BrowserScreenshotResult { + base64: result.base64, + format: result.format, + }) +} + +/// Get page source +#[tauri::command] +pub async fn browser_get_source( + state: State<'_, BrowserState>, + session_id: String, +) -> Result { + let client = state.client.read().await; + client.get_source(&session_id).await.map_err(|e| e.to_string()) +} + +// ============================================================================ +// High-Level Task Commands (for Hands integration) +// ============================================================================ + +/// Scrape page content +#[tauri::command] +pub async fn browser_scrape_page( + state: State<'_, BrowserState>, + session_id: String, + selectors: Vec, + wait_for: Option, + timeout_ms: Option, +) -> Result { + let client = state.client.read().await; + + // Wait for element if specified + if let Some(selector) = wait_for { + client + .wait_for_element(&session_id, &selector, timeout_ms.unwrap_or(10000)) + .await + .map_err(|e| e.to_string())?; + } + + // Extract content from all selectors + let mut results = serde_json::Map::new(); + + for selector in selectors { + if let Ok(elements) = client.find_elements(&session_id, &selector).await { + let texts: Vec = elements.iter().filter_map(|e| e.text.clone()).collect(); + results.insert(selector, serde_json::json!(texts)); + } + } + + Ok(serde_json::Value::Object(results)) +} + +/// Fill form +#[tauri::command] +pub async fn browser_fill_form( + state: State<'_, BrowserState>, + session_id: String, + fields: Vec, + submit_selector: Option, +) -> Result<(), String> { + let client = state.client.read().await; + + // Fill each field + for field in fields { + client + .clear_and_type(&session_id, &field.selector, &field.value) + .await + .map_err(|e| e.to_string())?; + } + + // Submit form if selector provided + if let Some(selector) = submit_selector { + client + .click(&session_id, &selector) + .await + .map_err(|e| e.to_string())?; + } + + Ok(()) +} + +// ============================================================================ +// Response Types +// ============================================================================ + +#[derive(Debug, Serialize)] +pub struct BrowserSessionResult { + pub session_id: String, +} + +#[derive(Debug, Serialize)] +pub struct BrowserSessionInfo { + pub id: String, + pub name: String, + pub current_url: Option, + pub title: Option, + pub status: String, + pub created_at: String, + pub last_activity: String, +} + +#[derive(Debug, Serialize)] +pub struct BrowserNavigationResult { + pub url: Option, + pub title: Option, +} + +#[derive(Debug, Serialize)] +pub struct BrowserElementInfo { + pub selector: String, + pub tag_name: Option, + pub text: Option, + pub is_displayed: bool, + pub is_enabled: bool, + pub is_selected: bool, + pub location: Option, + pub size: Option, +} + +#[derive(Debug, Serialize)] +pub struct BrowserElementLocation { + pub x: i32, + pub y: i32, +} + +#[derive(Debug, Serialize)] +pub struct BrowserElementSize { + pub width: u64, + pub height: u64, +} + +#[derive(Debug, Serialize)] +pub struct BrowserScreenshotResult { + pub base64: String, + pub format: String, +} + +#[derive(Debug, Deserialize)] +pub struct FormFieldData { + pub selector: String, + pub value: String, +} diff --git a/desktop/src-tauri/src/browser/error.rs b/desktop/src-tauri/src/browser/error.rs new file mode 100644 index 0000000..dafe6ee --- /dev/null +++ b/desktop/src-tauri/src/browser/error.rs @@ -0,0 +1,86 @@ +// Browser automation error types + +use serde::Serialize; +use thiserror::Error; + +#[derive(Debug, Error, Serialize)] +pub enum BrowserError { + #[error("WebDriver connection failed: {0}")] + ConnectionFailed(String), + + #[error("Session not found: {0}")] + SessionNotFound(String), + + #[error("Element not found: {selector}")] + ElementNotFound { selector: String }, + + #[error("Navigation failed: {url}")] + NavigationFailed { url: String }, + + #[error("Timeout waiting for element: {selector}")] + Timeout { selector: String }, + + #[error("Invalid selector: {selector}")] + InvalidSelector { selector: String }, + + #[error("JavaScript execution failed: {message}")] + ScriptError { message: String }, + + #[error("Screenshot failed: {reason}")] + ScreenshotFailed { reason: String }, + + #[error("Form interaction failed: {field}")] + FormError { field: String }, + + #[error("WebDriver not available: {reason}")] + DriverNotAvailable { reason: String }, + + #[error("Session already exists: {id}")] + SessionExists { id: String }, + + #[error("Operation cancelled by user")] + Cancelled, + + #[error("Configuration error: {0}")] + ConfigError(String), + + #[error("IO error: {0}")] + IoError(String), + + #[error("WebDriver command failed: {0}")] + CommandFailed(String), + + #[error("Unknown error: {0}")] + Unknown(String), +} + +// Manual conversion from fantoccini errors since the enum variants differ between versions +impl From for BrowserError { + fn from(e: fantoccini::error::NewSessionError) -> Self { + BrowserError::ConnectionFailed(e.to_string()) + } +} + +impl From for BrowserError { + fn from(e: fantoccini::error::CmdError) -> Self { + // Convert to string and wrap in appropriate error type + let msg = e.to_string(); + if msg.contains("not found") || msg.contains("no such element") { + BrowserError::ElementNotFound { selector: msg } + } else if msg.contains("timeout") || msg.contains("timed out") { + BrowserError::Timeout { selector: msg } + } else if msg.contains("script") || msg.contains("javascript") { + BrowserError::ScriptError { message: msg } + } else { + BrowserError::CommandFailed(msg) + } + } +} + +impl From for BrowserError { + fn from(e: std::io::Error) -> Self { + BrowserError::IoError(e.to_string()) + } +} + +pub type Result = std::result::Result; diff --git a/desktop/src-tauri/src/browser/mod.rs b/desktop/src-tauri/src/browser/mod.rs new file mode 100644 index 0000000..3b2d45c --- /dev/null +++ b/desktop/src-tauri/src/browser/mod.rs @@ -0,0 +1,13 @@ +// Browser automation module using Fantoccini +// Provides Browser Hand capabilities for ZCLAW + +pub mod client; +pub mod commands; +pub mod error; +pub mod session; +pub mod actions; + +pub use client::BrowserClient; +pub use error::{BrowserError, Result}; +pub use session::{BrowserSession, SessionConfig}; +pub use actions::{BrowserAction, ActionResult}; diff --git a/desktop/src-tauri/src/browser/session.rs b/desktop/src-tauri/src/browser/session.rs new file mode 100644 index 0000000..dc8272c --- /dev/null +++ b/desktop/src-tauri/src/browser/session.rs @@ -0,0 +1,187 @@ +// Browser session management + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock; +use chrono::{DateTime, Utc}; + +/// Browser session configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SessionConfig { + /// WebDriver URL (e.g., "http://localhost:4444") + pub webdriver_url: String, + + /// Browser type (chrome, firefox, etc.) + pub browser_type: BrowserType, + + /// Headless mode + pub headless: bool, + + /// Window size (width, height) + pub window_size: Option<(u32, u32)>, + + /// Page load timeout in seconds + pub page_load_timeout: u64, + + /// Script timeout in seconds + pub script_timeout: u64, + + /// Implicit wait timeout in milliseconds + pub implicit_wait_timeout: u64, + + /// Custom browser arguments + pub browser_args: Vec, +} + +impl Default for SessionConfig { + fn default() -> Self { + Self { + webdriver_url: "http://localhost:4444".to_string(), + browser_type: BrowserType::Chrome, + headless: true, + window_size: Some((1920, 1080)), + page_load_timeout: 30, + script_timeout: 30, + implicit_wait_timeout: 1000, + browser_args: vec![], + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum BrowserType { + Chrome, + Firefox, + Edge, + Safari, +} + +/// Active browser session +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrowserSession { + /// Unique session identifier + pub id: String, + + /// Session name for display + pub name: String, + + /// Current URL + pub current_url: Option, + + /// Page title + pub title: Option, + + /// Session status + pub status: SessionStatus, + + /// Creation timestamp + pub created_at: DateTime, + + /// Last activity timestamp + pub last_activity: DateTime, + + /// Session configuration + pub config: SessionConfig, + + /// Custom metadata + pub metadata: HashMap, +} + +impl BrowserSession { + pub fn new(id: String, config: SessionConfig) -> Self { + let now = Utc::now(); + Self { + id, + name: format!("Browser Session"), + current_url: None, + title: None, + status: SessionStatus::Connected, + created_at: now, + last_activity: now, + config, + metadata: HashMap::new(), + } + } + + pub fn touch(&mut self) { + self.last_activity = Utc::now(); + } + + pub fn update_location(&mut self, url: Option, title: Option) { + self.current_url = url; + self.title = title; + self.touch(); + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum SessionStatus { + Connecting, + Connected, + Active, + Idle, + Disconnected, + Error, +} + +/// Session manager for multiple browser instances +pub struct SessionManager { + sessions: Arc>>, +} + +impl SessionManager { + pub fn new() -> Self { + Self { + sessions: Arc::new(RwLock::new(HashMap::new())), + } + } + + pub async fn add_session(&self, session: BrowserSession) { + let mut sessions = self.sessions.write().await; + sessions.insert(session.id.clone(), session); + } + + pub async fn get_session(&self, id: &str) -> Option { + let sessions = self.sessions.read().await; + sessions.get(id).cloned() + } + + pub async fn update_session(&self, id: &str, updater: impl FnOnce(&mut BrowserSession)) { + let mut sessions = self.sessions.write().await; + if let Some(session) = sessions.get_mut(id) { + updater(session); + } + } + + pub async fn remove_session(&self, id: &str) -> Option { + let mut sessions = self.sessions.write().await; + sessions.remove(id) + } + + pub async fn list_sessions(&self) -> Vec { + let sessions = self.sessions.read().await; + sessions.values().cloned().collect() + } + + pub async fn session_count(&self) -> usize { + let sessions = self.sessions.read().await; + sessions.len() + } +} + +impl Default for SessionManager { + fn default() -> Self { + Self::new() + } +} + +impl Clone for SessionManager { + fn clone(&self) -> Self { + Self { + sessions: Arc::clone(&self.sessions), + } + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 0c14596..e8e761f 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -12,6 +12,9 @@ mod viking_server; mod memory; mod llm; +// Browser automation module (Fantoccini-based Browser Hand) +mod browser; + use serde::Serialize; use serde_json::{json, Value}; use std::fs; @@ -991,8 +994,12 @@ fn gateway_doctor(app: AppHandle) -> Result { #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { + // Initialize browser state + let browser_state = browser::commands::BrowserState::new(); + tauri::Builder::default() .plugin(tauri_plugin_opener::init()) + .manage(browser_state) .invoke_handler(tauri::generate_handler![ // OpenFang commands (new naming) openfang_status, @@ -1035,7 +1042,31 @@ pub fn run() { memory::extractor::extract_session_memories, memory::context_builder::estimate_content_tokens, // LLM commands (for extraction) - llm::llm_complete + llm::llm_complete, + // Browser automation commands (Fantoccini-based Browser Hand) + browser::commands::browser_create_session, + browser::commands::browser_close_session, + browser::commands::browser_list_sessions, + browser::commands::browser_get_session, + browser::commands::browser_navigate, + browser::commands::browser_back, + browser::commands::browser_forward, + browser::commands::browser_refresh, + browser::commands::browser_get_url, + browser::commands::browser_get_title, + browser::commands::browser_find_element, + browser::commands::browser_find_elements, + browser::commands::browser_click, + browser::commands::browser_type, + browser::commands::browser_get_text, + browser::commands::browser_get_attribute, + browser::commands::browser_wait_for_element, + browser::commands::browser_execute_script, + browser::commands::browser_screenshot, + browser::commands::browser_element_screenshot, + browser::commands::browser_get_source, + browser::commands::browser_scrape_page, + browser::commands::browser_fill_form ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index 8a5c6dc..7a6f304 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -13,6 +13,7 @@ import { useGatewayStore } from './store/gatewayStore'; import { useTeamStore } from './store/teamStore'; import { getStoredGatewayToken } from './lib/gateway-client'; import { pageVariants, defaultTransition, fadeInVariants } from './lib/animations'; +import { silentErrorHandler } from './lib/error-utils'; import { Bot, Users } from 'lucide-react'; import { EmptyState } from './components/ui'; @@ -33,7 +34,7 @@ function App() { useEffect(() => { if (connectionState === 'disconnected') { const gatewayToken = getStoredGatewayToken(); - connect(undefined, gatewayToken).catch(() => {}); + connect(undefined, gatewayToken).catch(silentErrorHandler('App')); } }, [connect, connectionState]); diff --git a/desktop/src/components/ActiveLearningPanel.tsx b/desktop/src/components/ActiveLearningPanel.tsx new file mode 100644 index 0000000..fbf3f6b --- /dev/null +++ b/desktop/src/components/ActiveLearningPanel.tsx @@ -0,0 +1,409 @@ +/** + * ActiveLearningPanel - 主动学习状态面板 + * + * 展示学习事件、模式和系统建议。 + */ + +import { useCallback, useEffect, useState } from 'react'; +import { motion, AnimatePresence } from 'framer-motion'; +import { + Brain, + TrendingUp, + Lightbulb, + Check, + X, + RefreshCw, + Download, + Upload, + Settings, + BarChart3, + Clock, + Zap, +} from 'lucide-react'; +import { Button, Badge, EmptyState } from './ui'; +import { + useActiveLearningStore, + type LearningEvent, + type LearningPattern, + type LearningSuggestion, + type LearningEventType, +} from '../store/activeLearningStore'; +import { useChatStore } from '../store/chatStore'; +import { cardHover, defaultTransition } from '../lib/animations'; + +// === Constants === + +const EVENT_TYPE_LABELS: Record = { + preference: { label: '偏好', color: 'text-amber-400' }, + correction: { label: '纠正', color: 'text-red-400' }, + context: { label: '上下文', color: 'text-purple-400' }, + feedback: { label: '反馈', color: 'text-blue-400' }, + behavior: { label: '行为', color: 'text-green-400' }, + implicit: { label: '隐式', color: 'text-gray-400' }, +}; + +const PATTERN_TYPE_LABELS: Record = { + preference: { label: '偏好模式', icon: '🎯' }, + rule: { label: '规则模式', icon: '📋' }, + context: { label: '上下文模式', icon: '🔗' }, + behavior: { label: '行为模式', icon: '⚡' }, +}; + +// === Sub-Components === + +interface EventItemProps { + event: LearningEvent; + onAcknowledge: () => void; +} + +function EventItem({ event, onAcknowledge }: EventItemProps) { + const typeInfo = EVENT_TYPE_LABELS[event.type]; + const timeAgo = getTimeAgo(event.timestamp); + + return ( + +
+
+
+ + {typeInfo.label} + + {timeAgo} +
+

{event.observation}

+ {event.inferredPreference && ( +

→ {event.inferredPreference}

+ )} +
+ + {!event.acknowledged && ( + + )} +
+ +
+ 置信度: {(event.confidence * 100).toFixed(0)}% + {event.appliedCount > 0 && ( + • 应用 {event.appliedCount} 次 + )} +
+
+ ); +} + +interface SuggestionCardProps { + suggestion: LearningSuggestion; + onApply: () => void; + onDismiss: () => void; +} + +function SuggestionCard({ suggestion, onApply, onDismiss }: SuggestionCardProps) { + const daysLeft = Math.ceil( + (suggestion.expiresAt.getTime() - Date.now()) / (1000 * 60 * 60 * 24) + ); + + return ( + +
+ +
+

{suggestion.suggestion}

+
+ 置信度: {(suggestion.confidence * 100).toFixed(0)}% + {daysLeft > 0 && • {daysLeft} 天后过期} +
+
+
+ +
+ + +
+ +
+ ); +} + +// === Main Component === + +interface ActiveLearningPanelProps { + className?: string; +} + +export function ActiveLearningPanel({ className = '' }: ActiveLearningPanelProps) { + const { currentAgent } = useChatStore(); + const agentId = currentAgent?.id || 'default'; + + const [activeTab, setActiveTab] = useState<'events' | 'patterns' | 'suggestions'>('suggestions'); + + const { + events, + patterns, + suggestions, + config, + isLoading, + acknowledgeEvent, + getPatterns, + getSuggestions, + applySuggestion, + dismissSuggestion, + getStats, + setConfig, + exportLearningData, + clearEvents, + } = useActiveLearningStore(); + + const stats = getStats(agentId); + const agentEvents = events.filter(e => e.agentId === agentId).slice(0, 20); + const agentPatterns = getPatterns(agentId); + const agentSuggestions = getSuggestions(agentId); + + // 处理确认事件 + const handleAcknowledge = useCallback((eventId: string) => { + acknowledgeEvent(eventId); + }, [acknowledgeEvent]); + + // 处理应用建议 + const handleApplySuggestion = useCallback((suggestionId: string) => { + applySuggestion(suggestionId); + }, [applySuggestion]); + + // 处理忽略建议 + const handleDismissSuggestion = useCallback((suggestionId: string) => { + dismissSuggestion(suggestionId); + }, [dismissSuggestion]); + + // 导出学习数据 + const handleExport = useCallback(async () => { + const data = await exportLearningData(agentId); + const blob = new Blob([data], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `zclaw-learning-${agentId}-${new Date().toISOString().slice(0, 10)}.json`; + a.click(); + URL.revokeObjectURL(url); + }, [agentId, exportLearningData]); + + // 清除学习数据 + const handleClear = useCallback(() => { + if (confirm('确定要清除所有学习数据吗?此操作不可撤销。')) { + clearEvents(agentId); + } + }, [agentId, clearEvents]); + + return ( +
+ {/* 夨览栏 */} +
+
+ +

主动学习

+
+ +
+ + + +
+
+ + {/* 统计概览 */} +
+
+
{stats.totalEvents}
+
学习事件
+
+
+
{stats.totalPatterns}
+
学习模式
+
+
+
{agentSuggestions.length}
+
待处理建议
+
+
+
+ {(stats.avgConfidence * 100).toFixed(0)}% +
+
平均置信度
+
+
+ + {/* Tab 切换 */} +
+ {(['suggestions', 'events', 'patterns'] as const).map(tab => ( + + ))} +
+ + {/* 内容区域 */} +
+ + {activeTab === 'suggestions' && ( + + {agentSuggestions.length === 0 ? ( + } + title="暂无学习建议" + description="系统会根据您的反馈自动生成改进建议" + /> + ) : ( + agentSuggestions.map(suggestion => ( + handleApplySuggestion(suggestion.id)} + onDismiss={() => handleDismissSuggestion(suggestion.id)} + /> + )) + )} + + )} + + {activeTab === 'events' && ( + + {agentEvents.length === 0 ? ( + } + title="暂无学习事件" + description="开始对话后,系统会自动记录学习事件" + /> + ) : ( + agentEvents.map(event => ( + handleAcknowledge(event.id)} + /> + )) + )} + + )} + + {activeTab === 'patterns' && ( + + {agentPatterns.length === 0 ? ( + } + title="暂无学习模式" + description="积累更多反馈后,系统会识别出行为模式" + /> + ) : ( + agentPatterns.map(pattern => { + const typeInfo = PATTERN_TYPE_LABELS[pattern.type] || { label: pattern.type, icon: '📊' }; + return ( +
+
+
+ {typeInfo.icon} + {typeInfo.label} +
+ + {(pattern.confidence * 100).toFixed(0)}% + +
+

{pattern.description}

+
+ {pattern.examples.length} 个示例 +
+
+ ); + }) + )} +
+ )} +
+
+ + {/* 底部操作栏 */} +
+
+ 上次更新: {agentEvents[0] ? getTimeAgo(agentEvents[0].timestamp) : '无'} +
+ +
+
+ ); +} + +// === Helpers === + +function getTimeAgo(timestamp: number): string { + const seconds = Math.floor((Date.now() - timestamp) / 1000); + + if (seconds < 60) return '刚刚'; + if (seconds < 3600) return `${Math.floor(seconds / 60)} 分钟前`; + if (seconds < 86400) return `${Math.floor(seconds / 3600)} 小时前`; + return `${Math.floor(seconds / 86400)} 天前`; +} + +export default ActiveLearningPanel; diff --git a/desktop/src/components/AgentOnboardingWizard.tsx b/desktop/src/components/AgentOnboardingWizard.tsx new file mode 100644 index 0000000..d807bca --- /dev/null +++ b/desktop/src/components/AgentOnboardingWizard.tsx @@ -0,0 +1,663 @@ +/** + * AgentOnboardingWizard - Guided Agent creation wizard + * + * A 5-step wizard for creating new Agents with personality settings. + * Inspired by OpenClaw's quick configuration modal. + */ +import { useState, useCallback, useEffect } from 'react'; +import { motion, AnimatePresence } from 'framer-motion'; +import { + X, + User, + Bot, + Sparkles, + Briefcase, + Folder, + ChevronLeft, + ChevronRight, + Check, + Loader2, + AlertCircle, +} from 'lucide-react'; +import { cn } from '../lib/utils'; +import { useAgentStore, type CloneCreateOptions } from '../store/agentStore'; +import { EmojiPicker } from './ui/EmojiPicker'; +import { PersonalitySelector } from './PersonalitySelector'; +import { ScenarioTags } from './ScenarioTags'; +import type { Clone } from '../store/agentStore'; + +// === Types === + +interface WizardFormData { + userName: string; + userRole: string; + agentName: string; + agentRole: string; + agentNickname: string; + emoji: string; + personality: string; + scenarios: string[]; + workspaceDir: string; + restrictFiles: boolean; + privacyOptIn: boolean; + notes: string; +} + +interface AgentOnboardingWizardProps { + isOpen: boolean; + onClose: () => void; + onSuccess?: (clone: Clone) => void; +} + +const initialFormData: WizardFormData = { + userName: '', + userRole: '', + agentName: '', + agentRole: '', + agentNickname: '', + emoji: '', + personality: '', + scenarios: [], + workspaceDir: '', + restrictFiles: true, + privacyOptIn: false, + notes: '', +}; + +// === Step Configuration === + +const steps = [ + { id: 1, title: '认识用户', description: '让我们了解一下您', icon: User }, + { id: 2, title: 'Agent 身份', description: '给助手起个名字', icon: Bot }, + { id: 3, title: '人格风格', description: '选择沟通风格', icon: Sparkles }, + { id: 4, title: '使用场景', description: '选择应用场景', icon: Briefcase }, + { id: 5, title: '工作环境', description: '配置工作目录', icon: Folder }, +]; + +// === Component === + +export function AgentOnboardingWizard({ isOpen, onClose, onSuccess }: AgentOnboardingWizardProps) { + const { createClone, isLoading, error, clearError } = useAgentStore(); + const [currentStep, setCurrentStep] = useState(1); + const [formData, setFormData] = useState(initialFormData); + const [errors, setErrors] = useState>({}); + const [submitStatus, setSubmitStatus] = useState<'idle' | 'success' | 'error'>('idle'); + + // Reset form when modal opens + useEffect(() => { + if (isOpen) { + setFormData(initialFormData); + setCurrentStep(1); + setErrors({}); + setSubmitStatus('idle'); + clearError(); + } + }, [isOpen, clearError]); + + // Update form field + const updateField = (field: K, value: WizardFormData[K]) => { + setFormData((prev) => ({ ...prev, [field]: value })); + if (errors[field]) { + setErrors((prev) => { + const newErrors = { ...prev }; + delete newErrors[field]; + return newErrors; + }); + } + }; + + // Validate current step + const validateStep = useCallback((step: number): boolean => { + const newErrors: Record = {}; + + switch (step) { + case 1: + if (!formData.userName.trim()) { + newErrors.userName = '请输入您的名字'; + } + break; + case 2: + if (!formData.agentName.trim()) { + newErrors.agentName = '请输入 Agent 名称'; + } + break; + case 3: + if (!formData.emoji) { + newErrors.emoji = '请选择一个 Emoji'; + } + if (!formData.personality) { + newErrors.personality = '请选择一个人格风格'; + } + break; + case 4: + if (formData.scenarios.length === 0) { + newErrors.scenarios = '请至少选择一个使用场景'; + } + break; + case 5: + // Optional step, no validation + break; + } + + setErrors(newErrors); + return Object.keys(newErrors).length === 0; + }, [formData]); + + // Navigate to next step + const nextStep = () => { + if (validateStep(currentStep)) { + setCurrentStep((prev) => Math.min(prev + 1, steps.length)); + } + }; + + // Navigate to previous step + const prevStep = () => { + setCurrentStep((prev) => Math.max(prev - 1, 1)); + }; + + // Handle form submission + const handleSubmit = async () => { + if (!validateStep(currentStep)) { + return; + } + + setSubmitStatus('idle'); + + try { + const createOptions: CloneCreateOptions = { + name: formData.agentName, + role: formData.agentRole || undefined, + nickname: formData.agentNickname || undefined, + userName: formData.userName, + userRole: formData.userRole || undefined, + scenarios: formData.scenarios, + workspaceDir: formData.workspaceDir || undefined, + restrictFiles: formData.restrictFiles, + privacyOptIn: formData.privacyOptIn, + emoji: formData.emoji, + personality: formData.personality, + notes: formData.notes || undefined, + }; + + const clone = await createClone(createOptions); + + if (clone) { + setSubmitStatus('success'); + setTimeout(() => { + onSuccess?.(clone); + onClose(); + }, 1500); + } else { + setSubmitStatus('error'); + } + } catch { + setSubmitStatus('error'); + } + }; + + if (!isOpen) return null; + + const CurrentStepIcon = steps[currentStep - 1]?.icon || Bot; + + return ( +
+ {/* Backdrop */} +
+ + {/* Modal */} +
+ {/* Header */} +
+
+
+ +
+
+

+ 创建新 Agent +

+

+ 步骤 {currentStep}/{steps.length}: {steps[currentStep - 1]?.title} +

+
+
+ +
+ + {/* Progress Bar */} +
+
+ {steps.map((step, index) => { + const StepIcon = step.icon; + const isActive = currentStep === step.id; + const isCompleted = currentStep > step.id; + return ( +
+ + {index < steps.length - 1 && ( +
+ )} +
+ ); + })} +
+
+ + {/* Content */} +
+ + + {/* Step 1: 认识用户 */} + {currentStep === 1 && ( + <> +
+

+ 让我们认识一下 +

+

+ 请告诉我们您的名字,让助手更好地为您服务 +

+
+ +
+ + updateField('userName', e.target.value)} + placeholder="例如:张三" + className={cn( + 'w-full px-3 py-2 text-sm border rounded-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-primary', + errors.userName ? 'border-red-500' : 'border-gray-300 dark:border-gray-600' + )} + /> + {errors.userName && ( +

+ + {errors.userName} +

+ )} +
+ +
+ + updateField('userRole', e.target.value)} + placeholder="例如:产品经理、开发工程师" + className="w-full px-3 py-2 text-sm border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+ + )} + + {/* Step 2: Agent 身份 */} + {currentStep === 2 && ( + <> +
+

+ 给您的助手起个名字 +

+

+ 这将是您助手的身份标识 +

+
+ +
+ + updateField('agentName', e.target.value)} + placeholder="例如:小龙助手" + className={cn( + 'w-full px-3 py-2 text-sm border rounded-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-primary', + errors.agentName ? 'border-red-500' : 'border-gray-300 dark:border-gray-600' + )} + /> + {errors.agentName && ( +

+ + {errors.agentName} +

+ )} +
+ +
+ + updateField('agentRole', e.target.value)} + placeholder="例如:编程助手、写作顾问" + className="w-full px-3 py-2 text-sm border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+ +
+ + updateField('agentNickname', e.target.value)} + placeholder="例如:小龙" + className="w-full px-3 py-2 text-sm border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+ + )} + + {/* Step 3: 人格风格 */} + {currentStep === 3 && ( + <> +
+

+ 选择人格风格 +

+

+ 这决定了助手的沟通方式和性格特点 +

+
+ +
+ + updateField('emoji', emoji)} + /> + {errors.emoji && ( +

+ + {errors.emoji} +

+ )} +
+ +
+ + updateField('personality', personality)} + /> + {errors.personality && ( +

+ + {errors.personality} +

+ )} +
+ + )} + + {/* Step 4: 使用场景 */} + {currentStep === 4 && ( + <> +
+

+ 选择使用场景 +

+

+ 选择您希望 Agent 协助的领域(最多5个) +

+
+ + updateField('scenarios', scenarios)} + maxSelections={5} + /> + {errors.scenarios && ( +

+ + {errors.scenarios} +

+ )} + + )} + + {/* Step 5: 工作环境 */} + {currentStep === 5 && ( + <> +
+

+ 配置工作环境 +

+

+ 设置 Agent 的工作目录和权限 +

+
+ +
+ + updateField('workspaceDir', e.target.value)} + placeholder="例如:/home/user/projects/myproject" + className="w-full px-3 py-2 text-sm border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-primary font-mono" + /> +

+ Agent 将在此目录下工作,留空则使用默认目录 +

+
+ +
+
+
+

+ 限制文件访问 +

+

+ 仅允许访问工作目录内的文件 +

+
+ +
+ +
+
+

+ 匿名使用数据 +

+

+ 允许收集匿名使用数据以改进产品 +

+
+ +
+
+ +
+ +