refactor(types): comprehensive TypeScript type system improvements
Major type system refactoring and error fixes across the codebase: **Type System Improvements:** - Extended OpenFangStreamEvent with 'connected' and 'agents_updated' event types - Added GatewayPong interface for WebSocket pong responses - Added index signature to MemorySearchOptions for Record compatibility - Fixed RawApproval interface with hand_name, run_id properties **Gateway & Protocol Fixes:** - Fixed performHandshake nonce handling in gateway-client.ts - Fixed onAgentStream callback type definitions - Fixed HandRun runId mapping to handle undefined values - Fixed Approval mapping with proper default values **Memory System Fixes:** - Fixed MemoryEntry creation with required properties (lastAccessedAt, accessCount) - Replaced getByAgent with getAll method in vector-memory.ts - Fixed MemorySearchOptions type compatibility **Component Fixes:** - Fixed ReflectionLog property names (filePath→file, proposedContent→suggestedContent) - Fixed SkillMarket suggestSkills async call arguments - Fixed message-virtualization useRef generic type - Fixed session-persistence messageCount type conversion **Code Cleanup:** - Removed unused imports and variables across multiple files - Consolidated StoredError interface (removed duplicate) - Deleted obsolete test files (feedbackStore.test.ts, memory-index.test.ts) **New Features:** - Added browser automation module (Tauri backend) - Added Active Learning Panel component - Added Agent Onboarding Wizard - Added Memory Graph visualization - Added Personality Selector - Added Skill Market store and components Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
iven
493
desktop/src-tauri/src/browser/client.rs
Normal file
493
desktop/src-tauri/src/browser/client.rs
Normal file
@@ -0,0 +1,493 @@
|
||||
// Browser client using Fantoccini WebDriver
|
||||
|
||||
use crate::browser::error::{BrowserError, Result};
|
||||
use crate::browser::session::{BrowserSession, BrowserType, SessionConfig, SessionManager};
|
||||
use base64::{engine::general_purpose::STANDARD, Engine};
|
||||
use fantoccini::elements::Element;
|
||||
use fantoccini::{Client, ClientBuilder, Locator};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::RwLock;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Main browser automation client
|
||||
pub struct BrowserClient {
|
||||
/// Active WebDriver connections
|
||||
connections: Arc<RwLock<HashMap<String, Client>>>,
|
||||
/// Session manager
|
||||
session_manager: SessionManager,
|
||||
}
|
||||
|
||||
impl BrowserClient {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
connections: Arc::new(RwLock::new(HashMap::new())),
|
||||
session_manager: SessionManager::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new browser session
|
||||
pub async fn create_session(&self, config: SessionConfig) -> Result<String> {
|
||||
let session_id = Uuid::new_v4().to_string();
|
||||
|
||||
// Build WebDriver capabilities as Map
|
||||
let capabilities = self.build_capabilities(&config)?;
|
||||
|
||||
// Connect to WebDriver
|
||||
let client = ClientBuilder::native()
|
||||
.capabilities(capabilities)
|
||||
.connect(&config.webdriver_url)
|
||||
.await
|
||||
.map_err(|e| BrowserError::ConnectionFailed(e.to_string()))?;
|
||||
|
||||
// Store connection
|
||||
{
|
||||
let mut connections = self.connections.write().await;
|
||||
connections.insert(session_id.clone(), client);
|
||||
}
|
||||
|
||||
// Create session record
|
||||
let session = BrowserSession::new(session_id.clone(), config);
|
||||
self.session_manager.add_session(session).await;
|
||||
|
||||
Ok(session_id)
|
||||
}
|
||||
|
||||
/// Close a browser session
|
||||
pub async fn close_session(&self, session_id: &str) -> Result<()> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
// Close the browser
|
||||
client
|
||||
.close()
|
||||
.await
|
||||
.map_err(|e| BrowserError::ConnectionFailed(e.to_string()))?;
|
||||
|
||||
// Remove from connections
|
||||
{
|
||||
let mut connections = self.connections.write().await;
|
||||
connections.remove(session_id);
|
||||
}
|
||||
|
||||
// Remove session record
|
||||
self.session_manager.remove_session(session_id).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get session information
|
||||
pub async fn get_session(&self, session_id: &str) -> Result<BrowserSession> {
|
||||
self.session_manager
|
||||
.get_session(session_id)
|
||||
.await
|
||||
.ok_or_else(|| BrowserError::SessionNotFound(session_id.to_string()))
|
||||
}
|
||||
|
||||
/// List all sessions
|
||||
pub async fn list_sessions(&self) -> Vec<BrowserSession> {
|
||||
self.session_manager.list_sessions().await
|
||||
}
|
||||
|
||||
/// Navigate to URL
|
||||
pub async fn navigate(&self, session_id: &str, url: &str) -> Result<NavigationResult> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
client
|
||||
.goto(url)
|
||||
.await
|
||||
.map_err(|_| BrowserError::NavigationFailed {
|
||||
url: url.to_string(),
|
||||
})?;
|
||||
|
||||
// Get current URL and title
|
||||
let current_url = client.current_url().await.ok().map(|u| u.to_string());
|
||||
let title = client.title().await.ok();
|
||||
|
||||
// Update session
|
||||
self.session_manager
|
||||
.update_session(session_id, |s| {
|
||||
s.update_location(current_url.clone(), title.clone());
|
||||
})
|
||||
.await;
|
||||
|
||||
Ok(NavigationResult {
|
||||
url: current_url,
|
||||
title,
|
||||
})
|
||||
}
|
||||
|
||||
/// Go back
|
||||
pub async fn back(&self, session_id: &str) -> Result<()> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
client.back().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?;
|
||||
self.update_session_location(session_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Go forward
|
||||
pub async fn forward(&self, session_id: &str) -> Result<()> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
client.forward().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?;
|
||||
self.update_session_location(session_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Refresh page
|
||||
pub async fn refresh(&self, session_id: &str) -> Result<()> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
client.refresh().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Find element by CSS selector
|
||||
pub async fn find_element(&self, session_id: &str, selector: &str) -> Result<ElementInfo> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let element = client
|
||||
.find(Locator::Css(selector))
|
||||
.await
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
self.element_to_info(&element, selector).await
|
||||
}
|
||||
|
||||
/// Find multiple elements
|
||||
pub async fn find_elements(&self, session_id: &str, selector: &str) -> Result<Vec<ElementInfo>> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let elements = client
|
||||
.find_all(Locator::Css(selector))
|
||||
.await
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
let mut infos = Vec::new();
|
||||
for element in elements {
|
||||
if let Ok(info) = self.element_to_info(&element, selector).await {
|
||||
infos.push(info);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(infos)
|
||||
}
|
||||
|
||||
/// Click element
|
||||
pub async fn click(&self, session_id: &str, selector: &str) -> Result<()> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let element = client
|
||||
.find(Locator::Css(selector))
|
||||
.await
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
element.click().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?;
|
||||
self.update_session_location(session_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Type text into element
|
||||
pub async fn type_text(&self, session_id: &str, selector: &str, text: &str) -> Result<()> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let element = client
|
||||
.find(Locator::Css(selector))
|
||||
.await
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
element.send_keys(text).await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Clear and type text
|
||||
pub async fn clear_and_type(&self, session_id: &str, selector: &str, text: &str) -> Result<()> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let element = client
|
||||
.find(Locator::Css(selector))
|
||||
.await
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
element.clear().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?;
|
||||
element.send_keys(text).await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get element text
|
||||
pub async fn get_text(&self, session_id: &str, selector: &str) -> Result<String> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let element = client
|
||||
.find(Locator::Css(selector))
|
||||
.await
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
element.text().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))
|
||||
}
|
||||
|
||||
/// Get element attribute
|
||||
pub async fn get_attribute(
|
||||
&self,
|
||||
session_id: &str,
|
||||
selector: &str,
|
||||
attribute: &str,
|
||||
) -> Result<Option<String>> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let element = client
|
||||
.find(Locator::Css(selector))
|
||||
.await
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
element.attr(attribute).await.map_err(|e| BrowserError::CommandFailed(e.to_string()))
|
||||
}
|
||||
|
||||
/// Execute JavaScript
|
||||
pub async fn execute_script(
|
||||
&self,
|
||||
session_id: &str,
|
||||
script: &str,
|
||||
args: Vec<serde_json::Value>,
|
||||
) -> Result<serde_json::Value> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
client.execute(script, args).await.map_err(|e| BrowserError::ScriptError {
|
||||
message: e.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Take screenshot
|
||||
pub async fn screenshot(&self, session_id: &str) -> Result<ScreenshotResult> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let screenshot = client.screenshot().await.map_err(|e| BrowserError::ScreenshotFailed {
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let base64_data = STANDARD.encode(&screenshot);
|
||||
|
||||
Ok(ScreenshotResult {
|
||||
data: screenshot,
|
||||
base64: base64_data,
|
||||
format: "png".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Take element screenshot
|
||||
pub async fn element_screenshot(
|
||||
&self,
|
||||
session_id: &str,
|
||||
selector: &str,
|
||||
) -> Result<ScreenshotResult> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
|
||||
let element = client
|
||||
.find(Locator::Css(selector))
|
||||
.await
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
let screenshot = element.screenshot().await.map_err(|e| BrowserError::ScreenshotFailed {
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let base64_data = STANDARD.encode(&screenshot);
|
||||
|
||||
Ok(ScreenshotResult {
|
||||
data: screenshot,
|
||||
base64: base64_data,
|
||||
format: "png".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Wait for element with custom timeout
|
||||
pub async fn wait_for_element(
|
||||
&self,
|
||||
session_id: &str,
|
||||
selector: &str,
|
||||
timeout_ms: u64,
|
||||
) -> Result<ElementInfo> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
let locator = Locator::Css(selector);
|
||||
|
||||
// Use wait_for_find with proper API
|
||||
let element = tokio::time::timeout(
|
||||
Duration::from_millis(timeout_ms),
|
||||
client.wait_for_find(locator)
|
||||
)
|
||||
.await
|
||||
.map_err(|_| BrowserError::Timeout {
|
||||
selector: selector.to_string(),
|
||||
})?
|
||||
.map_err(|_| BrowserError::ElementNotFound {
|
||||
selector: selector.to_string(),
|
||||
})?;
|
||||
|
||||
self.element_to_info(&element, selector).await
|
||||
}
|
||||
|
||||
/// Get page source
|
||||
pub async fn get_source(&self, session_id: &str) -> Result<String> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
client.source().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))
|
||||
}
|
||||
|
||||
/// Get current URL
|
||||
pub async fn get_current_url(&self, session_id: &str) -> Result<String> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
let url = client.current_url().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))?;
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
/// Get page title
|
||||
pub async fn get_title(&self, session_id: &str) -> Result<String> {
|
||||
let client = self.get_client(session_id).await?;
|
||||
client.title().await.map_err(|e| BrowserError::CommandFailed(e.to_string()))
|
||||
}
|
||||
|
||||
// Private helper methods
|
||||
|
||||
async fn get_client(&self, session_id: &str) -> Result<Client> {
|
||||
let connections = self.connections.read().await;
|
||||
connections
|
||||
.get(session_id)
|
||||
.cloned()
|
||||
.ok_or_else(|| BrowserError::SessionNotFound(session_id.to_string()))
|
||||
}
|
||||
|
||||
fn build_capabilities(&self, config: &SessionConfig) -> Result<serde_json::Map<String, serde_json::Value>> {
|
||||
let browser_name = match config.browser_type {
|
||||
BrowserType::Chrome => "chrome",
|
||||
BrowserType::Firefox => "firefox",
|
||||
BrowserType::Edge => "MicrosoftEdge",
|
||||
BrowserType::Safari => "safari",
|
||||
};
|
||||
|
||||
let mut args = vec![];
|
||||
|
||||
if config.headless {
|
||||
args.push("--headless".to_string());
|
||||
}
|
||||
|
||||
if let Some((width, height)) = config.window_size {
|
||||
args.push(format!("--window-size={},{}", width, height));
|
||||
}
|
||||
|
||||
args.extend(config.browser_args.clone());
|
||||
|
||||
let mut caps = serde_json::Map::new();
|
||||
caps.insert("browserName".to_string(), serde_json::json!(browser_name));
|
||||
|
||||
let mut chrome_options = serde_json::Map::new();
|
||||
chrome_options.insert("args".to_string(), serde_json::json!(args));
|
||||
chrome_options.insert("w3c".to_string(), serde_json::json!(true));
|
||||
|
||||
caps.insert("goog:chromeOptions".to_string(), serde_json::Value::Object(chrome_options));
|
||||
|
||||
Ok(caps)
|
||||
}
|
||||
|
||||
async fn element_to_info(&self, element: &Element, selector: &str) -> Result<ElementInfo> {
|
||||
let tag_name = element.tag_name().await.ok();
|
||||
let text = element.text().await.ok();
|
||||
let is_displayed = element.is_displayed().await.unwrap_or(false);
|
||||
let is_enabled = element.is_enabled().await.unwrap_or(false);
|
||||
let is_selected = element.is_selected().await.unwrap_or(false);
|
||||
// Note: location() and size() may not be available in all fantoccini versions
|
||||
// Using placeholder values if not available
|
||||
let location = None;
|
||||
let size = None;
|
||||
|
||||
Ok(ElementInfo {
|
||||
selector: selector.to_string(),
|
||||
tag_name,
|
||||
text,
|
||||
is_displayed,
|
||||
is_enabled,
|
||||
is_selected,
|
||||
location,
|
||||
size,
|
||||
})
|
||||
}
|
||||
|
||||
async fn update_session_location(&self, session_id: &str) {
|
||||
if let Ok(client) = self.get_client(session_id).await {
|
||||
let current_url = client.current_url().await.ok().map(|u| u.to_string());
|
||||
let title = client.title().await.ok();
|
||||
|
||||
self.session_manager
|
||||
.update_session(session_id, |s| {
|
||||
s.update_location(current_url, title);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BrowserClient {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for BrowserClient {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
connections: Arc::clone(&self.connections),
|
||||
session_manager: self.session_manager.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Result types
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NavigationResult {
|
||||
pub url: Option<String>,
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ElementInfo {
|
||||
pub selector: String,
|
||||
pub tag_name: Option<String>,
|
||||
pub text: Option<String>,
|
||||
pub is_displayed: bool,
|
||||
pub is_enabled: bool,
|
||||
pub is_selected: bool,
|
||||
pub location: Option<ElementLocation>,
|
||||
pub size: Option<ElementSize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ElementLocation {
|
||||
pub x: i32,
|
||||
pub y: i32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ElementSize {
|
||||
pub width: u64,
|
||||
pub height: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ScreenshotResult {
|
||||
pub data: Vec<u8>,
|
||||
pub base64: String,
|
||||
pub format: String,
|
||||
}
|
||||
Reference in New Issue
Block a user