feat: add internal ZCLAW kernel crates to git tracking

crates/zclaw-memory/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "zclaw-memory"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+description = "ZCLAW memory substrate with SQLite storage"
+
+[dependencies]
+zclaw-types = { workspace = true }
+
+tokio = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+uuid = { workspace = true }
+chrono = { workspace = true }
+tracing = { workspace = true }
+
+# SQLite
+sqlx = { workspace = true }
+
+# Async utilities
+futures = { workspace = true }

crates/zclaw-memory/src/lib.rs

@@ -0,0 +1,11 @@
+//! ZCLAW Memory Substrate
+//!
+//! SQLite-backed storage for agents, sessions, and memory.
+
+mod store;
+mod session;
+mod schema;
+
+pub use store::*;
+pub use session::*;
+pub use schema::*;

crates/zclaw-memory/src/schema.rs

@@ -0,0 +1,56 @@
+//! Database schema definitions
+
+/// Current schema version
+pub const SCHEMA_VERSION: i32 = 1;
+
+/// Schema creation SQL
+pub const CREATE_SCHEMA: &str = r#"
+-- Agents table
+CREATE TABLE IF NOT EXISTS agents (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    config TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL
+);
+
+-- Sessions table
+CREATE TABLE IF NOT EXISTS sessions (
+    id TEXT PRIMARY KEY,
+    agent_id TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE
+);
+
+-- Messages table
+CREATE TABLE IF NOT EXISTS messages (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id TEXT NOT NULL,
+    seq INTEGER NOT NULL,
+    content TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE,
+    UNIQUE(session_id, seq)
+);
+
+-- KV Store table
+CREATE TABLE IF NOT EXISTS kv_store (
+    agent_id TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (agent_id, key),
+    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE
+);
+
+-- Schema version table
+CREATE TABLE IF NOT EXISTS schema_version (
+    version INTEGER PRIMARY KEY
+);
+
+-- Indexes
+CREATE INDEX IF NOT EXISTS idx_sessions_agent ON sessions(agent_id);
+CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id);
+CREATE INDEX IF NOT EXISTS idx_kv_agent ON kv_store(agent_id);
+"#;

crates/zclaw-memory/src/session.rs

@@ -0,0 +1,96 @@
+//! Session management types
+
+use serde::{Deserialize, Serialize};
+use chrono::{DateTime, Utc};
+use zclaw_types::{SessionId, AgentId, Message};
+
+/// A conversation session
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Session {
+    pub id: SessionId,
+    pub agent_id: AgentId,
+    pub messages: Vec<Message>,
+    pub created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+    /// Token count estimate
+    pub token_count: usize,
+}
+
+impl Session {
+    pub fn new(agent_id: AgentId) -> Self {
+        Self {
+            id: SessionId::new(),
+            agent_id,
+            messages: Vec::new(),
+            created_at: Utc::now(),
+            updated_at: Utc::now(),
+            token_count: 0,
+        }
+    }
+
+    /// Add a message to the session
+    pub fn add_message(&mut self, message: Message) {
+        // Simple token estimation: ~4 chars per token
+        let tokens = self.estimate_tokens(&message);
+        self.messages.push(message);
+        self.token_count += tokens;
+        self.updated_at = Utc::now();
+    }
+
+    /// Estimate token count for a message
+    fn estimate_tokens(&self, message: &Message) -> usize {
+        let text = match message {
+            Message::User { content } => content,
+            Message::Assistant { content, thinking } => {
+                thinking.as_ref().map(|t| t.as_str()).unwrap_or("");
+                content
+            }
+            Message::System { content } => content,
+            Message::ToolUse { input, .. } => {
+                return serde_json::to_string(input).map(|s| s.len() / 4).unwrap_or(0);
+            }
+            Message::ToolResult { output, .. } => {
+                return serde_json::to_string(output).map(|s| s.len() / 4).unwrap_or(0);
+            }
+        };
+        text.len() / 4
+    }
+
+    /// Check if session exceeds context window
+    pub fn exceeds_threshold(&self, max_tokens: usize, threshold: f32) -> bool {
+        let threshold_tokens = (max_tokens as f32 * threshold) as usize;
+        self.token_count > threshold_tokens
+    }
+
+    /// Compact the session by keeping only recent messages
+    pub fn compact(&mut self, keep_last: usize) {
+        if self.messages.len() <= keep_last {
+            return;
+        }
+
+        // Keep system messages and last N messages
+        let system_messages: Vec<_> = self.messages.iter()
+            .filter(|m| matches!(m, Message::System { .. }))
+            .cloned()
+            .collect();
+
+        let recent_messages: Vec<_> = self.messages.iter()
+            .rev()
+            .take(keep_last)
+            .cloned()
+            .collect::<Vec<_>>()
+            .into_iter()
+            .rev()
+            .collect();
+
+        self.messages = [system_messages, recent_messages].concat();
+        self.recalculate_token_count();
+        self.updated_at = Utc::now();
+    }
+
+    fn recalculate_token_count(&mut self) {
+        self.token_count = self.messages.iter()
+            .map(|m| self.estimate_tokens(m))
+            .sum();
+    }
+}

crates/zclaw-memory/src/store.rs

@@ -0,0 +1,246 @@
+//! Memory store implementation
+
+use sqlx::SqlitePool;
+use zclaw_types::{AgentConfig, AgentId, SessionId, Message, Result, ZclawError};
+
+/// Memory store for persisting ZCLAW data
+pub struct MemoryStore {
+    pool: SqlitePool,
+}
+
+impl MemoryStore {
+    /// Create a new memory store with the given database path
+    pub async fn new(database_url: &str) -> Result<Self> {
+        let pool = SqlitePool::connect(database_url).await
+            .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+        let store = Self { pool };
+        store.run_migrations().await?;
+        Ok(store)
+    }
+
+    /// Create an in-memory database (for testing)
+    pub async fn in_memory() -> Result<Self> {
+        Self::new("sqlite::memory:").await
+    }
+
+    /// Run database migrations
+    async fn run_migrations(&self) -> Result<()> {
+        sqlx::query(crate::schema::CREATE_SCHEMA)
+            .execute(&self.pool)
+            .await
+            .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+        Ok(())
+    }
+
+    // === Agent CRUD ===
+
+    /// Save an agent configuration
+    pub async fn save_agent(&self, agent: &AgentConfig) -> Result<()> {
+        let config_json = serde_json::to_string(agent)?;
+        let id = agent.id.to_string();
+        let name = &agent.name;
+
+        sqlx::query(
+            r#"
+            INSERT INTO agents (id, name, config, created_at, updated_at)
+            VALUES (?, ?, ?, datetime('now'), datetime('now'))
+            ON CONFLICT(id) DO UPDATE SET
+                name = excluded.name,
+                config = excluded.config,
+                updated_at = datetime('now')
+            "#,
+        )
+        .bind(&id)
+        .bind(name)
+        .bind(&config_json)
+        .execute(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        Ok(())
+    }
+
+    /// Load an agent by ID
+    pub async fn load_agent(&self, id: &AgentId) -> Result<Option<AgentConfig>> {
+        let id_str = id.to_string();
+
+        let row = sqlx::query_as::<_, (String,)>(
+            "SELECT config FROM agents WHERE id = ?"
+        )
+        .bind(&id_str)
+        .fetch_optional(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        match row {
+            Some((config,)) => {
+                let agent: AgentConfig = serde_json::from_str(&config)?;
+                Ok(Some(agent))
+            }
+            None => Ok(None),
+        }
+    }
+
+    /// List all agents
+    pub async fn list_agents(&self) -> Result<Vec<AgentConfig>> {
+        let rows = sqlx::query_as::<_, (String,)>(
+            "SELECT config FROM agents"
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        let agents = rows
+            .into_iter()
+            .filter_map(|(config,)| serde_json::from_str(&config).ok())
+            .collect();
+        Ok(agents)
+    }
+
+    /// Delete an agent
+    pub async fn delete_agent(&self, id: &AgentId) -> Result<()> {
+        let id_str = id.to_string();
+
+        sqlx::query("DELETE FROM agents WHERE id = ?")
+            .bind(&id_str)
+            .execute(&self.pool)
+            .await
+            .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        Ok(())
+    }
+
+    // === Session Management ===
+
+    /// Create a new session for an agent
+    pub async fn create_session(&self, agent_id: &AgentId) -> Result<SessionId> {
+        let session_id = SessionId::new();
+        let session_str = session_id.to_string();
+        let agent_str = agent_id.to_string();
+
+        sqlx::query(
+            r#"
+            INSERT INTO sessions (id, agent_id, created_at, updated_at)
+            VALUES (?, ?, datetime('now'), datetime('now'))
+            "#,
+        )
+        .bind(&session_str)
+        .bind(&agent_str)
+        .execute(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        Ok(session_id)
+    }
+
+    /// Append a message to a session
+    pub async fn append_message(&self, session_id: &SessionId, message: &Message) -> Result<()> {
+        let session_str = session_id.to_string();
+        let message_json = serde_json::to_string(message)?;
+
+        sqlx::query(
+            r#"
+            INSERT INTO messages (session_id, seq, content, created_at)
+            SELECT ?, COALESCE(MAX(seq), 0) + 1, datetime('now')
+            FROM messages WHERE session_id = ?
+            "#,
+        )
+        .bind(&session_str)
+        .bind(&message_json)
+        .bind(&session_str)
+        .execute(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        // Update session updated_at
+        sqlx::query("UPDATE sessions SET updated_at = datetime('now') WHERE id = ?")
+            .bind(&session_str)
+            .execute(&self.pool)
+            .await
+            .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        Ok(())
+    }
+
+    /// Get all messages for a session
+    pub async fn get_messages(&self, session_id: &SessionId) -> Result<Vec<Message>> {
+        let session_str = session_id.to_string();
+
+        let rows = sqlx::query_as::<_, (String,)>(
+            "SELECT content FROM messages WHERE session_id = ? ORDER BY seq"
+        )
+        .bind(&session_str)
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        let messages = rows
+            .into_iter()
+            .filter_map(|(content,)| serde_json::from_str(&content).ok())
+            .collect();
+        Ok(messages)
+    }
+
+    // === KV Store ===
+
+    /// Store a key-value pair for an agent
+    pub async fn kv_store(&self, agent_id: &AgentId, key: &str, value: &serde_json::Value) -> Result<()> {
+        let agent_str = agent_id.to_string();
+        let value_json = serde_json::to_string(value)?;
+
+        sqlx::query(
+            r#"
+            INSERT INTO kv_store (agent_id, key, value, updated_at)
+            VALUES (?, ?, ?, datetime('now'))
+            ON CONFLICT(agent_id, key) DO UPDATE SET
+                value = excluded.value,
+                updated_at = datetime('now')
+            "#,
+        )
+        .bind(&agent_str)
+        .bind(key)
+        .bind(&value_json)
+        .execute(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        Ok(())
+    }
+
+    /// Recall a value by key
+    pub async fn kv_recall(&self, agent_id: &AgentId, key: &str) -> Result<Option<serde_json::Value>> {
+        let agent_str = agent_id.to_string();
+
+        let row = sqlx::query_as::<_, (String,)>(
+            "SELECT value FROM kv_store WHERE agent_id = ? AND key = ?"
+        )
+        .bind(&agent_str)
+        .bind(key)
+        .fetch_optional(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        match row {
+            Some((value,)) => {
+                let v: serde_json::Value = serde_json::from_str(&value)?;
+                Ok(Some(v))
+            }
+            None => Ok(None),
+        }
+    }
+
+    /// List all keys for an agent
+    pub async fn kv_list(&self, agent_id: &AgentId) -> Result<Vec<String>> {
+        let agent_str = agent_id.to_string();
+
+        let rows = sqlx::query_as::<_, (String,)>(
+            "SELECT key FROM kv_store WHERE agent_id = ?"
+        )
+        .bind(&agent_str)
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| ZclawError::StorageError(e.to_string()))?;
+
+        Ok(rows.into_iter().map(|(key,)| key).collect())
+    }
+}