refactor(crates): kernel/generation module split + DeerFlow optimizations + middleware + dead code cleanup
- Split zclaw-kernel/kernel.rs (1486 lines) into 9 domain modules - Split zclaw-kernel/generation.rs (1080 lines) into 3 modules - Add DeerFlow-inspired middleware: DanglingTool, SubagentLimit, ToolError, ToolOutputGuard - Add PromptBuilder for structured system prompt assembly - Add FactStore (zclaw-memory) for persistent fact extraction - Add task builtin tool for agent task management - Driver improvements: Anthropic/OpenAI extended thinking, Gemini safety settings - Replace let _ = with proper log::warn! across SaaS handlers - Remove unused dependency (url) from zclaw-hands
This commit is contained in:
@@ -291,6 +291,27 @@ impl sqlx::FromRow<'_, SqliteRow> for MemoryRow {
|
||||
|
||||
/// Private helper methods on SqliteStorage (NOT in impl VikingStorage block)
|
||||
impl SqliteStorage {
|
||||
/// Sanitize a user query for FTS5 MATCH syntax.
|
||||
///
|
||||
/// FTS5 treats several characters as operators (`+`, `-`, `*`, `"`, `(`, `)`, `:`).
|
||||
/// Strips these and keeps only alphanumeric + CJK tokens with length > 1,
|
||||
/// then joins them with `OR` for broad matching.
|
||||
fn sanitize_fts_query(query: &str) -> String {
|
||||
let terms: Vec<String> = query
|
||||
.to_lowercase()
|
||||
.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|s| !s.is_empty() && s.len() > 1)
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
|
||||
if terms.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// Join with OR so any term can match (broad recall, then rerank by similarity)
|
||||
terms.join(" OR ")
|
||||
}
|
||||
|
||||
/// Fetch memories by scope with importance-based ordering.
|
||||
/// Used internally by find() for scope-based queries.
|
||||
pub(crate) async fn fetch_by_scope_priv(&self, scope: Option<&str>, limit: usize) -> Result<Vec<MemoryRow>> {
|
||||
@@ -363,7 +384,10 @@ impl VikingStorage for SqliteStorage {
|
||||
let _ = sqlx::query("DELETE FROM memories_fts WHERE uri = ?")
|
||||
.bind(&entry.uri)
|
||||
.execute(&self.pool)
|
||||
.await;
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::warn!("[SqliteStorage] Failed to delete old FTS entry: {}", e);
|
||||
});
|
||||
|
||||
let keywords_text = entry.keywords.join(" ");
|
||||
let _ = sqlx::query(
|
||||
@@ -376,7 +400,10 @@ impl VikingStorage for SqliteStorage {
|
||||
.bind(&entry.content)
|
||||
.bind(&keywords_text)
|
||||
.execute(&self.pool)
|
||||
.await;
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::warn!("[SqliteStorage] Failed to insert FTS entry: {}", e);
|
||||
});
|
||||
|
||||
// Update semantic scorer (use embedding when available)
|
||||
let mut scorer = self.scorer.write().await;
|
||||
@@ -416,8 +443,21 @@ impl VikingStorage for SqliteStorage {
|
||||
|
||||
// Strategy: use FTS5 for initial filtering when query is non-empty,
|
||||
// then score candidates with TF-IDF / embedding for precise ranking.
|
||||
// Fallback to scope-only scan when query is empty (e.g., "list all").
|
||||
// When FTS5 returns nothing, we return empty — do NOT fall back to
|
||||
// scope scan (that returns irrelevant high-importance memories).
|
||||
let rows = if !query.is_empty() {
|
||||
// Sanitize query for FTS5: strip operators that cause syntax errors
|
||||
let sanitized = Self::sanitize_fts_query(query);
|
||||
|
||||
if sanitized.is_empty() {
|
||||
// Query had no meaningful terms after sanitization (e.g., "1+2")
|
||||
tracing::debug!(
|
||||
"[SqliteStorage] Query '{}' produced no FTS5-searchable terms, skipping",
|
||||
query.chars().take(50).collect::<String>()
|
||||
);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// FTS5-powered candidate retrieval (fast, index-based)
|
||||
let fts_candidates = if let Some(ref scope) = options.scope {
|
||||
sqlx::query_as::<_, MemoryRow>(
|
||||
@@ -432,7 +472,7 @@ impl VikingStorage for SqliteStorage {
|
||||
LIMIT ?
|
||||
"#
|
||||
)
|
||||
.bind(query)
|
||||
.bind(&sanitized)
|
||||
.bind(format!("{}%", scope))
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
@@ -449,7 +489,7 @@ impl VikingStorage for SqliteStorage {
|
||||
LIMIT ?
|
||||
"#
|
||||
)
|
||||
.bind(query)
|
||||
.bind(&sanitized)
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
@@ -457,11 +497,25 @@ impl VikingStorage for SqliteStorage {
|
||||
|
||||
match fts_candidates {
|
||||
Ok(rows) if !rows.is_empty() => rows,
|
||||
Ok(_) | Err(_) => {
|
||||
// FTS5 returned nothing or query syntax was invalid —
|
||||
// fallback to scope-based scan (no full table scan unless no scope)
|
||||
tracing::debug!("[SqliteStorage] FTS5 returned no results, falling back to scope scan");
|
||||
self.fetch_by_scope_priv(options.scope.as_deref(), limit).await?
|
||||
Ok(_) => {
|
||||
// FTS5 returned no results — memories are genuinely irrelevant.
|
||||
// Do NOT fall back to scope scan (that was the root cause of
|
||||
// injecting "广东光华" memories into "1+9" queries).
|
||||
tracing::debug!(
|
||||
"[SqliteStorage] FTS5 returned no results for query: '{}'",
|
||||
query.chars().take(50).collect::<String>()
|
||||
);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
Err(e) => {
|
||||
// FTS5 syntax error after sanitization — return empty rather
|
||||
// than falling back to irrelevant scope-based results.
|
||||
tracing::debug!(
|
||||
"[SqliteStorage] FTS5 query failed for '{}': {}",
|
||||
query.chars().take(50).collect::<String>(),
|
||||
e
|
||||
);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -557,7 +611,10 @@ impl VikingStorage for SqliteStorage {
|
||||
let _ = sqlx::query("DELETE FROM memories_fts WHERE uri = ?")
|
||||
.bind(uri)
|
||||
.execute(&self.pool)
|
||||
.await;
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::warn!("[SqliteStorage] Failed to delete FTS entry: {}", e);
|
||||
});
|
||||
|
||||
// Remove from in-memory scorer
|
||||
let mut scorer = self.scorer.write().await;
|
||||
|
||||
Reference in New Issue
Block a user