refactor(crates): kernel/generation module split + DeerFlow optimizations + middleware + dead code cleanup

- Split zclaw-kernel/kernel.rs (1486 lines) into 9 domain modules
- Split zclaw-kernel/generation.rs (1080 lines) into 3 modules
- Add DeerFlow-inspired middleware: DanglingTool, SubagentLimit, ToolError, ToolOutputGuard
- Add PromptBuilder for structured system prompt assembly
- Add FactStore (zclaw-memory) for persistent fact extraction
- Add task builtin tool for agent task management
- Driver improvements: Anthropic/OpenAI extended thinking, Gemini safety settings
- Replace let _ = with proper log::warn! across SaaS handlers
- Remove unused dependency (url) from zclaw-hands
This commit is contained in:
iven
2026-04-03 00:28:03 +08:00
parent 0a04b260a4
commit 52bdafa633
55 changed files with 4130 additions and 1959 deletions

View File

@@ -291,6 +291,27 @@ impl sqlx::FromRow<'_, SqliteRow> for MemoryRow {
/// Private helper methods on SqliteStorage (NOT in impl VikingStorage block)
impl SqliteStorage {
/// Sanitize a user query for FTS5 MATCH syntax.
///
/// FTS5 treats several characters as operators (`+`, `-`, `*`, `"`, `(`, `)`, `:`).
/// Strips these and keeps only alphanumeric + CJK tokens with length > 1,
/// then joins them with `OR` for broad matching.
fn sanitize_fts_query(query: &str) -> String {
let terms: Vec<String> = query
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
.filter(|s| !s.is_empty() && s.len() > 1)
.map(|s| s.to_string())
.collect();
if terms.is_empty() {
return String::new();
}
// Join with OR so any term can match (broad recall, then rerank by similarity)
terms.join(" OR ")
}
/// Fetch memories by scope with importance-based ordering.
/// Used internally by find() for scope-based queries.
pub(crate) async fn fetch_by_scope_priv(&self, scope: Option<&str>, limit: usize) -> Result<Vec<MemoryRow>> {
@@ -363,7 +384,10 @@ impl VikingStorage for SqliteStorage {
let _ = sqlx::query("DELETE FROM memories_fts WHERE uri = ?")
.bind(&entry.uri)
.execute(&self.pool)
.await;
.await
.map_err(|e| {
tracing::warn!("[SqliteStorage] Failed to delete old FTS entry: {}", e);
});
let keywords_text = entry.keywords.join(" ");
let _ = sqlx::query(
@@ -376,7 +400,10 @@ impl VikingStorage for SqliteStorage {
.bind(&entry.content)
.bind(&keywords_text)
.execute(&self.pool)
.await;
.await
.map_err(|e| {
tracing::warn!("[SqliteStorage] Failed to insert FTS entry: {}", e);
});
// Update semantic scorer (use embedding when available)
let mut scorer = self.scorer.write().await;
@@ -416,8 +443,21 @@ impl VikingStorage for SqliteStorage {
// Strategy: use FTS5 for initial filtering when query is non-empty,
// then score candidates with TF-IDF / embedding for precise ranking.
// Fallback to scope-only scan when query is empty (e.g., "list all").
// When FTS5 returns nothing, we return empty — do NOT fall back to
// scope scan (that returns irrelevant high-importance memories).
let rows = if !query.is_empty() {
// Sanitize query for FTS5: strip operators that cause syntax errors
let sanitized = Self::sanitize_fts_query(query);
if sanitized.is_empty() {
// Query had no meaningful terms after sanitization (e.g., "1+2")
tracing::debug!(
"[SqliteStorage] Query '{}' produced no FTS5-searchable terms, skipping",
query.chars().take(50).collect::<String>()
);
return Ok(Vec::new());
}
// FTS5-powered candidate retrieval (fast, index-based)
let fts_candidates = if let Some(ref scope) = options.scope {
sqlx::query_as::<_, MemoryRow>(
@@ -432,7 +472,7 @@ impl VikingStorage for SqliteStorage {
LIMIT ?
"#
)
.bind(query)
.bind(&sanitized)
.bind(format!("{}%", scope))
.bind(limit as i64)
.fetch_all(&self.pool)
@@ -449,7 +489,7 @@ impl VikingStorage for SqliteStorage {
LIMIT ?
"#
)
.bind(query)
.bind(&sanitized)
.bind(limit as i64)
.fetch_all(&self.pool)
.await
@@ -457,11 +497,25 @@ impl VikingStorage for SqliteStorage {
match fts_candidates {
Ok(rows) if !rows.is_empty() => rows,
Ok(_) | Err(_) => {
// FTS5 returned nothing or query syntax was invalid —
// fallback to scope-based scan (no full table scan unless no scope)
tracing::debug!("[SqliteStorage] FTS5 returned no results, falling back to scope scan");
self.fetch_by_scope_priv(options.scope.as_deref(), limit).await?
Ok(_) => {
// FTS5 returned no results — memories are genuinely irrelevant.
// Do NOT fall back to scope scan (that was the root cause of
// injecting "广东光华" memories into "1+9" queries).
tracing::debug!(
"[SqliteStorage] FTS5 returned no results for query: '{}'",
query.chars().take(50).collect::<String>()
);
return Ok(Vec::new());
}
Err(e) => {
// FTS5 syntax error after sanitization — return empty rather
// than falling back to irrelevant scope-based results.
tracing::debug!(
"[SqliteStorage] FTS5 query failed for '{}': {}",
query.chars().take(50).collect::<String>(),
e
);
return Ok(Vec::new());
}
}
} else {
@@ -557,7 +611,10 @@ impl VikingStorage for SqliteStorage {
let _ = sqlx::query("DELETE FROM memories_fts WHERE uri = ?")
.bind(uri)
.execute(&self.pool)
.await;
.await
.map_err(|e| {
tracing::warn!("[SqliteStorage] Failed to delete FTS entry: {}", e);
});
// Remove from in-memory scorer
let mut scorer = self.scorer.write().await;