fix(tests): resolve workspace compilation + CJK search failures
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled

- saas test harness: align WorkerDispatcher::new and AppState::new
  signatures with SpawnLimiter addition and init_db(&DatabaseConfig)
- growth sqlite: add CJK fallback (LIKE-based) when FTS5 unicode61
  tokenizer fails on Chinese queries (unicode61 doesn't index CJK)
This commit is contained in:
iven
2026-04-04 00:34:34 +08:00
parent 5db2907420
commit 8faefd6a61
2 changed files with 64 additions and 21 deletions

View File

@@ -497,25 +497,63 @@ impl VikingStorage for SqliteStorage {
match fts_candidates {
Ok(rows) if !rows.is_empty() => rows,
Ok(_) => {
// FTS5 returned no results — memories are genuinely irrelevant.
// Do NOT fall back to scope scan (that was the root cause of
// injecting "广东光华" memories into "1+9" queries).
Ok(_) | Err(_) => {
// FTS5 returned no results or failed — check if query contains CJK
// characters. unicode61 tokenizer doesn't index CJK, so fall back
// to LIKE-based search for CJK queries.
let has_cjk = query.chars().any(|c| {
matches!(c, '\u{4E00}'..='\u{9FFF}' | '\u{3400}'..='\u{4DBF}' | '\u{F900}'..='\u{FAFF}')
});
if !has_cjk {
tracing::debug!(
"[SqliteStorage] FTS5 returned no results for query: '{}'",
query.chars().take(50).collect::<String>()
);
return Ok(Vec::new());
}
tracing::debug!(
"[SqliteStorage] FTS5 returned no results for query: '{}'",
"[SqliteStorage] FTS5 miss for CJK query, falling back to LIKE: '{}'",
query.chars().take(50).collect::<String>()
);
return Ok(Vec::new());
}
Err(e) => {
// FTS5 syntax error after sanitization — return empty rather
// than falling back to irrelevant scope-based results.
tracing::debug!(
"[SqliteStorage] FTS5 query failed for '{}': {}",
query.chars().take(50).collect::<String>(),
e
);
return Ok(Vec::new());
let pattern = format!("%{}%", query);
if let Some(ref scope) = options.scope {
sqlx::query_as::<_, MemoryRow>(
r#"
SELECT uri, memory_type, content, keywords, importance,
access_count, created_at, last_accessed, overview, abstract_summary
FROM memories
WHERE content LIKE ?
AND uri LIKE ?
ORDER BY importance DESC, access_count DESC
LIMIT ?
"#
)
.bind(&pattern)
.bind(format!("{}%", scope))
.bind(limit as i64)
.fetch_all(&self.pool)
.await
.unwrap_or_default()
} else {
sqlx::query_as::<_, MemoryRow>(
r#"
SELECT uri, memory_type, content, keywords, importance,
access_count, created_at, last_accessed, overview, abstract_summary
FROM memories
WHERE content LIKE ?
ORDER BY importance DESC, access_count DESC
LIMIT ?
"#
)
.bind(&pattern)
.bind(limit as i64)
.fetch_all(&self.pool)
.await
.unwrap_or_default()
}
}
}
} else {

View File

@@ -24,9 +24,9 @@ use sqlx::PgPool;
use std::sync::atomic::{AtomicBool, Ordering};
use tokio_util::sync::CancellationToken;
use tower::ServiceExt;
use zclaw_saas::config::SaaSConfig;
use zclaw_saas::config::{DatabaseConfig, SaaSConfig};
use zclaw_saas::db::init_db;
use zclaw_saas::state::AppState;
use zclaw_saas::state::{AppState, SpawnLimiter};
use zclaw_saas::workers::WorkerDispatcher;
pub const MAX_BODY: usize = 2 * 1024 * 1024;
@@ -123,7 +123,11 @@ pub async fn build_test_app() -> (Router, PgPool) {
drop(truncate_pool);
// init_db: schema (IF NOT EXISTS, fast) + seed data
let pool = init_db(&db_url).await.expect("init_db failed");
let db_config = DatabaseConfig {
url: db_url,
..DatabaseConfig::default()
};
let pool = init_db(&db_config).await.expect("init_db failed");
let mut config = SaaSConfig::default();
config.auth.jwt_expiration_hours = 24;
@@ -131,9 +135,10 @@ pub async fn build_test_app() -> (Router, PgPool) {
config.rate_limit.requests_per_minute = 10_000;
config.rate_limit.burst = 1_000;
let dispatcher = WorkerDispatcher::new(pool.clone());
let worker_limiter = SpawnLimiter::new("test-worker", 20);
let dispatcher = WorkerDispatcher::new(pool.clone(), worker_limiter.clone());
let shutdown_token = CancellationToken::new();
let state = AppState::new(pool.clone(), config, dispatcher, shutdown_token).expect("AppState::new failed");
let state = AppState::new(pool.clone(), config, dispatcher, shutdown_token, worker_limiter).expect("AppState::new failed");
let router = build_router(state);
(router, pool)
}