fix(tests): resolve workspace compilation + CJK search failures
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
- saas test harness: align WorkerDispatcher::new and AppState::new signatures with SpawnLimiter addition and init_db(&DatabaseConfig) - growth sqlite: add CJK fallback (LIKE-based) when FTS5 unicode61 tokenizer fails on Chinese queries (unicode61 doesn't index CJK)
This commit is contained in:
@@ -497,25 +497,63 @@ impl VikingStorage for SqliteStorage {
|
||||
|
||||
match fts_candidates {
|
||||
Ok(rows) if !rows.is_empty() => rows,
|
||||
Ok(_) => {
|
||||
// FTS5 returned no results — memories are genuinely irrelevant.
|
||||
// Do NOT fall back to scope scan (that was the root cause of
|
||||
// injecting "广东光华" memories into "1+9" queries).
|
||||
Ok(_) | Err(_) => {
|
||||
// FTS5 returned no results or failed — check if query contains CJK
|
||||
// characters. unicode61 tokenizer doesn't index CJK, so fall back
|
||||
// to LIKE-based search for CJK queries.
|
||||
let has_cjk = query.chars().any(|c| {
|
||||
matches!(c, '\u{4E00}'..='\u{9FFF}' | '\u{3400}'..='\u{4DBF}' | '\u{F900}'..='\u{FAFF}')
|
||||
});
|
||||
|
||||
if !has_cjk {
|
||||
tracing::debug!(
|
||||
"[SqliteStorage] FTS5 returned no results for query: '{}'",
|
||||
query.chars().take(50).collect::<String>()
|
||||
);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
"[SqliteStorage] FTS5 returned no results for query: '{}'",
|
||||
"[SqliteStorage] FTS5 miss for CJK query, falling back to LIKE: '{}'",
|
||||
query.chars().take(50).collect::<String>()
|
||||
);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
Err(e) => {
|
||||
// FTS5 syntax error after sanitization — return empty rather
|
||||
// than falling back to irrelevant scope-based results.
|
||||
tracing::debug!(
|
||||
"[SqliteStorage] FTS5 query failed for '{}': {}",
|
||||
query.chars().take(50).collect::<String>(),
|
||||
e
|
||||
);
|
||||
return Ok(Vec::new());
|
||||
|
||||
let pattern = format!("%{}%", query);
|
||||
if let Some(ref scope) = options.scope {
|
||||
sqlx::query_as::<_, MemoryRow>(
|
||||
r#"
|
||||
SELECT uri, memory_type, content, keywords, importance,
|
||||
access_count, created_at, last_accessed, overview, abstract_summary
|
||||
FROM memories
|
||||
WHERE content LIKE ?
|
||||
AND uri LIKE ?
|
||||
ORDER BY importance DESC, access_count DESC
|
||||
LIMIT ?
|
||||
"#
|
||||
)
|
||||
.bind(&pattern)
|
||||
.bind(format!("{}%", scope))
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
sqlx::query_as::<_, MemoryRow>(
|
||||
r#"
|
||||
SELECT uri, memory_type, content, keywords, importance,
|
||||
access_count, created_at, last_accessed, overview, abstract_summary
|
||||
FROM memories
|
||||
WHERE content LIKE ?
|
||||
ORDER BY importance DESC, access_count DESC
|
||||
LIMIT ?
|
||||
"#
|
||||
)
|
||||
.bind(&pattern)
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -24,9 +24,9 @@ use sqlx::PgPool;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tower::ServiceExt;
|
||||
use zclaw_saas::config::SaaSConfig;
|
||||
use zclaw_saas::config::{DatabaseConfig, SaaSConfig};
|
||||
use zclaw_saas::db::init_db;
|
||||
use zclaw_saas::state::AppState;
|
||||
use zclaw_saas::state::{AppState, SpawnLimiter};
|
||||
use zclaw_saas::workers::WorkerDispatcher;
|
||||
|
||||
pub const MAX_BODY: usize = 2 * 1024 * 1024;
|
||||
@@ -123,7 +123,11 @@ pub async fn build_test_app() -> (Router, PgPool) {
|
||||
drop(truncate_pool);
|
||||
|
||||
// init_db: schema (IF NOT EXISTS, fast) + seed data
|
||||
let pool = init_db(&db_url).await.expect("init_db failed");
|
||||
let db_config = DatabaseConfig {
|
||||
url: db_url,
|
||||
..DatabaseConfig::default()
|
||||
};
|
||||
let pool = init_db(&db_config).await.expect("init_db failed");
|
||||
|
||||
let mut config = SaaSConfig::default();
|
||||
config.auth.jwt_expiration_hours = 24;
|
||||
@@ -131,9 +135,10 @@ pub async fn build_test_app() -> (Router, PgPool) {
|
||||
config.rate_limit.requests_per_minute = 10_000;
|
||||
config.rate_limit.burst = 1_000;
|
||||
|
||||
let dispatcher = WorkerDispatcher::new(pool.clone());
|
||||
let worker_limiter = SpawnLimiter::new("test-worker", 20);
|
||||
let dispatcher = WorkerDispatcher::new(pool.clone(), worker_limiter.clone());
|
||||
let shutdown_token = CancellationToken::new();
|
||||
let state = AppState::new(pool.clone(), config, dispatcher, shutdown_token).expect("AppState::new failed");
|
||||
let state = AppState::new(pool.clone(), config, dispatcher, shutdown_token, worker_limiter).expect("AppState::new failed");
|
||||
let router = build_router(state);
|
||||
(router, pool)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user