fix(tests): resolve workspace compilation + CJK search failures
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
- saas test harness: align WorkerDispatcher::new and AppState::new signatures with SpawnLimiter addition and init_db(&DatabaseConfig) - growth sqlite: add CJK fallback (LIKE-based) when FTS5 unicode61 tokenizer fails on Chinese queries (unicode61 doesn't index CJK)
This commit is contained in:
@@ -497,25 +497,63 @@ impl VikingStorage for SqliteStorage {
|
|||||||
|
|
||||||
match fts_candidates {
|
match fts_candidates {
|
||||||
Ok(rows) if !rows.is_empty() => rows,
|
Ok(rows) if !rows.is_empty() => rows,
|
||||||
Ok(_) => {
|
Ok(_) | Err(_) => {
|
||||||
// FTS5 returned no results — memories are genuinely irrelevant.
|
// FTS5 returned no results or failed — check if query contains CJK
|
||||||
// Do NOT fall back to scope scan (that was the root cause of
|
// characters. unicode61 tokenizer doesn't index CJK, so fall back
|
||||||
// injecting "广东光华" memories into "1+9" queries).
|
// to LIKE-based search for CJK queries.
|
||||||
|
let has_cjk = query.chars().any(|c| {
|
||||||
|
matches!(c, '\u{4E00}'..='\u{9FFF}' | '\u{3400}'..='\u{4DBF}' | '\u{F900}'..='\u{FAFF}')
|
||||||
|
});
|
||||||
|
|
||||||
|
if !has_cjk {
|
||||||
|
tracing::debug!(
|
||||||
|
"[SqliteStorage] FTS5 returned no results for query: '{}'",
|
||||||
|
query.chars().take(50).collect::<String>()
|
||||||
|
);
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
tracing::debug!(
|
tracing::debug!(
|
||||||
"[SqliteStorage] FTS5 returned no results for query: '{}'",
|
"[SqliteStorage] FTS5 miss for CJK query, falling back to LIKE: '{}'",
|
||||||
query.chars().take(50).collect::<String>()
|
query.chars().take(50).collect::<String>()
|
||||||
);
|
);
|
||||||
return Ok(Vec::new());
|
|
||||||
}
|
let pattern = format!("%{}%", query);
|
||||||
Err(e) => {
|
if let Some(ref scope) = options.scope {
|
||||||
// FTS5 syntax error after sanitization — return empty rather
|
sqlx::query_as::<_, MemoryRow>(
|
||||||
// than falling back to irrelevant scope-based results.
|
r#"
|
||||||
tracing::debug!(
|
SELECT uri, memory_type, content, keywords, importance,
|
||||||
"[SqliteStorage] FTS5 query failed for '{}': {}",
|
access_count, created_at, last_accessed, overview, abstract_summary
|
||||||
query.chars().take(50).collect::<String>(),
|
FROM memories
|
||||||
e
|
WHERE content LIKE ?
|
||||||
);
|
AND uri LIKE ?
|
||||||
return Ok(Vec::new());
|
ORDER BY importance DESC, access_count DESC
|
||||||
|
LIMIT ?
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.bind(&pattern)
|
||||||
|
.bind(format!("{}%", scope))
|
||||||
|
.bind(limit as i64)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default()
|
||||||
|
} else {
|
||||||
|
sqlx::query_as::<_, MemoryRow>(
|
||||||
|
r#"
|
||||||
|
SELECT uri, memory_type, content, keywords, importance,
|
||||||
|
access_count, created_at, last_accessed, overview, abstract_summary
|
||||||
|
FROM memories
|
||||||
|
WHERE content LIKE ?
|
||||||
|
ORDER BY importance DESC, access_count DESC
|
||||||
|
LIMIT ?
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.bind(&pattern)
|
||||||
|
.bind(limit as i64)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -24,9 +24,9 @@ use sqlx::PgPool;
|
|||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tower::ServiceExt;
|
use tower::ServiceExt;
|
||||||
use zclaw_saas::config::SaaSConfig;
|
use zclaw_saas::config::{DatabaseConfig, SaaSConfig};
|
||||||
use zclaw_saas::db::init_db;
|
use zclaw_saas::db::init_db;
|
||||||
use zclaw_saas::state::AppState;
|
use zclaw_saas::state::{AppState, SpawnLimiter};
|
||||||
use zclaw_saas::workers::WorkerDispatcher;
|
use zclaw_saas::workers::WorkerDispatcher;
|
||||||
|
|
||||||
pub const MAX_BODY: usize = 2 * 1024 * 1024;
|
pub const MAX_BODY: usize = 2 * 1024 * 1024;
|
||||||
@@ -123,7 +123,11 @@ pub async fn build_test_app() -> (Router, PgPool) {
|
|||||||
drop(truncate_pool);
|
drop(truncate_pool);
|
||||||
|
|
||||||
// init_db: schema (IF NOT EXISTS, fast) + seed data
|
// init_db: schema (IF NOT EXISTS, fast) + seed data
|
||||||
let pool = init_db(&db_url).await.expect("init_db failed");
|
let db_config = DatabaseConfig {
|
||||||
|
url: db_url,
|
||||||
|
..DatabaseConfig::default()
|
||||||
|
};
|
||||||
|
let pool = init_db(&db_config).await.expect("init_db failed");
|
||||||
|
|
||||||
let mut config = SaaSConfig::default();
|
let mut config = SaaSConfig::default();
|
||||||
config.auth.jwt_expiration_hours = 24;
|
config.auth.jwt_expiration_hours = 24;
|
||||||
@@ -131,9 +135,10 @@ pub async fn build_test_app() -> (Router, PgPool) {
|
|||||||
config.rate_limit.requests_per_minute = 10_000;
|
config.rate_limit.requests_per_minute = 10_000;
|
||||||
config.rate_limit.burst = 1_000;
|
config.rate_limit.burst = 1_000;
|
||||||
|
|
||||||
let dispatcher = WorkerDispatcher::new(pool.clone());
|
let worker_limiter = SpawnLimiter::new("test-worker", 20);
|
||||||
|
let dispatcher = WorkerDispatcher::new(pool.clone(), worker_limiter.clone());
|
||||||
let shutdown_token = CancellationToken::new();
|
let shutdown_token = CancellationToken::new();
|
||||||
let state = AppState::new(pool.clone(), config, dispatcher, shutdown_token).expect("AppState::new failed");
|
let state = AppState::new(pool.clone(), config, dispatcher, shutdown_token, worker_limiter).expect("AppState::new failed");
|
||||||
let router = build_router(state);
|
let router = build_router(state);
|
||||||
(router, pool)
|
(router, pool)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user