iven/zclaw_openfang
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor(saas): 架构重构 + 性能优化 — 借鉴 loco-rs 模式

Browse Source 
Phase 0: 知识库
- docs/knowledge-base/loco-rs-patterns.md — loco-rs 10 个可借鉴模式研究

Phase 1: 数据层重构
- crates/zclaw-saas/src/models/ — 15 个 FromRow 类型化模型
- Login 3 次查询合并为 1 次 AccountLoginRow 查询
- 所有 service 文件从元组解构迁移到 FromRow 结构体

Phase 2: Worker + Scheduler 系统
- crates/zclaw-saas/src/workers/ — Worker trait + 5 个具体实现
- crates/zclaw-saas/src/scheduler.rs — TOML 声明式调度器
- crates/zclaw-saas/src/tasks/ — CLI 任务系统

Phase 3: 性能修复
- Relay N+1 查询 → 精准 SQL (relay/handlers.rs)
- Config RwLock → AtomicU32 无锁 rate limit (state.rs, middleware.rs)
- SSE std::sync::Mutex → tokio::sync::Mutex (relay/service.rs)
- /auth/refresh 阻塞清理 → Scheduler 定期执行

Phase 4: 多环境配置
- config/saas-{development,production,test}.toml
- ZCLAW_ENV 环境选择 + ZCLAW_SAAS_CONFIG 精确覆盖
- scheduler 配置集成到 TOML
This commit is contained in:
iven
2026-03-29 19:21:48 +08:00
parent 5fdf96c3f5
commit 8b9d506893
64 changed files with 3348 additions and 520 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
crates/zclaw-saas/src/telemetry/service.rs
View File

@@ -2,9 +2,12 @@
use sqlx::PgPool;
use crate::error::SaasResult;
use crate::models::{TelemetryModelStatsRow, TelemetryDailyStatsRow};
use super::types::*;
/// 批量写入遥测记录
const CHUNK_SIZE: usize = 100;
/// 批量写入遥测记录（分块多行 INSERT，每 chunk 100 条）
pub async fn ingest_telemetry(
db: &PgPool,
account_id: &str,
@@ -12,54 +15,73 @@ pub async fn ingest_telemetry(
app_version: &str,
entries: &[TelemetryEntry],
) -> SaasResult<TelemetryReportResponse> {
let mut accepted = 0usize;
// 预验证所有条目，分离有效/无效
let now = chrono::Utc::now().to_rfc3339();
let mut rejected = 0usize;
for entry in entries {
// 基本验证
if entry.input_tokens < 0 || entry.output_tokens < 0 {
let valid: Vec<&TelemetryEntry> = entries.iter().filter(|e| {
if e.input_tokens < 0 || e.output_tokens < 0 || e.model_id.is_empty() {
rejected += 1;
continue;
false
} else {
true
}
if entry.model_id.is_empty() {
rejected += 1;
continue;
}).collect();
if valid.is_empty() {
return Ok(TelemetryReportResponse { accepted: 0, rejected });
}
let mut tx = db.begin().await?;
let mut accepted = 0usize;
let cols = 13;
for chunk in valid.chunks(CHUNK_SIZE) {
// 预分配所有参数（拥有所有权）
let ids: Vec<String> = (0..chunk.len()).map(|_| uuid::Uuid::new_v4().to_string()).collect();
// 构建 VALUES 占位符
let placeholders: Vec<String> = (0..chunk.len())
.map(|i| {
let base = i * cols + 1;
format!("(${},${},${},${},${},${},${},${},${},${},${},${},${})",
base, base+1, base+2, base+3, base+4, base+5, base+6,
base+7, base+8, base+9, base+10, base+11, base+12)
}).collect();
let sql = format!(
"INSERT INTO telemetry_reports \
(id, account_id, device_id, app_version, model_id, input_tokens, output_tokens, \
latency_ms, success, error_type, connection_mode, reported_at, created_at) VALUES {}",
placeholders.join(", ")
);
let mut query = sqlx::query(&sql);
for (i, entry) in chunk.iter().enumerate() {
query = query
.bind(&ids[i])
.bind(account_id)
.bind(device_id)
.bind(app_version)
.bind(&entry.model_id)
.bind(entry.input_tokens)
.bind(entry.output_tokens)
.bind(entry.latency_ms)
.bind(entry.success)
.bind(&entry.error_type)
.bind(&entry.connection_mode)
.bind(&entry.timestamp)
.bind(&now);
}
let id = uuid::Uuid::new_v4().to_string();
let now = chrono::Utc::now().to_rfc3339();
let result = sqlx::query(
"INSERT INTO telemetry_reports
(id, account_id, device_id, app_version, model_id, input_tokens, output_tokens,
latency_ms, success, error_type, connection_mode, reported_at, created_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)"
)
.bind(&id)
.bind(account_id)
.bind(device_id)
.bind(app_version)
.bind(&entry.model_id)
.bind(entry.input_tokens)
.bind(entry.output_tokens)
.bind(entry.latency_ms)
.bind(entry.success)
.bind(&entry.error_type)
.bind(&entry.connection_mode)
.bind(&entry.timestamp)
.bind(&now)
.execute(db)
.await;
match result {
Ok(_) => accepted += 1,
match query.execute(&mut *tx).await {
Ok(result) => accepted += result.rows_affected() as usize,
Err(e) => {
tracing::warn!("Failed to insert telemetry entry: {}", e);
rejected += 1;
tracing::warn!("Failed to insert telemetry chunk: {}", e);
rejected += chunk.len();
}
}
}
tx.commit().await?;
Ok(TelemetryReportResponse { accepted, rejected })
}
@@ -116,7 +138,7 @@ pub async fn get_model_stats(
where_sql
);
let mut query_builder = sqlx::query_as::<_, (String, i64, i64, i64, Option<f64>, Option<f64>)>(&sql);
let mut query_builder = sqlx::query_as::<_, TelemetryModelStatsRow>(&sql);
for p in &params {
query_builder = query_builder.bind(p);
}
@@ -125,14 +147,14 @@ pub async fn get_model_stats(
let stats: Vec<ModelUsageStat> = rows
.into_iter()
.map(|(model_id, request_count, input_tokens, output_tokens, avg_latency_ms, success_rate)| {
.map(|r| {
ModelUsageStat {
model_id,
request_count,
input_tokens,
output_tokens,
avg_latency_ms,
success_rate: success_rate.unwrap_or(0.0),
model_id: r.model_id,
request_count: r.request_count,
input_tokens: r.input_tokens,
output_tokens: r.output_tokens,
avg_latency_ms: r.avg_latency_ms,
success_rate: r.success_rate.unwrap_or(0.0),
}
})
.collect();
@@ -140,84 +162,107 @@ pub async fn get_model_stats(
Ok(stats)
}
/// 写入审计日志摘要（批量写入 operation_logs）
/// 写入审计日志摘要（分块多行 INSERT，每 chunk 100 条）
pub async fn ingest_audit_summary(
db: &PgPool,
account_id: &str,
device_id: &str,
entries: &[AuditSummaryEntry],
) -> SaasResult<usize> {
// 预过滤空 action
let valid: Vec<_> = entries.iter().filter(|e| !e.action.is_empty()).collect();
if valid.is_empty() {
return Ok(0);
}
let mut tx = db.begin().await?;
let mut written = 0usize;
for entry in entries {
if entry.action.is_empty() {
continue;
// 每行 6 列参数
let cols = 6;
for chunk in valid.chunks(CHUNK_SIZE) {
let mut sql = String::from(
"INSERT INTO operation_logs (account_id, action, target_type, target_id, details, created_at) VALUES "
);
let placeholders: Vec<String> = (0..chunk.len())
.map(|i| {
let base = i * cols + 1;
format!("(${},${},${},${},${},${})", base, base+1, base+2, base+3, base+4, base+5)
}).collect();
sql.push_str(&placeholders.join(", "));
// 预收集 details（拥有所有权），避免借用生命周期问题
let details_list: Vec<serde_json::Value> = chunk.iter().map(|entry| {
serde_json::json!({
"source": "desktop",
"device_id": device_id,
"result": entry.result,
})
}).collect();
let mut query = sqlx::query(&sql);
for (i, entry) in chunk.iter().enumerate() {
query = query
.bind(account_id)
.bind(&entry.action)
.bind("desktop_audit")
.bind(&entry.target)
.bind(&details_list[i])
.bind(&entry.timestamp);
}
// 审计详情仅包含操作类型和目标，不包含用户内容
let details = serde_json::json!({
"source": "desktop",
"device_id": device_id,
"result": entry.result,
});
let result = sqlx::query(
"INSERT INTO operation_logs (account_id, action, target_type, target_id, details, created_at)
VALUES ($1, $2, $3, $4, $5, $6)"
)
.bind(account_id)
.bind(&entry.action)
.bind("desktop_audit")
.bind(&entry.target)
.bind(&details)
.bind(&entry.timestamp)
.execute(db)
.await;
match result {
Ok(_) => written += 1,
match query.execute(&mut *tx).await {
Ok(result) => written += result.rows_affected() as usize,
Err(e) => {
tracing::warn!("Failed to insert audit summary entry: {}", e);
tracing::warn!("Failed to insert audit summary chunk: {}", e);
}
}
}
tx.commit().await?;
Ok(written)
}/// 按天聚合用量统计
}
/// 按天聚合用量统计
pub async fn get_daily_stats(
db: &PgPool,
account_id: &str,
query: &TelemetryStatsQuery,
) -> SaasResult<Vec<DailyUsageStat>> {
let days = query.days.unwrap_or(30).min(90).max(1);
let days = query.days.unwrap_or(30).min(90).max(1) as i64;
let sql = format!(
"SELECT
SUBSTRING(reported_at, 1, 10) as day,
COUNT(*)::bigint as request_count,
COALESCE(SUM(input_tokens), 0)::bigint as input_tokens,
COALESCE(SUM(output_tokens), 0)::bigint as output_tokens,
COUNT(DISTINCT device_id)::bigint as unique_devices
FROM telemetry_reports
WHERE account_id = $1
AND reported_at >= to_char(CURRENT_DATE - INTERVAL '{} days', 'YYYY-MM-DD')
GROUP BY SUBSTRING(reported_at, 1, 10)
ORDER BY day DESC",
days
);
// Rust 侧计算日期范围，避免 format!() 拼 SQL
let from_ts = (chrono::Utc::now() - chrono::Duration::days(days))
.date_naive()
.and_hms_opt(0, 0, 0).unwrap()
.and_utc()
.to_rfc3339();
let rows: Vec<(String, i64, i64, i64, i64)> =
sqlx::query_as(&sql).bind(account_id).fetch_all(db).await?;
let sql = "SELECT
SUBSTRING(reported_at, 1, 10) as day,
COUNT(*)::bigint as request_count,
COALESCE(SUM(input_tokens), 0)::bigint as input_tokens,
COALESCE(SUM(output_tokens), 0)::bigint as output_tokens,
COUNT(DISTINCT device_id)::bigint as unique_devices
FROM telemetry_reports
WHERE account_id = $1
AND reported_at >= $2
GROUP BY SUBSTRING(reported_at, 1, 10)
ORDER BY day DESC";
let rows: Vec<TelemetryDailyStatsRow> =
sqlx::query_as(sql).bind(account_id).bind(&from_ts).fetch_all(db).await?;
let stats: Vec<DailyUsageStat> = rows
.into_iter()
.map(|(day, request_count, input_tokens, output_tokens, unique_devices)| {
.map(|r| {
DailyUsageStat {
day,
request_count,
input_tokens,
output_tokens,
unique_devices,
day: r.day,
request_count: r.request_count,
input_tokens: r.input_tokens,
output_tokens: r.output_tokens,
unique_devices: r.unique_devices,
}
})
.collect();