fix(saas): relay_tasks 超时自动清理 — 每5分钟扫描 processing >10min 标记 failed
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
- scheduler.rs: 新增 start_db_cleanup_tasks 中的 relay 超时清理定时任务 - status=processing 且 updated_at 超过 10 分钟的 relay_task 自动标记为 failed - 避免 Provider key 禁用后 relay_task 永久停留在 processing 状态
This commit is contained in:
@@ -82,6 +82,7 @@ pub fn start_scheduler(config: &SchedulerConfig, _db: PgPool, dispatcher: Worker
|
|||||||
pub fn start_db_cleanup_tasks(db: PgPool) {
|
pub fn start_db_cleanup_tasks(db: PgPool) {
|
||||||
let db_devices = db.clone();
|
let db_devices = db.clone();
|
||||||
let db_key_pool = db.clone();
|
let db_key_pool = db.clone();
|
||||||
|
let db_relay = db.clone();
|
||||||
|
|
||||||
// 每 24 小时清理不活跃设备
|
// 每 24 小时清理不活跃设备
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
@@ -128,6 +129,28 @@ pub fn start_db_cleanup_tasks(db: PgPool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// 每 5 分钟清理超时的 relay_tasks(status=processing 且 updated_at 超过 10 分钟)
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut interval = tokio::time::interval(Duration::from_secs(300));
|
||||||
|
loop {
|
||||||
|
interval.tick().await;
|
||||||
|
match sqlx::query(
|
||||||
|
"UPDATE relay_tasks SET status = 'failed', error_message = 'timeout: upstream not responding', completed_at = NOW() \
|
||||||
|
WHERE status = 'processing' AND updated_at < NOW() - INTERVAL '10 minutes'"
|
||||||
|
)
|
||||||
|
.execute(&db_relay)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(result) => {
|
||||||
|
if result.rows_affected() > 0 {
|
||||||
|
tracing::warn!("Cleaned up {} timed-out relay tasks (>10m processing)", result.rows_affected());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => tracing::error!("Relay task timeout cleanup failed: {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 用户任务调度器
|
/// 用户任务调度器
|
||||||
|
|||||||
Reference in New Issue
Block a user