feat(saas): add model groups for cross-provider failover

Model Groups provide logical model names that map to multiple physical
models across providers, with automatic failover when one provider's
key pool is exhausted.

Backend:
- New model_groups + model_group_members tables with FK constraints
- Full CRUD API (7 endpoints) with admin-only write permissions
- Cache layer: DashMap-backed CachedModelGroup with load_from_db
- Relay integration: ModelResolution enum for Direct/Group routing
- Cross-provider failover: sort_candidates_by_quota + OnceLock cache
- Relay failure path: record failure usage + relay_dequeue (fixes
  queue counter leak that caused connection pool exhaustion)
- add_group_member: validate model_id exists before insert

Frontend:
- saas-relay-client: accept getModel() callback for dynamic model selection
- connectionStore: prefer conversationStore.currentModel over first available

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

crates/zclaw-saas/src/model_config/handlers.rs

@@ -274,3 +274,105 @@ pub async fn list_provider_models(
 ) -> SaasResult<Json<PaginatedResponse<ModelInfo>>> {
     service::list_models(&state.db, Some(&provider_id), None, None).await.map(Json)
 }
+
+// ============ Model Groups ============
+
+/// GET /api/v1/model-groups
+pub async fn list_model_groups(
+    State(state): State<AppState>,
+    _ctx: Extension<AuthContext>,
+) -> SaasResult<Json<Vec<ModelGroupInfo>>> {
+    service::list_model_groups(&state.db).await.map(Json)
+}
+
+/// POST /api/v1/model-groups (admin only)
+pub async fn create_model_group(
+    State(state): State<AppState>,
+    Extension(ctx): Extension<AuthContext>,
+    Json(req): Json<CreateModelGroupRequest>,
+) -> SaasResult<(StatusCode, Json<ModelGroupInfo>)> {
+    check_permission(&ctx, "model:manage")?;
+    if req.name.trim().is_empty() {
+        return Err(SaasError::InvalidInput("name 不能为空".into()));
+    }
+    let group = service::create_model_group(&state.db, &req).await?;
+    log_operation(&state.db, &ctx.account_id, "model_group.create", "model_group", &group.id,
+        Some(serde_json::json!({"name": &req.name})), ctx.client_ip.as_deref()).await?;
+    if let Err(e) = state.cache.load_from_db(&state.db).await {
+        tracing::warn!("Cache reload failed after model_group.create: {}", e);
+    }
+    Ok((StatusCode::CREATED, Json(group)))
+}
+
+/// GET /api/v1/model-groups/:id
+pub async fn get_model_group(
+    State(state): State<AppState>,
+    Path(id): Path<String>,
+    _ctx: Extension<AuthContext>,
+) -> SaasResult<Json<ModelGroupInfo>> {
+    service::get_model_group(&state.db, &id).await.map(Json)
+}
+
+/// PATCH /api/v1/model-groups/:id (admin only)
+pub async fn update_model_group(
+    State(state): State<AppState>,
+    Path(id): Path<String>,
+    Extension(ctx): Extension<AuthContext>,
+    Json(req): Json<UpdateModelGroupRequest>,
+) -> SaasResult<Json<ModelGroupInfo>> {
+    check_permission(&ctx, "model:manage")?;
+    let group = service::update_model_group(&state.db, &id, &req).await?;
+    log_operation(&state.db, &ctx.account_id, "model_group.update", "model_group", &id, None, ctx.client_ip.as_deref()).await?;
+    if let Err(e) = state.cache.load_from_db(&state.db).await {
+        tracing::warn!("Cache reload failed after model_group.update: {}", e);
+    }
+    Ok(Json(group))
+}
+
+/// DELETE /api/v1/model-groups/:id (admin only)
+pub async fn delete_model_group(
+    State(state): State<AppState>,
+    Path(id): Path<String>,
+    Extension(ctx): Extension<AuthContext>,
+) -> SaasResult<Json<serde_json::Value>> {
+    check_permission(&ctx, "model:manage")?;
+    service::delete_model_group(&state.db, &id).await?;
+    log_operation(&state.db, &ctx.account_id, "model_group.delete", "model_group", &id, None, ctx.client_ip.as_deref()).await?;
+    if let Err(e) = state.cache.load_from_db(&state.db).await {
+        tracing::warn!("Cache reload failed after model_group.delete: {}", e);
+    }
+    Ok(Json(serde_json::json!({"ok": true})))
+}
+
+/// POST /api/v1/model-groups/:id/members (admin only)
+pub async fn add_group_member(
+    State(state): State<AppState>,
+    Path(id): Path<String>,
+    Extension(ctx): Extension<AuthContext>,
+    Json(req): Json<AddGroupMemberRequest>,
+) -> SaasResult<(StatusCode, Json<ModelGroupMemberInfo>)> {
+    check_permission(&ctx, "model:manage")?;
+    let member = service::add_group_member(&state.db, &id, &req).await?;
+    log_operation(&state.db, &ctx.account_id, "model_group.add_member", "model_group", &id,
+        Some(serde_json::json!({"provider_id": &req.provider_id, "model_id": &req.model_id})), ctx.client_ip.as_deref()).await?;
+    if let Err(e) = state.cache.load_from_db(&state.db).await {
+        tracing::warn!("Cache reload failed after add_group_member: {}", e);
+    }
+    Ok((StatusCode::CREATED, Json(member)))
+}
+
+/// DELETE /api/v1/model-groups/:id/members/:mid (admin only)
+pub async fn remove_group_member(
+    State(state): State<AppState>,
+    Path((id, mid)): Path<(String, String)>,
+    Extension(ctx): Extension<AuthContext>,
+) -> SaasResult<Json<serde_json::Value>> {
+    check_permission(&ctx, "model:manage")?;
+    service::remove_group_member(&state.db, &mid).await?;
+    log_operation(&state.db, &ctx.account_id, "model_group.remove_member", "model_group", &id,
+        Some(serde_json::json!({"member_id": mid})), ctx.client_ip.as_deref()).await?;
+    if let Err(e) = state.cache.load_from_db(&state.db).await {
+        tracing::warn!("Cache reload failed after remove_group_member: {}", e);
+    }
+    Ok(Json(serde_json::json!({"ok": true})))
+}

crates/zclaw-saas/src/model_config/mod.rs

@@ -17,6 +17,11 @@ pub fn routes() -> axum::Router<AppState> {
         // Models
         .route("/api/v1/models", get(handlers::list_models).post(handlers::create_model))
         .route("/api/v1/models/:id", get(handlers::get_model).patch(handlers::update_model).delete(handlers::delete_model))
+        // Model Groups
+        .route("/api/v1/model-groups", get(handlers::list_model_groups).post(handlers::create_model_group))
+        .route("/api/v1/model-groups/:id", get(handlers::get_model_group).patch(handlers::update_model_group).delete(handlers::delete_model_group))
+        .route("/api/v1/model-groups/:id/members", post(handlers::add_group_member))
+        .route("/api/v1/model-groups/:id/members/:mid", delete(handlers::remove_group_member))
         // Account API Keys
         .route("/api/v1/keys", get(handlers::list_api_keys).post(handlers::create_api_key))
         .route("/api/v1/keys/:id", delete(handlers::revoke_api_key))

crates/zclaw-saas/src/model_config/service.rs

@@ -491,3 +491,167 @@ fn mask_api_key(key: &str) -> String {
     }
     format!("{}...{}", &key[..4], &key[key.len()-4..])
 }
+
+// ============ Model Groups ============
+
+pub async fn list_model_groups(db: &PgPool) -> SaasResult<Vec<ModelGroupInfo>> {
+    let group_rows: Vec<(String, String, String, String, bool, String, String, String)> = sqlx::query_as(
+        "SELECT id, name, display_name, COALESCE(description, ''), enabled,
+                COALESCE(failover_strategy, 'quota_aware'), created_at, updated_at
+         FROM model_groups ORDER BY name"
+    ).fetch_all(db).await?;
+
+    let member_rows: Vec<(String, String, String, String, i32, bool)> = sqlx::query_as(
+        "SELECT id, group_id, provider_id, model_id, priority, enabled
+         FROM model_group_members ORDER BY priority ASC"
+    ).fetch_all(db).await?;
+
+    let groups = group_rows.into_iter().map(|(id, name, display_name, description, enabled, failover_strategy, created_at, updated_at)| {
+        let members: Vec<ModelGroupMemberInfo> = member_rows.iter()
+            .filter(|(_, gid, _, _, _, _)| gid == &id)
+            .map(|(mid, _, pid, mid2, pri, en)| ModelGroupMemberInfo {
+                id: mid.clone(),
+                provider_id: pid.clone(),
+                model_id: mid2.clone(),
+                priority: *pri,
+                enabled: *en,
+            })
+            .collect();
+        ModelGroupInfo { id, name, display_name, description, enabled, failover_strategy, members, created_at, updated_at }
+    }).collect();
+
+    Ok(groups)
+}
+
+pub async fn get_model_group(db: &PgPool, group_id: &str) -> SaasResult<ModelGroupInfo> {
+    let row: Option<(String, String, String, String, bool, String, String, String)> = sqlx::query_as(
+        "SELECT id, name, display_name, COALESCE(description, ''), enabled,
+                COALESCE(failover_strategy, 'quota_aware'), created_at, updated_at
+         FROM model_groups WHERE id = $1"
+    ).bind(group_id).fetch_optional(db).await?;
+
+    let (id, name, display_name, description, enabled, failover_strategy, created_at, updated_at) =
+        row.ok_or_else(|| SaasError::NotFound(format!("模型组 {} 不存在", group_id)))?;
+
+    let member_rows: Vec<(String, String, String, String, i32, bool)> = sqlx::query_as(
+        "SELECT id, group_id, provider_id, model_id, priority, enabled
+         FROM model_group_members WHERE group_id = $1 ORDER BY priority ASC"
+    ).bind(group_id).fetch_all(db).await?;
+
+    let members = member_rows.into_iter()
+        .map(|(mid, _, pid, mid2, pri, en)| ModelGroupMemberInfo {
+            id: mid, provider_id: pid, model_id: mid2, priority: pri, enabled: en,
+        })
+        .collect();
+
+    Ok(ModelGroupInfo { id, name, display_name, description, enabled, failover_strategy, members, created_at, updated_at })
+}
+
+pub async fn create_model_group(db: &PgPool, req: &CreateModelGroupRequest) -> SaasResult<ModelGroupInfo> {
+    let id = uuid::Uuid::new_v4().to_string();
+    let now = chrono::Utc::now().to_rfc3339();
+
+    // 检查名称唯一性
+    let existing: Option<(String,)> = sqlx::query_as("SELECT id FROM model_groups WHERE name = $1")
+        .bind(&req.name).fetch_optional(db).await?;
+    if existing.is_some() {
+        return Err(SaasError::AlreadyExists(format!("模型组 '{}' 已存在", req.name)));
+    }
+
+    // 名称不能和已有 model_id 冲突（避免路由歧义）
+    let model_conflict: Option<(String,)> = sqlx::query_as("SELECT model_id FROM models WHERE model_id = $1")
+        .bind(&req.name).fetch_optional(db).await?;
+    if model_conflict.is_some() {
+        return Err(SaasError::InvalidInput(
+            format!("模型组名称 '{}' 与已有模型 ID 冲突，请使用不同的名称", req.name)
+        ));
+    }
+
+    sqlx::query(
+        "INSERT INTO model_groups (id, name, display_name, description, failover_strategy, created_at, updated_at)
+         VALUES ($1, $2, $3, $4, $5, $6, $6)"
+    )
+    .bind(&id).bind(&req.name).bind(&req.display_name).bind(&req.description)
+    .bind(&req.failover_strategy).bind(&now)
+    .execute(db).await?;
+
+    get_model_group(db, &id).await
+}
+
+pub async fn update_model_group(
+    db: &PgPool, group_id: &str, req: &UpdateModelGroupRequest,
+) -> SaasResult<ModelGroupInfo> {
+    let now = chrono::Utc::now().to_rfc3339();
+
+    sqlx::query(
+        "UPDATE model_groups SET
+            display_name = COALESCE($1, display_name),
+            description = COALESCE($2, description),
+            enabled = COALESCE($3, enabled),
+            failover_strategy = COALESCE($4, failover_strategy),
+            updated_at = $5
+         WHERE id = $6"
+    )
+    .bind(req.display_name.as_deref())
+    .bind(req.description.as_deref())
+    .bind(req.enabled)
+    .bind(req.failover_strategy.as_deref())
+    .bind(&now)
+    .bind(group_id)
+    .execute(db).await?;
+
+    get_model_group(db, group_id).await
+}
+
+pub async fn delete_model_group(db: &PgPool, group_id: &str) -> SaasResult<()> {
+    let result = sqlx::query("DELETE FROM model_groups WHERE id = $1")
+        .bind(group_id).execute(db).await?;
+    if result.rows_affected() == 0 {
+        return Err(SaasError::NotFound(format!("模型组 {} 不存在", group_id)));
+    }
+    Ok(())
+}
+
+pub async fn add_group_member(
+    db: &PgPool, group_id: &str, req: &AddGroupMemberRequest,
+) -> SaasResult<ModelGroupMemberInfo> {
+    // 验证 group 存在
+    sqlx::query_scalar::<_, String>("SELECT id FROM model_groups WHERE id = $1")
+        .bind(group_id).fetch_optional(db).await?
+        .ok_or_else(|| SaasError::NotFound(format!("模型组 {} 不存在", group_id)))?;
+
+    // 验证 provider 存在
+    sqlx::query_scalar::<_, String>("SELECT id FROM providers WHERE id = $1")
+        .bind(&req.provider_id).fetch_optional(db).await?
+        .ok_or_else(|| SaasError::NotFound(format!("Provider {} 不存在", req.provider_id)))?;
+
+    // 验证 model 存在（避免插入无效 model_id 导致 relay 运行时找不到模型）
+    sqlx::query_scalar::<_, String>("SELECT model_id FROM models WHERE model_id = $1")
+        .bind(&req.model_id).fetch_optional(db).await?
+        .ok_or_else(|| SaasError::NotFound(format!("模型 {} 不存在", req.model_id)))?;
+
+    let id = uuid::Uuid::new_v4().to_string();
+    sqlx::query(
+        "INSERT INTO model_group_members (id, group_id, provider_id, model_id, priority, created_at, updated_at)
+         VALUES ($1, $2, $3, $4, $5, NOW(), NOW())"
+    )
+    .bind(&id).bind(group_id).bind(&req.provider_id).bind(&req.model_id).bind(req.priority)
+    .execute(db).await?;
+
+    Ok(ModelGroupMemberInfo {
+        id,
+        provider_id: req.provider_id.clone(),
+        model_id: req.model_id.clone(),
+        priority: req.priority,
+        enabled: true,
+    })
+}
+
+pub async fn remove_group_member(db: &PgPool, member_id: &str) -> SaasResult<()> {
+    let result = sqlx::query("DELETE FROM model_group_members WHERE id = $1")
+        .bind(member_id).execute(db).await?;
+    if result.rows_affected() == 0 {
+        return Err(SaasError::NotFound(format!("成员 {} 不存在", member_id)));
+    }
+    Ok(())
+}

crates/zclaw-saas/src/model_config/types.rs

@@ -155,6 +155,58 @@ pub struct UsageQuery {
     pub days: Option<i32>,
 }
 
+// --- Model Groups ---
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ModelGroupInfo {
+    pub id: String,
+    pub name: String,
+    pub display_name: String,
+    pub description: String,
+    pub enabled: bool,
+    pub failover_strategy: String,
+    pub members: Vec<ModelGroupMemberInfo>,
+    pub created_at: String,
+    pub updated_at: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ModelGroupMemberInfo {
+    pub id: String,
+    pub provider_id: String,
+    pub model_id: String,
+    pub priority: i32,
+    pub enabled: bool,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CreateModelGroupRequest {
+    pub name: String,
+    pub display_name: String,
+    #[serde(default)]
+    pub description: String,
+    #[serde(default = "default_failover_strategy")]
+    pub failover_strategy: String,
+}
+
+fn default_failover_strategy() -> String { "quota_aware".into() }
+
+#[derive(Debug, Deserialize)]
+pub struct UpdateModelGroupRequest {
+    pub display_name: Option<String>,
+    pub description: Option<String>,
+    pub enabled: Option<bool>,
+    pub failover_strategy: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AddGroupMemberRequest {
+    pub provider_id: String,
+    pub model_id: String,
+    #[serde(default)]
+    pub priority: i32,
+}
+
 // --- Seed Data ---
 
 #[derive(Debug, Deserialize)]