6d2bedcfd7
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
4 new Playwright spec files covering all 10 planned E2E scenarios: - user-scenarios-core.spec.ts (14 tests): Onboarding, multi-turn dialogue, model switching — covers scenarios 1-3 - user-scenarios-automation.spec.ts (16 tests): Hands CRUD/trigger/approval, Pipeline workflow, automation triggers — covers scenarios 4, 6, 9 - user-scenarios-saas-memory.spec.ts (16 tests): Memory system, settings config, SaaS integration, butler panel — covers scenarios 5, 7, 8, 10 - user-scenarios-live.spec.ts (1 test): 100+ round real LLM conversation with context recall verification — uses live backend
423 lines
15 KiB
TypeScript
423 lines
15 KiB
TypeScript
/**
|
||
* ZCLAW User Scenario E2E Tests — Core Scenarios
|
||
*
|
||
* Scenario 1: 新用户首次体验 (First-time user experience)
|
||
* Scenario 2: 多轮对话 (Multi-turn dialogue with mock LLM)
|
||
* Scenario 3: 模型切换 (Model switching)
|
||
*
|
||
* Uses mock gateway for deterministic responses.
|
||
*/
|
||
|
||
import { test, expect } from '@playwright/test';
|
||
import {
|
||
setupMockGateway,
|
||
setupMockGatewayWithWebSocket,
|
||
} from '../fixtures/mock-gateway';
|
||
import { storeInspectors, STORE_NAMES } from '../fixtures/store-inspectors';
|
||
import { skipOnboarding, waitForAppReady, navigateToTab } from '../utils/user-actions';
|
||
|
||
const BASE_URL = 'http://localhost:1420';
|
||
test.setTimeout(120000);
|
||
|
||
/** Helper: click send button */
|
||
async function clickSend(page: import('@playwright/test').Page) {
|
||
const sendButton = page.getByRole('button', { name: '发送消息' }).or(
|
||
page.locator('button.bg-orange-500').first()
|
||
);
|
||
await sendButton.first().click();
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════════════════
|
||
// Scenario 1: 新用户首次体验
|
||
// ═══════════════════════════════════════════════════════════════════
|
||
|
||
test.describe('Scenario 1: 新用户首次体验', () => {
|
||
test.describe.configure({ mode: 'parallel' });
|
||
|
||
test('S1-01: 首次打开应显示引导或主界面', async ({ page }) => {
|
||
await setupMockGateway(page);
|
||
await page.goto(BASE_URL);
|
||
await page.waitForLoadState('networkidle', { timeout: 30000 }).catch(() => {});
|
||
|
||
const body = page.locator('body');
|
||
await expect(body).not.toBeEmpty({ timeout: 10000 });
|
||
|
||
await page.screenshot({ path: 'test-results/screenshots/s1-01-first-load.png' });
|
||
});
|
||
|
||
test('S1-02: 跳过引导后应显示主界面', async ({ page }) => {
|
||
await setupMockGateway(page);
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const sidebar = page.locator('aside');
|
||
await expect(sidebar).toBeVisible({ timeout: 10000 });
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await expect(chatInput).toBeVisible({ timeout: 10000 });
|
||
|
||
await page.screenshot({ path: 'test-results/screenshots/s1-02-main-ui.png' });
|
||
});
|
||
|
||
test('S1-03: 发送第一条消息应收到回复', async ({ page }) => {
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: { responseContent: '你好！我是 ZCLAW，很高兴为您服务。', streaming: true },
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill('你好');
|
||
await clickSend(page);
|
||
await page.waitForTimeout(3000);
|
||
|
||
const userMsg = page.locator('text=你好').first();
|
||
await expect(userMsg).toBeVisible({ timeout: 10000 });
|
||
|
||
await page.screenshot({ path: 'test-results/screenshots/s1-03-first-message.png' });
|
||
});
|
||
|
||
test('S1-04: 消息持久化 — 切换 Tab 后消息仍在', async ({ page }) => {
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: { responseContent: '消息持久化测试回复', streaming: true },
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
await storeInspectors.clearStore(page, STORE_NAMES.CHAT);
|
||
await page.waitForTimeout(500);
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill('这条消息应该被持久化');
|
||
await clickSend(page);
|
||
await page.waitForTimeout(3000);
|
||
|
||
const stateBefore = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string }>;
|
||
}>(page);
|
||
const countBefore = stateBefore?.messages?.length ?? 0;
|
||
|
||
if (countBefore === 0) {
|
||
test.skip();
|
||
return;
|
||
}
|
||
|
||
// 切换到其他 Tab 再切回
|
||
await navigateToTab(page, '自动化');
|
||
await page.waitForTimeout(1000);
|
||
await navigateToTab(page, '分身');
|
||
await page.waitForTimeout(1000);
|
||
|
||
const stateAfter = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string }>;
|
||
}>(page);
|
||
expect(stateAfter?.messages?.length).toBeGreaterThanOrEqual(countBefore);
|
||
});
|
||
|
||
test('S1-05: 侧边栏显示默认 Agent', async ({ page }) => {
|
||
await setupMockGateway(page);
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const sidebar = page.locator('aside');
|
||
await expect(sidebar).toBeVisible();
|
||
|
||
const sidebarContent = await sidebar.textContent();
|
||
expect(sidebarContent).toBeTruthy();
|
||
expect(sidebarContent!.length).toBeGreaterThan(0);
|
||
});
|
||
});
|
||
|
||
// ═══════════════════════════════════════════════════════════════════
|
||
// Scenario 2: 多轮对话 (Mock LLM)
|
||
// ═══════════════════════════════════════════════════════════════════
|
||
|
||
test.describe('Scenario 2: 多轮对话', () => {
|
||
test.describe.configure({ mode: 'serial' });
|
||
|
||
test('S2-01: 连续多轮对话应保持上下文', async ({ page }) => {
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: { responseContent: '这是模拟回复，我会记住您说的话。', streaming: true },
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
await storeInspectors.clearStore(page, STORE_NAMES.CHAT);
|
||
await page.waitForTimeout(500);
|
||
|
||
const userMessages = [
|
||
'你好，我是玩具厂老板',
|
||
'我们主要做塑料玩具，ABS材质的',
|
||
'最近ABS原料价格波动很大，你帮我分析一下趋势',
|
||
'那现在适合囤货吗？',
|
||
'澄海这边有什么好的供应商推荐吗？',
|
||
];
|
||
|
||
for (const msg of userMessages) {
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill(msg);
|
||
await clickSend(page);
|
||
await page.waitForTimeout(2000);
|
||
}
|
||
|
||
const state = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string; role: string }>;
|
||
}>(page);
|
||
const userMsgCount = state?.messages?.filter(m => m.role === 'user').length ?? 0;
|
||
expect(userMsgCount).toBeGreaterThanOrEqual(3);
|
||
});
|
||
|
||
test('S2-02: Token 统计应正常累加', async ({ page }) => {
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: { responseContent: 'Token统计测试回复', streaming: true },
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const initialState = await storeInspectors.getChatState<{
|
||
totalInputTokens?: number;
|
||
totalOutputTokens?: number;
|
||
}>(page);
|
||
const initialInput = initialState?.totalInputTokens ?? 0;
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill('Token test message');
|
||
await clickSend(page);
|
||
await page.waitForTimeout(3000);
|
||
|
||
const newState = await storeInspectors.getChatState<{
|
||
totalInputTokens?: number;
|
||
totalOutputTokens?: number;
|
||
}>(page);
|
||
const newInput = newState?.totalInputTokens ?? 0;
|
||
|
||
expect(newInput).toBeGreaterThanOrEqual(initialInput);
|
||
});
|
||
|
||
test('S2-03: 流式响应应稳定不中断', async ({ page }) => {
|
||
const longResponse = '这是一个较长的流式回复测试，用于验证流式传输的稳定性。'.repeat(15);
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: { responseContent: longResponse, streaming: true, chunkDelay: 30 },
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill('请写一段详细的分析');
|
||
await clickSend(page);
|
||
|
||
// 等待流式完成
|
||
await page.waitForFunction(
|
||
() => {
|
||
const stored = localStorage.getItem('zclaw-chat-storage');
|
||
if (!stored) return true;
|
||
try {
|
||
const state = JSON.parse(stored).state;
|
||
return state.isStreaming === false;
|
||
} catch {
|
||
return true;
|
||
}
|
||
},
|
||
{ timeout: 30000 }
|
||
).catch(() => {});
|
||
|
||
await expect(page.locator('textarea').first()).toBeVisible({ timeout: 5000 });
|
||
});
|
||
|
||
test('S2-04: 发送空消息应被阻止', async ({ page }) => {
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: { responseContent: '空消息测试', streaming: true },
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const stateBefore = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string }>;
|
||
}>(page);
|
||
const countBefore = stateBefore?.messages?.length ?? 0;
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill('');
|
||
await clickSend(page).catch(() => {});
|
||
await page.waitForTimeout(1000);
|
||
|
||
const stateAfter = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string }>;
|
||
}>(page);
|
||
expect(stateAfter?.messages?.length).toBeLessThanOrEqual(countBefore);
|
||
});
|
||
|
||
test('S2-05: 取消流式响应应停止生成', async ({ page }) => {
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: {
|
||
responseContent: '这条回复比较长，用于测试取消功能。'.repeat(10),
|
||
streaming: true,
|
||
chunkDelay: 200,
|
||
},
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill('写一篇长文');
|
||
await clickSend(page);
|
||
await page.waitForTimeout(500);
|
||
|
||
// 查找取消按钮
|
||
const cancelButton = page.getByRole('button', { name: /停止|取消|stop|cancel/i }).or(
|
||
page.locator('button').filter({ hasText: /停止|取消/ })
|
||
);
|
||
|
||
if (await cancelButton.isVisible({ timeout: 2000 }).catch(() => false)) {
|
||
await cancelButton.click();
|
||
await page.waitForTimeout(1000);
|
||
}
|
||
|
||
await expect(page.locator('textarea').first()).toBeVisible();
|
||
});
|
||
});
|
||
|
||
// ═══════════════════════════════════════════════════════════════════
|
||
// Scenario 3: 模型切换
|
||
// ═══════════════════════════════════════════════════════════════════
|
||
|
||
test.describe('Scenario 3: 模型切换', () => {
|
||
test.describe.configure({ mode: 'serial' });
|
||
|
||
test('S3-01: 模型列表应可加载', async ({ page }) => {
|
||
await setupMockGateway(page);
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const modelsResponse = await page.evaluate(async () => {
|
||
try {
|
||
const response = await fetch('/api/models');
|
||
return await response.json();
|
||
} catch {
|
||
return null;
|
||
}
|
||
});
|
||
|
||
expect(Array.isArray(modelsResponse)).toBe(true);
|
||
expect(modelsResponse.length).toBeGreaterThan(0);
|
||
});
|
||
|
||
test('S3-02: 切换模型后发送消息应正常', async ({ page }) => {
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: { responseContent: '模型切换后的回复', streaming: true },
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill('模型A的消息');
|
||
await clickSend(page);
|
||
await page.waitForTimeout(3000);
|
||
|
||
const stateBefore = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string }>;
|
||
}>(page);
|
||
const countBefore = stateBefore?.messages?.length ?? 0;
|
||
|
||
// 通过 store 切换模型
|
||
await page.evaluate(() => {
|
||
const stores = (window as any).__ZCLAW_STORES__;
|
||
if (stores?.chat?.setState) {
|
||
stores.chat.setState({ currentModel: 'gpt-4o' });
|
||
}
|
||
});
|
||
|
||
await chatInput.fill('模型B的消息');
|
||
await clickSend(page);
|
||
await page.waitForTimeout(3000);
|
||
|
||
const stateAfter = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string }>;
|
||
}>(page);
|
||
expect(stateAfter?.messages?.length).toBeGreaterThan(countBefore);
|
||
});
|
||
|
||
test('S3-03: 切换回原模型上下文不丢', async ({ page }) => {
|
||
await setupMockGatewayWithWebSocket(page, {
|
||
wsConfig: { responseContent: '上下文保持测试回复', streaming: true },
|
||
});
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
await storeInspectors.clearStore(page, STORE_NAMES.CHAT);
|
||
|
||
const chatInput = page.locator('textarea').first();
|
||
await chatInput.fill('上下文测试消息');
|
||
await clickSend(page);
|
||
await page.waitForTimeout(3000);
|
||
|
||
const state = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string }>;
|
||
}>(page);
|
||
const originalCount = state?.messages?.length ?? 0;
|
||
|
||
if (originalCount === 0) {
|
||
test.skip();
|
||
return;
|
||
}
|
||
|
||
await page.evaluate(() => {
|
||
const stores = (window as any).__ZCLAW_STORES__;
|
||
if (stores?.chat?.setState) {
|
||
stores.chat.setState({ currentModel: 'gpt-4o' });
|
||
}
|
||
});
|
||
|
||
await page.evaluate(() => {
|
||
const stores = (window as any).__ZCLAW_STORES__;
|
||
if (stores?.chat?.setState) {
|
||
stores.chat.setState({ currentModel: 'claude-sonnet-4-20250514' });
|
||
}
|
||
});
|
||
|
||
const stateAfter = await storeInspectors.getChatState<{
|
||
messages: Array<{ content: string }>;
|
||
}>(page);
|
||
expect(stateAfter?.messages?.length).toBeGreaterThanOrEqual(originalCount);
|
||
});
|
||
|
||
test('S3-04: 不存在的模型应优雅处理', async ({ page }) => {
|
||
await setupMockGateway(page);
|
||
await skipOnboarding(page);
|
||
await page.goto(BASE_URL);
|
||
await waitForAppReady(page);
|
||
|
||
await page.evaluate(() => {
|
||
const stores = (window as any).__ZCLAW_STORES__;
|
||
if (stores?.chat?.setState) {
|
||
stores.chat.setState({ currentModel: 'nonexistent-model-xyz' });
|
||
}
|
||
});
|
||
|
||
await expect(page.locator('textarea').first()).toBeVisible({ timeout: 5000 });
|
||
});
|
||
});
|
||
|
||
// ═══════════════════════════════════════════════════════════════════
|
||
// Test Report
|
||
// ═══════════════════════════════════════════════════════════════════
|
||
|
||
test.afterAll(async ({}, testInfo) => {
|
||
console.log('\n========================================');
|
||
console.log('ZCLAW User Scenario Core Tests Complete');
|
||
console.log(`Test Time: ${new Date().toISOString()}`);
|
||
console.log('========================================\n');
|
||
});
|