feat(pipeline): implement Pipeline DSL system for automated workflows
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
Some checks failed
CI / Lint & TypeCheck (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Build Frontend (push) Has been cancelled
CI / Rust Check (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
CI / E2E Tests (push) Has been cancelled
Add complete Pipeline DSL system including:
- Rust backend (zclaw-pipeline crate) with parser, executor, and state management
- Frontend components: PipelinesPanel, PipelineResultPreview, ClassroomPreviewer
- Pipeline recommender for Agent conversation integration
- 5 pipeline templates: education, marketing, legal, research, productivity
- Documentation for Pipeline DSL architecture
Pipeline DSL enables declarative workflow definitions with:
- YAML-based configuration
- Expression resolution (${inputs.topic}, ${steps.step1.output})
- LLM integration, parallel execution, file export
- Agent smart recommendations in conversations
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
211
crates/zclaw-pipeline/src/parser.rs
Normal file
211
crates/zclaw-pipeline/src/parser.rs
Normal file
@@ -0,0 +1,211 @@
|
||||
//! Pipeline DSL Parser
|
||||
//!
|
||||
//! Parses YAML pipeline definitions into Pipeline structs.
|
||||
|
||||
use std::path::Path;
|
||||
use serde_yaml;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::types::{Pipeline, API_VERSION};
|
||||
|
||||
/// Parser errors
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ParseError {
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("YAML parse error: {0}")]
|
||||
Yaml(#[from] serde_yaml::Error),
|
||||
|
||||
#[error("Invalid API version: expected '{expected}', got '{actual}'")]
|
||||
InvalidVersion { expected: String, actual: String },
|
||||
|
||||
#[error("Invalid kind: expected 'Pipeline', got '{0}'")]
|
||||
InvalidKind(String),
|
||||
|
||||
#[error("Missing required field: {0}")]
|
||||
MissingField(String),
|
||||
|
||||
#[error("Invalid action type: {0}")]
|
||||
InvalidAction(String),
|
||||
|
||||
#[error("Validation error: {0}")]
|
||||
Validation(String),
|
||||
}
|
||||
|
||||
/// Pipeline parser
|
||||
pub struct PipelineParser;
|
||||
|
||||
impl PipelineParser {
|
||||
/// Parse a pipeline from YAML string
|
||||
pub fn parse(yaml: &str) -> Result<Pipeline, ParseError> {
|
||||
let pipeline: Pipeline = serde_yaml::from_str(yaml)?;
|
||||
|
||||
// Validate API version
|
||||
if pipeline.api_version != API_VERSION {
|
||||
return Err(ParseError::InvalidVersion {
|
||||
expected: API_VERSION.to_string(),
|
||||
actual: pipeline.api_version.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
// Validate kind
|
||||
if pipeline.kind != "Pipeline" {
|
||||
return Err(ParseError::InvalidKind(pipeline.kind.clone()));
|
||||
}
|
||||
|
||||
// Validate required fields
|
||||
if pipeline.metadata.name.is_empty() {
|
||||
return Err(ParseError::MissingField("metadata.name".to_string()));
|
||||
}
|
||||
|
||||
if pipeline.spec.steps.is_empty() {
|
||||
return Err(ParseError::Validation("Pipeline must have at least one step".to_string()));
|
||||
}
|
||||
|
||||
// Validate step IDs are unique
|
||||
let mut seen_ids = std::collections::HashSet::new();
|
||||
for step in &pipeline.spec.steps {
|
||||
if !seen_ids.insert(&step.id) {
|
||||
return Err(ParseError::Validation(
|
||||
format!("Duplicate step ID: {}", step.id)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Validate input names are unique
|
||||
let mut seen_inputs = std::collections::HashSet::new();
|
||||
for input in &pipeline.spec.inputs {
|
||||
if !seen_inputs.insert(&input.name) {
|
||||
return Err(ParseError::Validation(
|
||||
format!("Duplicate input name: {}", input.name)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(pipeline)
|
||||
}
|
||||
|
||||
/// Parse a pipeline from file
|
||||
pub fn parse_file(path: &Path) -> Result<Pipeline, ParseError> {
|
||||
let content = std::fs::read_to_string(path)?;
|
||||
Self::parse(&content)
|
||||
}
|
||||
|
||||
/// Parse and validate all pipelines in a directory
|
||||
pub fn parse_directory(dir: &Path) -> Result<Vec<(String, Pipeline)>, ParseError> {
|
||||
let mut pipelines = Vec::new();
|
||||
|
||||
if !dir.exists() {
|
||||
return Ok(pipelines);
|
||||
}
|
||||
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.extension().map(|e| e == "yaml" || e == "yml").unwrap_or(false) {
|
||||
match Self::parse_file(&path) {
|
||||
Ok(pipeline) => {
|
||||
let filename = path.file_stem()
|
||||
.map(|s| s.to_string_lossy().to_string())
|
||||
.unwrap_or_default();
|
||||
pipelines.push((filename, pipeline));
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to parse pipeline {:?}: {}", path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(pipelines)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_valid_pipeline() {
|
||||
let yaml = r#"
|
||||
apiVersion: zclaw/v1
|
||||
kind: Pipeline
|
||||
metadata:
|
||||
name: test-pipeline
|
||||
spec:
|
||||
steps:
|
||||
- id: step1
|
||||
action:
|
||||
type: llm_generate
|
||||
template: "test"
|
||||
"#;
|
||||
let pipeline = PipelineParser::parse(yaml).unwrap();
|
||||
assert_eq!(pipeline.metadata.name, "test-pipeline");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_invalid_version() {
|
||||
let yaml = r#"
|
||||
apiVersion: invalid/v1
|
||||
kind: Pipeline
|
||||
metadata:
|
||||
name: test
|
||||
spec:
|
||||
steps: []
|
||||
"#;
|
||||
let result = PipelineParser::parse(yaml);
|
||||
assert!(matches!(result, Err(ParseError::InvalidVersion { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_invalid_kind() {
|
||||
let yaml = r#"
|
||||
apiVersion: zclaw/v1
|
||||
kind: NotPipeline
|
||||
metadata:
|
||||
name: test
|
||||
spec:
|
||||
steps: []
|
||||
"#;
|
||||
let result = PipelineParser::parse(yaml);
|
||||
assert!(matches!(result, Err(ParseError::InvalidKind(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_empty_steps() {
|
||||
let yaml = r#"
|
||||
apiVersion: zclaw/v1
|
||||
kind: Pipeline
|
||||
metadata:
|
||||
name: test
|
||||
spec:
|
||||
steps: []
|
||||
"#;
|
||||
let result = PipelineParser::parse(yaml);
|
||||
assert!(matches!(result, Err(ParseError::Validation(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_duplicate_step_ids() {
|
||||
let yaml = r#"
|
||||
apiVersion: zclaw/v1
|
||||
kind: Pipeline
|
||||
metadata:
|
||||
name: test
|
||||
spec:
|
||||
steps:
|
||||
- id: step1
|
||||
action:
|
||||
type: llm_generate
|
||||
template: "test"
|
||||
- id: step1
|
||||
action:
|
||||
type: llm_generate
|
||||
template: "test2"
|
||||
"#;
|
||||
let result = PipelineParser::parse(yaml);
|
||||
assert!(matches!(result, Err(ParseError::Validation(_))));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user