utils/stream-parser/src/assistant_text.rs
130 lines
use crate::CitationStreamParser;use crate::ProposedPlanParser;use crate::ProposedPlanSegment;use crate::StreamTextChunk;use crate::StreamTextParser;#[derive(Debug, Clone, PartialEq, Eq, Default)]pub struct AssistantTextChunk { pub visible_text: String, pub citations: Vec<String>, pub plan_segments: Vec<ProposedPlanSegment>,}impl AssistantTextChunk { pub fn is_empty(&self) -> bool { self.visible_text.is_empty() && self.citations.is_empty() && self.plan_segments.is_empty() }}/// Parses assistant text streaming markup in one pass:/// - strips `<oai-mem-citation>` tags and extracts citation payloads/// - in plan mode, also strips `<proposed_plan>` blocks and emits plan segments#[derive(Debug, Default)]pub struct AssistantTextStreamParser { plan_mode: bool, citations: CitationStreamParser, plan: ProposedPlanParser,}impl AssistantTextStreamParser { pub fn new(plan_mode: bool) -> Self { Self { plan_mode, ..Self::default() } } pub fn push_str(&mut self, chunk: &str) -> AssistantTextChunk { let citation_chunk = self.citations.push_str(chunk); let mut out = self.parse_visible_text(citation_chunk.visible_text); out.citations = citation_chunk.extracted; out } pub fn finish(&mut self) -> AssistantTextChunk { let citation_chunk = self.citations.finish(); let mut out = self.parse_visible_text(citation_chunk.visible_text); if self.plan_mode { let mut tail = self.plan.finish(); if !tail.is_empty() { out.visible_text.push_str(&tail.visible_text); out.plan_segments.append(&mut tail.extracted); } } out.citations = citation_chunk.extracted; out } fn parse_visible_text(&mut self, visible_text: String) -> AssistantTextChunk { if !self.plan_mode { return AssistantTextChunk { visible_text, ..AssistantTextChunk::default() }; } let plan_chunk: StreamTextChunk<ProposedPlanSegment> = self.plan.push_str(&visible_text); AssistantTextChunk { visible_text: plan_chunk.visible_text, plan_segments: plan_chunk.extracted, ..AssistantTextChunk::default() } }}#[cfg(test)]mod tests { use super::AssistantTextStreamParser; use crate::ProposedPlanSegment; use pretty_assertions::assert_eq; #[test] fn parses_citations_across_seed_and_delta_boundaries() { let mut parser = AssistantTextStreamParser::new(/*plan_mode*/ false); let seeded = parser.push_str("hello <oai-mem-citation>doc"); let parsed = parser.push_str("1</oai-mem-citation> world"); let tail = parser.finish(); assert_eq!(seeded.visible_text, "hello "); assert_eq!(seeded.citations, Vec::<String>::new()); assert_eq!(parsed.visible_text, " world"); assert_eq!(parsed.citations, vec!["doc1".to_string()]); assert_eq!(tail.visible_text, ""); assert_eq!(tail.citations, Vec::<String>::new()); } #[test] fn parses_plan_segments_after_citation_stripping() { let mut parser = AssistantTextStreamParser::new(/*plan_mode*/ true); let seeded = parser.push_str("Intro\n<proposed"); let parsed = parser.push_str("_plan>\n- step <oai-mem-citation>doc</oai-mem-citation>\n"); let tail = parser.push_str("</proposed_plan>\nOutro"); let finish = parser.finish(); assert_eq!(seeded.visible_text, "Intro\n"); assert_eq!( seeded.plan_segments, vec![ProposedPlanSegment::Normal("Intro\n".to_string())] ); assert_eq!(parsed.visible_text, ""); assert_eq!(parsed.citations, vec!["doc".to_string()]); assert_eq!( parsed.plan_segments, vec![ ProposedPlanSegment::ProposedPlanStart, ProposedPlanSegment::ProposedPlanDelta("- step \n".to_string()), ] ); assert_eq!(tail.visible_text, "Outro"); assert_eq!( tail.plan_segments, vec![ ProposedPlanSegment::ProposedPlanEnd, ProposedPlanSegment::Normal("Outro".to_string()), ] ); assert!(finish.is_empty()); }}