//! This module is responsible for parsing & validating a patch into a list of "hunks".//! (It does not attempt to actually check that the patch can be applied to the filesystem.)//!//! The official Lark grammar for the apply-patch format is://!//! start: begin_patch environment_id? hunk+ end_patch//! begin_patch: "*** Begin Patch" LF//! environment_id: "*** Environment ID: " filename LF//! end_patch: "*** End Patch" LF?//!//! hunk: add_hunk | delete_hunk | update_hunk//! add_hunk: "*** Add File: " filename LF add_line+//! delete_hunk: "*** Delete File: " filename LF//! update_hunk: "*** Update File: " filename LF change_move? change?//! filename: /(.+)///! add_line: "+" /(.+)/ LF -> line//!//! change_move: "*** Move to: " filename LF//! change: (change_context | change_line)+ eof_line?//! change_context: ("@@" | "@@ " /(.+)/) LF//! change_line: ("+" | "-" | " ") /(.+)/ LF//! eof_line: "*** End of File" LF//!//! The parser below is a little more lenient than the explicit spec and allows for//! leading/trailing whitespace around patch markers.use crate::ApplyPatchArgs;use crate::streaming_parser::StreamingPatchParser;use codex_utils_absolute_path::AbsolutePathBuf;#[cfg(test)]use codex_utils_absolute_path::test_support::PathBufExt;use std::path::Path;use std::path::PathBuf;use thiserror::Error;pub(crate) const BEGIN_PATCH_MARKER: &str = "*** Begin Patch";pub(crate) const END_PATCH_MARKER: &str = "*** End Patch";pub(crate) const ADD_FILE_MARKER: &str = "*** Add File: ";pub(crate) const DELETE_FILE_MARKER: &str = "*** Delete File: ";pub(crate) const UPDATE_FILE_MARKER: &str = "*** Update File: ";pub(crate) const MOVE_TO_MARKER: &str = "*** Move to: ";pub(crate) const EOF_MARKER: &str = "*** End of File";pub(crate) const CHANGE_CONTEXT_MARKER: &str = "@@ ";pub(crate) const EMPTY_CHANGE_CONTEXT_MARKER: &str = "@@";/// Currently, the only OpenAI model that knowingly requires lenient parsing is/// gpt-4.1. While we could try to require everyone to pass in a strictness/// param when invoking apply_patch, it is a pain to thread it through all of/// the call sites, so we resign ourselves allowing lenient parsing for all/// models. See [`ParseMode::Lenient`] for details on the exceptions we make for/// gpt-4.1.const PARSE_IN_STRICT_MODE: bool = false;#[derive(Debug, PartialEq, Error, Clone)]pub enum ParseError {    #[error("invalid patch: {0}")]    InvalidPatchError(String),    #[error("invalid hunk at line {line_number}, {message}")]    InvalidHunkError { message: String, line_number: usize },}use ParseError::*;#[derive(Debug, PartialEq, Clone)]#[allow(clippy::enum_variant_names)]pub enum Hunk {    AddFile {        path: PathBuf,        contents: String,    },    DeleteFile {        path: PathBuf,    },    UpdateFile {        path: PathBuf,        move_path: Option<PathBuf>,        /// Chunks should be in order, i.e. the `change_context` of one chunk        /// should occur later in the file than the previous chunk.        chunks: Vec<UpdateFileChunk>,    },}impl Hunk {    pub fn resolve_path(&self, cwd: &AbsolutePathBuf) -> AbsolutePathBuf {        let path = match self {            Hunk::UpdateFile { path, .. } => path,            Hunk::AddFile { .. } | Hunk::DeleteFile { .. } => self.path(),        };        AbsolutePathBuf::resolve_path_against_base(path, cwd)    }    /// Returns the path affected by this hunk, using the move destination for rename hunks.    pub fn path(&self) -> &Path {        match self {            Hunk::AddFile { path, .. } => path,            Hunk::DeleteFile { path } => path,            Hunk::UpdateFile {                move_path: Some(path),                ..            } => path,            Hunk::UpdateFile {                path,                move_path: None,                ..            } => path,        }    }}#[cfg(test)]use Hunk::*;#[derive(Debug, PartialEq, Clone)]pub struct UpdateFileChunk {    /// A single line of context used to narrow down the position of the chunk    /// (this is usually a class, method, or function definition.)    pub change_context: Option<String>,    /// A contiguous block of lines that should be replaced with `new_lines`.    /// `old_lines` must occur strictly after `change_context`.    pub old_lines: Vec<String>,    pub new_lines: Vec<String>,    /// If set to true, `old_lines` must occur at the end of the source file.    /// (Tolerance around trailing newlines should be encouraged.)    pub is_end_of_file: bool,}pub fn parse_patch(patch: &str) -> Result<ApplyPatchArgs, ParseError> {    let mode = if PARSE_IN_STRICT_MODE {        ParseMode::Strict    } else {        ParseMode::Lenient    };    parse_patch_text(patch, mode)}enum ParseMode {    /// Parse the patch text argument as is.    Strict,    /// GPT-4.1 is known to formulate the `command` array for the `local_shell`    /// tool call for `apply_patch` call using something like the following:    ///    /// ```json    /// [    ///   "apply_patch",    ///   "<<'EOF'\n*** Begin Patch\n*** Update File: README.md\n@@...\n*** End Patch\nEOF\n",    /// ]    /// ```    ///    /// This is a problem because `local_shell` is a bit of a misnomer: the    /// `command` is not invoked by passing the arguments to a shell like Bash,    /// but are invoked using something akin to `execvpe(3)`.    ///    /// This is significant in this case because where a shell would interpret    /// `<<'EOF'...` as a heredoc and pass the contents via stdin (which is    /// fine, as `apply_patch` is specified to read from stdin if no argument is    /// passed), `execvpe(3)` interprets the heredoc as a literal string. To get    /// the `local_shell` tool to run a command the way shell would, the    /// `command` array must be something like:    ///    /// ```json    /// [    ///   "bash",    ///   "-lc",    ///   "apply_patch <<'EOF'\n*** Begin Patch\n*** Update File: README.md\n@@...\n*** End Patch\nEOF\n",    /// ]    /// ```    ///    /// In lenient mode, we check if the argument to `apply_patch` starts with    /// `<<'EOF'` and ends with `EOF\n`. If so, we strip off these markers,    /// trim() the result, and treat what is left as the patch text.    Lenient,}fn parse_patch_text(patch: &str, mode: ParseMode) -> Result<ApplyPatchArgs, ParseError> {    let lines: Vec<&str> = patch.trim().lines().collect();    let patch_lines = match mode {        ParseMode::Strict => check_patch_boundaries_strict(&lines)?,        ParseMode::Lenient => check_patch_boundaries_lenient(&lines)?,    };    let patch = patch_lines.join("\n");    let mut parser = StreamingPatchParser::default();    parser.push_delta(&patch)?;    let hunks = parser.finish()?;    let environment_id = parser.environment_id().map(str::to_owned);    Ok(ApplyPatchArgs {        hunks,        patch,        workdir: None,        environment_id,    })}/// Checks the start and end lines of the patch text for `apply_patch`,/// returning an error if they do not match the expected markers.fn check_patch_boundaries_strict<'a>(lines: &'a [&'a str]) -> Result<&'a [&'a str], ParseError> {    let (first_line, last_line) = match lines {        [] => (None, None),        [first] => (Some(first), Some(first)),        [first, .., last] => (Some(first), Some(last)),    };    check_start_and_end_lines_strict(first_line, last_line)?;    Ok(lines)}/// If we are in lenient mode, we check if the first line starts with `<<EOF`/// (possibly quoted) and the last line ends with `EOF`. There must be at least/// 4 lines total because the heredoc markers take up 2 lines and the patch text/// must have at least 2 lines.////// If successful, returns the lines of the patch text that contain the patch/// contents, excluding the heredoc markers.fn check_patch_boundaries_lenient<'a>(    original_lines: &'a [&'a str],) -> Result<&'a [&'a str], ParseError> {    let original_parse_error = match check_patch_boundaries_strict(original_lines) {        Ok(lines) => return Ok(lines),        Err(e) => e,    };    match original_lines {        [first, .., last] => {            if (first == &"<<EOF" || first == &"<<'EOF'" || first == &"<<\"EOF\"")                && last.ends_with("EOF")                && original_lines.len() >= 4            {                let inner_lines = &original_lines[1..original_lines.len() - 1];                check_patch_boundaries_strict(inner_lines)            } else {                Err(original_parse_error)            }        }        _ => Err(original_parse_error),    }}fn check_start_and_end_lines_strict(    first_line: Option<&&str>,    last_line: Option<&&str>,) -> Result<(), ParseError> {    let first_line = first_line.map(|line| line.trim());    let last_line = last_line.map(|line| line.trim());    match (first_line, last_line) {        (Some(first), Some(last)) if first == BEGIN_PATCH_MARKER && last == END_PATCH_MARKER => {            Ok(())        }        (Some(first), _) if first != BEGIN_PATCH_MARKER => Err(InvalidPatchError(String::from(            "The first line of the patch must be '*** Begin Patch'",        ))),        _ => Err(InvalidPatchError(String::from(            "The last line of the patch must be '*** End Patch'",        ))),    }}#[test]fn test_parse_patch() {    assert_eq!(        parse_patch_text("bad", ParseMode::Strict),        Err(InvalidPatchError(            "The first line of the patch must be '*** Begin Patch'".to_string()        ))    );    assert_eq!(        parse_patch_text("*** Begin Patch\nbad", ParseMode::Strict),        Err(InvalidPatchError(            "The last line of the patch must be '*** End Patch'".to_string()        ))    );    assert_eq!(        parse_patch_text(            concat!(                "*** Begin Patch",                " ",                "\n*** Add File: foo\n+hi\n",                " ",                "*** End Patch"            ),            ParseMode::Strict        )        .unwrap()        .hunks,        vec![AddFile {            path: PathBuf::from("foo"),            contents: "hi\n".to_string()        }]    );    assert_eq!(        parse_patch_text(            "*** Begin Patch\n\             *** Update File: test.py\n\             *** End Patch",            ParseMode::Strict        ),        Err(InvalidHunkError {            message: "Update file hunk for path 'test.py' is empty".to_string(),            line_number: 2,        })    );    assert_eq!(        parse_patch_text(            "*** Begin Patch\n\             *** End Patch",            ParseMode::Strict        )        .unwrap()        .hunks,        Vec::new()    );    assert_eq!(        parse_patch_text(            "*** Begin Patch\n\             *** Add File: path/add.py\n\             +abc\n\             +def\n\             *** Delete File: path/delete.py\n\             *** Update File: path/update.py\n\             *** Move to: path/update2.py\n\             @@ def f():\n\             -    pass\n\             +    return 123\n\             *** End Patch",            ParseMode::Strict        )        .unwrap()        .hunks,        vec![            AddFile {                path: PathBuf::from("path/add.py"),                contents: "abc\ndef\n".to_string()            },            DeleteFile {                path: PathBuf::from("path/delete.py")            },            UpdateFile {                path: PathBuf::from("path/update.py"),                move_path: Some(PathBuf::from("path/update2.py")),                chunks: vec![UpdateFileChunk {                    change_context: Some("def f():".to_string()),                    old_lines: vec!["    pass".to_string()],                    new_lines: vec!["    return 123".to_string()],                    is_end_of_file: false                }]            }        ]    );    // Update hunk followed by another hunk (Add File).    assert_eq!(        parse_patch_text(            "*** Begin Patch\n\             *** Update File: file.py\n\             @@\n\             +line\n\             *** Add File: other.py\n\             +content\n\             *** End Patch",            ParseMode::Strict        )        .unwrap()        .hunks,        vec![            UpdateFile {                path: PathBuf::from("file.py"),                move_path: None,                chunks: vec![UpdateFileChunk {                    change_context: None,                    old_lines: vec![],                    new_lines: vec!["line".to_string()],                    is_end_of_file: false                }],            },            AddFile {                path: PathBuf::from("other.py"),                contents: "content\n".to_string()            }        ]    );    // Update hunk without an explicit @@ header for the first chunk should parse.    // Use a raw string to preserve the leading space diff marker on the context line.    assert_eq!(        parse_patch_text(            r#"*** Begin Patch*** Update File: file2.py import foo+bar*** End Patch"#,            ParseMode::Strict        )        .unwrap()        .hunks,        vec![UpdateFile {            path: PathBuf::from("file2.py"),            move_path: None,            chunks: vec![UpdateFileChunk {                change_context: None,                old_lines: vec!["import foo".to_string()],                new_lines: vec!["import foo".to_string(), "bar".to_string()],                is_end_of_file: false,            }],        }]    );}#[test]fn test_parse_patch_preserves_end_of_file_marker() {    let patch =        "*** Begin Patch\n*** Update File: file.txt\n@@\n+quux\n*** End of File\n\n*** End Patch";    assert_eq!(        parse_patch(patch),        Ok(ApplyPatchArgs {            hunks: vec![UpdateFile {                path: PathBuf::from("file.txt"),                move_path: None,                chunks: vec![UpdateFileChunk {                    change_context: None,                    old_lines: Vec::new(),                    new_lines: vec!["quux".to_string()],                    is_end_of_file: true,                }],            }],            patch: patch.to_string(),            workdir: None,            environment_id: None,        })    );}#[test]fn test_parse_patch_accepts_relative_and_absolute_hunk_paths() {    let dir = tempfile::tempdir().unwrap();    let absolute_delete = dir.path().join("absolute-delete.py").abs();    let absolute_update = dir.path().join("absolute-update.py").abs();    let patch_text = format!(        r#"*** Begin Patch*** Add File: relative-add.py+content*** Delete File: {}*** Update File: {}@@-old+new*** End Patch"#,        absolute_delete.display(),        absolute_update.display()    );    assert_eq!(        parse_patch_text(&patch_text, ParseMode::Strict)            .unwrap()            .hunks,        vec![            AddFile {                path: PathBuf::from("relative-add.py"),                contents: "content\n".to_string()            },            DeleteFile {                path: absolute_delete.to_path_buf()            },            UpdateFile {                path: absolute_update.to_path_buf(),                move_path: None,                chunks: vec![UpdateFileChunk {                    change_context: None,                    old_lines: vec!["old".to_string()],                    new_lines: vec!["new".to_string()],                    is_end_of_file: false                }]            },        ]    );}#[test]fn test_hunk_resolve_path_accepts_relative_and_absolute_paths() {    let cwd_dir = tempfile::tempdir().unwrap();    let cwd = cwd_dir.path().to_path_buf().abs();    let absolute_dir = tempfile::tempdir().unwrap();    let absolute_add = absolute_dir.path().join("absolute-add.py").abs();    let absolute_delete = absolute_dir.path().join("absolute-delete.py").abs();    let absolute_update = absolute_dir.path().join("absolute-update.py").abs();    for (hunk, expected_path) in [        (            AddFile {                path: PathBuf::from("relative-add.py"),                contents: String::new(),            },            cwd.join("relative-add.py"),        ),        (            DeleteFile {                path: PathBuf::from("relative-delete.py"),            },            cwd.join("relative-delete.py"),        ),        (            UpdateFile {                path: PathBuf::from("relative-update.py"),                move_path: None,                chunks: Vec::new(),            },            cwd.join("relative-update.py"),        ),        (            AddFile {                path: absolute_add.to_path_buf(),                contents: String::new(),            },            absolute_add,        ),        (            DeleteFile {                path: absolute_delete.to_path_buf(),            },            absolute_delete,        ),        (            UpdateFile {                path: absolute_update.to_path_buf(),                move_path: None,                chunks: Vec::new(),            },            absolute_update,        ),    ] {        assert_eq!(hunk.resolve_path(&cwd), expected_path);    }}#[test]fn test_parse_patch_lenient() {    let patch_text = r#"*** Begin Patch*** Update File: file2.py import foo+bar*** End Patch"#;    let expected_patch = vec![UpdateFile {        path: PathBuf::from("file2.py"),        move_path: None,        chunks: vec![UpdateFileChunk {            change_context: None,            old_lines: vec!["import foo".to_string()],            new_lines: vec!["import foo".to_string(), "bar".to_string()],            is_end_of_file: false,        }],    }];    let expected_error =        InvalidPatchError("The first line of the patch must be '*** Begin Patch'".to_string());    let patch_text_in_heredoc = format!("<<EOF\n{patch_text}\nEOF\n");    assert_eq!(        parse_patch_text(&patch_text_in_heredoc, ParseMode::Strict),        Err(expected_error.clone())    );    assert_eq!(        parse_patch_text(&patch_text_in_heredoc, ParseMode::Lenient),        Ok(ApplyPatchArgs {            hunks: expected_patch.clone(),            patch: patch_text.to_string(),            workdir: None,            environment_id: None,        })    );    let patch_text_in_single_quoted_heredoc = format!("<<'EOF'\n{patch_text}\nEOF\n");    assert_eq!(        parse_patch_text(&patch_text_in_single_quoted_heredoc, ParseMode::Strict),        Err(expected_error.clone())    );    assert_eq!(        parse_patch_text(&patch_text_in_single_quoted_heredoc, ParseMode::Lenient),        Ok(ApplyPatchArgs {            hunks: expected_patch.clone(),            patch: patch_text.to_string(),            workdir: None,            environment_id: None,        })    );    let patch_text_in_double_quoted_heredoc = format!("<<\"EOF\"\n{patch_text}\nEOF\n");    assert_eq!(        parse_patch_text(&patch_text_in_double_quoted_heredoc, ParseMode::Strict),        Err(expected_error.clone())    );    assert_eq!(        parse_patch_text(&patch_text_in_double_quoted_heredoc, ParseMode::Lenient),        Ok(ApplyPatchArgs {            hunks: expected_patch,            patch: patch_text.to_string(),            workdir: None,            environment_id: None,        })    );    let patch_text_in_mismatched_quotes_heredoc = format!("<<\"EOF'\n{patch_text}\nEOF\n");    assert_eq!(        parse_patch_text(&patch_text_in_mismatched_quotes_heredoc, ParseMode::Strict),        Err(expected_error.clone())    );    assert_eq!(        parse_patch_text(&patch_text_in_mismatched_quotes_heredoc, ParseMode::Lenient),        Err(expected_error.clone())    );    let patch_text_with_missing_closing_heredoc =        "<<EOF\n*** Begin Patch\n*** Update File: file2.py\nEOF\n".to_string();    assert_eq!(        parse_patch_text(&patch_text_with_missing_closing_heredoc, ParseMode::Strict),        Err(expected_error)    );    assert_eq!(        parse_patch_text(&patch_text_with_missing_closing_heredoc, ParseMode::Lenient),        Err(InvalidPatchError(            "The last line of the patch must be '*** End Patch'".to_string()        ))    );}#[test]fn test_parse_patch_environment_id_preamble() {    assert_eq!(        parse_patch_text(            "*** Begin Patch\n\             *** Environment ID: remote\n\             *** Add File: hello.txt\n\             +hello\n\             *** End Patch",            ParseMode::Strict        ),        Ok(ApplyPatchArgs {            hunks: vec![AddFile {                path: PathBuf::from("hello.txt"),                contents: "hello\n".to_string(),            }],            patch: "*** Begin Patch\n*** Environment ID: remote\n*** Add File: hello.txt\n+hello\n*** End Patch".to_string(),            workdir: None,            environment_id: Some("remote".to_string()),        })    );    assert_eq!(        parse_patch_text(            "*** Begin Patch\n\             *** Environment ID:   \n\             *** Add File: hello.txt\n\             +hello\n\             *** End Patch",            ParseMode::Strict        ),        Err(InvalidPatchError(            "apply_patch environment_id cannot be empty".to_string()        ))    );}