core/src/image_preparation.rs
122 lines
use codex_protocol::models::ContentItem;use codex_protocol::models::FunctionCallOutputContentItem;use codex_protocol::models::ImageDetail;use codex_protocol::models::ResponseItem;use codex_utils_image::ImageProcessingError;use codex_utils_image::PromptImageMode;use codex_utils_image::PromptImageResizeLimits;use codex_utils_image::load_data_url_for_prompt;use tracing::warn;pub(crate) const IMAGE_PROCESSING_ERROR_PLACEHOLDER: &str = "image content omitted because it could not be processed";const IMAGE_TOO_LARGE_PLACEHOLDER: &str = "image content omitted because it exceeded the supported size limit; use a smaller image";const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because detail 'low' is not supported; use 'high', 'original', or 'auto'";const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { max_dimension: 2048, max_patches: 2_500,};const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { max_dimension: 6000, max_patches: 10_000,};#[derive(Debug, thiserror::Error)]enum ImagePreparationError { #[error("image detail `low` is not supported")] UnsupportedLowDetail, #[error(transparent)] Processing(#[from] ImageProcessingError),}impl ImagePreparationError { fn placeholder(&self) -> &'static str { match self { ImagePreparationError::UnsupportedLowDetail => UNSUPPORTED_LOW_DETAIL_PLACEHOLDER, ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge { .. }) => { IMAGE_TOO_LARGE_PLACEHOLDER } ImagePreparationError::Processing(_) => IMAGE_PROCESSING_ERROR_PLACEHOLDER, } }}pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) { for item in items { match item { ResponseItem::Message { content, .. } => prepare_message_content(content), ResponseItem::FunctionCallOutput { output, .. } | ResponseItem::CustomToolCallOutput { output, .. } => { if let Some(content) = output.content_items_mut() { prepare_tool_output_content(content); } } ResponseItem::Reasoning { .. } | ResponseItem::AgentMessage { .. } | ResponseItem::LocalShellCall { .. } | ResponseItem::FunctionCall { .. } | ResponseItem::ToolSearchCall { .. } | ResponseItem::CustomToolCall { .. } | ResponseItem::ToolSearchOutput { .. } | ResponseItem::WebSearchCall { .. } | ResponseItem::ImageGenerationCall { .. } | ResponseItem::Compaction { .. } | ResponseItem::CompactionTrigger { .. } | ResponseItem::ContextCompaction { .. } | ResponseItem::Other => {} } }}fn prepare_message_content(items: &mut [ContentItem]) { for item in items { if let ContentItem::InputImage { image_url, detail } = item && is_data_url(image_url) && let Err(error) = prepare_image(image_url, *detail) { warn!(%error, "failed to prepare message image"); *item = ContentItem::InputText { text: error.placeholder().to_string(), }; } }}fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) { for item in items { if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item && is_data_url(image_url) && let Err(error) = prepare_image(image_url, *detail) { warn!(%error, "failed to prepare tool output image"); *item = FunctionCallOutputContentItem::InputText { text: error.placeholder().to_string(), }; } }}fn is_data_url(image_url: &str) -> bool { image_url .get(.."data:".len()) .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))}fn prepare_image( image_url: &mut String, detail: Option<ImageDetail>,) -> Result<(), ImagePreparationError> { let limits = match detail { None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS, Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS, Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail), }; let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?; *image_url = image.into_data_url(); Ok(())}#[cfg(test)]#[path = "image_preparation_tests.rs"]mod tests;