From dc27988bc2830e2380efb3d92a43ed7c242512bf Mon Sep 17 00:00:00 2001 From: bherbruck Date: Mon, 19 Oct 2026 00:02:29 -0400 Subject: [PATCH] Agent chat: PDFs as documents, any other file by reference (#2812) (#2858) A PDF goes whole to agents that take embedded documents; to the others, or past 12 MB, the message names it with its type, size and place. Any file that is not a picture, text or CAD file is attached by reference instead of refused. Chips show the type's extension; the file dialog offers All files. --- apps/solidcraft/src/main.rs | 3 +- assets/locales/en.json | 3 +- assets/locales/ja.json | 3 +- assets/locales/zh.json | 3 +- crates/ui-egui/src/agent_chat/acp.rs | 14 ++ crates/ui-egui/src/agent_chat/acp_client.rs | 4 + crates/ui-egui/src/agent_chat/api_client.rs | 8 +- crates/ui-egui/src/agent_chat/attach.rs | 232 ++++++++++++++++++-- crates/ui-egui/src/agent_chat/backend.rs | 28 +++ crates/ui-egui/src/agent_chat/mod.rs | 17 +- crates/ui-egui/src/agent_chat/panel.rs | 7 +- 11 files changed, 280 insertions(+), 42 deletions(-) diff --git a/apps/solidcraft/src/main.rs b/apps/solidcraft/src/main.rs index 014a58428..d346b1f2d 100644 --- a/apps/solidcraft/src/main.rs +++ b/apps/solidcraft/src/main.rs @@ -179,7 +179,8 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services { } } let all: Vec<&str> = all.iter().map(String::as_str).collect(); - let future = file_parent.clone().add_filter(label, &all).pick_file(); + // Any other file attaches by reference, so the list is not the only choice. + let future = file_parent.clone().add_filter(label, &all).add_filter("All files", &["*"]).pick_file(); Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) }) })), pick_program: Some(Box::new(move || { diff --git a/assets/locales/en.json b/assets/locales/en.json index 17b92389b..6c2253456 100644 --- a/assets/locales/en.json +++ b/assets/locales/en.json @@ -859,7 +859,7 @@ "ui.agent_chat_approval": "Allow the agent to do this?", "ui.agent_chat_attach_view": "Attach view", "ui.agent_chat_attach_view_tip": "Send a picture of the current view with the message", - "ui.agent_chat_attachable": "Pictures, text and CAD files", + "ui.agent_chat_attachable": "Pictures, PDFs, text and CAD files", "ui.agent_chat_auth_agent": "Sign in with {name}", "ui.agent_chat_auth_env": "Set {vars} in your environment and restart SolidCraft.", "ui.agent_chat_auth_needed": "The agent asks you to sign in. You sign in with the agent itself; SolidCraft never sees your password or keys.", @@ -891,7 +891,6 @@ "ui.agent_chat_file_error": "{name}: {error}", "ui.agent_chat_file_heic": "{name}: HEIC pictures can't be read here; export it as JPEG or PNG first", "ui.agent_chat_file_importing": "Importing into the design…", - "ui.agent_chat_file_kinds": "{name}: pictures (PNG, JPEG, GIF, WebP, BMP, TIFF), text files (SVG too) and CAD files (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) can be attached", "ui.agent_chat_file_loading": "Reading the file…", "ui.agent_chat_file_not_a_file": "{name}: not a file", "ui.agent_chat_file_not_a_picture": "{name}: not a picture", diff --git a/assets/locales/ja.json b/assets/locales/ja.json index 1a25af5d6..396aebaee 100644 --- a/assets/locales/ja.json +++ b/assets/locales/ja.json @@ -859,7 +859,7 @@ "ui.agent_chat_approval": "エージェントにこの操作を許可しますか?", "ui.agent_chat_attach_view": "ビューを添付", "ui.agent_chat_attach_view_tip": "現在のビューの画像をメッセージと一緒に送ります", - "ui.agent_chat_attachable": "画像・テキスト・CAD ファイル", + "ui.agent_chat_attachable": "画像・PDF・テキスト・CAD ファイル", "ui.agent_chat_auth_agent": "{name} でサインイン", "ui.agent_chat_auth_env": "環境変数 {vars} を設定して SolidCraft を再起動してください。", "ui.agent_chat_auth_needed": "エージェントがサインインを求めています。サインインはエージェント自身で行います。SolidCraft がパスワードやキーを見ることはありません。", @@ -891,7 +891,6 @@ "ui.agent_chat_file_error": "{name}: {error}", "ui.agent_chat_file_heic": "{name}: HEIC 画像はここでは読めません。先に JPEG か PNG に書き出してください", "ui.agent_chat_file_importing": "デザインにインポート中…", - "ui.agent_chat_file_kinds": "{name}: 添付できるのは画像 (PNG、JPEG、GIF、WebP、BMP、TIFF)、テキストファイル (SVG も)、CAD ファイル (STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF) です", "ui.agent_chat_file_loading": "ファイルを読み込み中…", "ui.agent_chat_file_not_a_file": "{name}: ファイルではありません", "ui.agent_chat_file_not_a_picture": "{name}: 画像ではありません", diff --git a/assets/locales/zh.json b/assets/locales/zh.json index 6519da022..c9eafe6ec 100644 --- a/assets/locales/zh.json +++ b/assets/locales/zh.json @@ -859,7 +859,7 @@ "ui.agent_chat_approval": "允许智能体执行此操作吗?", "ui.agent_chat_attach_view": "附加视图", "ui.agent_chat_attach_view_tip": "随消息发送当前视图的图片", - "ui.agent_chat_attachable": "图片、文本和 CAD 文件", + "ui.agent_chat_attachable": "图片、PDF、文本和 CAD 文件", "ui.agent_chat_auth_agent": "使用 {name} 登录", "ui.agent_chat_auth_env": "请在环境中设置 {vars},然后重启 SolidCraft。", "ui.agent_chat_auth_needed": "智能体要求你登录。登录在智能体本身完成;SolidCraft 从不接触你的密码或密钥。", @@ -891,7 +891,6 @@ "ui.agent_chat_file_error": "{name}:{error}", "ui.agent_chat_file_heic": "{name}:这里无法读取 HEIC 图片;请先导出为 JPEG 或 PNG", "ui.agent_chat_file_importing": "正在导入设计…", - "ui.agent_chat_file_kinds": "{name}:可以附加图片(PNG、JPEG、GIF、WebP、BMP、TIFF)、文本文件(包括 SVG)和 CAD 文件(STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF)", "ui.agent_chat_file_loading": "正在读取文件…", "ui.agent_chat_file_not_a_file": "{name}:不是文件", "ui.agent_chat_file_not_a_picture": "{name}:不是图片", diff --git a/crates/ui-egui/src/agent_chat/acp.rs b/crates/ui-egui/src/agent_chat/acp.rs index 28315f8d6..56c159edd 100644 --- a/crates/ui-egui/src/agent_chat/acp.rs +++ b/crates/ui-egui/src/agent_chat/acp.rs @@ -149,6 +149,12 @@ pub fn set_model_params(session: &str, model: &str) -> Value { json!({"sessionId": session, "modelId": model}) } +/// Whether the agent takes embedded documents in a message +/// (`agentCapabilities.promptCapabilities.embeddedContext`: it reads `resource` blocks). +pub fn takes_documents(init: &Value) -> bool { + init.pointer("/agentCapabilities/promptCapabilities/embeddedContext").and_then(Value::as_bool).unwrap_or(false) +} + /// Whether the agent takes pictures in a message (`agentCapabilities.promptCapabilities.image`). pub fn takes_images(init: &Value) -> bool { init.pointer("/agentCapabilities/promptCapabilities/image").and_then(Value::as_bool).unwrap_or(false) @@ -461,6 +467,14 @@ pub fn split_command(line: &str) -> Vec { mod tests { use super::*; + #[test] + fn documents_are_taken_only_when_the_agent_says_so() { + let yes = json!({"agentCapabilities": {"promptCapabilities": {"image": true, "embeddedContext": true}}}); + let no = json!({"agentCapabilities": {"promptCapabilities": {"image": true}}}); + assert!(takes_documents(&yes) && takes_images(&yes)); + assert!(!takes_documents(&no) && !takes_documents(&json!({}))); + } + #[test] fn commands_split_like_a_shell_without_one() { assert_eq!(split_command("gemini --acp"), ["gemini", "--acp"]); diff --git a/crates/ui-egui/src/agent_chat/acp_client.rs b/crates/ui-egui/src/agent_chat/acp_client.rs index ca9e16b76..82918e90f 100644 --- a/crates/ui-egui/src/agent_chat/acp_client.rs +++ b/crates/ui-egui/src/agent_chat/acp_client.rs @@ -33,6 +33,7 @@ pub struct AcpBackend { session: Option, agent: String, images: bool, + documents: bool, auth: Vec, /// Open permission questions: our key → the agent's request id. permissions: HashMap, @@ -54,6 +55,7 @@ impl AcpBackend { session: None, agent: String::new(), images: false, + documents: false, auth: Vec::new(), permissions: HashMap::new(), next_key: 0, @@ -82,6 +84,7 @@ impl AcpBackend { (Pending::Initialize, Ok(v)) => { self.agent = acp::agent_info(&v); self.images = acp::takes_images(&v); + self.documents = acp::takes_documents(&v); self.auth = acp::auth_methods(&v); self.new_session(); } @@ -94,6 +97,7 @@ impl AcpBackend { if let Some((current, available)) = acp::models(&v) { out.push(Event::Models { current, available }); } + out.push(Event::Documents(self.documents)); out.push(Event::Ready { agent: self.agent.clone(), images: self.images }); } None => out.push(Event::Error("session/new: no sessionId".into())), diff --git a/crates/ui-egui/src/agent_chat/api_client.rs b/crates/ui-egui/src/agent_chat/api_client.rs index 8c50fd316..39704695a 100644 --- a/crates/ui-egui/src/agent_chat/api_client.rs +++ b/crates/ui-egui/src/agent_chat/api_client.rs @@ -404,9 +404,11 @@ fn user_message(parts: &[PromptPart]) -> Value { } let content: Vec = parts .iter() - .map(|p| match p { - PromptPart::Text(t) => json!({"type": "text", "text": t}), - PromptPart::Image { mime, base64 } => json!({"type": "image_url", "image_url": {"url": format!("data:{mime};base64,{base64}")}}), + .filter_map(|p| match p { + PromptPart::Text(t) => Some(json!({"type": "text", "text": t})), + PromptPart::Image { mime, base64 } => Some(json!({"type": "image_url", "image_url": {"url": format!("data:{mime};base64,{base64}")}})), + // This endpoint takes no documents (the chat doesn't send any); the message names the file. + PromptPart::Document { .. } => None, }) .collect(); json!({"role": "user", "content": content}) diff --git a/crates/ui-egui/src/agent_chat/attach.rs b/crates/ui-egui/src/agent_chat/attach.rs index 5926e8cae..926aae003 100644 --- a/crates/ui-egui/src/agent_chat/attach.rs +++ b/crates/ui-egui/src/agent_chat/attach.rs @@ -1,9 +1,10 @@ //! Files a person adds to a message (the + button, a drop on the chat, a paste): pictures go to the -//! agent with it, text files go in as text, and CAD files are imported into the design, the agent -//! being told what arrived. Each file is checked and bounded before it is kept; a refused one says -//! why. +//! agent with it, text files go in as text, PDFs go as documents to agents that take them, CAD +//! files are imported into the design, and any other file is named by reference (its type, size +//! and place on this machine) for the agent to read with its own tools. The agent is told what +//! arrived. Each file is checked and bounded before it is kept; a refused one says why. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use super::backend::{self, PromptPart}; @@ -27,6 +28,8 @@ pub const MAX_FILES: usize = 8; const MAX_PICTURE_BYTES: u64 = 40 * 1024 * 1024; /// Largest text file sent in full. const MAX_TEXT_BYTES: u64 = 256 * 1024; +/// Largest PDF sent as a document; a bigger one goes by reference. +const MAX_PDF_BYTES: u64 = 12 * 1024 * 1024; /// A picture's longer side is scaled down to this before it is sent. pub const MAX_SIDE: u32 = 2048; /// Largest picture sent, as it came or re-encoded: its base64 (a third larger) stays under 5 MB, @@ -47,9 +50,9 @@ const TEXT_EXTS: &[&str] = &["svg", "txt", "md", "markdown", "json", "csv", "tsv /// importers themselves). const CAD_EXTS: &[&str] = &["step", "stp", "igs", "iges", "FCStd", "3dm", "stl", "3mf", "obj", "glb", "gltf"]; -/// Every extension that can be attached, pictures first (the file picker's list). +/// Every extension the file picker lists first, pictures first (any other file can be chosen too). pub fn extensions() -> Vec<&'static str> { - PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).copied().collect() + PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).chain(&["pdf"]).copied().collect() } /// What a file is, by its extension. @@ -58,6 +61,8 @@ pub enum Kind { Picture, Text, Cad, + Pdf, + /// Anything else, attached by reference. Other, } @@ -66,6 +71,7 @@ pub fn kind(path: &Path) -> Kind { match ext.as_str() { e if PICTURE_EXTS.contains(&e) => Kind::Picture, e if TEXT_EXTS.contains(&e) => Kind::Text, + "pdf" => Kind::Pdf, _ => { let p = path.to_string_lossy(); if solidcraft_engine::io::is_step_path(&p) || solidcraft_engine::io::is_mesh_path(&p) { Kind::Cad } else { Kind::Other } @@ -82,6 +88,17 @@ pub enum Content { size: [u32; 2], }, Text(Arc), + /// A PDF, sent whole as a document to an agent that takes them. + Pdf { + bytes: Arc>, + path: PathBuf, + }, + /// A file the agent is pointed to, not given: what it is, its size and where it is. + Reference { + path: PathBuf, + size: u64, + mime: &'static str, + }, /// A CAD file imported into the design as its own component. Imported, } @@ -104,6 +121,70 @@ impl Attachment { pub fn is_picture(&self) -> bool { matches!(self.content, Content::Picture { .. }) } + /// The chip's mark for a file with no thumbnail: its extension, in capitals, cut to fit. + pub fn mark(&self) -> String { + let ext = Path::new(&self.name).extension().and_then(|e| e.to_str()).unwrap_or(""); + ext.chars().take(4).collect::().to_uppercase() + } +} + +/// What a file is called to the agent, by its extension. +fn mime_of(path: &Path) -> &'static str { + let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).unwrap_or_default(); + match ext.as_str() { + "pdf" => "application/pdf", + "zip" => "application/zip", + "7z" => "application/x-7z-compressed", + "gz" | "tgz" => "application/gzip", + "dxf" => "image/vnd.dxf", + "dwg" => "image/vnd.dwg", + "step" | "stp" => "model/step", + "iges" | "igs" => "model/iges", + "stl" => "model/stl", + "obj" => "model/obj", + "3mf" => "model/3mf", + "gltf" => "model/gltf+json", + "glb" => "model/gltf-binary", + "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "mp4" | "mov" | "webm" => "video/mp4", + "wav" => "audio/wav", + "mp3" => "audio/mpeg", + _ => "application/octet-stream", + } +} + +/// A size for people: `1.5 MB`. +fn size_text(bytes: u64) -> String { + const KB: f64 = 1024.0; + let b = bytes as f64; + if b < KB { + format!("{bytes} bytes") + } else if b < KB * KB { + format!("{:.0} KB", b / KB) + } else if b < KB * KB * KB { + format!("{:.1} MB", b / KB / KB) + } else { + format!("{:.1} GB", b / KB / KB / KB) + } +} + +/// A `file://` link to a path, for a document sent to an agent. +fn file_uri(path: &Path) -> String { + let mut s = path.to_string_lossy().replace('\\', "/"); + if !s.starts_with('/') { + s.insert(0, '/'); + } + let mut out = String::from("file://"); + for b in s.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'/' | b'-' | b'_' | b'.' | b'~' | b':') { + out.push(char::from(b)); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out } pub(crate) fn name_of(path: &Path) -> String { @@ -134,13 +215,29 @@ pub fn read(path: &Path) -> Result { Ok(Attachment { id: 0, name, content: Content::Text(Arc::new(text)), thumb: None }) } Kind::Cad => Err(refused("ui.agent_chat_file_cad_not_read", &name, &[])), + Kind::Pdf => { + let path = absolute(path); + // Too big to send: the agent is pointed to it instead. + if meta.len() > MAX_PDF_BYTES { + return Ok(Attachment { id: 0, name, content: Content::Reference { path, size: meta.len(), mime: "application/pdf" }, thumb: None }); + } + let bytes = std::fs::read(&path).map_err(|e| failed(&name, e))?; + Ok(Attachment { id: 0, name, content: Content::Pdf { bytes: Arc::new(bytes), path }, thumb: None }) + } Kind::Other if matches!(path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("heic" | "heif")) => { Err(refused("ui.agent_chat_file_heic", &name, &[])) } - Kind::Other => Err(refused("ui.agent_chat_file_kinds", &name, &[])), + Kind::Other => { + Ok(Attachment { id: 0, name, content: Content::Reference { path: absolute(path), size: meta.len(), mime: mime_of(path) }, thumb: None }) + } } } +/// The path as an absolute one, so the agent (working in another folder) finds the file. +fn absolute(path: &Path) -> PathBuf { + std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf()) +} + /// A picture from a file's bytes: decoded within limits, and sent as it came when the agents read /// that format (PNG, JPEG, GIF, WebP), it is at most [`MAX_SIDE`] on a side and at most /// [`MAX_SENT_BYTES`]; otherwise re-encoded (see [`encoded`]). @@ -239,10 +336,10 @@ fn thumbnail(img: &image::DynamicImage) -> Option> { /// What the message says about its files, after the person's words: the text files' text, and the /// pictures and imported parts by name (the agent sees the pictures themselves as images). -/// `pictures_sent`: whether this agent takes pictures. The text files share `budget` characters +/// `pictures_sent` and `documents_sent`: whether this agent takes pictures and PDFs. The text files share `budget` characters /// in order: one that doesn't fit is cut and says how much is missing, and one past the budget /// is named only, so the message stays within the prompt cap. -pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String { +pub fn note(files: &[Attachment], pictures_sent: bool, documents_sent: bool, budget: usize) -> String { let mut out = String::new(); let pictures: Vec<&str> = files.iter().filter(|f| f.is_picture()).map(|f| f.name.as_str()).collect(); if !pictures.is_empty() { @@ -270,6 +367,31 @@ pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String left = 0; } } + Content::Pdf { path, bytes } if documents_sent => { + out += &format!( + "\n\nAttached PDF `{}` ({}, at {}): sent with this message as a document.", + f.name, + size_text(bytes.len() as u64), + path.display() + ); + } + // An agent that takes no documents is pointed to the file, as for any other file. + Content::Pdf { path, bytes } => { + out += &format!( + "\n\nAttached file `{}` (PDF, {}, at {}): read it with your own tools; it is not in this message.", + f.name, + size_text(bytes.len() as u64), + path.display() + ); + } + Content::Reference { path, size, mime } => { + out += &format!( + "\n\nAttached file `{}` ({mime}, {}, at {}): read it with your own tools; it is not in this message.", + f.name, + size_text(*size), + path.display() + ); + } Content::Imported => out += &format!("\n\n(I added `{}` to the design as a component of its own.)", f.name), Content::Picture { .. } => {} } @@ -277,12 +399,18 @@ pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String out } -/// The pictures as image parts of the message. -pub fn parts(files: &[Attachment]) -> Vec { +/// The pictures and PDFs as parts of the message, to the agents that take them. +pub fn parts(files: &[Attachment], pictures: bool, documents: bool) -> Vec { files .iter() .filter_map(|f| match &f.content { - Content::Picture { mime, bytes, .. } => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }), + Content::Picture { mime, bytes, .. } if pictures => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }), + Content::Pdf { bytes, path } if documents => Some(PromptPart::Document { + name: f.name.clone(), + uri: file_uri(path), + mime: "application/pdf".to_string(), + base64: backend::base64(bytes), + }), _ => None, }) .collect() @@ -390,9 +518,6 @@ mod tests { let binary = d.join("bin.json"); std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap(); assert!(read(&binary).unwrap_err().contains("UTF-8")); - let other = d.join("movie.mov"); - std::fs::write(&other, b"x").unwrap(); - assert!(read(&other).unwrap_err().contains("can be attached")); assert!(read(&d.join("missing.png")).is_err()); assert_eq!(kind(Path::new("part.STEP")), Kind::Cad); assert_eq!(kind(Path::new("mount.stl")), Kind::Cad); @@ -411,12 +536,9 @@ mod tests { let d = dir("lang"); let binary = d.join("bin.json"); std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap(); - let other = d.join("movie.mov"); - std::fs::write(&other, b"x").unwrap(); { let _ja = crate::i18n::scope("ja"); assert!(read(&binary).unwrap_err().contains("UTF-8 のテキストではありません")); - assert!(read(&other).unwrap_err().starts_with("movie.mov: 添付できるのは")); } { let _zh = crate::i18n::scope("zh"); @@ -467,12 +589,12 @@ mod tests { fn attached_text_is_kept_within_the_prompt_budget() { let file = |n: &str, len: usize| Attachment { id: 0, name: n.into(), content: Content::Text(Arc::new("x".repeat(len))), thumb: None }; let files = vec![file("a.md", 600), file("b.md", 600), file("c.md", 600)]; - let n = note(&files, true, 1000); + let n = note(&files, true, false, 1000); assert!(n.contains("`a.md`") && n.contains("`b.md`") && n.contains("`c.md`"), "every file is named: {n}"); assert!(n.matches('x').count() <= 1000, "the files' text stays within the budget"); assert!(n.contains("200 more characters"), "a cut file says how much is missing: {n}"); assert!(n.contains("left out"), "a file past the budget is named only: {n}"); - assert_eq!(note(&files, true, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole"); + assert_eq!(note(&files, true, false, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole"); } #[test] @@ -481,13 +603,77 @@ mod tests { let text = Attachment { id: 0, name: "brief.md".into(), content: Content::Text(Arc::new("wall 2 mm\n".into())), thumb: None }; let part = Attachment::imported("motor.step".into()); let files = vec![pic, text, part]; - let n = note(&files, true, usize::MAX); + let n = note(&files, true, false, usize::MAX); assert!(n.contains("Attached pictures: target.png."), "{n}"); assert!(n.contains("Attached file `brief.md`:\n```\nwall 2 mm\n```"), "{n}"); assert!(n.contains("`motor.step` to the design"), "{n}"); - assert!(note(&files, false, usize::MAX).contains("not sent")); - let p = parts(&files); + assert!(note(&files, false, false, usize::MAX).contains("not sent")); + let p = parts(&files, true, true); assert_eq!(p.len(), 1); assert!(matches!(&p[0], PromptPart::Image { mime, base64 } if mime == "image/png" && base64.starts_with("iVBORw0KGgo"))); } + + /// Any file that is neither a picture, text, CAD nor a PDF is named by reference (what it is, + /// its size and its place) instead of refused; the agent reads it with its own tools. + #[test] + fn other_files_go_by_reference() { + let d = dir("reference"); + let zip = d.join("fixtures.zip"); + std::fs::write(&zip, vec![0u8; 3000]).unwrap(); + let dxf = d.join("outline.DXF"); + std::fs::write(&dxf, b"0\nSECTION\n").unwrap(); + let a = read(&zip).unwrap(); + let Content::Reference { path, size, mime } = &a.content else { panic!("a reference") }; + assert_eq!((*size, *mime), (3000, "application/zip")); + assert!(path.is_absolute() && path.ends_with("fixtures.zip")); + assert!(a.thumb.is_none() && a.mark() == "ZIP"); + assert!(matches!(read(&dxf).unwrap().content, Content::Reference { mime: "image/vnd.dxf", .. }), "by extension, in any case"); + let bare = d.join("LICENSE_BLOB"); + std::fs::write(&bare, b"x").unwrap(); + assert!(matches!(read(&bare).unwrap().content, Content::Reference { mime: "application/octet-stream", .. }), "no extension"); + let files = vec![a]; + let n = note(&files, true, true, usize::MAX); + assert!(n.contains("Attached file `fixtures.zip` (application/zip, 3 KB, at "), "{n}"); + assert!(n.contains("fixtures.zip): read it with your own tools"), "{n}"); + assert!(parts(&files, true, true).is_empty(), "a reference sends nothing but the note"); + assert!(read(&d).unwrap_err().contains("not a file"), "a folder is refused"); + assert!(read(&d.join("missing.bin")).is_err()); + let _ = std::fs::remove_dir_all(&d); + } + + /// A PDF goes whole as a document to an agent that takes them (and is named with its place); + /// to one that doesn't, and when it is too big to send, the agent is pointed to the file. + #[test] + fn pdfs_go_as_documents_or_by_reference() { + let d = dir("pdf"); + let pdf = d.join("brief 1.pdf"); + std::fs::write(&pdf, b"%PDF-1.4\n%%EOF\n").unwrap(); + let a = read(&pdf).unwrap(); + assert!(matches!(&a.content, Content::Pdf { bytes, .. } if bytes.starts_with(b"%PDF"))); + assert_eq!(a.mark(), "PDF"); + let files = vec![a]; + let sent = parts(&files, false, true); + let [PromptPart::Document { name, uri, mime, base64 }] = sent.as_slice() else { panic!("one document: {sent:?}") }; + assert_eq!((name.as_str(), mime.as_str()), ("brief 1.pdf", "application/pdf")); + assert!(uri.starts_with("file://") && uri.ends_with("/brief%201.pdf"), "{uri}"); + assert!(base64.starts_with("JVBERi0"), "the PDF's bytes: {base64}"); + assert!(note(&files, false, true, usize::MAX).contains("sent with this message as a document")); + assert!(parts(&files, true, false).is_empty(), "an agent with no documents gets none"); + let n = note(&files, true, false, usize::MAX); + assert!(n.contains("(PDF, 15 bytes, at ") && n.contains("read it with your own tools"), "{n}"); + let big = d.join("drawing set.pdf"); + std::fs::write(&big, vec![b'x'; usize::try_from(MAX_PDF_BYTES).unwrap() + 1]).unwrap(); + let b = read(&big).unwrap(); + assert!(matches!(b.content, Content::Reference { mime: "application/pdf", .. }), "too big to send: by reference"); + assert!(parts(&[b], true, true).is_empty()); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn links_to_files_are_escaped() { + assert_eq!(file_uri(Path::new("/home/a b/ü.pdf")), "file:///home/a%20b/%C3%BC.pdf"); + assert_eq!(file_uri(Path::new("C:\\Users\\a\\x.pdf")), "file:///C:/Users/a/x.pdf"); + assert_eq!(size_text(1536), "2 KB"); + assert_eq!(size_text(5 * 1024 * 1024 + 512 * 1024), "5.5 MB"); + } } diff --git a/crates/ui-egui/src/agent_chat/backend.rs b/crates/ui-egui/src/agent_chat/backend.rs index a44a54826..a470ccad9 100644 --- a/crates/ui-egui/src/agent_chat/backend.rs +++ b/crates/ui-egui/src/agent_chat/backend.rs @@ -19,11 +19,21 @@ pub enum PromptPart { mime: String, base64: String, }, + /// A file the agent reads as a document (a PDF), sent only to agents that take documents. + Document { + name: String, + /// Where it is on this machine (a `file://` link). + uri: String, + mime: String, + base64: String, + }, } /// What a backend reports to the panel. #[derive(Clone, Debug, PartialEq)] pub enum Event { + /// Whether the agent takes documents (PDFs) in a message; sent before [`Event::Ready`]. + Documents(bool), /// Ready for a message. `agent`: its name and version; `images`: it takes pictures. Ready { agent: String, images: bool }, /// Text it streams (`thought`: its reasoning, shown dimmed). @@ -136,6 +146,9 @@ pub fn content_blocks(parts: &[PromptPart]) -> Vec { .map(|p| match p { PromptPart::Text(t) => serde_json::json!({"type": "text", "text": t}), PromptPart::Image { mime, base64 } => serde_json::json!({"type": "image", "mimeType": mime, "data": base64}), + PromptPart::Document { uri, mime, base64, .. } => { + serde_json::json!({"type": "resource", "resource": {"uri": uri, "mimeType": mime, "blob": base64}}) + } }) .collect() } @@ -158,4 +171,19 @@ mod tests { assert_eq!(b[1]["type"], "image"); assert_eq!(b[1]["mimeType"], "image/png"); } + + #[test] + fn documents_go_as_embedded_resources() { + let doc = PromptPart::Document { + name: "brief.pdf".into(), + uri: "file:///a/brief.pdf".into(), + mime: "application/pdf".into(), + base64: "AA==".into(), + }; + let b = content_blocks(&[doc]); + assert_eq!(b[0]["type"], "resource"); + assert_eq!(b[0]["resource"]["uri"], "file:///a/brief.pdf"); + assert_eq!(b[0]["resource"]["mimeType"], "application/pdf"); + assert_eq!(b[0]["resource"]["blob"], "AA=="); + } } diff --git a/crates/ui-egui/src/agent_chat/mod.rs b/crates/ui-egui/src/agent_chat/mod.rs index 42b0ecca4..4dad5c60c 100644 --- a/crates/ui-egui/src/agent_chat/mod.rs +++ b/crates/ui-egui/src/agent_chat/mod.rs @@ -303,6 +303,8 @@ pub struct AgentChat { state: Option, agent: String, images: bool, + /// The agent takes documents (PDFs) in a message. + documents: bool, /// Send a picture of the view with the next message. pub attach_view: bool, auth: Vec, @@ -512,6 +514,7 @@ impl AgentChat { /// Take in what the backend reported. fn on_event(&mut self, e: Event) -> Option { match e { + Event::Documents(documents) => self.documents = documents, Event::Ready { agent, images } => { self.agent = agent; self.images = images; @@ -960,14 +963,14 @@ fn start_turn(app: &mut SolveApp, text: String, image: bool, at: Option, /// files are named after the text (text files in full), and pictures go as images to an agent /// that takes them. `at` is the message's place in the conversation, where its picture is kept. fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option, files: Vec, note: Option<&str>) { - let pictures = app.agent_chat.images; + let (pictures, documents) = (app.agent_chat.images, app.agent_chat.documents); let lead = match note { Some(n) => format!("{n}\n\n{text}"), None => text.clone(), }; // The whole message stays within the cap: the files' text gets what the words leave. let budget = MAX_PROMPT.saturating_sub(lead.chars().count() + FILE_NAMES_ROOM); - let about_files = attach::note(&files, pictures, budget); + let about_files = attach::note(&files, pictures, documents, budget); let sent: String = format!("{lead}{about_files}").chars().take(MAX_PROMPT).collect(); let mut parts = vec![PromptPart::Text(sent)]; if image { @@ -979,9 +982,7 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option log::warn!("the view's picture is left out of the prompt: {e}"), } } - if pictures { - parts.extend(attach::parts(&files)); - } + parts.extend(attach::parts(&files, pictures, documents)); // The undo step is named after the message, or its first file when it has no words. let first_line = text.lines().find(|l| !l.trim().is_empty()).or_else(|| files.first().map(|f| f.name.as_str())).unwrap_or(""); let label: String = first_line.chars().take(48).collect(); @@ -1017,9 +1018,10 @@ pub fn attach_path(app: &mut SolveApp, path: &str) -> std::result::Result Value { "state": c.state().id(), "agent": c.agent, "images": c.images, + "documents": c.documents, "authStatus": c.auth_status, "authMethods": c.auth.iter().map(|m| json!({"id": m.id, "name": m.name, "kind": match &m.kind { acp::AuthKind::Agent => "agent", acp::AuthKind::Terminal { .. } => "terminal", acp::AuthKind::EnvVar { .. } => "env_var" }})).collect::>(), "toolsPort": c.tools_port, diff --git a/crates/ui-egui/src/agent_chat/panel.rs b/crates/ui-egui/src/agent_chat/panel.rs index 7ad6cd836..354185a2d 100644 --- a/crates/ui-egui/src/agent_chat/panel.rs +++ b/crates/ui-egui/src/agent_chat/panel.rs @@ -1278,8 +1278,11 @@ fn chip(ui: &mut egui::Ui, thumbs: &mut std::collections::HashMap { - let (r, _) = ui.allocate_exact_size(vec2(16.0, 20.0), Sense::hover()); - ui.painter().rect_stroke(r.shrink2(vec2(2.0, 3.0)), 2.0, Stroke::new(1.2, t.text_dim), egui::StrokeKind::Inside); + // A page marked with the file's type. + let (r, _) = ui.allocate_exact_size(vec2(28.0, 20.0), Sense::hover()); + let color = if matches!(f.content, attach::Content::Pdf { .. }) { t.error } else { t.text_dim }; + ui.painter().rect_stroke(r.shrink2(vec2(1.0, 1.0)), 2.0, Stroke::new(1.2, color), egui::StrokeKind::Inside); + ui.painter().text(r.center(), Align2::CENTER_CENTER, f.mark(), egui::FontId::proportional(8.5), color); r.width() } };