Agent chat: PDFs as documents, any other file by reference (#2812) (#2858)

A PDF goes whole to agents that take embedded documents; to the others, or past 12 MB, the message names it with its type, size and place. Any file that is not a picture, text or CAD file is attached by reference instead of refused. Chips show the type's extension; the file dialog offers All files.
This commit is contained in:
bherbruck authored and GitHub committed 2026-10-19 00:02:29 -04:00
1 parent e3831b05d4
commit dc27988bc2
11 files changed
+280 -42

No files matched your search

+2 -1
View File
@@ -179,7 +179,8 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services {
}
}
let all: Vec<&str> = all.iter().map(String::as_str).collect();
let future = file_parent.clone().add_filter(label, &all).pick_file();
// Any other file attaches by reference, so the list is not the only choice.
let future = file_parent.clone().add_filter(label, &all).add_filter("All files", &["*"]).pick_file();
Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) })
})),
pick_program: Some(Box::new(move || {
+1 -2
View File
@@ -859,7 +859,7 @@
"ui.agent_chat_approval": "Allow the agent to do this?",
"ui.agent_chat_attach_view": "Attach view",
"ui.agent_chat_attach_view_tip": "Send a picture of the current view with the message",
"ui.agent_chat_attachable": "Pictures, text and CAD files",
"ui.agent_chat_attachable": "Pictures, PDFs, text and CAD files",
"ui.agent_chat_auth_agent": "Sign in with {name}",
"ui.agent_chat_auth_env": "Set {vars} in your environment and restart SolidCraft.",
"ui.agent_chat_auth_needed": "The agent asks you to sign in. You sign in with the agent itself; SolidCraft never sees your password or keys.",
@@ -891,7 +891,6 @@
"ui.agent_chat_file_error": "{name}: {error}",
"ui.agent_chat_file_heic": "{name}: HEIC pictures can't be read here; export it as JPEG or PNG first",
"ui.agent_chat_file_importing": "Importing into the design…",
"ui.agent_chat_file_kinds": "{name}: pictures (PNG, JPEG, GIF, WebP, BMP, TIFF), text files (SVG too) and CAD files (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) can be attached",
"ui.agent_chat_file_loading": "Reading the file…",
"ui.agent_chat_file_not_a_file": "{name}: not a file",
"ui.agent_chat_file_not_a_picture": "{name}: not a picture",
+1 -2
View File
@@ -859,7 +859,7 @@
"ui.agent_chat_approval": "エージェントにこの操作を許可しますか?",
"ui.agent_chat_attach_view": "ビューを添付",
"ui.agent_chat_attach_view_tip": "現在のビューの画像をメッセージと一緒に送ります",
"ui.agent_chat_attachable": "画像・テキスト・CAD ファイル",
"ui.agent_chat_attachable": "画像・PDF・テキスト・CAD ファイル",
"ui.agent_chat_auth_agent": "{name} でサインイン",
"ui.agent_chat_auth_env": "環境変数 {vars} を設定して SolidCraft を再起動してください。",
"ui.agent_chat_auth_needed": "エージェントがサインインを求めています。サインインはエージェント自身で行います。SolidCraft がパスワードやキーを見ることはありません。",
@@ -891,7 +891,6 @@
"ui.agent_chat_file_error": "{name}: {error}",
"ui.agent_chat_file_heic": "{name}: HEIC 画像はここでは読めません。先に JPEG か PNG に書き出してください",
"ui.agent_chat_file_importing": "デザインにインポート中…",
"ui.agent_chat_file_kinds": "{name}: 添付できるのは画像 (PNG、JPEG、GIF、WebP、BMP、TIFF)、テキストファイル (SVG も)、CAD ファイル (STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF) です",
"ui.agent_chat_file_loading": "ファイルを読み込み中…",
"ui.agent_chat_file_not_a_file": "{name}: ファイルではありません",
"ui.agent_chat_file_not_a_picture": "{name}: 画像ではありません",
+1 -2
View File
@@ -859,7 +859,7 @@
"ui.agent_chat_approval": "允许智能体执行此操作吗?",
"ui.agent_chat_attach_view": "附加视图",
"ui.agent_chat_attach_view_tip": "随消息发送当前视图的图片",
"ui.agent_chat_attachable": "图片、文本和 CAD 文件",
"ui.agent_chat_attachable": "图片、PDF、文本和 CAD 文件",
"ui.agent_chat_auth_agent": "使用 {name} 登录",
"ui.agent_chat_auth_env": "请在环境中设置 {vars},然后重启 SolidCraft。",
"ui.agent_chat_auth_needed": "智能体要求你登录。登录在智能体本身完成;SolidCraft 从不接触你的密码或密钥。",
@@ -891,7 +891,6 @@
"ui.agent_chat_file_error": "{name}:{error}",
"ui.agent_chat_file_heic": "{name}:这里无法读取 HEIC 图片;请先导出为 JPEG 或 PNG",
"ui.agent_chat_file_importing": "正在导入设计…",
"ui.agent_chat_file_kinds": "{name}:可以附加图片(PNG、JPEG、GIF、WebP、BMP、TIFF)、文本文件(包括 SVG)和 CAD 文件(STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF)",
"ui.agent_chat_file_loading": "正在读取文件…",
"ui.agent_chat_file_not_a_file": "{name}:不是文件",
"ui.agent_chat_file_not_a_picture": "{name}:不是图片",
+14
View File
@@ -149,6 +149,12 @@ pub fn set_model_params(session: &str, model: &str) -> Value {
json!({"sessionId": session, "modelId": model})
}
/// Whether the agent takes embedded documents in a message
/// (`agentCapabilities.promptCapabilities.embeddedContext`: it reads `resource` blocks).
pub fn takes_documents(init: &Value) -> bool {
init.pointer("/agentCapabilities/promptCapabilities/embeddedContext").and_then(Value::as_bool).unwrap_or(false)
}
/// Whether the agent takes pictures in a message (`agentCapabilities.promptCapabilities.image`).
pub fn takes_images(init: &Value) -> bool {
init.pointer("/agentCapabilities/promptCapabilities/image").and_then(Value::as_bool).unwrap_or(false)
@@ -461,6 +467,14 @@ pub fn split_command(line: &str) -> Vec<String> {
mod tests {
use super::*;
#[test]
fn documents_are_taken_only_when_the_agent_says_so() {
let yes = json!({"agentCapabilities": {"promptCapabilities": {"image": true, "embeddedContext": true}}});
let no = json!({"agentCapabilities": {"promptCapabilities": {"image": true}}});
assert!(takes_documents(&yes) && takes_images(&yes));
assert!(!takes_documents(&no) && !takes_documents(&json!({})));
}
#[test]
fn commands_split_like_a_shell_without_one() {
assert_eq!(split_command("gemini --acp"), ["gemini", "--acp"]);
@@ -33,6 +33,7 @@ pub struct AcpBackend {
session: Option<String>,
agent: String,
images: bool,
documents: bool,
auth: Vec<acp::AuthMethod>,
/// Open permission questions: our key → the agent's request id.
permissions: HashMap<u64, Value>,
@@ -54,6 +55,7 @@ impl AcpBackend {
session: None,
agent: String::new(),
images: false,
documents: false,
auth: Vec::new(),
permissions: HashMap::new(),
next_key: 0,
@@ -82,6 +84,7 @@ impl AcpBackend {
(Pending::Initialize, Ok(v)) => {
self.agent = acp::agent_info(&v);
self.images = acp::takes_images(&v);
self.documents = acp::takes_documents(&v);
self.auth = acp::auth_methods(&v);
self.new_session();
}
@@ -94,6 +97,7 @@ impl AcpBackend {
if let Some((current, available)) = acp::models(&v) {
out.push(Event::Models { current, available });
}
out.push(Event::Documents(self.documents));
out.push(Event::Ready { agent: self.agent.clone(), images: self.images });
}
None => out.push(Event::Error("session/new: no sessionId".into())),
+5 -3
View File
@@ -404,9 +404,11 @@ fn user_message(parts: &[PromptPart]) -> Value {
}
let content: Vec<Value> = parts
.iter()
.map(|p| match p {
PromptPart::Text(t) => json!({"type": "text", "text": t}),
PromptPart::Image { mime, base64 } => json!({"type": "image_url", "image_url": {"url": format!("data:{mime};base64,{base64}")}}),
.filter_map(|p| match p {
PromptPart::Text(t) => Some(json!({"type": "text", "text": t})),
PromptPart::Image { mime, base64 } => Some(json!({"type": "image_url", "image_url": {"url": format!("data:{mime};base64,{base64}")}})),
// This endpoint takes no documents (the chat doesn't send any); the message names the file.
PromptPart::Document { .. } => None,
})
.collect();
json!({"role": "user", "content": content})
+209 -23
View File
@@ -1,9 +1,10 @@
//! Files a person adds to a message (the + button, a drop on the chat, a paste): pictures go to the
//! agent with it, text files go in as text, and CAD files are imported into the design, the agent
//! being told what arrived. Each file is checked and bounded before it is kept; a refused one says
//! why.
//! agent with it, text files go in as text, PDFs go as documents to agents that take them, CAD
//! files are imported into the design, and any other file is named by reference (its type, size
//! and place on this machine) for the agent to read with its own tools. The agent is told what
//! arrived. Each file is checked and bounded before it is kept; a refused one says why.
use std::path::Path;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::backend::{self, PromptPart};
@@ -27,6 +28,8 @@ pub const MAX_FILES: usize = 8;
const MAX_PICTURE_BYTES: u64 = 40 * 1024 * 1024;
/// Largest text file sent in full.
const MAX_TEXT_BYTES: u64 = 256 * 1024;
/// Largest PDF sent as a document; a bigger one goes by reference.
const MAX_PDF_BYTES: u64 = 12 * 1024 * 1024;
/// A picture's longer side is scaled down to this before it is sent.
pub const MAX_SIDE: u32 = 2048;
/// Largest picture sent, as it came or re-encoded: its base64 (a third larger) stays under 5 MB,
@@ -47,9 +50,9 @@ const TEXT_EXTS: &[&str] = &["svg", "txt", "md", "markdown", "json", "csv", "tsv
/// importers themselves).
const CAD_EXTS: &[&str] = &["step", "stp", "igs", "iges", "FCStd", "3dm", "stl", "3mf", "obj", "glb", "gltf"];
/// Every extension that can be attached, pictures first (the file picker's list).
/// Every extension the file picker lists first, pictures first (any other file can be chosen too).
pub fn extensions() -> Vec<&'static str> {
PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).copied().collect()
PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).chain(&["pdf"]).copied().collect()
}
/// What a file is, by its extension.
@@ -58,6 +61,8 @@ pub enum Kind {
Picture,
Text,
Cad,
Pdf,
/// Anything else, attached by reference.
Other,
}
@@ -66,6 +71,7 @@ pub fn kind(path: &Path) -> Kind {
match ext.as_str() {
e if PICTURE_EXTS.contains(&e) => Kind::Picture,
e if TEXT_EXTS.contains(&e) => Kind::Text,
"pdf" => Kind::Pdf,
_ => {
let p = path.to_string_lossy();
if solidcraft_engine::io::is_step_path(&p) || solidcraft_engine::io::is_mesh_path(&p) { Kind::Cad } else { Kind::Other }
@@ -82,6 +88,17 @@ pub enum Content {
size: [u32; 2],
},
Text(Arc<String>),
/// A PDF, sent whole as a document to an agent that takes them.
Pdf {
bytes: Arc<Vec<u8>>,
path: PathBuf,
},
/// A file the agent is pointed to, not given: what it is, its size and where it is.
Reference {
path: PathBuf,
size: u64,
mime: &'static str,
},
/// A CAD file imported into the design as its own component.
Imported,
}
@@ -104,6 +121,70 @@ impl Attachment {
pub fn is_picture(&self) -> bool {
matches!(self.content, Content::Picture { .. })
}
/// The chip's mark for a file with no thumbnail: its extension, in capitals, cut to fit.
pub fn mark(&self) -> String {
let ext = Path::new(&self.name).extension().and_then(|e| e.to_str()).unwrap_or("");
ext.chars().take(4).collect::<String>().to_uppercase()
}
}
/// What a file is called to the agent, by its extension.
fn mime_of(path: &Path) -> &'static str {
let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).unwrap_or_default();
match ext.as_str() {
"pdf" => "application/pdf",
"zip" => "application/zip",
"7z" => "application/x-7z-compressed",
"gz" | "tgz" => "application/gzip",
"dxf" => "image/vnd.dxf",
"dwg" => "image/vnd.dwg",
"step" | "stp" => "model/step",
"iges" | "igs" => "model/iges",
"stl" => "model/stl",
"obj" => "model/obj",
"3mf" => "model/3mf",
"gltf" => "model/gltf+json",
"glb" => "model/gltf-binary",
"xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
"mp4" | "mov" | "webm" => "video/mp4",
"wav" => "audio/wav",
"mp3" => "audio/mpeg",
_ => "application/octet-stream",
}
}
/// A size for people: `1.5 MB`.
fn size_text(bytes: u64) -> String {
const KB: f64 = 1024.0;
let b = bytes as f64;
if b < KB {
format!("{bytes} bytes")
} else if b < KB * KB {
format!("{:.0} KB", b / KB)
} else if b < KB * KB * KB {
format!("{:.1} MB", b / KB / KB)
} else {
format!("{:.1} GB", b / KB / KB / KB)
}
}
/// A `file://` link to a path, for a document sent to an agent.
fn file_uri(path: &Path) -> String {
let mut s = path.to_string_lossy().replace('\\', "/");
if !s.starts_with('/') {
s.insert(0, '/');
}
let mut out = String::from("file://");
for b in s.bytes() {
if b.is_ascii_alphanumeric() || matches!(b, b'/' | b'-' | b'_' | b'.' | b'~' | b':') {
out.push(char::from(b));
} else {
out.push_str(&format!("%{b:02X}"));
}
}
out
}
pub(crate) fn name_of(path: &Path) -> String {
@@ -134,13 +215,29 @@ pub fn read(path: &Path) -> Result<Attachment, String> {
Ok(Attachment { id: 0, name, content: Content::Text(Arc::new(text)), thumb: None })
}
Kind::Cad => Err(refused("ui.agent_chat_file_cad_not_read", &name, &[])),
Kind::Pdf => {
let path = absolute(path);
// Too big to send: the agent is pointed to it instead.
if meta.len() > MAX_PDF_BYTES {
return Ok(Attachment { id: 0, name, content: Content::Reference { path, size: meta.len(), mime: "application/pdf" }, thumb: None });
}
let bytes = std::fs::read(&path).map_err(|e| failed(&name, e))?;
Ok(Attachment { id: 0, name, content: Content::Pdf { bytes: Arc::new(bytes), path }, thumb: None })
}
Kind::Other if matches!(path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("heic" | "heif")) => {
Err(refused("ui.agent_chat_file_heic", &name, &[]))
}
Kind::Other => Err(refused("ui.agent_chat_file_kinds", &name, &[])),
Kind::Other => {
Ok(Attachment { id: 0, name, content: Content::Reference { path: absolute(path), size: meta.len(), mime: mime_of(path) }, thumb: None })
}
}
}
/// The path as an absolute one, so the agent (working in another folder) finds the file.
fn absolute(path: &Path) -> PathBuf {
std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf())
}
/// A picture from a file's bytes: decoded within limits, and sent as it came when the agents read
/// that format (PNG, JPEG, GIF, WebP), it is at most [`MAX_SIDE`] on a side and at most
/// [`MAX_SENT_BYTES`]; otherwise re-encoded (see [`encoded`]).
@@ -239,10 +336,10 @@ fn thumbnail(img: &image::DynamicImage) -> Option<Arc<egui::ColorImage>> {
/// What the message says about its files, after the person's words: the text files' text, and the
/// pictures and imported parts by name (the agent sees the pictures themselves as images).
/// `pictures_sent`: whether this agent takes pictures. The text files share `budget` characters
/// `pictures_sent` and `documents_sent`: whether this agent takes pictures and PDFs. The text files share `budget` characters
/// in order: one that doesn't fit is cut and says how much is missing, and one past the budget
/// is named only, so the message stays within the prompt cap.
pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String {
pub fn note(files: &[Attachment], pictures_sent: bool, documents_sent: bool, budget: usize) -> String {
let mut out = String::new();
let pictures: Vec<&str> = files.iter().filter(|f| f.is_picture()).map(|f| f.name.as_str()).collect();
if !pictures.is_empty() {
@@ -270,6 +367,31 @@ pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String
left = 0;
}
}
Content::Pdf { path, bytes } if documents_sent => {
out += &format!(
"\n\nAttached PDF `{}` ({}, at {}): sent with this message as a document.",
f.name,
size_text(bytes.len() as u64),
path.display()
);
}
// An agent that takes no documents is pointed to the file, as for any other file.
Content::Pdf { path, bytes } => {
out += &format!(
"\n\nAttached file `{}` (PDF, {}, at {}): read it with your own tools; it is not in this message.",
f.name,
size_text(bytes.len() as u64),
path.display()
);
}
Content::Reference { path, size, mime } => {
out += &format!(
"\n\nAttached file `{}` ({mime}, {}, at {}): read it with your own tools; it is not in this message.",
f.name,
size_text(*size),
path.display()
);
}
Content::Imported => out += &format!("\n\n(I added `{}` to the design as a component of its own.)", f.name),
Content::Picture { .. } => {}
}
@@ -277,12 +399,18 @@ pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String
out
}
/// The pictures as image parts of the message.
pub fn parts(files: &[Attachment]) -> Vec<PromptPart> {
/// The pictures and PDFs as parts of the message, to the agents that take them.
pub fn parts(files: &[Attachment], pictures: bool, documents: bool) -> Vec<PromptPart> {
files
.iter()
.filter_map(|f| match &f.content {
Content::Picture { mime, bytes, .. } => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),
Content::Picture { mime, bytes, .. } if pictures => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),
Content::Pdf { bytes, path } if documents => Some(PromptPart::Document {
name: f.name.clone(),
uri: file_uri(path),
mime: "application/pdf".to_string(),
base64: backend::base64(bytes),
}),
_ => None,
})
.collect()
@@ -390,9 +518,6 @@ mod tests {
let binary = d.join("bin.json");
std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();
assert!(read(&binary).unwrap_err().contains("UTF-8"));
let other = d.join("movie.mov");
std::fs::write(&other, b"x").unwrap();
assert!(read(&other).unwrap_err().contains("can be attached"));
assert!(read(&d.join("missing.png")).is_err());
assert_eq!(kind(Path::new("part.STEP")), Kind::Cad);
assert_eq!(kind(Path::new("mount.stl")), Kind::Cad);
@@ -411,12 +536,9 @@ mod tests {
let d = dir("lang");
let binary = d.join("bin.json");
std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();
let other = d.join("movie.mov");
std::fs::write(&other, b"x").unwrap();
{
let _ja = crate::i18n::scope("ja");
assert!(read(&binary).unwrap_err().contains("UTF-8 のテキストではありません"));
assert!(read(&other).unwrap_err().starts_with("movie.mov: 添付できるのは"));
}
{
let _zh = crate::i18n::scope("zh");
@@ -467,12 +589,12 @@ mod tests {
fn attached_text_is_kept_within_the_prompt_budget() {
let file = |n: &str, len: usize| Attachment { id: 0, name: n.into(), content: Content::Text(Arc::new("x".repeat(len))), thumb: None };
let files = vec![file("a.md", 600), file("b.md", 600), file("c.md", 600)];
let n = note(&files, true, 1000);
let n = note(&files, true, false, 1000);
assert!(n.contains("`a.md`") && n.contains("`b.md`") && n.contains("`c.md`"), "every file is named: {n}");
assert!(n.matches('x').count() <= 1000, "the files' text stays within the budget");
assert!(n.contains("200 more characters"), "a cut file says how much is missing: {n}");
assert!(n.contains("left out"), "a file past the budget is named only: {n}");
assert_eq!(note(&files, true, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole");
assert_eq!(note(&files, true, false, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole");
}
#[test]
@@ -481,13 +603,77 @@ mod tests {
let text = Attachment { id: 0, name: "brief.md".into(), content: Content::Text(Arc::new("wall 2 mm\n".into())), thumb: None };
let part = Attachment::imported("motor.step".into());
let files = vec![pic, text, part];
let n = note(&files, true, usize::MAX);
let n = note(&files, true, false, usize::MAX);
assert!(n.contains("Attached pictures: target.png."), "{n}");
assert!(n.contains("Attached file `brief.md`:\n```\nwall 2 mm\n```"), "{n}");
assert!(n.contains("`motor.step` to the design"), "{n}");
assert!(note(&files, false, usize::MAX).contains("not sent"));
let p = parts(&files);
assert!(note(&files, false, false, usize::MAX).contains("not sent"));
let p = parts(&files, true, true);
assert_eq!(p.len(), 1);
assert!(matches!(&p[0], PromptPart::Image { mime, base64 } if mime == "image/png" && base64.starts_with("iVBORw0KGgo")));
}
/// Any file that is neither a picture, text, CAD nor a PDF is named by reference (what it is,
/// its size and its place) instead of refused; the agent reads it with its own tools.
#[test]
fn other_files_go_by_reference() {
let d = dir("reference");
let zip = d.join("fixtures.zip");
std::fs::write(&zip, vec![0u8; 3000]).unwrap();
let dxf = d.join("outline.DXF");
std::fs::write(&dxf, b"0\nSECTION\n").unwrap();
let a = read(&zip).unwrap();
let Content::Reference { path, size, mime } = &a.content else { panic!("a reference") };
assert_eq!((*size, *mime), (3000, "application/zip"));
assert!(path.is_absolute() && path.ends_with("fixtures.zip"));
assert!(a.thumb.is_none() && a.mark() == "ZIP");
assert!(matches!(read(&dxf).unwrap().content, Content::Reference { mime: "image/vnd.dxf", .. }), "by extension, in any case");
let bare = d.join("LICENSE_BLOB");
std::fs::write(&bare, b"x").unwrap();
assert!(matches!(read(&bare).unwrap().content, Content::Reference { mime: "application/octet-stream", .. }), "no extension");
let files = vec![a];
let n = note(&files, true, true, usize::MAX);
assert!(n.contains("Attached file `fixtures.zip` (application/zip, 3 KB, at "), "{n}");
assert!(n.contains("fixtures.zip): read it with your own tools"), "{n}");
assert!(parts(&files, true, true).is_empty(), "a reference sends nothing but the note");
assert!(read(&d).unwrap_err().contains("not a file"), "a folder is refused");
assert!(read(&d.join("missing.bin")).is_err());
let _ = std::fs::remove_dir_all(&d);
}
/// A PDF goes whole as a document to an agent that takes them (and is named with its place);
/// to one that doesn't, and when it is too big to send, the agent is pointed to the file.
#[test]
fn pdfs_go_as_documents_or_by_reference() {
let d = dir("pdf");
let pdf = d.join("brief 1.pdf");
std::fs::write(&pdf, b"%PDF-1.4\n%%EOF\n").unwrap();
let a = read(&pdf).unwrap();
assert!(matches!(&a.content, Content::Pdf { bytes, .. } if bytes.starts_with(b"%PDF")));
assert_eq!(a.mark(), "PDF");
let files = vec![a];
let sent = parts(&files, false, true);
let [PromptPart::Document { name, uri, mime, base64 }] = sent.as_slice() else { panic!("one document: {sent:?}") };
assert_eq!((name.as_str(), mime.as_str()), ("brief 1.pdf", "application/pdf"));
assert!(uri.starts_with("file://") && uri.ends_with("/brief%201.pdf"), "{uri}");
assert!(base64.starts_with("JVBERi0"), "the PDF's bytes: {base64}");
assert!(note(&files, false, true, usize::MAX).contains("sent with this message as a document"));
assert!(parts(&files, true, false).is_empty(), "an agent with no documents gets none");
let n = note(&files, true, false, usize::MAX);
assert!(n.contains("(PDF, 15 bytes, at ") && n.contains("read it with your own tools"), "{n}");
let big = d.join("drawing set.pdf");
std::fs::write(&big, vec![b'x'; usize::try_from(MAX_PDF_BYTES).unwrap() + 1]).unwrap();
let b = read(&big).unwrap();
assert!(matches!(b.content, Content::Reference { mime: "application/pdf", .. }), "too big to send: by reference");
assert!(parts(&[b], true, true).is_empty());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn links_to_files_are_escaped() {
assert_eq!(file_uri(Path::new("/home/a b/ü.pdf")), "file:///home/a%20b/%C3%BC.pdf");
assert_eq!(file_uri(Path::new("C:\\Users\\a\\x.pdf")), "file:///C:/Users/a/x.pdf");
assert_eq!(size_text(1536), "2 KB");
assert_eq!(size_text(5 * 1024 * 1024 + 512 * 1024), "5.5 MB");
}
}
+28
View File
@@ -19,11 +19,21 @@ pub enum PromptPart {
mime: String,
base64: String,
},
/// A file the agent reads as a document (a PDF), sent only to agents that take documents.
Document {
name: String,
/// Where it is on this machine (a `file://` link).
uri: String,
mime: String,
base64: String,
},
}
/// What a backend reports to the panel.
#[derive(Clone, Debug, PartialEq)]
pub enum Event {
/// Whether the agent takes documents (PDFs) in a message; sent before [`Event::Ready`].
Documents(bool),
/// Ready for a message. `agent`: its name and version; `images`: it takes pictures.
Ready { agent: String, images: bool },
/// Text it streams (`thought`: its reasoning, shown dimmed).
@@ -136,6 +146,9 @@ pub fn content_blocks(parts: &[PromptPart]) -> Vec<Value> {
.map(|p| match p {
PromptPart::Text(t) => serde_json::json!({"type": "text", "text": t}),
PromptPart::Image { mime, base64 } => serde_json::json!({"type": "image", "mimeType": mime, "data": base64}),
PromptPart::Document { uri, mime, base64, .. } => {
serde_json::json!({"type": "resource", "resource": {"uri": uri, "mimeType": mime, "blob": base64}})
}
})
.collect()
}
@@ -158,4 +171,19 @@ mod tests {
assert_eq!(b[1]["type"], "image");
assert_eq!(b[1]["mimeType"], "image/png");
}
#[test]
fn documents_go_as_embedded_resources() {
let doc = PromptPart::Document {
name: "brief.pdf".into(),
uri: "file:///a/brief.pdf".into(),
mime: "application/pdf".into(),
base64: "AA==".into(),
};
let b = content_blocks(&[doc]);
assert_eq!(b[0]["type"], "resource");
assert_eq!(b[0]["resource"]["uri"], "file:///a/brief.pdf");
assert_eq!(b[0]["resource"]["mimeType"], "application/pdf");
assert_eq!(b[0]["resource"]["blob"], "AA==");
}
}
+10 -7
View File
@@ -303,6 +303,8 @@ pub struct AgentChat {
state: Option<State>,
agent: String,
images: bool,
/// The agent takes documents (PDFs) in a message.
documents: bool,
/// Send a picture of the view with the next message.
pub attach_view: bool,
auth: Vec<acp::AuthMethod>,
@@ -512,6 +514,7 @@ impl AgentChat {
/// Take in what the backend reported.
fn on_event(&mut self, e: Event) -> Option<Outcome> {
match e {
Event::Documents(documents) => self.documents = documents,
Event::Ready { agent, images } => {
self.agent = agent;
self.images = images;
@@ -960,14 +963,14 @@ fn start_turn(app: &mut SolveApp, text: String, image: bool, at: Option<usize>,
/// files are named after the text (text files in full), and pictures go as images to an agent
/// that takes them. `at` is the message's place in the conversation, where its picture is kept.
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>, note: Option<&str>) {
let pictures = app.agent_chat.images;
let (pictures, documents) = (app.agent_chat.images, app.agent_chat.documents);
let lead = match note {
Some(n) => format!("{n}\n\n{text}"),
None => text.clone(),
};
// The whole message stays within the cap: the files' text gets what the words leave.
let budget = MAX_PROMPT.saturating_sub(lead.chars().count() + FILE_NAMES_ROOM);
let about_files = attach::note(&files, pictures, budget);
let about_files = attach::note(&files, pictures, documents, budget);
let sent: String = format!("{lead}{about_files}").chars().take(MAX_PROMPT).collect();
let mut parts = vec![PromptPart::Text(sent)];
if image {
@@ -979,9 +982,7 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usi
Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"),
}
}
if pictures {
parts.extend(attach::parts(&files));
}
parts.extend(attach::parts(&files, pictures, documents));
// The undo step is named after the message, or its first file when it has no words.
let first_line = text.lines().find(|l| !l.trim().is_empty()).or_else(|| files.first().map(|f| f.name.as_str())).unwrap_or("");
let label: String = first_line.chars().take(48).collect();
@@ -1017,9 +1018,10 @@ pub fn attach_path(app: &mut SolveApp, path: &str) -> std::result::Result<String
crate::file_load::begin_request(app, request).map_err(|e| trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &e)]))?;
return Ok(app.agent_chat.importing(name, rx));
} else {
// A picture is decoded off the window thread: a large one takes long enough to stop it.
// A picture is decoded and a PDF read off the window thread: a large one takes long enough
// to stop it.
#[cfg(not(target_arch = "wasm32"))]
if attach::kind(p) == attach::Kind::Picture {
if matches!(attach::kind(p), attach::Kind::Picture | attach::Kind::Pdf) {
// A missing file is said at once; reading and decoding wait for the worker.
if !p.is_file() {
return Err(trf("ui.agent_chat_file_not_a_file", &[("name", &attach::name_of(p))]));
@@ -1654,6 +1656,7 @@ pub fn status(app: &SolveApp) -> Value {
"state": c.state().id(),
"agent": c.agent,
"images": c.images,
"documents": c.documents,
"authStatus": c.auth_status,
"authMethods": c.auth.iter().map(|m| json!({"id": m.id, "name": m.name, "kind": match &m.kind { acp::AuthKind::Agent => "agent", acp::AuthKind::Terminal { .. } => "terminal", acp::AuthKind::EnvVar { .. } => "env_var" }})).collect::<Vec<_>>(),
"toolsPort": c.tools_port,
+5 -2
View File
@@ -1278,8 +1278,11 @@ fn chip(ui: &mut egui::Ui, thumbs: &mut std::collections::HashMap<u64, egui::Tex
ui.image((tex.id(), size * (22.0 / size.x.max(size.y).max(1.0)))).rect.width()
}
None => {
let (r, _) = ui.allocate_exact_size(vec2(16.0, 20.0), Sense::hover());
ui.painter().rect_stroke(r.shrink2(vec2(2.0, 3.0)), 2.0, Stroke::new(1.2, t.text_dim), egui::StrokeKind::Inside);
// A page marked with the file's type.
let (r, _) = ui.allocate_exact_size(vec2(28.0, 20.0), Sense::hover());
let color = if matches!(f.content, attach::Content::Pdf { .. }) { t.error } else { t.text_dim };
ui.painter().rect_stroke(r.shrink2(vec2(1.0, 1.0)), 2.0, Stroke::new(1.2, color), egui::StrokeKind::Inside);
ui.painter().text(r.center(), Align2::CENTER_CENTER, f.mark(), egui::FontId::proportional(8.5), color);
r.width()
}
};