A PDF goes whole to agents that take embedded documents; to the others, or past 12 MB, the message names it with its type, size and place. Any file that is not a picture, text or CAD file is attached by reference instead of refused. Chips show the type's extension; the file dialog offers All files.
This commit is contained in:
1 parent
e3831b05d4
commit
dc27988bc2
11 files changed
+280
-42
No files matched your search
@@ -179,7 +179,8 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services {
|
||||
}
|
||||
}
|
||||
let all: Vec<&str> = all.iter().map(String::as_str).collect();
|
||||
let future = file_parent.clone().add_filter(label, &all).pick_file();
|
||||
// Any other file attaches by reference, so the list is not the only choice.
|
||||
let future = file_parent.clone().add_filter(label, &all).add_filter("All files", &["*"]).pick_file();
|
||||
Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) })
|
||||
})),
|
||||
pick_program: Some(Box::new(move || {
|
||||
|
||||
@@ -859,7 +859,7 @@
|
||||
"ui.agent_chat_approval": "Allow the agent to do this?",
|
||||
"ui.agent_chat_attach_view": "Attach view",
|
||||
"ui.agent_chat_attach_view_tip": "Send a picture of the current view with the message",
|
||||
"ui.agent_chat_attachable": "Pictures, text and CAD files",
|
||||
"ui.agent_chat_attachable": "Pictures, PDFs, text and CAD files",
|
||||
"ui.agent_chat_auth_agent": "Sign in with {name}",
|
||||
"ui.agent_chat_auth_env": "Set {vars} in your environment and restart SolidCraft.",
|
||||
"ui.agent_chat_auth_needed": "The agent asks you to sign in. You sign in with the agent itself; SolidCraft never sees your password or keys.",
|
||||
@@ -891,7 +891,6 @@
|
||||
"ui.agent_chat_file_error": "{name}: {error}",
|
||||
"ui.agent_chat_file_heic": "{name}: HEIC pictures can't be read here; export it as JPEG or PNG first",
|
||||
"ui.agent_chat_file_importing": "Importing into the design…",
|
||||
"ui.agent_chat_file_kinds": "{name}: pictures (PNG, JPEG, GIF, WebP, BMP, TIFF), text files (SVG too) and CAD files (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) can be attached",
|
||||
"ui.agent_chat_file_loading": "Reading the file…",
|
||||
"ui.agent_chat_file_not_a_file": "{name}: not a file",
|
||||
"ui.agent_chat_file_not_a_picture": "{name}: not a picture",
|
||||
|
||||
@@ -859,7 +859,7 @@
|
||||
"ui.agent_chat_approval": "エージェントにこの操作を許可しますか?",
|
||||
"ui.agent_chat_attach_view": "ビューを添付",
|
||||
"ui.agent_chat_attach_view_tip": "現在のビューの画像をメッセージと一緒に送ります",
|
||||
"ui.agent_chat_attachable": "画像・テキスト・CAD ファイル",
|
||||
"ui.agent_chat_attachable": "画像・PDF・テキスト・CAD ファイル",
|
||||
"ui.agent_chat_auth_agent": "{name} でサインイン",
|
||||
"ui.agent_chat_auth_env": "環境変数 {vars} を設定して SolidCraft を再起動してください。",
|
||||
"ui.agent_chat_auth_needed": "エージェントがサインインを求めています。サインインはエージェント自身で行います。SolidCraft がパスワードやキーを見ることはありません。",
|
||||
@@ -891,7 +891,6 @@
|
||||
"ui.agent_chat_file_error": "{name}: {error}",
|
||||
"ui.agent_chat_file_heic": "{name}: HEIC 画像はここでは読めません。先に JPEG か PNG に書き出してください",
|
||||
"ui.agent_chat_file_importing": "デザインにインポート中…",
|
||||
"ui.agent_chat_file_kinds": "{name}: 添付できるのは画像 (PNG、JPEG、GIF、WebP、BMP、TIFF)、テキストファイル (SVG も)、CAD ファイル (STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF) です",
|
||||
"ui.agent_chat_file_loading": "ファイルを読み込み中…",
|
||||
"ui.agent_chat_file_not_a_file": "{name}: ファイルではありません",
|
||||
"ui.agent_chat_file_not_a_picture": "{name}: 画像ではありません",
|
||||
|
||||
@@ -859,7 +859,7 @@
|
||||
"ui.agent_chat_approval": "允许智能体执行此操作吗?",
|
||||
"ui.agent_chat_attach_view": "附加视图",
|
||||
"ui.agent_chat_attach_view_tip": "随消息发送当前视图的图片",
|
||||
"ui.agent_chat_attachable": "图片、文本和 CAD 文件",
|
||||
"ui.agent_chat_attachable": "图片、PDF、文本和 CAD 文件",
|
||||
"ui.agent_chat_auth_agent": "使用 {name} 登录",
|
||||
"ui.agent_chat_auth_env": "请在环境中设置 {vars},然后重启 SolidCraft。",
|
||||
"ui.agent_chat_auth_needed": "智能体要求你登录。登录在智能体本身完成;SolidCraft 从不接触你的密码或密钥。",
|
||||
@@ -891,7 +891,6 @@
|
||||
"ui.agent_chat_file_error": "{name}:{error}",
|
||||
"ui.agent_chat_file_heic": "{name}:这里无法读取 HEIC 图片;请先导出为 JPEG 或 PNG",
|
||||
"ui.agent_chat_file_importing": "正在导入设计…",
|
||||
"ui.agent_chat_file_kinds": "{name}:可以附加图片(PNG、JPEG、GIF、WebP、BMP、TIFF)、文本文件(包括 SVG)和 CAD 文件(STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF)",
|
||||
"ui.agent_chat_file_loading": "正在读取文件…",
|
||||
"ui.agent_chat_file_not_a_file": "{name}:不是文件",
|
||||
"ui.agent_chat_file_not_a_picture": "{name}:不是图片",
|
||||
|
||||
@@ -149,6 +149,12 @@ pub fn set_model_params(session: &str, model: &str) -> Value {
|
||||
json!({"sessionId": session, "modelId": model})
|
||||
}
|
||||
|
||||
/// Whether the agent takes embedded documents in a message
|
||||
/// (`agentCapabilities.promptCapabilities.embeddedContext`: it reads `resource` blocks).
|
||||
pub fn takes_documents(init: &Value) -> bool {
|
||||
init.pointer("/agentCapabilities/promptCapabilities/embeddedContext").and_then(Value::as_bool).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether the agent takes pictures in a message (`agentCapabilities.promptCapabilities.image`).
|
||||
pub fn takes_images(init: &Value) -> bool {
|
||||
init.pointer("/agentCapabilities/promptCapabilities/image").and_then(Value::as_bool).unwrap_or(false)
|
||||
@@ -461,6 +467,14 @@ pub fn split_command(line: &str) -> Vec<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn documents_are_taken_only_when_the_agent_says_so() {
|
||||
let yes = json!({"agentCapabilities": {"promptCapabilities": {"image": true, "embeddedContext": true}}});
|
||||
let no = json!({"agentCapabilities": {"promptCapabilities": {"image": true}}});
|
||||
assert!(takes_documents(&yes) && takes_images(&yes));
|
||||
assert!(!takes_documents(&no) && !takes_documents(&json!({})));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commands_split_like_a_shell_without_one() {
|
||||
assert_eq!(split_command("gemini --acp"), ["gemini", "--acp"]);
|
||||
|
||||
@@ -33,6 +33,7 @@ pub struct AcpBackend {
|
||||
session: Option<String>,
|
||||
agent: String,
|
||||
images: bool,
|
||||
documents: bool,
|
||||
auth: Vec<acp::AuthMethod>,
|
||||
/// Open permission questions: our key → the agent's request id.
|
||||
permissions: HashMap<u64, Value>,
|
||||
@@ -54,6 +55,7 @@ impl AcpBackend {
|
||||
session: None,
|
||||
agent: String::new(),
|
||||
images: false,
|
||||
documents: false,
|
||||
auth: Vec::new(),
|
||||
permissions: HashMap::new(),
|
||||
next_key: 0,
|
||||
@@ -82,6 +84,7 @@ impl AcpBackend {
|
||||
(Pending::Initialize, Ok(v)) => {
|
||||
self.agent = acp::agent_info(&v);
|
||||
self.images = acp::takes_images(&v);
|
||||
self.documents = acp::takes_documents(&v);
|
||||
self.auth = acp::auth_methods(&v);
|
||||
self.new_session();
|
||||
}
|
||||
@@ -94,6 +97,7 @@ impl AcpBackend {
|
||||
if let Some((current, available)) = acp::models(&v) {
|
||||
out.push(Event::Models { current, available });
|
||||
}
|
||||
out.push(Event::Documents(self.documents));
|
||||
out.push(Event::Ready { agent: self.agent.clone(), images: self.images });
|
||||
}
|
||||
None => out.push(Event::Error("session/new: no sessionId".into())),
|
||||
|
||||
@@ -404,9 +404,11 @@ fn user_message(parts: &[PromptPart]) -> Value {
|
||||
}
|
||||
let content: Vec<Value> = parts
|
||||
.iter()
|
||||
.map(|p| match p {
|
||||
PromptPart::Text(t) => json!({"type": "text", "text": t}),
|
||||
PromptPart::Image { mime, base64 } => json!({"type": "image_url", "image_url": {"url": format!("data:{mime};base64,{base64}")}}),
|
||||
.filter_map(|p| match p {
|
||||
PromptPart::Text(t) => Some(json!({"type": "text", "text": t})),
|
||||
PromptPart::Image { mime, base64 } => Some(json!({"type": "image_url", "image_url": {"url": format!("data:{mime};base64,{base64}")}})),
|
||||
// This endpoint takes no documents (the chat doesn't send any); the message names the file.
|
||||
PromptPart::Document { .. } => None,
|
||||
})
|
||||
.collect();
|
||||
json!({"role": "user", "content": content})
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
//! Files a person adds to a message (the + button, a drop on the chat, a paste): pictures go to the
|
||||
//! agent with it, text files go in as text, and CAD files are imported into the design, the agent
|
||||
//! being told what arrived. Each file is checked and bounded before it is kept; a refused one says
|
||||
//! why.
|
||||
//! agent with it, text files go in as text, PDFs go as documents to agents that take them, CAD
|
||||
//! files are imported into the design, and any other file is named by reference (its type, size
|
||||
//! and place on this machine) for the agent to read with its own tools. The agent is told what
|
||||
//! arrived. Each file is checked and bounded before it is kept; a refused one says why.
|
||||
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::backend::{self, PromptPart};
|
||||
@@ -27,6 +28,8 @@ pub const MAX_FILES: usize = 8;
|
||||
const MAX_PICTURE_BYTES: u64 = 40 * 1024 * 1024;
|
||||
/// Largest text file sent in full.
|
||||
const MAX_TEXT_BYTES: u64 = 256 * 1024;
|
||||
/// Largest PDF sent as a document; a bigger one goes by reference.
|
||||
const MAX_PDF_BYTES: u64 = 12 * 1024 * 1024;
|
||||
/// A picture's longer side is scaled down to this before it is sent.
|
||||
pub const MAX_SIDE: u32 = 2048;
|
||||
/// Largest picture sent, as it came or re-encoded: its base64 (a third larger) stays under 5 MB,
|
||||
@@ -47,9 +50,9 @@ const TEXT_EXTS: &[&str] = &["svg", "txt", "md", "markdown", "json", "csv", "tsv
|
||||
/// importers themselves).
|
||||
const CAD_EXTS: &[&str] = &["step", "stp", "igs", "iges", "FCStd", "3dm", "stl", "3mf", "obj", "glb", "gltf"];
|
||||
|
||||
/// Every extension that can be attached, pictures first (the file picker's list).
|
||||
/// Every extension the file picker lists first, pictures first (any other file can be chosen too).
|
||||
pub fn extensions() -> Vec<&'static str> {
|
||||
PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).copied().collect()
|
||||
PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).chain(&["pdf"]).copied().collect()
|
||||
}
|
||||
|
||||
/// What a file is, by its extension.
|
||||
@@ -58,6 +61,8 @@ pub enum Kind {
|
||||
Picture,
|
||||
Text,
|
||||
Cad,
|
||||
Pdf,
|
||||
/// Anything else, attached by reference.
|
||||
Other,
|
||||
}
|
||||
|
||||
@@ -66,6 +71,7 @@ pub fn kind(path: &Path) -> Kind {
|
||||
match ext.as_str() {
|
||||
e if PICTURE_EXTS.contains(&e) => Kind::Picture,
|
||||
e if TEXT_EXTS.contains(&e) => Kind::Text,
|
||||
"pdf" => Kind::Pdf,
|
||||
_ => {
|
||||
let p = path.to_string_lossy();
|
||||
if solidcraft_engine::io::is_step_path(&p) || solidcraft_engine::io::is_mesh_path(&p) { Kind::Cad } else { Kind::Other }
|
||||
@@ -82,6 +88,17 @@ pub enum Content {
|
||||
size: [u32; 2],
|
||||
},
|
||||
Text(Arc<String>),
|
||||
/// A PDF, sent whole as a document to an agent that takes them.
|
||||
Pdf {
|
||||
bytes: Arc<Vec<u8>>,
|
||||
path: PathBuf,
|
||||
},
|
||||
/// A file the agent is pointed to, not given: what it is, its size and where it is.
|
||||
Reference {
|
||||
path: PathBuf,
|
||||
size: u64,
|
||||
mime: &'static str,
|
||||
},
|
||||
/// A CAD file imported into the design as its own component.
|
||||
Imported,
|
||||
}
|
||||
@@ -104,6 +121,70 @@ impl Attachment {
|
||||
pub fn is_picture(&self) -> bool {
|
||||
matches!(self.content, Content::Picture { .. })
|
||||
}
|
||||
/// The chip's mark for a file with no thumbnail: its extension, in capitals, cut to fit.
|
||||
pub fn mark(&self) -> String {
|
||||
let ext = Path::new(&self.name).extension().and_then(|e| e.to_str()).unwrap_or("");
|
||||
ext.chars().take(4).collect::<String>().to_uppercase()
|
||||
}
|
||||
}
|
||||
|
||||
/// What a file is called to the agent, by its extension.
|
||||
fn mime_of(path: &Path) -> &'static str {
|
||||
let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).unwrap_or_default();
|
||||
match ext.as_str() {
|
||||
"pdf" => "application/pdf",
|
||||
"zip" => "application/zip",
|
||||
"7z" => "application/x-7z-compressed",
|
||||
"gz" | "tgz" => "application/gzip",
|
||||
"dxf" => "image/vnd.dxf",
|
||||
"dwg" => "image/vnd.dwg",
|
||||
"step" | "stp" => "model/step",
|
||||
"iges" | "igs" => "model/iges",
|
||||
"stl" => "model/stl",
|
||||
"obj" => "model/obj",
|
||||
"3mf" => "model/3mf",
|
||||
"gltf" => "model/gltf+json",
|
||||
"glb" => "model/gltf-binary",
|
||||
"xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"mp4" | "mov" | "webm" => "video/mp4",
|
||||
"wav" => "audio/wav",
|
||||
"mp3" => "audio/mpeg",
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
|
||||
/// A size for people: `1.5 MB`.
|
||||
fn size_text(bytes: u64) -> String {
|
||||
const KB: f64 = 1024.0;
|
||||
let b = bytes as f64;
|
||||
if b < KB {
|
||||
format!("{bytes} bytes")
|
||||
} else if b < KB * KB {
|
||||
format!("{:.0} KB", b / KB)
|
||||
} else if b < KB * KB * KB {
|
||||
format!("{:.1} MB", b / KB / KB)
|
||||
} else {
|
||||
format!("{:.1} GB", b / KB / KB / KB)
|
||||
}
|
||||
}
|
||||
|
||||
/// A `file://` link to a path, for a document sent to an agent.
|
||||
fn file_uri(path: &Path) -> String {
|
||||
let mut s = path.to_string_lossy().replace('\\', "/");
|
||||
if !s.starts_with('/') {
|
||||
s.insert(0, '/');
|
||||
}
|
||||
let mut out = String::from("file://");
|
||||
for b in s.bytes() {
|
||||
if b.is_ascii_alphanumeric() || matches!(b, b'/' | b'-' | b'_' | b'.' | b'~' | b':') {
|
||||
out.push(char::from(b));
|
||||
} else {
|
||||
out.push_str(&format!("%{b:02X}"));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(crate) fn name_of(path: &Path) -> String {
|
||||
@@ -134,13 +215,29 @@ pub fn read(path: &Path) -> Result<Attachment, String> {
|
||||
Ok(Attachment { id: 0, name, content: Content::Text(Arc::new(text)), thumb: None })
|
||||
}
|
||||
Kind::Cad => Err(refused("ui.agent_chat_file_cad_not_read", &name, &[])),
|
||||
Kind::Pdf => {
|
||||
let path = absolute(path);
|
||||
// Too big to send: the agent is pointed to it instead.
|
||||
if meta.len() > MAX_PDF_BYTES {
|
||||
return Ok(Attachment { id: 0, name, content: Content::Reference { path, size: meta.len(), mime: "application/pdf" }, thumb: None });
|
||||
}
|
||||
let bytes = std::fs::read(&path).map_err(|e| failed(&name, e))?;
|
||||
Ok(Attachment { id: 0, name, content: Content::Pdf { bytes: Arc::new(bytes), path }, thumb: None })
|
||||
}
|
||||
Kind::Other if matches!(path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("heic" | "heif")) => {
|
||||
Err(refused("ui.agent_chat_file_heic", &name, &[]))
|
||||
}
|
||||
Kind::Other => Err(refused("ui.agent_chat_file_kinds", &name, &[])),
|
||||
Kind::Other => {
|
||||
Ok(Attachment { id: 0, name, content: Content::Reference { path: absolute(path), size: meta.len(), mime: mime_of(path) }, thumb: None })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The path as an absolute one, so the agent (working in another folder) finds the file.
|
||||
fn absolute(path: &Path) -> PathBuf {
|
||||
std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf())
|
||||
}
|
||||
|
||||
/// A picture from a file's bytes: decoded within limits, and sent as it came when the agents read
|
||||
/// that format (PNG, JPEG, GIF, WebP), it is at most [`MAX_SIDE`] on a side and at most
|
||||
/// [`MAX_SENT_BYTES`]; otherwise re-encoded (see [`encoded`]).
|
||||
@@ -239,10 +336,10 @@ fn thumbnail(img: &image::DynamicImage) -> Option<Arc<egui::ColorImage>> {
|
||||
|
||||
/// What the message says about its files, after the person's words: the text files' text, and the
|
||||
/// pictures and imported parts by name (the agent sees the pictures themselves as images).
|
||||
/// `pictures_sent`: whether this agent takes pictures. The text files share `budget` characters
|
||||
/// `pictures_sent` and `documents_sent`: whether this agent takes pictures and PDFs. The text files share `budget` characters
|
||||
/// in order: one that doesn't fit is cut and says how much is missing, and one past the budget
|
||||
/// is named only, so the message stays within the prompt cap.
|
||||
pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String {
|
||||
pub fn note(files: &[Attachment], pictures_sent: bool, documents_sent: bool, budget: usize) -> String {
|
||||
let mut out = String::new();
|
||||
let pictures: Vec<&str> = files.iter().filter(|f| f.is_picture()).map(|f| f.name.as_str()).collect();
|
||||
if !pictures.is_empty() {
|
||||
@@ -270,6 +367,31 @@ pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String
|
||||
left = 0;
|
||||
}
|
||||
}
|
||||
Content::Pdf { path, bytes } if documents_sent => {
|
||||
out += &format!(
|
||||
"\n\nAttached PDF `{}` ({}, at {}): sent with this message as a document.",
|
||||
f.name,
|
||||
size_text(bytes.len() as u64),
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
// An agent that takes no documents is pointed to the file, as for any other file.
|
||||
Content::Pdf { path, bytes } => {
|
||||
out += &format!(
|
||||
"\n\nAttached file `{}` (PDF, {}, at {}): read it with your own tools; it is not in this message.",
|
||||
f.name,
|
||||
size_text(bytes.len() as u64),
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
Content::Reference { path, size, mime } => {
|
||||
out += &format!(
|
||||
"\n\nAttached file `{}` ({mime}, {}, at {}): read it with your own tools; it is not in this message.",
|
||||
f.name,
|
||||
size_text(*size),
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
Content::Imported => out += &format!("\n\n(I added `{}` to the design as a component of its own.)", f.name),
|
||||
Content::Picture { .. } => {}
|
||||
}
|
||||
@@ -277,12 +399,18 @@ pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String
|
||||
out
|
||||
}
|
||||
|
||||
/// The pictures as image parts of the message.
|
||||
pub fn parts(files: &[Attachment]) -> Vec<PromptPart> {
|
||||
/// The pictures and PDFs as parts of the message, to the agents that take them.
|
||||
pub fn parts(files: &[Attachment], pictures: bool, documents: bool) -> Vec<PromptPart> {
|
||||
files
|
||||
.iter()
|
||||
.filter_map(|f| match &f.content {
|
||||
Content::Picture { mime, bytes, .. } => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),
|
||||
Content::Picture { mime, bytes, .. } if pictures => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),
|
||||
Content::Pdf { bytes, path } if documents => Some(PromptPart::Document {
|
||||
name: f.name.clone(),
|
||||
uri: file_uri(path),
|
||||
mime: "application/pdf".to_string(),
|
||||
base64: backend::base64(bytes),
|
||||
}),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
@@ -390,9 +518,6 @@ mod tests {
|
||||
let binary = d.join("bin.json");
|
||||
std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();
|
||||
assert!(read(&binary).unwrap_err().contains("UTF-8"));
|
||||
let other = d.join("movie.mov");
|
||||
std::fs::write(&other, b"x").unwrap();
|
||||
assert!(read(&other).unwrap_err().contains("can be attached"));
|
||||
assert!(read(&d.join("missing.png")).is_err());
|
||||
assert_eq!(kind(Path::new("part.STEP")), Kind::Cad);
|
||||
assert_eq!(kind(Path::new("mount.stl")), Kind::Cad);
|
||||
@@ -411,12 +536,9 @@ mod tests {
|
||||
let d = dir("lang");
|
||||
let binary = d.join("bin.json");
|
||||
std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();
|
||||
let other = d.join("movie.mov");
|
||||
std::fs::write(&other, b"x").unwrap();
|
||||
{
|
||||
let _ja = crate::i18n::scope("ja");
|
||||
assert!(read(&binary).unwrap_err().contains("UTF-8 のテキストではありません"));
|
||||
assert!(read(&other).unwrap_err().starts_with("movie.mov: 添付できるのは"));
|
||||
}
|
||||
{
|
||||
let _zh = crate::i18n::scope("zh");
|
||||
@@ -467,12 +589,12 @@ mod tests {
|
||||
fn attached_text_is_kept_within_the_prompt_budget() {
|
||||
let file = |n: &str, len: usize| Attachment { id: 0, name: n.into(), content: Content::Text(Arc::new("x".repeat(len))), thumb: None };
|
||||
let files = vec![file("a.md", 600), file("b.md", 600), file("c.md", 600)];
|
||||
let n = note(&files, true, 1000);
|
||||
let n = note(&files, true, false, 1000);
|
||||
assert!(n.contains("`a.md`") && n.contains("`b.md`") && n.contains("`c.md`"), "every file is named: {n}");
|
||||
assert!(n.matches('x').count() <= 1000, "the files' text stays within the budget");
|
||||
assert!(n.contains("200 more characters"), "a cut file says how much is missing: {n}");
|
||||
assert!(n.contains("left out"), "a file past the budget is named only: {n}");
|
||||
assert_eq!(note(&files, true, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole");
|
||||
assert_eq!(note(&files, true, false, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -481,13 +603,77 @@ mod tests {
|
||||
let text = Attachment { id: 0, name: "brief.md".into(), content: Content::Text(Arc::new("wall 2 mm\n".into())), thumb: None };
|
||||
let part = Attachment::imported("motor.step".into());
|
||||
let files = vec![pic, text, part];
|
||||
let n = note(&files, true, usize::MAX);
|
||||
let n = note(&files, true, false, usize::MAX);
|
||||
assert!(n.contains("Attached pictures: target.png."), "{n}");
|
||||
assert!(n.contains("Attached file `brief.md`:\n```\nwall 2 mm\n```"), "{n}");
|
||||
assert!(n.contains("`motor.step` to the design"), "{n}");
|
||||
assert!(note(&files, false, usize::MAX).contains("not sent"));
|
||||
let p = parts(&files);
|
||||
assert!(note(&files, false, false, usize::MAX).contains("not sent"));
|
||||
let p = parts(&files, true, true);
|
||||
assert_eq!(p.len(), 1);
|
||||
assert!(matches!(&p[0], PromptPart::Image { mime, base64 } if mime == "image/png" && base64.starts_with("iVBORw0KGgo")));
|
||||
}
|
||||
|
||||
/// Any file that is neither a picture, text, CAD nor a PDF is named by reference (what it is,
|
||||
/// its size and its place) instead of refused; the agent reads it with its own tools.
|
||||
#[test]
|
||||
fn other_files_go_by_reference() {
|
||||
let d = dir("reference");
|
||||
let zip = d.join("fixtures.zip");
|
||||
std::fs::write(&zip, vec![0u8; 3000]).unwrap();
|
||||
let dxf = d.join("outline.DXF");
|
||||
std::fs::write(&dxf, b"0\nSECTION\n").unwrap();
|
||||
let a = read(&zip).unwrap();
|
||||
let Content::Reference { path, size, mime } = &a.content else { panic!("a reference") };
|
||||
assert_eq!((*size, *mime), (3000, "application/zip"));
|
||||
assert!(path.is_absolute() && path.ends_with("fixtures.zip"));
|
||||
assert!(a.thumb.is_none() && a.mark() == "ZIP");
|
||||
assert!(matches!(read(&dxf).unwrap().content, Content::Reference { mime: "image/vnd.dxf", .. }), "by extension, in any case");
|
||||
let bare = d.join("LICENSE_BLOB");
|
||||
std::fs::write(&bare, b"x").unwrap();
|
||||
assert!(matches!(read(&bare).unwrap().content, Content::Reference { mime: "application/octet-stream", .. }), "no extension");
|
||||
let files = vec![a];
|
||||
let n = note(&files, true, true, usize::MAX);
|
||||
assert!(n.contains("Attached file `fixtures.zip` (application/zip, 3 KB, at "), "{n}");
|
||||
assert!(n.contains("fixtures.zip): read it with your own tools"), "{n}");
|
||||
assert!(parts(&files, true, true).is_empty(), "a reference sends nothing but the note");
|
||||
assert!(read(&d).unwrap_err().contains("not a file"), "a folder is refused");
|
||||
assert!(read(&d.join("missing.bin")).is_err());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// A PDF goes whole as a document to an agent that takes them (and is named with its place);
|
||||
/// to one that doesn't, and when it is too big to send, the agent is pointed to the file.
|
||||
#[test]
|
||||
fn pdfs_go_as_documents_or_by_reference() {
|
||||
let d = dir("pdf");
|
||||
let pdf = d.join("brief 1.pdf");
|
||||
std::fs::write(&pdf, b"%PDF-1.4\n%%EOF\n").unwrap();
|
||||
let a = read(&pdf).unwrap();
|
||||
assert!(matches!(&a.content, Content::Pdf { bytes, .. } if bytes.starts_with(b"%PDF")));
|
||||
assert_eq!(a.mark(), "PDF");
|
||||
let files = vec![a];
|
||||
let sent = parts(&files, false, true);
|
||||
let [PromptPart::Document { name, uri, mime, base64 }] = sent.as_slice() else { panic!("one document: {sent:?}") };
|
||||
assert_eq!((name.as_str(), mime.as_str()), ("brief 1.pdf", "application/pdf"));
|
||||
assert!(uri.starts_with("file://") && uri.ends_with("/brief%201.pdf"), "{uri}");
|
||||
assert!(base64.starts_with("JVBERi0"), "the PDF's bytes: {base64}");
|
||||
assert!(note(&files, false, true, usize::MAX).contains("sent with this message as a document"));
|
||||
assert!(parts(&files, true, false).is_empty(), "an agent with no documents gets none");
|
||||
let n = note(&files, true, false, usize::MAX);
|
||||
assert!(n.contains("(PDF, 15 bytes, at ") && n.contains("read it with your own tools"), "{n}");
|
||||
let big = d.join("drawing set.pdf");
|
||||
std::fs::write(&big, vec![b'x'; usize::try_from(MAX_PDF_BYTES).unwrap() + 1]).unwrap();
|
||||
let b = read(&big).unwrap();
|
||||
assert!(matches!(b.content, Content::Reference { mime: "application/pdf", .. }), "too big to send: by reference");
|
||||
assert!(parts(&[b], true, true).is_empty());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn links_to_files_are_escaped() {
|
||||
assert_eq!(file_uri(Path::new("/home/a b/ü.pdf")), "file:///home/a%20b/%C3%BC.pdf");
|
||||
assert_eq!(file_uri(Path::new("C:\\Users\\a\\x.pdf")), "file:///C:/Users/a/x.pdf");
|
||||
assert_eq!(size_text(1536), "2 KB");
|
||||
assert_eq!(size_text(5 * 1024 * 1024 + 512 * 1024), "5.5 MB");
|
||||
}
|
||||
}
|
||||
@@ -19,11 +19,21 @@ pub enum PromptPart {
|
||||
mime: String,
|
||||
base64: String,
|
||||
},
|
||||
/// A file the agent reads as a document (a PDF), sent only to agents that take documents.
|
||||
Document {
|
||||
name: String,
|
||||
/// Where it is on this machine (a `file://` link).
|
||||
uri: String,
|
||||
mime: String,
|
||||
base64: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// What a backend reports to the panel.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Event {
|
||||
/// Whether the agent takes documents (PDFs) in a message; sent before [`Event::Ready`].
|
||||
Documents(bool),
|
||||
/// Ready for a message. `agent`: its name and version; `images`: it takes pictures.
|
||||
Ready { agent: String, images: bool },
|
||||
/// Text it streams (`thought`: its reasoning, shown dimmed).
|
||||
@@ -136,6 +146,9 @@ pub fn content_blocks(parts: &[PromptPart]) -> Vec<Value> {
|
||||
.map(|p| match p {
|
||||
PromptPart::Text(t) => serde_json::json!({"type": "text", "text": t}),
|
||||
PromptPart::Image { mime, base64 } => serde_json::json!({"type": "image", "mimeType": mime, "data": base64}),
|
||||
PromptPart::Document { uri, mime, base64, .. } => {
|
||||
serde_json::json!({"type": "resource", "resource": {"uri": uri, "mimeType": mime, "blob": base64}})
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -158,4 +171,19 @@ mod tests {
|
||||
assert_eq!(b[1]["type"], "image");
|
||||
assert_eq!(b[1]["mimeType"], "image/png");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn documents_go_as_embedded_resources() {
|
||||
let doc = PromptPart::Document {
|
||||
name: "brief.pdf".into(),
|
||||
uri: "file:///a/brief.pdf".into(),
|
||||
mime: "application/pdf".into(),
|
||||
base64: "AA==".into(),
|
||||
};
|
||||
let b = content_blocks(&[doc]);
|
||||
assert_eq!(b[0]["type"], "resource");
|
||||
assert_eq!(b[0]["resource"]["uri"], "file:///a/brief.pdf");
|
||||
assert_eq!(b[0]["resource"]["mimeType"], "application/pdf");
|
||||
assert_eq!(b[0]["resource"]["blob"], "AA==");
|
||||
}
|
||||
}
|
||||
@@ -303,6 +303,8 @@ pub struct AgentChat {
|
||||
state: Option<State>,
|
||||
agent: String,
|
||||
images: bool,
|
||||
/// The agent takes documents (PDFs) in a message.
|
||||
documents: bool,
|
||||
/// Send a picture of the view with the next message.
|
||||
pub attach_view: bool,
|
||||
auth: Vec<acp::AuthMethod>,
|
||||
@@ -512,6 +514,7 @@ impl AgentChat {
|
||||
/// Take in what the backend reported.
|
||||
fn on_event(&mut self, e: Event) -> Option<Outcome> {
|
||||
match e {
|
||||
Event::Documents(documents) => self.documents = documents,
|
||||
Event::Ready { agent, images } => {
|
||||
self.agent = agent;
|
||||
self.images = images;
|
||||
@@ -960,14 +963,14 @@ fn start_turn(app: &mut SolveApp, text: String, image: bool, at: Option<usize>,
|
||||
/// files are named after the text (text files in full), and pictures go as images to an agent
|
||||
/// that takes them. `at` is the message's place in the conversation, where its picture is kept.
|
||||
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>, note: Option<&str>) {
|
||||
let pictures = app.agent_chat.images;
|
||||
let (pictures, documents) = (app.agent_chat.images, app.agent_chat.documents);
|
||||
let lead = match note {
|
||||
Some(n) => format!("{n}\n\n{text}"),
|
||||
None => text.clone(),
|
||||
};
|
||||
// The whole message stays within the cap: the files' text gets what the words leave.
|
||||
let budget = MAX_PROMPT.saturating_sub(lead.chars().count() + FILE_NAMES_ROOM);
|
||||
let about_files = attach::note(&files, pictures, budget);
|
||||
let about_files = attach::note(&files, pictures, documents, budget);
|
||||
let sent: String = format!("{lead}{about_files}").chars().take(MAX_PROMPT).collect();
|
||||
let mut parts = vec![PromptPart::Text(sent)];
|
||||
if image {
|
||||
@@ -979,9 +982,7 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usi
|
||||
Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"),
|
||||
}
|
||||
}
|
||||
if pictures {
|
||||
parts.extend(attach::parts(&files));
|
||||
}
|
||||
parts.extend(attach::parts(&files, pictures, documents));
|
||||
// The undo step is named after the message, or its first file when it has no words.
|
||||
let first_line = text.lines().find(|l| !l.trim().is_empty()).or_else(|| files.first().map(|f| f.name.as_str())).unwrap_or("");
|
||||
let label: String = first_line.chars().take(48).collect();
|
||||
@@ -1017,9 +1018,10 @@ pub fn attach_path(app: &mut SolveApp, path: &str) -> std::result::Result<String
|
||||
crate::file_load::begin_request(app, request).map_err(|e| trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &e)]))?;
|
||||
return Ok(app.agent_chat.importing(name, rx));
|
||||
} else {
|
||||
// A picture is decoded off the window thread: a large one takes long enough to stop it.
|
||||
// A picture is decoded and a PDF read off the window thread: a large one takes long enough
|
||||
// to stop it.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if attach::kind(p) == attach::Kind::Picture {
|
||||
if matches!(attach::kind(p), attach::Kind::Picture | attach::Kind::Pdf) {
|
||||
// A missing file is said at once; reading and decoding wait for the worker.
|
||||
if !p.is_file() {
|
||||
return Err(trf("ui.agent_chat_file_not_a_file", &[("name", &attach::name_of(p))]));
|
||||
@@ -1654,6 +1656,7 @@ pub fn status(app: &SolveApp) -> Value {
|
||||
"state": c.state().id(),
|
||||
"agent": c.agent,
|
||||
"images": c.images,
|
||||
"documents": c.documents,
|
||||
"authStatus": c.auth_status,
|
||||
"authMethods": c.auth.iter().map(|m| json!({"id": m.id, "name": m.name, "kind": match &m.kind { acp::AuthKind::Agent => "agent", acp::AuthKind::Terminal { .. } => "terminal", acp::AuthKind::EnvVar { .. } => "env_var" }})).collect::<Vec<_>>(),
|
||||
"toolsPort": c.tools_port,
|
||||
|
||||
@@ -1278,8 +1278,11 @@ fn chip(ui: &mut egui::Ui, thumbs: &mut std::collections::HashMap<u64, egui::Tex
|
||||
ui.image((tex.id(), size * (22.0 / size.x.max(size.y).max(1.0)))).rect.width()
|
||||
}
|
||||
None => {
|
||||
let (r, _) = ui.allocate_exact_size(vec2(16.0, 20.0), Sense::hover());
|
||||
ui.painter().rect_stroke(r.shrink2(vec2(2.0, 3.0)), 2.0, Stroke::new(1.2, t.text_dim), egui::StrokeKind::Inside);
|
||||
// A page marked with the file's type.
|
||||
let (r, _) = ui.allocate_exact_size(vec2(28.0, 20.0), Sense::hover());
|
||||
let color = if matches!(f.content, attach::Content::Pdf { .. }) { t.error } else { t.text_dim };
|
||||
ui.painter().rect_stroke(r.shrink2(vec2(1.0, 1.0)), 2.0, Stroke::new(1.2, color), egui::StrokeKind::Inside);
|
||||
ui.painter().text(r.center(), Align2::CENTER_CENTER, f.mark(), egui::FontId::proportional(8.5), color);
|
||||
r.width()
|
||||
}
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user