Agent chat: keep Send in view for a long message, open "View attached" (#2224) (#2225)

Thanks @Rec0iL.
This commit is contained in:
Rec0iL authored and GitHub committed 2026-10-18 10:53:31 -04:00
1 parent 55bfeb71a7
commit f498d23a79
4 files changed
+429 -85

No files matched your search

+84 -15
View File
@@ -56,6 +56,8 @@ const FILE_NAMES_ROOM: usize = 2_000;
const APPROVAL_TIMEOUT_MS: f64 = 240_000.0;
/// Size of the picture of the view sent with a message.
const VIEW_IMAGE: (u32, u32) = (960, 600);
/// A picture kept for the user to open later, at most this big.
const MAX_PICTURE_BYTES: usize = 4 << 20;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum State {
@@ -93,11 +95,58 @@ impl State {
}
}
/// A picture shown with a message: what the agent got (a picture of the view, or any other
/// attachment) as PNG, and a note shown with it in the viewer. The thumbnail and the viewer share
/// one texture, made when the picture is first drawn.
#[derive(Clone)]
pub struct Picture {
png: Arc<Vec<u8>>,
note: String,
texture: Arc<std::sync::OnceLock<Option<egui::TextureHandle>>>,
}
impl Picture {
pub fn new(png: Vec<u8>, note: impl Into<String>) -> Self {
Self { png: Arc::new(png), note: note.into(), texture: Arc::default() }
}
/// The picture as a texture, `None` when it can't be decoded.
#[cfg(not(target_arch = "wasm32"))]
fn texture(&self, ctx: &egui::Context) -> Option<egui::TextureHandle> {
self.texture
.get_or_init(|| {
let rgba = image::load_from_memory_with_format(&self.png, image::ImageFormat::Png).ok()?.to_rgba8();
let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?];
Some(ctx.load_texture(
"sc_agent_chat_picture",
egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()),
egui::TextureOptions::LINEAR,
))
})
.clone()
}
}
impl std::fmt::Debug for Picture {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Picture({} bytes, {:?})", self.png.len(), self.note)
}
}
/// The viewer: the pictures of one message, and the one shown.
#[derive(Clone, Debug)]
pub struct Gallery {
pictures: Vec<Picture>,
at: usize,
}
#[derive(Clone, Debug)]
pub enum Item {
User {
text: String,
image: bool,
/// The view's picture, kept once it is taken (when the agent starts on the message).
pictures: Vec<Picture>,
/// The names of the files that went with it.
files: Vec<String>,
},
@@ -202,6 +251,8 @@ pub struct AgentChat {
pub open: bool,
pub items: Vec<Item>,
pub input: String,
/// The viewer, open on one message's pictures.
gallery: Option<Gallery>,
state: Option<State>,
agent: String,
images: bool,
@@ -209,8 +260,9 @@ pub struct AgentChat {
pub attach_view: bool,
auth: Vec<acp::AuthMethod>,
auth_status: Option<String>,
/// A message typed before the agent was ready, with its files.
queued: Option<(String, bool, Vec<attach::Attachment>)>,
/// A message typed before the agent was ready: its text, whether the view goes with it, its
/// place in the conversation, and its files.
queued: Option<(String, bool, Option<usize>, Vec<attach::Attachment>)>,
/// Messages typed while the agent works, in order. They go as one prompt when the turn ends
/// (or at once with Send now); after Stop they wait for Send.
pub pending: Vec<Pending>,
@@ -231,7 +283,7 @@ pub struct AgentChat {
/// Where the panel was drawn last (for its shortcuts).
pub panel_rect: Option<egui::Rect>,
/// The last message sent, kept to send again after a sign-in (its files too).
last: Option<(String, bool, Vec<attach::Attachment>)>,
last: Option<(String, bool, Option<usize>, Vec<attach::Attachment>)>,
/// Files for the next message (pictures, text files, imported parts), shown as chips.
pub(crate) attachments: Vec<attach::Attachment>,
/// Pictures pasted so far, to name the next one.
@@ -347,6 +399,17 @@ impl AgentChat {
}
self.items.push(item);
}
/// Keep the picture just sent with the message at `at` in the conversation (the user can open
/// it). A message sent again (after a sign-in) has its picture already: nothing is added, and
/// never to another message.
fn keep_picture(&mut self, at: Option<usize>, png: Vec<u8>) {
if let Some(Item::User { image: true, pictures, .. }) = at.and_then(|i| self.items.get_mut(i))
&& pictures.is_empty()
&& png.len() <= MAX_PICTURE_BYTES
{
pictures.push(Picture::new(png, ""));
}
}
fn notice(&mut self, text: impl Into<String>, error: bool) {
self.push(Item::Notice { text: text.into(), error });
}
@@ -704,11 +767,13 @@ pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) {
/// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it
/// cut in on its previous step).
fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec<attach::Attachment>, interrupted: bool) {
app.agent_chat.push(Item::User { text: text.clone(), image, files: files.iter().map(|f| f.name.clone()).collect() });
app.agent_chat.push(Item::User { text: text.clone(), image, pictures: Vec::new(), files: files.iter().map(|f| f.name.clone()).collect() });
// The message's place in the conversation: its picture is kept there when it is taken.
let at = app.agent_chat.items.len().checked_sub(1);
match app.agent_chat.state() {
State::Ready => start_turn_with(app, text, image, files, interrupted.then_some(INTERRUPT_NOTE)),
State::Ready => start_turn_with(app, text, image, at, files, interrupted.then_some(INTERRUPT_NOTE)),
s => {
app.agent_chat.queued = Some((text, image, files));
app.agent_chat.queued = Some((text, image, at, files));
#[cfg(not(target_arch = "wasm32"))]
if !s.running() {
connect(app, ctx);
@@ -775,6 +840,7 @@ pub fn new_chat(app: &mut SolveApp) {
let chat = &mut app.agent_chat;
chat.confirm_new = false;
chat.items.clear();
chat.gallery = None;
chat.pending.clear();
chat.attachments.clear();
chat.loading.clear();
@@ -823,14 +889,14 @@ fn after_turn(app: &mut SolveApp, ctx: &egui::Context) {
}
}
fn start_turn(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>) {
start_turn_with(app, text, image, files, None);
fn start_turn(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>) {
start_turn_with(app, text, image, at, files, None);
}
/// Start a message; `note` goes before the text for the agent, not into the conversation. The
/// files are named after the text (text files in full), and pictures go as images to an agent
/// that takes them.
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>, note: Option<&str>) {
/// that takes them. `at` is the message's place in the conversation, where its picture is kept.
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>, note: Option<&str>) {
let pictures = app.agent_chat.images;
let lead = match note {
Some(n) => format!("{n}\n\n{text}"),
@@ -843,7 +909,10 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<att
let mut parts = vec![PromptPart::Text(sent)];
if image {
match crate::control::view_png(app, VIEW_IMAGE.0, VIEW_IMAGE.1) {
Ok(png) => parts.push(PromptPart::Image { mime: "image/png".into(), base64: backend::base64(&png) }),
Ok(png) => {
parts.push(PromptPart::Image { mime: "image/png".into(), base64: backend::base64(&png) });
app.agent_chat.keep_picture(at, png);
}
Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"),
}
}
@@ -855,7 +924,7 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<att
let label: String = first_line.chars().take(48).collect();
app.agent_chat.turn =
Some(Turn { start_doc: app.session.doc.clone(), start_undo: app.session.undo.len(), changes: 0, requests: 0, limit_told: false, label });
app.agent_chat.last = Some((text, image, files));
app.agent_chat.last = Some((text, image, at, files));
app.agent_chat.state = Some(State::Busy);
app.agent_chat.stopped = false;
app.agent_chat.steering = false;
@@ -1258,8 +1327,8 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
if ready {
apply_saved_model(app);
}
if ready && let Some((text, image, files)) = app.agent_chat.queued.take() {
start_turn(app, text, image, files);
if ready && let Some((text, image, at, files)) = app.agent_chat.queued.take() {
start_turn(app, text, image, at, files);
}
}
pump_tools(app);
@@ -1478,7 +1547,7 @@ pub fn status(app: &SolveApp) -> Value {
.items
.iter()
.map(|i| match i {
Item::User { text, image, files } => json!({"user": text, "image": image, "files": files}),
Item::User { text, image, pictures, files } => json!({"user": text, "image": image, "pictures": pictures.len(), "files": files}),
Item::Agent { thought, text } => json!({"agent": text, "thought": thought}),
Item::Tool { id, title, kind, status, output } => {
json!({"tool": id, "title": title, "kind": kind, "status": status.map(|s| format!("{s:?}")), "output": output})
+175 -65
View File
@@ -18,6 +18,12 @@ fn panel_id(left: bool) -> Id {
Id::new(if left { "sc_agent_chat_left" } else { "sc_agent_chat_right" })
}
/// The largest thumbnail of a message's picture, and its largest preview while hovered (points).
#[cfg(not(target_arch = "wasm32"))]
const THUMB_MAX: (f32, f32) = (150.0, 56.0);
#[cfg(not(target_arch = "wasm32"))]
const PREVIEW_MAX: (f32, f32) = (560.0, 380.0);
/// Forget the panel's own idea of its width, so the preference applies on the next frame.
pub fn forget_width(ctx: &egui::Context) {
ctx.data_mut(|d| {
@@ -105,6 +111,8 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
.show(ui, |ui| body(app, ui, left));
crate::scenario::publish_rect("agent_chat:panel", r.response.rect);
app.agent_chat.panel_rect = Some(r.response.rect);
#[cfg(not(target_arch = "wasm32"))]
gallery_window(app, &ctx);
app.agent_chat.drop_rect = Some(r.response.rect);
show_drop_hint(app, &ctx);
let w = r.response.rect.width();
@@ -113,6 +121,88 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
}
}
/// The pictures of a message as small thumbnails in a row. Hovering one shows it bigger for as long
/// as the pointer stays; a click returns its place in the row, to open the viewer there.
#[cfg(not(target_arch = "wasm32"))]
fn picture_strip(ui: &mut egui::Ui, n: usize, pictures: &[Picture]) -> Option<usize> {
let t = Tokens::get();
let mut clicked = None;
ui.horizontal_wrapped(|ui| {
for (i, picture) in pictures.iter().enumerate() {
let Some(texture) = picture.texture(ui.ctx()) else {
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
continue;
};
let size = fit(texture.size_vec2(), vec2(THUMB_MAX.0, THUMB_MAX.1));
let thumb = ui.add(egui::Image::new((texture.id(), size)).corner_radius(4.0).sense(Sense::click()));
crate::scenario::publish_handle(&format!("agent_chat:picture:{n}:{i}"), thumb.rect.center());
let hover_max = (ui.ctx().content_rect().size() * 0.5).min(vec2(PREVIEW_MAX.0, PREVIEW_MAX.1));
let thumb = thumb.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_ui(|ui| {
ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), hover_max))).corner_radius(4.0));
if !picture.note.is_empty() {
ui.label(&picture.note);
}
});
if thumb.clicked() {
clicked = Some(i);
}
}
});
clicked
}
/// `size` scaled to fit in `max`, keeping its shape (never below one point).
#[cfg(not(target_arch = "wasm32"))]
fn fit(size: egui::Vec2, max: egui::Vec2) -> egui::Vec2 {
let scale = (max.x / size.x.max(1.0)).min(max.y / size.y.max(1.0)).min(2.0);
vec2((size.x * scale).max(1.0), (size.y * scale).max(1.0))
}
/// The viewer: a message's pictures over the app, one at a time with a counter, Previous and Next
/// (also the arrow keys), closed with the button, Esc or a click outside.
#[cfg(not(target_arch = "wasm32"))]
fn gallery_window(app: &mut SolveApp, ctx: &egui::Context) {
let Some(mut gallery) = app.agent_chat.gallery.take() else { return };
let count = gallery.pictures.len();
gallery.at = gallery.at.min(count.saturating_sub(1));
let (left, right) =
ctx.input_mut(|i| (i.consume_key(egui::Modifiers::NONE, egui::Key::ArrowLeft), i.consume_key(egui::Modifiers::NONE, egui::Key::ArrowRight)));
let mut step: isize = if left { -1 } else { isize::from(right) };
let t = Tokens::get();
let max = ctx.content_rect().size() * vec2(0.8, 0.7);
let modal = egui::Modal::new(Id::new("sc_agent_chat_gallery")).show(ctx, |ui| {
ui.set_max_width(max.x);
ui.horizontal(|ui| {
let prev = ui.add_enabled(count > 1, egui::Button::new("‹"));
crate::scenario::publish_handle("agent_chat:gallery_prev", prev.rect.center());
let next = ui.add_enabled(count > 1, egui::Button::new("›"));
crate::scenario::publish_handle("agent_chat:gallery_next", next.rect.center());
if prev.clicked() {
step = -1;
}
if next.clicked() {
step = 1;
}
ui.label(RichText::new(format!("{} / {}", gallery.at + 1, count)).color(t.text_dim));
});
if let Some(picture) = gallery.pictures.get(gallery.at) {
if let Some(texture) = picture.texture(ui.ctx()) {
let resp = ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), max))).corner_radius(4.0));
crate::scenario::publish_rect("agent_chat:gallery_picture", resp.rect);
}
if !picture.note.is_empty() {
ui.add(egui::Label::new(&picture.note).wrap());
}
}
});
if count > 0 && step != 0 {
gallery.at = (gallery.at as isize + step).rem_euclid(count as isize) as usize;
}
if !modal.should_close() {
app.agent_chat.gallery = Some(gallery);
}
}
/// While files are dragged over the window: if they would drop on the chat, say so over it.
fn show_drop_hint(app: &SolveApp, ctx: &egui::Context) {
#[cfg(not(target_arch = "wasm32"))]
@@ -193,6 +283,8 @@ fn window(app: &mut SolveApp, ctx: &egui::Context) {
let t = Tokens::get();
let frame = egui::Frame::NONE.fill(t.panel).inner_margin(egui::Margin::symmetric(8, 6));
let r = egui::CentralPanel::default().frame(frame).show(ui, |ui| body(app, ui, false));
// The picture viewer opens over this window, where the click was, not over the main one.
gallery_window(app, ui.ctx());
if class == egui::ViewportClass::EmbeddedWindow {
// Floating inside the app: its handles are the main window's, as the panel's are.
crate::scenario::publish_rect("agent_chat:window", r.response.rect);
@@ -468,21 +560,12 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
auth_box(app, ui);
}
ui.separator();
// The input sits at the bottom (with a row per line of file chips); the conversation takes
// the rest.
let input_h = 92.0 + chip_rows(app.agent_chat.files(), ui.available_width(), ui.spacing().item_spacing.x) as f32 * 32.0;
let avail = (ui.available_height() - input_h).max(80.0);
egui::ScrollArea::vertical().id_salt("sc_agent_chat_scroll").max_height(avail).auto_shrink([false, false]).stick_to_bottom(true).show(ui, |ui| {
conversation(app, ui);
});
ui.separator();
// The input sits at the bottom of the panel and grows upwards: a long message makes its box
// taller up to `edit_max`, then it scrolls, and the conversation takes what is left above. It
// is a panel of its own at the bottom, as high as its content this frame, so the buttons under
// the box are always in view.
let edit_max = (ui.available_height() * 0.4).clamp(48.0, 160.0);
let busy = state == State::Busy;
attachment_chips(app, ui);
let edit = egui::TextEdit::multiline(&mut app.agent_chat.input)
.hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" }))
.desired_rows(2)
.desired_width(f32::INFINITY)
.id_salt("sc_agent_chat_input");
// Enter sends (queues while the agent works); Ctrl/Cmd+Enter sends now; Esc stops. They are
// read before the field sees them so no newline is typed.
let focused = ui.memory(|m| m.has_focus(Id::new("sc_agent_chat_input")));
@@ -491,52 +574,77 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
} else {
(false, false)
};
let r = ui.add(edit);
crate::scenario::publish_rect("agent_chat:input", r.rect);
// macOS shows a window's cursors only while it is the active window: over the text box of a
// chat window that isn't, the pointer stays an arrow. Coming from the main window, the chat
// window becomes active, so the text cursor shows and the first click places the caret.
let (chat_active, main_active) = (ui.ctx().input(|i| i.viewport().focused), ui.ctx().input_for(egui::ViewportId::ROOT, |i| i.viewport().focused));
if take_focus_for_text(app.preferences.agent_chat_popped, r.hovered(), chat_active, main_active) {
ui.ctx().send_viewport_cmd(egui::ViewportCommand::Focus);
}
let enter = r.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && !i.modifiers.shift);
if esc {
let (mut send_clicked, mut stop_clicked, mut enter) = (false, false, false);
egui::Panel::bottom(Id::new("sc_agent_chat_composer")).resizable(false).show_separator_line(false).frame(egui::Frame::NONE).show(ui, |ui| {
ui.separator();
// The files for the next message (#1744), as chips of their own.
attachment_chips(app, ui);
let edit = egui::TextEdit::multiline(&mut app.agent_chat.input)
.hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" }))
.desired_rows(2)
.desired_width(f32::INFINITY)
.id_salt("sc_agent_chat_input");
let r = egui::ScrollArea::vertical()
.id_salt("sc_agent_chat_input_scroll")
.max_height(edit_max)
.auto_shrink([false, true])
.show(ui, |ui| ui.add(edit))
.inner;
crate::scenario::publish_rect("agent_chat:input", r.rect);
// macOS shows a window's cursors only while it is the active window: over the text box of a
// chat window that isn't, the pointer stays an arrow. Coming from the main window, the chat
// window becomes active, so the text cursor shows and the first click places the caret.
let (chat_active, main_active) =
(ui.ctx().input(|i| i.viewport().focused), ui.ctx().input_for(egui::ViewportId::ROOT, |i| i.viewport().focused));
if take_focus_for_text(app.preferences.agent_chat_popped, r.hovered(), chat_active, main_active) {
ui.ctx().send_viewport_cmd(egui::ViewportCommand::Focus);
}
enter = r.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && !i.modifiers.shift);
ui.horizontal(|ui| {
add_menu(app, ui);
// A message waits for files still being read.
let reading = !app.agent_chat.loading.is_empty();
let send_b = ui
.add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })))
.on_disabled_hover_text(tr("ui.agent_chat_files_loading"));
crate::scenario::publish_handle("agent_chat:send", send_b.rect.center());
send_clicked = send_b.clicked();
stop_clicked = busy && glyph_button(ui, "stopturn", tr("ui.agent_chat_stop_turn_tip")).clicked();
// A picture of the view goes with the message to agents that take pictures.
let images = app.agent_chat.images || !state.running();
let attach = ui
.add_enabled(images, egui::Checkbox::new(&mut app.agent_chat.attach_view, tr("ui.agent_chat_attach_view")))
.on_hover_text(tr("ui.agent_chat_attach_view_tip"))
.on_disabled_hover_text(tr("ui.agent_chat_no_images"));
crate::scenario::publish_handle("agent_chat:attach", attach.rect.center());
});
});
egui::ScrollArea::vertical()
.id_salt("sc_agent_chat_scroll")
.max_height(ui.available_height())
.auto_shrink([false, false])
.stick_to_bottom(true)
.show(ui, |ui| {
conversation(app, ui);
});
if esc || stop_clicked {
cancel(app);
}
ui.horizontal(|ui| {
add_menu(app, ui);
// A message waits for files still being read.
let reading = !app.agent_chat.loading.is_empty();
let send_b = ui
.add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })))
.on_disabled_hover_text(tr("ui.agent_chat_files_loading"));
crate::scenario::publish_handle("agent_chat:send", send_b.rect.center());
// Files alone make a message.
let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty();
if reading {
// Enter waits with the button.
} else if (send_b.clicked() || enter || steer) && something {
let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string();
send(app, ui.ctx(), text);
if steer {
send_now(app, ui.ctx());
}
} else if steer {
// Nothing typed: Ctrl+Enter sends what is already queued.
// A message waits for files still being read, and files alone make a message.
let reading = !app.agent_chat.loading.is_empty();
let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty();
if reading {
// Enter waits with the button.
} else if (send_clicked || enter || steer) && something {
let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string();
send(app, ui.ctx(), text);
if steer {
send_now(app, ui.ctx());
}
if busy && glyph_button(ui, "stopturn", tr("ui.agent_chat_stop_turn_tip")).clicked() {
cancel(app);
}
// A picture of the view goes with the message to agents that take pictures.
let images = app.agent_chat.images || !state.running();
let attach = ui
.add_enabled(images, egui::Checkbox::new(&mut app.agent_chat.attach_view, tr("ui.agent_chat_attach_view")))
.on_hover_text(tr("ui.agent_chat_attach_view_tip"))
.on_disabled_hover_text(tr("ui.agent_chat_no_images"));
crate::scenario::publish_handle("agent_chat:attach", attach.rect.center());
});
} else if steer {
// Nothing typed: Ctrl+Enter sends what is already queued.
send_now(app, ui.ctx());
}
}
#[cfg(not(target_arch = "wasm32"))]
@@ -789,17 +897,23 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
// Allow (true), Deny (false), Always allow (None).
let mut approval: Option<(u64, Option<bool>)> = None;
let mut undo: Option<usize> = None;
let mut open_gallery: Option<Gallery> = None;
let current_doc = app.session.doc.clone();
for (n, item) in app.agent_chat.items.iter().enumerate() {
match item {
Item::User { text, image, files } => {
Item::User { text, image, pictures, files } => {
ui.add_space(6.0);
egui::Frame::NONE.fill(t.accent_soft).corner_radius(6.0).inner_margin(egui::Margin::same(6)).show(ui, |ui| {
ui.set_width(ui.available_width());
if !text.is_empty() {
ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap());
}
if *image {
if !pictures.is_empty() {
if let Some(at) = picture_strip(ui, n, pictures) {
open_gallery = Some(Gallery { pictures: pictures.clone(), at });
}
} else if *image {
// Taken when the agent starts on the message: nothing to show yet.
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
}
files_line(ui, files.iter().map(String::as_str));
@@ -1001,6 +1115,9 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
if send_queue_clicked {
send_now(app, ui.ctx());
}
if open_gallery.is_some() {
app.agent_chat.gallery = open_gallery;
}
if let Some((n, o)) = answer {
answer_permission(app, n, o.as_deref());
}
@@ -1027,13 +1144,6 @@ fn files_line<'a>(ui: &mut egui::Ui, names: impl Iterator<Item = &'a str>) {
/// The widest a file chip gets; a narrower chat makes them narrower.
const CHIP_W: f32 = 170.0;
/// How many rows `n` chips take in a chat `avail` wide (`gap` between them).
fn chip_rows(n: usize, avail: f32, gap: f32) -> usize {
let avail = avail.max(1.0);
let per_row = (((avail + gap) / (CHIP_W.min(avail) + gap)).floor() as usize).max(1);
n.div_ceil(per_row)
}
/// The files for the next message, as chips in rows that fit the chat: a picture's thumbnail (or
/// a file mark), the name cut to fit, and a × that takes the file off. Laid out by hand: egui
/// wraps plain widgets, not framed groups, and one row wider than the chat widens all below it.
+165 -4
View File
@@ -26,12 +26,12 @@ fn streamed_text_joins_and_tool_updates_change_their_call() {
#[test]
fn a_sign_in_request_keeps_the_question_for_after_the_login() {
let mut c = chat();
c.last = Some(("make a cube".into(), false, vec![attach::Attachment::imported("motor.step".into())]));
c.last = Some(("make a cube".into(), false, None, vec![attach::Attachment::imported("motor.step".into())]));
c.state = Some(State::Busy);
assert!(c.on_event(Event::TurnEnded { reason: "auth".into() }).is_some());
c.on_event(Event::AuthRequired { methods: vec![] });
assert_eq!(c.state(), State::NeedsAuth);
let (text, image, files) = c.queued.clone().expect("queued for after the login");
let (text, image, _, files) = c.queued.clone().expect("queued for after the login");
assert_eq!((text.as_str(), image), ("make a cube", false));
assert_eq!(files.iter().map(|f| f.name.as_str()).collect::<Vec<_>>(), ["motor.step"], "its files go again too");
}
@@ -39,7 +39,7 @@ fn a_sign_in_request_keeps_the_question_for_after_the_login() {
#[test]
fn a_plan_update_replaces_the_turns_plan() {
let mut c = chat();
c.push(Item::User { text: "go".into(), image: false, files: Vec::new() });
c.push(Item::User { text: "go".into(), image: false, pictures: Vec::new(), files: Vec::new() });
c.on_event(Event::Plan(vec![("a".into(), "pending".into())]));
c.on_event(Event::Text { thought: false, text: "working".into() });
c.on_event(Event::Plan(vec![("a".into(), "completed".into())]));
@@ -68,7 +68,7 @@ fn app_in_a_turn() -> (SolveApp, Sender<ControlRequest>) {
let (tx, rx) = channel();
app.agent_chat.tools_rx = Some(rx);
app.agent_chat.state = Some(State::Busy);
start_turn(&mut app, "two boxes".into(), false, Vec::new());
start_turn(&mut app, "two boxes".into(), false, None, Vec::new());
(app, tx)
}
@@ -1380,3 +1380,164 @@ fn an_agents_markdown_is_drawn_as_blocks() {
h.app.agent_chat.items.push(Item::Agent { thought: false, text: "*".repeat(50_000) + &"[".repeat(5_000) + &"`a".repeat(5_000) });
h.frames(3);
}
/// A long message scrolls inside its box: the buttons under the box stay in the window (#2224).
#[test]
fn a_long_message_keeps_the_buttons_in_the_window() {
let mut h = crate::scenario::Harness::new();
h.call("ui.resize", json!({"width": 1600.0, "height": 1000.0}));
h.call("ui.agent_chat", json!({"open": true}));
h.frames(8);
let [_, rest] = h.widget("Send").expect("the Send button is drawn");
let box_before = h.app.agent_chat.panel_rect.map(|r| r.bottom());
h.app.agent_chat.input = "Please make the plate thicker and then add a fillet on every outside edge.\n".repeat(60);
h.frames(8);
let window = 1000.0;
for name in ["Send", "Attach view"] {
let [_, y] = h.widget(name).unwrap_or_else(|| panic!("{name} is drawn"));
assert!(y < window, "{name} is at {y}, below the window's bottom at {window}");
// The box is pinned to the bottom and grows upwards: the buttons do not move.
assert!((y - rest).abs() < 1.0, "{name} moved from {rest} to {y}");
}
assert_eq!(h.app.agent_chat.panel_rect.map(|r| r.bottom()), box_before, "the panel keeps its size");
}
fn test_png(colour: [u8; 4], width: u32) -> Vec<u8> {
let mut png = Vec::new();
let image = image::RgbaImage::from_pixel(width, 4, image::Rgba(colour));
image::DynamicImage::ImageRgba8(image).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).expect("a PNG");
png
}
/// A message's pictures show as thumbnails; a click opens the viewer on that one, where Next, the
/// arrow keys and Esc work (#2224). A message whose picture is not taken yet keeps a plain caption.
#[test]
fn a_messages_pictures_open_in_a_viewer() {
let mut h = crate::scenario::Harness::new();
h.call("ui.agent_chat", json!({"open": true}));
h.app.agent_chat.items.push(Item::User { text: "waiting".into(), image: true, pictures: Vec::new(), files: Vec::new() });
h.frames(3);
let [x, y] = h.widget("View attached").expect("the plain caption is drawn");
h.call("ui.click", json!({"x": x, "y": y}));
h.frames(3);
assert!(h.app.agent_chat.gallery.is_none(), "no picture yet: nothing opens");
h.app.agent_chat.items.clear();
let pictures = vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view"), Picture::new(test_png([30, 30, 200, 255], 16), "")];
h.app.agent_chat.items.push(Item::User { text: "look".into(), image: true, pictures, files: Vec::new() });
h.frames(4);
assert!(h.widget("View attached").is_none(), "thumbnails replace the caption");
let point = |name: &str| {
let p = crate::scenario::handle_at(name).unwrap_or_else(|| panic!("{name} is drawn"));
json!({"x": p.x, "y": p.y})
};
// Hovering is not clicking.
let second = point("agent_chat:picture:0:1");
h.call("ui.move", second.clone());
h.frames(3);
assert!(h.app.agent_chat.gallery.is_none());
h.call("ui.click", second);
h.frames(4);
assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(1), "the click opens the viewer on that picture");
assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "the picture is drawn big");
// Next wraps round; the arrow keys step too.
let next = point("agent_chat:gallery_next");
h.call("ui.click", next);
h.frames(3);
assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(0));
h.call("ui.key", json!({"key": "ArrowRight"}));
h.frames(3);
assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(1));
let state = h.call("ui.agent_chat", json!({}));
assert_eq!(state["result"]["items"][0]["pictures"], 2);
h.call("ui.key", json!({"key": "Escape"}));
h.frames(3);
assert!(h.app.agent_chat.gallery.is_none(), "Esc closes the viewer");
// New chat closes it with the conversation.
h.app.agent_chat.gallery = Some(Gallery { pictures: vec![Picture::new(test_png([0, 0, 0, 255], 4), "")], at: 0 });
super::new_chat(&mut h.app);
assert!(h.app.agent_chat.gallery.is_none());
}
/// A message's picture is kept on exactly the message it was taken for: not an earlier one that
/// also wanted a picture, not one that has it already, and nothing that isn't such a message.
#[test]
fn the_picture_goes_to_the_message_it_was_taken_for() {
let mut chat = AgentChat::default();
chat.push(Item::User { text: "no picture".into(), image: false, pictures: Vec::new(), files: Vec::new() });
chat.push(Item::User { text: "first".into(), image: true, pictures: vec![Picture::new(vec![1], "")], files: Vec::new() });
chat.push(Item::User { text: "second".into(), image: true, pictures: Vec::new(), files: Vec::new() });
chat.push(Item::User { text: "third".into(), image: true, pictures: Vec::new(), files: Vec::new() });
let sizes = |chat: &AgentChat| -> Vec<usize> {
chat.items
.iter()
.filter_map(|i| match i {
Item::User { pictures, .. } => Some(pictures.iter().map(|p| p.png.len()).sum()),
_ => None,
})
.collect()
};
// The picture of the third message goes to the third, though the second still waits for one.
chat.keep_picture(Some(3), vec![2, 3]);
assert_eq!(sizes(&chat), [0, 1, 0, 2]);
// The same message again (it was sent again): nothing is added, nor moved to an older message.
chat.keep_picture(Some(3), vec![9; 5]);
assert_eq!(sizes(&chat), [0, 1, 0, 2]);
// A message that did not want one, no message, or a place that is gone.
chat.keep_picture(Some(0), vec![1]);
chat.keep_picture(None, vec![1]);
chat.keep_picture(Some(99), vec![1]);
assert_eq!(sizes(&chat), [0, 1, 0, 2]);
chat.keep_picture(Some(2), vec![0; super::MAX_PICTURE_BYTES + 1]);
assert_eq!(sizes(&chat), [0, 1, 0, 2], "an oversized picture is not kept");
}
/// A message sent again after a sign-in is still the message it was: its turn points at its own
/// place in the conversation, so its picture is kept there, once, and never on an older message that
/// wanted one (#2224 review). The picture itself can't be drawn without a renderer in a test, so the
/// place a turn is started for is what is checked.
#[test]
fn a_message_sent_again_after_sign_in_stays_the_message_it_was() {
let (mut app, script, ctx) = app_with_script();
app.agent_chat.images = true;
// An older message that wanted a picture and has none.
app.agent_chat.push(Item::User { text: "older".into(), image: true, pictures: Vec::new(), files: Vec::new() });
app.agent_chat.attach_view = true;
send(&mut app, &ctx, "the new one".into());
let place = app.agent_chat.items.len() - 1;
assert_eq!(app.agent_chat.last, Some(("the new one".into(), true, Some(place), Vec::new())), "the turn knows its message");
// The agent asked for a sign-in; after it the message goes again, as it is done on a sign-in.
ended(&script, "");
poll(&mut app, &ctx);
app.agent_chat.queued = app.agent_chat.last.take();
script.report(Event::Ready { agent: "fake".into(), images: true });
poll(&mut app, &ctx);
assert_eq!(app.agent_chat.last, Some(("the new one".into(), true, Some(place), Vec::new())), "sent again, still for the same message");
assert_eq!(app.agent_chat.items.iter().filter(|i| matches!(i, Item::User { .. })).count(), 2, "not a new message");
// Its picture goes there, once; the older message gets none.
app.agent_chat.keep_picture(Some(place), test_png([1, 2, 3, 255], 4));
app.agent_chat.keep_picture(Some(place), test_png([1, 2, 3, 255], 4));
let counts: Vec<usize> =
app.agent_chat.items.iter().filter_map(|i| if let Item::User { pictures, .. } = i { Some(pictures.len()) } else { None }).collect();
assert_eq!(counts, [0, 1]);
}
/// The picture viewer opens over the chat's own window too, not only over the docked panel (#1745).
#[test]
fn the_viewer_opens_in_the_popped_out_window() {
let mut h = crate::scenario::Harness::new();
h.call("ui.agent_chat", json!({"open": true}));
h.app.preferences.agent_chat_popped = true;
h.app.agent_chat.items.push(Item::User {
text: "look".into(),
image: true,
pictures: vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view")],
files: Vec::new(),
});
h.frames(4);
let at = crate::scenario::handle_at("agent_chat:picture:0:0").expect("the thumbnail is drawn in the window");
h.call("ui.click", json!({"x": at.x, "y": at.y}));
h.frames(4);
assert!(h.app.agent_chat.gallery.is_some(), "the click opened the viewer");
assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "and it is drawn");
}
+5 -1
View File
@@ -195,7 +195,11 @@ The agent reaches the open design only through SolidCraft's own tools (the
(with **Copy**), bold, italic, `code` and links (web links only; each link is a button that opens
it in your browser). What you type, and the agent's thoughts, stay plain text.
- **Attach view:** tick it to send a picture of the current view with your message, for agents
that take pictures.
that take pictures. The picture is taken when the agent starts on the message and then shows as
a small thumbnail under it. Hold the pointer over the thumbnail for a bigger preview that stays
while you do; click it to open the viewer (the message's pictures one at a time, with **‹** and
**›** or the arrow keys, closed with Esc or a click outside). A long message scrolls inside its
box, so **Send** and the buttons beside it stay in view.
## Permissions