diff --git a/crates/ui-egui/src/agent_chat/mod.rs b/crates/ui-egui/src/agent_chat/mod.rs index 52276420f..88d62f820 100644 --- a/crates/ui-egui/src/agent_chat/mod.rs +++ b/crates/ui-egui/src/agent_chat/mod.rs @@ -56,6 +56,8 @@ const FILE_NAMES_ROOM: usize = 2_000; const APPROVAL_TIMEOUT_MS: f64 = 240_000.0; /// Size of the picture of the view sent with a message. const VIEW_IMAGE: (u32, u32) = (960, 600); +/// A picture kept for the user to open later, at most this big. +const MAX_PICTURE_BYTES: usize = 4 << 20; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum State { @@ -93,11 +95,58 @@ impl State { } } +/// A picture shown with a message: what the agent got (a picture of the view, or any other +/// attachment) as PNG, and a note shown with it in the viewer. The thumbnail and the viewer share +/// one texture, made when the picture is first drawn. +#[derive(Clone)] +pub struct Picture { + png: Arc>, + note: String, + texture: Arc>>, +} + +impl Picture { + pub fn new(png: Vec, note: impl Into) -> Self { + Self { png: Arc::new(png), note: note.into(), texture: Arc::default() } + } + + /// The picture as a texture, `None` when it can't be decoded. + #[cfg(not(target_arch = "wasm32"))] + fn texture(&self, ctx: &egui::Context) -> Option { + self.texture + .get_or_init(|| { + let rgba = image::load_from_memory_with_format(&self.png, image::ImageFormat::Png).ok()?.to_rgba8(); + let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?]; + Some(ctx.load_texture( + "sc_agent_chat_picture", + egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()), + egui::TextureOptions::LINEAR, + )) + }) + .clone() + } +} + +impl std::fmt::Debug for Picture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Picture({} bytes, {:?})", self.png.len(), self.note) + } +} + +/// The viewer: the pictures of one message, and the one shown. +#[derive(Clone, Debug)] +pub struct Gallery { + pictures: Vec, + at: usize, +} + #[derive(Clone, Debug)] pub enum Item { User { text: String, image: bool, + /// The view's picture, kept once it is taken (when the agent starts on the message). + pictures: Vec, /// The names of the files that went with it. files: Vec, }, @@ -202,6 +251,8 @@ pub struct AgentChat { pub open: bool, pub items: Vec, pub input: String, + /// The viewer, open on one message's pictures. + gallery: Option, state: Option, agent: String, images: bool, @@ -209,8 +260,9 @@ pub struct AgentChat { pub attach_view: bool, auth: Vec, auth_status: Option, - /// A message typed before the agent was ready, with its files. - queued: Option<(String, bool, Vec)>, + /// A message typed before the agent was ready: its text, whether the view goes with it, its + /// place in the conversation, and its files. + queued: Option<(String, bool, Option, Vec)>, /// Messages typed while the agent works, in order. They go as one prompt when the turn ends /// (or at once with Send now); after Stop they wait for Send. pub pending: Vec, @@ -231,7 +283,7 @@ pub struct AgentChat { /// Where the panel was drawn last (for its shortcuts). pub panel_rect: Option, /// The last message sent, kept to send again after a sign-in (its files too). - last: Option<(String, bool, Vec)>, + last: Option<(String, bool, Option, Vec)>, /// Files for the next message (pictures, text files, imported parts), shown as chips. pub(crate) attachments: Vec, /// Pictures pasted so far, to name the next one. @@ -347,6 +399,17 @@ impl AgentChat { } self.items.push(item); } + /// Keep the picture just sent with the message at `at` in the conversation (the user can open + /// it). A message sent again (after a sign-in) has its picture already: nothing is added, and + /// never to another message. + fn keep_picture(&mut self, at: Option, png: Vec) { + if let Some(Item::User { image: true, pictures, .. }) = at.and_then(|i| self.items.get_mut(i)) + && pictures.is_empty() + && png.len() <= MAX_PICTURE_BYTES + { + pictures.push(Picture::new(png, "")); + } + } fn notice(&mut self, text: impl Into, error: bool) { self.push(Item::Notice { text: text.into(), error }); } @@ -704,11 +767,13 @@ pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) { /// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it /// cut in on its previous step). fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec, interrupted: bool) { - app.agent_chat.push(Item::User { text: text.clone(), image, files: files.iter().map(|f| f.name.clone()).collect() }); + app.agent_chat.push(Item::User { text: text.clone(), image, pictures: Vec::new(), files: files.iter().map(|f| f.name.clone()).collect() }); + // The message's place in the conversation: its picture is kept there when it is taken. + let at = app.agent_chat.items.len().checked_sub(1); match app.agent_chat.state() { - State::Ready => start_turn_with(app, text, image, files, interrupted.then_some(INTERRUPT_NOTE)), + State::Ready => start_turn_with(app, text, image, at, files, interrupted.then_some(INTERRUPT_NOTE)), s => { - app.agent_chat.queued = Some((text, image, files)); + app.agent_chat.queued = Some((text, image, at, files)); #[cfg(not(target_arch = "wasm32"))] if !s.running() { connect(app, ctx); @@ -775,6 +840,7 @@ pub fn new_chat(app: &mut SolveApp) { let chat = &mut app.agent_chat; chat.confirm_new = false; chat.items.clear(); + chat.gallery = None; chat.pending.clear(); chat.attachments.clear(); chat.loading.clear(); @@ -823,14 +889,14 @@ fn after_turn(app: &mut SolveApp, ctx: &egui::Context) { } } -fn start_turn(app: &mut SolveApp, text: String, image: bool, files: Vec) { - start_turn_with(app, text, image, files, None); +fn start_turn(app: &mut SolveApp, text: String, image: bool, at: Option, files: Vec) { + start_turn_with(app, text, image, at, files, None); } /// Start a message; `note` goes before the text for the agent, not into the conversation. The /// files are named after the text (text files in full), and pictures go as images to an agent -/// that takes them. -fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec, note: Option<&str>) { +/// that takes them. `at` is the message's place in the conversation, where its picture is kept. +fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option, files: Vec, note: Option<&str>) { let pictures = app.agent_chat.images; let lead = match note { Some(n) => format!("{n}\n\n{text}"), @@ -843,7 +909,10 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec parts.push(PromptPart::Image { mime: "image/png".into(), base64: backend::base64(&png) }), + Ok(png) => { + parts.push(PromptPart::Image { mime: "image/png".into(), base64: backend::base64(&png) }); + app.agent_chat.keep_picture(at, png); + } Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"), } } @@ -855,7 +924,7 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec Value { .items .iter() .map(|i| match i { - Item::User { text, image, files } => json!({"user": text, "image": image, "files": files}), + Item::User { text, image, pictures, files } => json!({"user": text, "image": image, "pictures": pictures.len(), "files": files}), Item::Agent { thought, text } => json!({"agent": text, "thought": thought}), Item::Tool { id, title, kind, status, output } => { json!({"tool": id, "title": title, "kind": kind, "status": status.map(|s| format!("{s:?}")), "output": output}) diff --git a/crates/ui-egui/src/agent_chat/panel.rs b/crates/ui-egui/src/agent_chat/panel.rs index 0b75e283d..66358816c 100644 --- a/crates/ui-egui/src/agent_chat/panel.rs +++ b/crates/ui-egui/src/agent_chat/panel.rs @@ -18,6 +18,12 @@ fn panel_id(left: bool) -> Id { Id::new(if left { "sc_agent_chat_left" } else { "sc_agent_chat_right" }) } +/// The largest thumbnail of a message's picture, and its largest preview while hovered (points). +#[cfg(not(target_arch = "wasm32"))] +const THUMB_MAX: (f32, f32) = (150.0, 56.0); +#[cfg(not(target_arch = "wasm32"))] +const PREVIEW_MAX: (f32, f32) = (560.0, 380.0); + /// Forget the panel's own idea of its width, so the preference applies on the next frame. pub fn forget_width(ctx: &egui::Context) { ctx.data_mut(|d| { @@ -105,6 +111,8 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) { .show(ui, |ui| body(app, ui, left)); crate::scenario::publish_rect("agent_chat:panel", r.response.rect); app.agent_chat.panel_rect = Some(r.response.rect); + #[cfg(not(target_arch = "wasm32"))] + gallery_window(app, &ctx); app.agent_chat.drop_rect = Some(r.response.rect); show_drop_hint(app, &ctx); let w = r.response.rect.width(); @@ -113,6 +121,88 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) { } } +/// The pictures of a message as small thumbnails in a row. Hovering one shows it bigger for as long +/// as the pointer stays; a click returns its place in the row, to open the viewer there. +#[cfg(not(target_arch = "wasm32"))] +fn picture_strip(ui: &mut egui::Ui, n: usize, pictures: &[Picture]) -> Option { + let t = Tokens::get(); + let mut clicked = None; + ui.horizontal_wrapped(|ui| { + for (i, picture) in pictures.iter().enumerate() { + let Some(texture) = picture.texture(ui.ctx()) else { + ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0)); + continue; + }; + let size = fit(texture.size_vec2(), vec2(THUMB_MAX.0, THUMB_MAX.1)); + let thumb = ui.add(egui::Image::new((texture.id(), size)).corner_radius(4.0).sense(Sense::click())); + crate::scenario::publish_handle(&format!("agent_chat:picture:{n}:{i}"), thumb.rect.center()); + let hover_max = (ui.ctx().content_rect().size() * 0.5).min(vec2(PREVIEW_MAX.0, PREVIEW_MAX.1)); + let thumb = thumb.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_ui(|ui| { + ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), hover_max))).corner_radius(4.0)); + if !picture.note.is_empty() { + ui.label(&picture.note); + } + }); + if thumb.clicked() { + clicked = Some(i); + } + } + }); + clicked +} + +/// `size` scaled to fit in `max`, keeping its shape (never below one point). +#[cfg(not(target_arch = "wasm32"))] +fn fit(size: egui::Vec2, max: egui::Vec2) -> egui::Vec2 { + let scale = (max.x / size.x.max(1.0)).min(max.y / size.y.max(1.0)).min(2.0); + vec2((size.x * scale).max(1.0), (size.y * scale).max(1.0)) +} + +/// The viewer: a message's pictures over the app, one at a time with a counter, Previous and Next +/// (also the arrow keys), closed with the button, Esc or a click outside. +#[cfg(not(target_arch = "wasm32"))] +fn gallery_window(app: &mut SolveApp, ctx: &egui::Context) { + let Some(mut gallery) = app.agent_chat.gallery.take() else { return }; + let count = gallery.pictures.len(); + gallery.at = gallery.at.min(count.saturating_sub(1)); + let (left, right) = + ctx.input_mut(|i| (i.consume_key(egui::Modifiers::NONE, egui::Key::ArrowLeft), i.consume_key(egui::Modifiers::NONE, egui::Key::ArrowRight))); + let mut step: isize = if left { -1 } else { isize::from(right) }; + let t = Tokens::get(); + let max = ctx.content_rect().size() * vec2(0.8, 0.7); + let modal = egui::Modal::new(Id::new("sc_agent_chat_gallery")).show(ctx, |ui| { + ui.set_max_width(max.x); + ui.horizontal(|ui| { + let prev = ui.add_enabled(count > 1, egui::Button::new("‹")); + crate::scenario::publish_handle("agent_chat:gallery_prev", prev.rect.center()); + let next = ui.add_enabled(count > 1, egui::Button::new("›")); + crate::scenario::publish_handle("agent_chat:gallery_next", next.rect.center()); + if prev.clicked() { + step = -1; + } + if next.clicked() { + step = 1; + } + ui.label(RichText::new(format!("{} / {}", gallery.at + 1, count)).color(t.text_dim)); + }); + if let Some(picture) = gallery.pictures.get(gallery.at) { + if let Some(texture) = picture.texture(ui.ctx()) { + let resp = ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), max))).corner_radius(4.0)); + crate::scenario::publish_rect("agent_chat:gallery_picture", resp.rect); + } + if !picture.note.is_empty() { + ui.add(egui::Label::new(&picture.note).wrap()); + } + } + }); + if count > 0 && step != 0 { + gallery.at = (gallery.at as isize + step).rem_euclid(count as isize) as usize; + } + if !modal.should_close() { + app.agent_chat.gallery = Some(gallery); + } +} + /// While files are dragged over the window: if they would drop on the chat, say so over it. fn show_drop_hint(app: &SolveApp, ctx: &egui::Context) { #[cfg(not(target_arch = "wasm32"))] @@ -193,6 +283,8 @@ fn window(app: &mut SolveApp, ctx: &egui::Context) { let t = Tokens::get(); let frame = egui::Frame::NONE.fill(t.panel).inner_margin(egui::Margin::symmetric(8, 6)); let r = egui::CentralPanel::default().frame(frame).show(ui, |ui| body(app, ui, false)); + // The picture viewer opens over this window, where the click was, not over the main one. + gallery_window(app, ui.ctx()); if class == egui::ViewportClass::EmbeddedWindow { // Floating inside the app: its handles are the main window's, as the panel's are. crate::scenario::publish_rect("agent_chat:window", r.response.rect); @@ -468,21 +560,12 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) { auth_box(app, ui); } ui.separator(); - // The input sits at the bottom (with a row per line of file chips); the conversation takes - // the rest. - let input_h = 92.0 + chip_rows(app.agent_chat.files(), ui.available_width(), ui.spacing().item_spacing.x) as f32 * 32.0; - let avail = (ui.available_height() - input_h).max(80.0); - egui::ScrollArea::vertical().id_salt("sc_agent_chat_scroll").max_height(avail).auto_shrink([false, false]).stick_to_bottom(true).show(ui, |ui| { - conversation(app, ui); - }); - ui.separator(); + // The input sits at the bottom of the panel and grows upwards: a long message makes its box + // taller up to `edit_max`, then it scrolls, and the conversation takes what is left above. It + // is a panel of its own at the bottom, as high as its content this frame, so the buttons under + // the box are always in view. + let edit_max = (ui.available_height() * 0.4).clamp(48.0, 160.0); let busy = state == State::Busy; - attachment_chips(app, ui); - let edit = egui::TextEdit::multiline(&mut app.agent_chat.input) - .hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" })) - .desired_rows(2) - .desired_width(f32::INFINITY) - .id_salt("sc_agent_chat_input"); // Enter sends (queues while the agent works); Ctrl/Cmd+Enter sends now; Esc stops. They are // read before the field sees them so no newline is typed. let focused = ui.memory(|m| m.has_focus(Id::new("sc_agent_chat_input"))); @@ -491,52 +574,77 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) { } else { (false, false) }; - let r = ui.add(edit); - crate::scenario::publish_rect("agent_chat:input", r.rect); - // macOS shows a window's cursors only while it is the active window: over the text box of a - // chat window that isn't, the pointer stays an arrow. Coming from the main window, the chat - // window becomes active, so the text cursor shows and the first click places the caret. - let (chat_active, main_active) = (ui.ctx().input(|i| i.viewport().focused), ui.ctx().input_for(egui::ViewportId::ROOT, |i| i.viewport().focused)); - if take_focus_for_text(app.preferences.agent_chat_popped, r.hovered(), chat_active, main_active) { - ui.ctx().send_viewport_cmd(egui::ViewportCommand::Focus); - } - let enter = r.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && !i.modifiers.shift); - if esc { + let (mut send_clicked, mut stop_clicked, mut enter) = (false, false, false); + egui::Panel::bottom(Id::new("sc_agent_chat_composer")).resizable(false).show_separator_line(false).frame(egui::Frame::NONE).show(ui, |ui| { + ui.separator(); + // The files for the next message (#1744), as chips of their own. + attachment_chips(app, ui); + let edit = egui::TextEdit::multiline(&mut app.agent_chat.input) + .hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" })) + .desired_rows(2) + .desired_width(f32::INFINITY) + .id_salt("sc_agent_chat_input"); + let r = egui::ScrollArea::vertical() + .id_salt("sc_agent_chat_input_scroll") + .max_height(edit_max) + .auto_shrink([false, true]) + .show(ui, |ui| ui.add(edit)) + .inner; + crate::scenario::publish_rect("agent_chat:input", r.rect); + // macOS shows a window's cursors only while it is the active window: over the text box of a + // chat window that isn't, the pointer stays an arrow. Coming from the main window, the chat + // window becomes active, so the text cursor shows and the first click places the caret. + let (chat_active, main_active) = + (ui.ctx().input(|i| i.viewport().focused), ui.ctx().input_for(egui::ViewportId::ROOT, |i| i.viewport().focused)); + if take_focus_for_text(app.preferences.agent_chat_popped, r.hovered(), chat_active, main_active) { + ui.ctx().send_viewport_cmd(egui::ViewportCommand::Focus); + } + enter = r.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && !i.modifiers.shift); + ui.horizontal(|ui| { + add_menu(app, ui); + // A message waits for files still being read. + let reading = !app.agent_chat.loading.is_empty(); + let send_b = ui + .add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" }))) + .on_disabled_hover_text(tr("ui.agent_chat_files_loading")); + crate::scenario::publish_handle("agent_chat:send", send_b.rect.center()); + send_clicked = send_b.clicked(); + stop_clicked = busy && glyph_button(ui, "stopturn", tr("ui.agent_chat_stop_turn_tip")).clicked(); + // A picture of the view goes with the message to agents that take pictures. + let images = app.agent_chat.images || !state.running(); + let attach = ui + .add_enabled(images, egui::Checkbox::new(&mut app.agent_chat.attach_view, tr("ui.agent_chat_attach_view"))) + .on_hover_text(tr("ui.agent_chat_attach_view_tip")) + .on_disabled_hover_text(tr("ui.agent_chat_no_images")); + crate::scenario::publish_handle("agent_chat:attach", attach.rect.center()); + }); + }); + egui::ScrollArea::vertical() + .id_salt("sc_agent_chat_scroll") + .max_height(ui.available_height()) + .auto_shrink([false, false]) + .stick_to_bottom(true) + .show(ui, |ui| { + conversation(app, ui); + }); + if esc || stop_clicked { cancel(app); } - ui.horizontal(|ui| { - add_menu(app, ui); - // A message waits for files still being read. - let reading = !app.agent_chat.loading.is_empty(); - let send_b = ui - .add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" }))) - .on_disabled_hover_text(tr("ui.agent_chat_files_loading")); - crate::scenario::publish_handle("agent_chat:send", send_b.rect.center()); - // Files alone make a message. - let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty(); - if reading { - // Enter waits with the button. - } else if (send_b.clicked() || enter || steer) && something { - let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string(); - send(app, ui.ctx(), text); - if steer { - send_now(app, ui.ctx()); - } - } else if steer { - // Nothing typed: Ctrl+Enter sends what is already queued. + // A message waits for files still being read, and files alone make a message. + let reading = !app.agent_chat.loading.is_empty(); + let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty(); + if reading { + // Enter waits with the button. + } else if (send_clicked || enter || steer) && something { + let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string(); + send(app, ui.ctx(), text); + if steer { send_now(app, ui.ctx()); } - if busy && glyph_button(ui, "stopturn", tr("ui.agent_chat_stop_turn_tip")).clicked() { - cancel(app); - } - // A picture of the view goes with the message to agents that take pictures. - let images = app.agent_chat.images || !state.running(); - let attach = ui - .add_enabled(images, egui::Checkbox::new(&mut app.agent_chat.attach_view, tr("ui.agent_chat_attach_view"))) - .on_hover_text(tr("ui.agent_chat_attach_view_tip")) - .on_disabled_hover_text(tr("ui.agent_chat_no_images")); - crate::scenario::publish_handle("agent_chat:attach", attach.rect.center()); - }); + } else if steer { + // Nothing typed: Ctrl+Enter sends what is already queued. + send_now(app, ui.ctx()); + } } #[cfg(not(target_arch = "wasm32"))] @@ -789,17 +897,23 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) { // Allow (true), Deny (false), Always allow (None). let mut approval: Option<(u64, Option)> = None; let mut undo: Option = None; + let mut open_gallery: Option = None; let current_doc = app.session.doc.clone(); for (n, item) in app.agent_chat.items.iter().enumerate() { match item { - Item::User { text, image, files } => { + Item::User { text, image, pictures, files } => { ui.add_space(6.0); egui::Frame::NONE.fill(t.accent_soft).corner_radius(6.0).inner_margin(egui::Margin::same(6)).show(ui, |ui| { ui.set_width(ui.available_width()); if !text.is_empty() { ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap()); } - if *image { + if !pictures.is_empty() { + if let Some(at) = picture_strip(ui, n, pictures) { + open_gallery = Some(Gallery { pictures: pictures.clone(), at }); + } + } else if *image { + // Taken when the agent starts on the message: nothing to show yet. ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0)); } files_line(ui, files.iter().map(String::as_str)); @@ -1001,6 +1115,9 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) { if send_queue_clicked { send_now(app, ui.ctx()); } + if open_gallery.is_some() { + app.agent_chat.gallery = open_gallery; + } if let Some((n, o)) = answer { answer_permission(app, n, o.as_deref()); } @@ -1027,13 +1144,6 @@ fn files_line<'a>(ui: &mut egui::Ui, names: impl Iterator) { /// The widest a file chip gets; a narrower chat makes them narrower. const CHIP_W: f32 = 170.0; -/// How many rows `n` chips take in a chat `avail` wide (`gap` between them). -fn chip_rows(n: usize, avail: f32, gap: f32) -> usize { - let avail = avail.max(1.0); - let per_row = (((avail + gap) / (CHIP_W.min(avail) + gap)).floor() as usize).max(1); - n.div_ceil(per_row) -} - /// The files for the next message, as chips in rows that fit the chat: a picture's thumbnail (or /// a file mark), the name cut to fit, and a × that takes the file off. Laid out by hand: egui /// wraps plain widgets, not framed groups, and one row wider than the chat widens all below it. diff --git a/crates/ui-egui/src/agent_chat/tests.rs b/crates/ui-egui/src/agent_chat/tests.rs index 9da7d9372..6881b0da7 100644 --- a/crates/ui-egui/src/agent_chat/tests.rs +++ b/crates/ui-egui/src/agent_chat/tests.rs @@ -26,12 +26,12 @@ fn streamed_text_joins_and_tool_updates_change_their_call() { #[test] fn a_sign_in_request_keeps_the_question_for_after_the_login() { let mut c = chat(); - c.last = Some(("make a cube".into(), false, vec![attach::Attachment::imported("motor.step".into())])); + c.last = Some(("make a cube".into(), false, None, vec![attach::Attachment::imported("motor.step".into())])); c.state = Some(State::Busy); assert!(c.on_event(Event::TurnEnded { reason: "auth".into() }).is_some()); c.on_event(Event::AuthRequired { methods: vec![] }); assert_eq!(c.state(), State::NeedsAuth); - let (text, image, files) = c.queued.clone().expect("queued for after the login"); + let (text, image, _, files) = c.queued.clone().expect("queued for after the login"); assert_eq!((text.as_str(), image), ("make a cube", false)); assert_eq!(files.iter().map(|f| f.name.as_str()).collect::>(), ["motor.step"], "its files go again too"); } @@ -39,7 +39,7 @@ fn a_sign_in_request_keeps_the_question_for_after_the_login() { #[test] fn a_plan_update_replaces_the_turns_plan() { let mut c = chat(); - c.push(Item::User { text: "go".into(), image: false, files: Vec::new() }); + c.push(Item::User { text: "go".into(), image: false, pictures: Vec::new(), files: Vec::new() }); c.on_event(Event::Plan(vec![("a".into(), "pending".into())])); c.on_event(Event::Text { thought: false, text: "working".into() }); c.on_event(Event::Plan(vec![("a".into(), "completed".into())])); @@ -68,7 +68,7 @@ fn app_in_a_turn() -> (SolveApp, Sender) { let (tx, rx) = channel(); app.agent_chat.tools_rx = Some(rx); app.agent_chat.state = Some(State::Busy); - start_turn(&mut app, "two boxes".into(), false, Vec::new()); + start_turn(&mut app, "two boxes".into(), false, None, Vec::new()); (app, tx) } @@ -1380,3 +1380,164 @@ fn an_agents_markdown_is_drawn_as_blocks() { h.app.agent_chat.items.push(Item::Agent { thought: false, text: "*".repeat(50_000) + &"[".repeat(5_000) + &"`a".repeat(5_000) }); h.frames(3); } + +/// A long message scrolls inside its box: the buttons under the box stay in the window (#2224). +#[test] +fn a_long_message_keeps_the_buttons_in_the_window() { + let mut h = crate::scenario::Harness::new(); + h.call("ui.resize", json!({"width": 1600.0, "height": 1000.0})); + h.call("ui.agent_chat", json!({"open": true})); + h.frames(8); + let [_, rest] = h.widget("Send").expect("the Send button is drawn"); + let box_before = h.app.agent_chat.panel_rect.map(|r| r.bottom()); + h.app.agent_chat.input = "Please make the plate thicker and then add a fillet on every outside edge.\n".repeat(60); + h.frames(8); + let window = 1000.0; + for name in ["Send", "Attach view"] { + let [_, y] = h.widget(name).unwrap_or_else(|| panic!("{name} is drawn")); + assert!(y < window, "{name} is at {y}, below the window's bottom at {window}"); + // The box is pinned to the bottom and grows upwards: the buttons do not move. + assert!((y - rest).abs() < 1.0, "{name} moved from {rest} to {y}"); + } + assert_eq!(h.app.agent_chat.panel_rect.map(|r| r.bottom()), box_before, "the panel keeps its size"); +} + +fn test_png(colour: [u8; 4], width: u32) -> Vec { + let mut png = Vec::new(); + let image = image::RgbaImage::from_pixel(width, 4, image::Rgba(colour)); + image::DynamicImage::ImageRgba8(image).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).expect("a PNG"); + png +} + +/// A message's pictures show as thumbnails; a click opens the viewer on that one, where Next, the +/// arrow keys and Esc work (#2224). A message whose picture is not taken yet keeps a plain caption. +#[test] +fn a_messages_pictures_open_in_a_viewer() { + let mut h = crate::scenario::Harness::new(); + h.call("ui.agent_chat", json!({"open": true})); + h.app.agent_chat.items.push(Item::User { text: "waiting".into(), image: true, pictures: Vec::new(), files: Vec::new() }); + h.frames(3); + let [x, y] = h.widget("View attached").expect("the plain caption is drawn"); + h.call("ui.click", json!({"x": x, "y": y})); + h.frames(3); + assert!(h.app.agent_chat.gallery.is_none(), "no picture yet: nothing opens"); + + h.app.agent_chat.items.clear(); + let pictures = vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view"), Picture::new(test_png([30, 30, 200, 255], 16), "")]; + h.app.agent_chat.items.push(Item::User { text: "look".into(), image: true, pictures, files: Vec::new() }); + h.frames(4); + assert!(h.widget("View attached").is_none(), "thumbnails replace the caption"); + let point = |name: &str| { + let p = crate::scenario::handle_at(name).unwrap_or_else(|| panic!("{name} is drawn")); + json!({"x": p.x, "y": p.y}) + }; + // Hovering is not clicking. + let second = point("agent_chat:picture:0:1"); + h.call("ui.move", second.clone()); + h.frames(3); + assert!(h.app.agent_chat.gallery.is_none()); + h.call("ui.click", second); + h.frames(4); + assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(1), "the click opens the viewer on that picture"); + assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "the picture is drawn big"); + // Next wraps round; the arrow keys step too. + let next = point("agent_chat:gallery_next"); + h.call("ui.click", next); + h.frames(3); + assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(0)); + h.call("ui.key", json!({"key": "ArrowRight"})); + h.frames(3); + assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(1)); + let state = h.call("ui.agent_chat", json!({})); + assert_eq!(state["result"]["items"][0]["pictures"], 2); + h.call("ui.key", json!({"key": "Escape"})); + h.frames(3); + assert!(h.app.agent_chat.gallery.is_none(), "Esc closes the viewer"); + // New chat closes it with the conversation. + h.app.agent_chat.gallery = Some(Gallery { pictures: vec![Picture::new(test_png([0, 0, 0, 255], 4), "")], at: 0 }); + super::new_chat(&mut h.app); + assert!(h.app.agent_chat.gallery.is_none()); +} + +/// A message's picture is kept on exactly the message it was taken for: not an earlier one that +/// also wanted a picture, not one that has it already, and nothing that isn't such a message. +#[test] +fn the_picture_goes_to_the_message_it_was_taken_for() { + let mut chat = AgentChat::default(); + chat.push(Item::User { text: "no picture".into(), image: false, pictures: Vec::new(), files: Vec::new() }); + chat.push(Item::User { text: "first".into(), image: true, pictures: vec![Picture::new(vec![1], "")], files: Vec::new() }); + chat.push(Item::User { text: "second".into(), image: true, pictures: Vec::new(), files: Vec::new() }); + chat.push(Item::User { text: "third".into(), image: true, pictures: Vec::new(), files: Vec::new() }); + let sizes = |chat: &AgentChat| -> Vec { + chat.items + .iter() + .filter_map(|i| match i { + Item::User { pictures, .. } => Some(pictures.iter().map(|p| p.png.len()).sum()), + _ => None, + }) + .collect() + }; + // The picture of the third message goes to the third, though the second still waits for one. + chat.keep_picture(Some(3), vec![2, 3]); + assert_eq!(sizes(&chat), [0, 1, 0, 2]); + // The same message again (it was sent again): nothing is added, nor moved to an older message. + chat.keep_picture(Some(3), vec![9; 5]); + assert_eq!(sizes(&chat), [0, 1, 0, 2]); + // A message that did not want one, no message, or a place that is gone. + chat.keep_picture(Some(0), vec![1]); + chat.keep_picture(None, vec![1]); + chat.keep_picture(Some(99), vec![1]); + assert_eq!(sizes(&chat), [0, 1, 0, 2]); + chat.keep_picture(Some(2), vec![0; super::MAX_PICTURE_BYTES + 1]); + assert_eq!(sizes(&chat), [0, 1, 0, 2], "an oversized picture is not kept"); +} + +/// A message sent again after a sign-in is still the message it was: its turn points at its own +/// place in the conversation, so its picture is kept there, once, and never on an older message that +/// wanted one (#2224 review). The picture itself can't be drawn without a renderer in a test, so the +/// place a turn is started for is what is checked. +#[test] +fn a_message_sent_again_after_sign_in_stays_the_message_it_was() { + let (mut app, script, ctx) = app_with_script(); + app.agent_chat.images = true; + // An older message that wanted a picture and has none. + app.agent_chat.push(Item::User { text: "older".into(), image: true, pictures: Vec::new(), files: Vec::new() }); + app.agent_chat.attach_view = true; + send(&mut app, &ctx, "the new one".into()); + let place = app.agent_chat.items.len() - 1; + assert_eq!(app.agent_chat.last, Some(("the new one".into(), true, Some(place), Vec::new())), "the turn knows its message"); + // The agent asked for a sign-in; after it the message goes again, as it is done on a sign-in. + ended(&script, ""); + poll(&mut app, &ctx); + app.agent_chat.queued = app.agent_chat.last.take(); + script.report(Event::Ready { agent: "fake".into(), images: true }); + poll(&mut app, &ctx); + assert_eq!(app.agent_chat.last, Some(("the new one".into(), true, Some(place), Vec::new())), "sent again, still for the same message"); + assert_eq!(app.agent_chat.items.iter().filter(|i| matches!(i, Item::User { .. })).count(), 2, "not a new message"); + // Its picture goes there, once; the older message gets none. + app.agent_chat.keep_picture(Some(place), test_png([1, 2, 3, 255], 4)); + app.agent_chat.keep_picture(Some(place), test_png([1, 2, 3, 255], 4)); + let counts: Vec = + app.agent_chat.items.iter().filter_map(|i| if let Item::User { pictures, .. } = i { Some(pictures.len()) } else { None }).collect(); + assert_eq!(counts, [0, 1]); +} + +/// The picture viewer opens over the chat's own window too, not only over the docked panel (#1745). +#[test] +fn the_viewer_opens_in_the_popped_out_window() { + let mut h = crate::scenario::Harness::new(); + h.call("ui.agent_chat", json!({"open": true})); + h.app.preferences.agent_chat_popped = true; + h.app.agent_chat.items.push(Item::User { + text: "look".into(), + image: true, + pictures: vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view")], + files: Vec::new(), + }); + h.frames(4); + let at = crate::scenario::handle_at("agent_chat:picture:0:0").expect("the thumbnail is drawn in the window"); + h.call("ui.click", json!({"x": at.x, "y": at.y})); + h.frames(4); + assert!(h.app.agent_chat.gallery.is_some(), "the click opened the viewer"); + assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "and it is drawn"); +} diff --git a/docs/agent-chat.md b/docs/agent-chat.md index c9e39630f..9a03b6908 100644 --- a/docs/agent-chat.md +++ b/docs/agent-chat.md @@ -195,7 +195,11 @@ The agent reaches the open design only through SolidCraft's own tools (the (with **Copy**), bold, italic, `code` and links (web links only; each link is a button that opens it in your browser). What you type, and the agent's thoughts, stay plain text. - **Attach view:** tick it to send a picture of the current view with your message, for agents - that take pictures. + that take pictures. The picture is taken when the agent starts on the message and then shows as + a small thumbnail under it. Hold the pointer over the thumbnail for a bigger preview that stays + while you do; click it to open the viewer (the message's pictures one at a time, with **‹** and + **›** or the arrow keys, closed with Esc or a click outside). A long message scrolls inside its + box, so **Send** and the buttons beside it stay in view. ## Permissions