Thanks @Rec0iL.
This commit is contained in:
1 parent
55bfeb71a7
commit
f498d23a79
4 files changed
+429
-85
No files matched your search
@@ -56,6 +56,8 @@ const FILE_NAMES_ROOM: usize = 2_000;
|
||||
const APPROVAL_TIMEOUT_MS: f64 = 240_000.0;
|
||||
/// Size of the picture of the view sent with a message.
|
||||
const VIEW_IMAGE: (u32, u32) = (960, 600);
|
||||
/// A picture kept for the user to open later, at most this big.
|
||||
const MAX_PICTURE_BYTES: usize = 4 << 20;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum State {
|
||||
@@ -93,11 +95,58 @@ impl State {
|
||||
}
|
||||
}
|
||||
|
||||
/// A picture shown with a message: what the agent got (a picture of the view, or any other
|
||||
/// attachment) as PNG, and a note shown with it in the viewer. The thumbnail and the viewer share
|
||||
/// one texture, made when the picture is first drawn.
|
||||
#[derive(Clone)]
|
||||
pub struct Picture {
|
||||
png: Arc<Vec<u8>>,
|
||||
note: String,
|
||||
texture: Arc<std::sync::OnceLock<Option<egui::TextureHandle>>>,
|
||||
}
|
||||
|
||||
impl Picture {
|
||||
pub fn new(png: Vec<u8>, note: impl Into<String>) -> Self {
|
||||
Self { png: Arc::new(png), note: note.into(), texture: Arc::default() }
|
||||
}
|
||||
|
||||
/// The picture as a texture, `None` when it can't be decoded.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn texture(&self, ctx: &egui::Context) -> Option<egui::TextureHandle> {
|
||||
self.texture
|
||||
.get_or_init(|| {
|
||||
let rgba = image::load_from_memory_with_format(&self.png, image::ImageFormat::Png).ok()?.to_rgba8();
|
||||
let size = [usize::try_from(rgba.width()).ok()?, usize::try_from(rgba.height()).ok()?];
|
||||
Some(ctx.load_texture(
|
||||
"sc_agent_chat_picture",
|
||||
egui::ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()),
|
||||
egui::TextureOptions::LINEAR,
|
||||
))
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Picture {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "Picture({} bytes, {:?})", self.png.len(), self.note)
|
||||
}
|
||||
}
|
||||
|
||||
/// The viewer: the pictures of one message, and the one shown.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Gallery {
|
||||
pictures: Vec<Picture>,
|
||||
at: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Item {
|
||||
User {
|
||||
text: String,
|
||||
image: bool,
|
||||
/// The view's picture, kept once it is taken (when the agent starts on the message).
|
||||
pictures: Vec<Picture>,
|
||||
/// The names of the files that went with it.
|
||||
files: Vec<String>,
|
||||
},
|
||||
@@ -202,6 +251,8 @@ pub struct AgentChat {
|
||||
pub open: bool,
|
||||
pub items: Vec<Item>,
|
||||
pub input: String,
|
||||
/// The viewer, open on one message's pictures.
|
||||
gallery: Option<Gallery>,
|
||||
state: Option<State>,
|
||||
agent: String,
|
||||
images: bool,
|
||||
@@ -209,8 +260,9 @@ pub struct AgentChat {
|
||||
pub attach_view: bool,
|
||||
auth: Vec<acp::AuthMethod>,
|
||||
auth_status: Option<String>,
|
||||
/// A message typed before the agent was ready, with its files.
|
||||
queued: Option<(String, bool, Vec<attach::Attachment>)>,
|
||||
/// A message typed before the agent was ready: its text, whether the view goes with it, its
|
||||
/// place in the conversation, and its files.
|
||||
queued: Option<(String, bool, Option<usize>, Vec<attach::Attachment>)>,
|
||||
/// Messages typed while the agent works, in order. They go as one prompt when the turn ends
|
||||
/// (or at once with Send now); after Stop they wait for Send.
|
||||
pub pending: Vec<Pending>,
|
||||
@@ -231,7 +283,7 @@ pub struct AgentChat {
|
||||
/// Where the panel was drawn last (for its shortcuts).
|
||||
pub panel_rect: Option<egui::Rect>,
|
||||
/// The last message sent, kept to send again after a sign-in (its files too).
|
||||
last: Option<(String, bool, Vec<attach::Attachment>)>,
|
||||
last: Option<(String, bool, Option<usize>, Vec<attach::Attachment>)>,
|
||||
/// Files for the next message (pictures, text files, imported parts), shown as chips.
|
||||
pub(crate) attachments: Vec<attach::Attachment>,
|
||||
/// Pictures pasted so far, to name the next one.
|
||||
@@ -347,6 +399,17 @@ impl AgentChat {
|
||||
}
|
||||
self.items.push(item);
|
||||
}
|
||||
/// Keep the picture just sent with the message at `at` in the conversation (the user can open
|
||||
/// it). A message sent again (after a sign-in) has its picture already: nothing is added, and
|
||||
/// never to another message.
|
||||
fn keep_picture(&mut self, at: Option<usize>, png: Vec<u8>) {
|
||||
if let Some(Item::User { image: true, pictures, .. }) = at.and_then(|i| self.items.get_mut(i))
|
||||
&& pictures.is_empty()
|
||||
&& png.len() <= MAX_PICTURE_BYTES
|
||||
{
|
||||
pictures.push(Picture::new(png, ""));
|
||||
}
|
||||
}
|
||||
fn notice(&mut self, text: impl Into<String>, error: bool) {
|
||||
self.push(Item::Notice { text: text.into(), error });
|
||||
}
|
||||
@@ -704,11 +767,13 @@ pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) {
|
||||
/// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it
|
||||
/// cut in on its previous step).
|
||||
fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec<attach::Attachment>, interrupted: bool) {
|
||||
app.agent_chat.push(Item::User { text: text.clone(), image, files: files.iter().map(|f| f.name.clone()).collect() });
|
||||
app.agent_chat.push(Item::User { text: text.clone(), image, pictures: Vec::new(), files: files.iter().map(|f| f.name.clone()).collect() });
|
||||
// The message's place in the conversation: its picture is kept there when it is taken.
|
||||
let at = app.agent_chat.items.len().checked_sub(1);
|
||||
match app.agent_chat.state() {
|
||||
State::Ready => start_turn_with(app, text, image, files, interrupted.then_some(INTERRUPT_NOTE)),
|
||||
State::Ready => start_turn_with(app, text, image, at, files, interrupted.then_some(INTERRUPT_NOTE)),
|
||||
s => {
|
||||
app.agent_chat.queued = Some((text, image, files));
|
||||
app.agent_chat.queued = Some((text, image, at, files));
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if !s.running() {
|
||||
connect(app, ctx);
|
||||
@@ -775,6 +840,7 @@ pub fn new_chat(app: &mut SolveApp) {
|
||||
let chat = &mut app.agent_chat;
|
||||
chat.confirm_new = false;
|
||||
chat.items.clear();
|
||||
chat.gallery = None;
|
||||
chat.pending.clear();
|
||||
chat.attachments.clear();
|
||||
chat.loading.clear();
|
||||
@@ -823,14 +889,14 @@ fn after_turn(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
}
|
||||
}
|
||||
|
||||
fn start_turn(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>) {
|
||||
start_turn_with(app, text, image, files, None);
|
||||
fn start_turn(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>) {
|
||||
start_turn_with(app, text, image, at, files, None);
|
||||
}
|
||||
|
||||
/// Start a message; `note` goes before the text for the agent, not into the conversation. The
|
||||
/// files are named after the text (text files in full), and pictures go as images to an agent
|
||||
/// that takes them.
|
||||
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>, note: Option<&str>) {
|
||||
/// that takes them. `at` is the message's place in the conversation, where its picture is kept.
|
||||
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, at: Option<usize>, files: Vec<attach::Attachment>, note: Option<&str>) {
|
||||
let pictures = app.agent_chat.images;
|
||||
let lead = match note {
|
||||
Some(n) => format!("{n}\n\n{text}"),
|
||||
@@ -843,7 +909,10 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<att
|
||||
let mut parts = vec![PromptPart::Text(sent)];
|
||||
if image {
|
||||
match crate::control::view_png(app, VIEW_IMAGE.0, VIEW_IMAGE.1) {
|
||||
Ok(png) => parts.push(PromptPart::Image { mime: "image/png".into(), base64: backend::base64(&png) }),
|
||||
Ok(png) => {
|
||||
parts.push(PromptPart::Image { mime: "image/png".into(), base64: backend::base64(&png) });
|
||||
app.agent_chat.keep_picture(at, png);
|
||||
}
|
||||
Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"),
|
||||
}
|
||||
}
|
||||
@@ -855,7 +924,7 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<att
|
||||
let label: String = first_line.chars().take(48).collect();
|
||||
app.agent_chat.turn =
|
||||
Some(Turn { start_doc: app.session.doc.clone(), start_undo: app.session.undo.len(), changes: 0, requests: 0, limit_told: false, label });
|
||||
app.agent_chat.last = Some((text, image, files));
|
||||
app.agent_chat.last = Some((text, image, at, files));
|
||||
app.agent_chat.state = Some(State::Busy);
|
||||
app.agent_chat.stopped = false;
|
||||
app.agent_chat.steering = false;
|
||||
@@ -1258,8 +1327,8 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
if ready {
|
||||
apply_saved_model(app);
|
||||
}
|
||||
if ready && let Some((text, image, files)) = app.agent_chat.queued.take() {
|
||||
start_turn(app, text, image, files);
|
||||
if ready && let Some((text, image, at, files)) = app.agent_chat.queued.take() {
|
||||
start_turn(app, text, image, at, files);
|
||||
}
|
||||
}
|
||||
pump_tools(app);
|
||||
@@ -1478,7 +1547,7 @@ pub fn status(app: &SolveApp) -> Value {
|
||||
.items
|
||||
.iter()
|
||||
.map(|i| match i {
|
||||
Item::User { text, image, files } => json!({"user": text, "image": image, "files": files}),
|
||||
Item::User { text, image, pictures, files } => json!({"user": text, "image": image, "pictures": pictures.len(), "files": files}),
|
||||
Item::Agent { thought, text } => json!({"agent": text, "thought": thought}),
|
||||
Item::Tool { id, title, kind, status, output } => {
|
||||
json!({"tool": id, "title": title, "kind": kind, "status": status.map(|s| format!("{s:?}")), "output": output})
|
||||
|
||||
@@ -18,6 +18,12 @@ fn panel_id(left: bool) -> Id {
|
||||
Id::new(if left { "sc_agent_chat_left" } else { "sc_agent_chat_right" })
|
||||
}
|
||||
|
||||
/// The largest thumbnail of a message's picture, and its largest preview while hovered (points).
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
const THUMB_MAX: (f32, f32) = (150.0, 56.0);
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
const PREVIEW_MAX: (f32, f32) = (560.0, 380.0);
|
||||
|
||||
/// Forget the panel's own idea of its width, so the preference applies on the next frame.
|
||||
pub fn forget_width(ctx: &egui::Context) {
|
||||
ctx.data_mut(|d| {
|
||||
@@ -105,6 +111,8 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
.show(ui, |ui| body(app, ui, left));
|
||||
crate::scenario::publish_rect("agent_chat:panel", r.response.rect);
|
||||
app.agent_chat.panel_rect = Some(r.response.rect);
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
gallery_window(app, &ctx);
|
||||
app.agent_chat.drop_rect = Some(r.response.rect);
|
||||
show_drop_hint(app, &ctx);
|
||||
let w = r.response.rect.width();
|
||||
@@ -113,6 +121,88 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
}
|
||||
}
|
||||
|
||||
/// The pictures of a message as small thumbnails in a row. Hovering one shows it bigger for as long
|
||||
/// as the pointer stays; a click returns its place in the row, to open the viewer there.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn picture_strip(ui: &mut egui::Ui, n: usize, pictures: &[Picture]) -> Option<usize> {
|
||||
let t = Tokens::get();
|
||||
let mut clicked = None;
|
||||
ui.horizontal_wrapped(|ui| {
|
||||
for (i, picture) in pictures.iter().enumerate() {
|
||||
let Some(texture) = picture.texture(ui.ctx()) else {
|
||||
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
|
||||
continue;
|
||||
};
|
||||
let size = fit(texture.size_vec2(), vec2(THUMB_MAX.0, THUMB_MAX.1));
|
||||
let thumb = ui.add(egui::Image::new((texture.id(), size)).corner_radius(4.0).sense(Sense::click()));
|
||||
crate::scenario::publish_handle(&format!("agent_chat:picture:{n}:{i}"), thumb.rect.center());
|
||||
let hover_max = (ui.ctx().content_rect().size() * 0.5).min(vec2(PREVIEW_MAX.0, PREVIEW_MAX.1));
|
||||
let thumb = thumb.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_ui(|ui| {
|
||||
ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), hover_max))).corner_radius(4.0));
|
||||
if !picture.note.is_empty() {
|
||||
ui.label(&picture.note);
|
||||
}
|
||||
});
|
||||
if thumb.clicked() {
|
||||
clicked = Some(i);
|
||||
}
|
||||
}
|
||||
});
|
||||
clicked
|
||||
}
|
||||
|
||||
/// `size` scaled to fit in `max`, keeping its shape (never below one point).
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn fit(size: egui::Vec2, max: egui::Vec2) -> egui::Vec2 {
|
||||
let scale = (max.x / size.x.max(1.0)).min(max.y / size.y.max(1.0)).min(2.0);
|
||||
vec2((size.x * scale).max(1.0), (size.y * scale).max(1.0))
|
||||
}
|
||||
|
||||
/// The viewer: a message's pictures over the app, one at a time with a counter, Previous and Next
|
||||
/// (also the arrow keys), closed with the button, Esc or a click outside.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn gallery_window(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
let Some(mut gallery) = app.agent_chat.gallery.take() else { return };
|
||||
let count = gallery.pictures.len();
|
||||
gallery.at = gallery.at.min(count.saturating_sub(1));
|
||||
let (left, right) =
|
||||
ctx.input_mut(|i| (i.consume_key(egui::Modifiers::NONE, egui::Key::ArrowLeft), i.consume_key(egui::Modifiers::NONE, egui::Key::ArrowRight)));
|
||||
let mut step: isize = if left { -1 } else { isize::from(right) };
|
||||
let t = Tokens::get();
|
||||
let max = ctx.content_rect().size() * vec2(0.8, 0.7);
|
||||
let modal = egui::Modal::new(Id::new("sc_agent_chat_gallery")).show(ctx, |ui| {
|
||||
ui.set_max_width(max.x);
|
||||
ui.horizontal(|ui| {
|
||||
let prev = ui.add_enabled(count > 1, egui::Button::new("‹"));
|
||||
crate::scenario::publish_handle("agent_chat:gallery_prev", prev.rect.center());
|
||||
let next = ui.add_enabled(count > 1, egui::Button::new("›"));
|
||||
crate::scenario::publish_handle("agent_chat:gallery_next", next.rect.center());
|
||||
if prev.clicked() {
|
||||
step = -1;
|
||||
}
|
||||
if next.clicked() {
|
||||
step = 1;
|
||||
}
|
||||
ui.label(RichText::new(format!("{} / {}", gallery.at + 1, count)).color(t.text_dim));
|
||||
});
|
||||
if let Some(picture) = gallery.pictures.get(gallery.at) {
|
||||
if let Some(texture) = picture.texture(ui.ctx()) {
|
||||
let resp = ui.add(egui::Image::new((texture.id(), fit(texture.size_vec2(), max))).corner_radius(4.0));
|
||||
crate::scenario::publish_rect("agent_chat:gallery_picture", resp.rect);
|
||||
}
|
||||
if !picture.note.is_empty() {
|
||||
ui.add(egui::Label::new(&picture.note).wrap());
|
||||
}
|
||||
}
|
||||
});
|
||||
if count > 0 && step != 0 {
|
||||
gallery.at = (gallery.at as isize + step).rem_euclid(count as isize) as usize;
|
||||
}
|
||||
if !modal.should_close() {
|
||||
app.agent_chat.gallery = Some(gallery);
|
||||
}
|
||||
}
|
||||
|
||||
/// While files are dragged over the window: if they would drop on the chat, say so over it.
|
||||
fn show_drop_hint(app: &SolveApp, ctx: &egui::Context) {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
@@ -193,6 +283,8 @@ fn window(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
let t = Tokens::get();
|
||||
let frame = egui::Frame::NONE.fill(t.panel).inner_margin(egui::Margin::symmetric(8, 6));
|
||||
let r = egui::CentralPanel::default().frame(frame).show(ui, |ui| body(app, ui, false));
|
||||
// The picture viewer opens over this window, where the click was, not over the main one.
|
||||
gallery_window(app, ui.ctx());
|
||||
if class == egui::ViewportClass::EmbeddedWindow {
|
||||
// Floating inside the app: its handles are the main window's, as the panel's are.
|
||||
crate::scenario::publish_rect("agent_chat:window", r.response.rect);
|
||||
@@ -468,21 +560,12 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
auth_box(app, ui);
|
||||
}
|
||||
ui.separator();
|
||||
// The input sits at the bottom (with a row per line of file chips); the conversation takes
|
||||
// the rest.
|
||||
let input_h = 92.0 + chip_rows(app.agent_chat.files(), ui.available_width(), ui.spacing().item_spacing.x) as f32 * 32.0;
|
||||
let avail = (ui.available_height() - input_h).max(80.0);
|
||||
egui::ScrollArea::vertical().id_salt("sc_agent_chat_scroll").max_height(avail).auto_shrink([false, false]).stick_to_bottom(true).show(ui, |ui| {
|
||||
conversation(app, ui);
|
||||
});
|
||||
ui.separator();
|
||||
// The input sits at the bottom of the panel and grows upwards: a long message makes its box
|
||||
// taller up to `edit_max`, then it scrolls, and the conversation takes what is left above. It
|
||||
// is a panel of its own at the bottom, as high as its content this frame, so the buttons under
|
||||
// the box are always in view.
|
||||
let edit_max = (ui.available_height() * 0.4).clamp(48.0, 160.0);
|
||||
let busy = state == State::Busy;
|
||||
attachment_chips(app, ui);
|
||||
let edit = egui::TextEdit::multiline(&mut app.agent_chat.input)
|
||||
.hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" }))
|
||||
.desired_rows(2)
|
||||
.desired_width(f32::INFINITY)
|
||||
.id_salt("sc_agent_chat_input");
|
||||
// Enter sends (queues while the agent works); Ctrl/Cmd+Enter sends now; Esc stops. They are
|
||||
// read before the field sees them so no newline is typed.
|
||||
let focused = ui.memory(|m| m.has_focus(Id::new("sc_agent_chat_input")));
|
||||
@@ -491,52 +574,77 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
} else {
|
||||
(false, false)
|
||||
};
|
||||
let r = ui.add(edit);
|
||||
crate::scenario::publish_rect("agent_chat:input", r.rect);
|
||||
// macOS shows a window's cursors only while it is the active window: over the text box of a
|
||||
// chat window that isn't, the pointer stays an arrow. Coming from the main window, the chat
|
||||
// window becomes active, so the text cursor shows and the first click places the caret.
|
||||
let (chat_active, main_active) = (ui.ctx().input(|i| i.viewport().focused), ui.ctx().input_for(egui::ViewportId::ROOT, |i| i.viewport().focused));
|
||||
if take_focus_for_text(app.preferences.agent_chat_popped, r.hovered(), chat_active, main_active) {
|
||||
ui.ctx().send_viewport_cmd(egui::ViewportCommand::Focus);
|
||||
}
|
||||
let enter = r.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && !i.modifiers.shift);
|
||||
if esc {
|
||||
let (mut send_clicked, mut stop_clicked, mut enter) = (false, false, false);
|
||||
egui::Panel::bottom(Id::new("sc_agent_chat_composer")).resizable(false).show_separator_line(false).frame(egui::Frame::NONE).show(ui, |ui| {
|
||||
ui.separator();
|
||||
// The files for the next message (#1744), as chips of their own.
|
||||
attachment_chips(app, ui);
|
||||
let edit = egui::TextEdit::multiline(&mut app.agent_chat.input)
|
||||
.hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" }))
|
||||
.desired_rows(2)
|
||||
.desired_width(f32::INFINITY)
|
||||
.id_salt("sc_agent_chat_input");
|
||||
let r = egui::ScrollArea::vertical()
|
||||
.id_salt("sc_agent_chat_input_scroll")
|
||||
.max_height(edit_max)
|
||||
.auto_shrink([false, true])
|
||||
.show(ui, |ui| ui.add(edit))
|
||||
.inner;
|
||||
crate::scenario::publish_rect("agent_chat:input", r.rect);
|
||||
// macOS shows a window's cursors only while it is the active window: over the text box of a
|
||||
// chat window that isn't, the pointer stays an arrow. Coming from the main window, the chat
|
||||
// window becomes active, so the text cursor shows and the first click places the caret.
|
||||
let (chat_active, main_active) =
|
||||
(ui.ctx().input(|i| i.viewport().focused), ui.ctx().input_for(egui::ViewportId::ROOT, |i| i.viewport().focused));
|
||||
if take_focus_for_text(app.preferences.agent_chat_popped, r.hovered(), chat_active, main_active) {
|
||||
ui.ctx().send_viewport_cmd(egui::ViewportCommand::Focus);
|
||||
}
|
||||
enter = r.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && !i.modifiers.shift);
|
||||
ui.horizontal(|ui| {
|
||||
add_menu(app, ui);
|
||||
// A message waits for files still being read.
|
||||
let reading = !app.agent_chat.loading.is_empty();
|
||||
let send_b = ui
|
||||
.add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })))
|
||||
.on_disabled_hover_text(tr("ui.agent_chat_files_loading"));
|
||||
crate::scenario::publish_handle("agent_chat:send", send_b.rect.center());
|
||||
send_clicked = send_b.clicked();
|
||||
stop_clicked = busy && glyph_button(ui, "stopturn", tr("ui.agent_chat_stop_turn_tip")).clicked();
|
||||
// A picture of the view goes with the message to agents that take pictures.
|
||||
let images = app.agent_chat.images || !state.running();
|
||||
let attach = ui
|
||||
.add_enabled(images, egui::Checkbox::new(&mut app.agent_chat.attach_view, tr("ui.agent_chat_attach_view")))
|
||||
.on_hover_text(tr("ui.agent_chat_attach_view_tip"))
|
||||
.on_disabled_hover_text(tr("ui.agent_chat_no_images"));
|
||||
crate::scenario::publish_handle("agent_chat:attach", attach.rect.center());
|
||||
});
|
||||
});
|
||||
egui::ScrollArea::vertical()
|
||||
.id_salt("sc_agent_chat_scroll")
|
||||
.max_height(ui.available_height())
|
||||
.auto_shrink([false, false])
|
||||
.stick_to_bottom(true)
|
||||
.show(ui, |ui| {
|
||||
conversation(app, ui);
|
||||
});
|
||||
if esc || stop_clicked {
|
||||
cancel(app);
|
||||
}
|
||||
ui.horizontal(|ui| {
|
||||
add_menu(app, ui);
|
||||
// A message waits for files still being read.
|
||||
let reading = !app.agent_chat.loading.is_empty();
|
||||
let send_b = ui
|
||||
.add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })))
|
||||
.on_disabled_hover_text(tr("ui.agent_chat_files_loading"));
|
||||
crate::scenario::publish_handle("agent_chat:send", send_b.rect.center());
|
||||
// Files alone make a message.
|
||||
let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty();
|
||||
if reading {
|
||||
// Enter waits with the button.
|
||||
} else if (send_b.clicked() || enter || steer) && something {
|
||||
let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string();
|
||||
send(app, ui.ctx(), text);
|
||||
if steer {
|
||||
send_now(app, ui.ctx());
|
||||
}
|
||||
} else if steer {
|
||||
// Nothing typed: Ctrl+Enter sends what is already queued.
|
||||
// A message waits for files still being read, and files alone make a message.
|
||||
let reading = !app.agent_chat.loading.is_empty();
|
||||
let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty();
|
||||
if reading {
|
||||
// Enter waits with the button.
|
||||
} else if (send_clicked || enter || steer) && something {
|
||||
let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string();
|
||||
send(app, ui.ctx(), text);
|
||||
if steer {
|
||||
send_now(app, ui.ctx());
|
||||
}
|
||||
if busy && glyph_button(ui, "stopturn", tr("ui.agent_chat_stop_turn_tip")).clicked() {
|
||||
cancel(app);
|
||||
}
|
||||
// A picture of the view goes with the message to agents that take pictures.
|
||||
let images = app.agent_chat.images || !state.running();
|
||||
let attach = ui
|
||||
.add_enabled(images, egui::Checkbox::new(&mut app.agent_chat.attach_view, tr("ui.agent_chat_attach_view")))
|
||||
.on_hover_text(tr("ui.agent_chat_attach_view_tip"))
|
||||
.on_disabled_hover_text(tr("ui.agent_chat_no_images"));
|
||||
crate::scenario::publish_handle("agent_chat:attach", attach.rect.center());
|
||||
});
|
||||
} else if steer {
|
||||
// Nothing typed: Ctrl+Enter sends what is already queued.
|
||||
send_now(app, ui.ctx());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
@@ -789,17 +897,23 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
// Allow (true), Deny (false), Always allow (None).
|
||||
let mut approval: Option<(u64, Option<bool>)> = None;
|
||||
let mut undo: Option<usize> = None;
|
||||
let mut open_gallery: Option<Gallery> = None;
|
||||
let current_doc = app.session.doc.clone();
|
||||
for (n, item) in app.agent_chat.items.iter().enumerate() {
|
||||
match item {
|
||||
Item::User { text, image, files } => {
|
||||
Item::User { text, image, pictures, files } => {
|
||||
ui.add_space(6.0);
|
||||
egui::Frame::NONE.fill(t.accent_soft).corner_radius(6.0).inner_margin(egui::Margin::same(6)).show(ui, |ui| {
|
||||
ui.set_width(ui.available_width());
|
||||
if !text.is_empty() {
|
||||
ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap());
|
||||
}
|
||||
if *image {
|
||||
if !pictures.is_empty() {
|
||||
if let Some(at) = picture_strip(ui, n, pictures) {
|
||||
open_gallery = Some(Gallery { pictures: pictures.clone(), at });
|
||||
}
|
||||
} else if *image {
|
||||
// Taken when the agent starts on the message: nothing to show yet.
|
||||
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
|
||||
}
|
||||
files_line(ui, files.iter().map(String::as_str));
|
||||
@@ -1001,6 +1115,9 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
if send_queue_clicked {
|
||||
send_now(app, ui.ctx());
|
||||
}
|
||||
if open_gallery.is_some() {
|
||||
app.agent_chat.gallery = open_gallery;
|
||||
}
|
||||
if let Some((n, o)) = answer {
|
||||
answer_permission(app, n, o.as_deref());
|
||||
}
|
||||
@@ -1027,13 +1144,6 @@ fn files_line<'a>(ui: &mut egui::Ui, names: impl Iterator<Item = &'a str>) {
|
||||
/// The widest a file chip gets; a narrower chat makes them narrower.
|
||||
const CHIP_W: f32 = 170.0;
|
||||
|
||||
/// How many rows `n` chips take in a chat `avail` wide (`gap` between them).
|
||||
fn chip_rows(n: usize, avail: f32, gap: f32) -> usize {
|
||||
let avail = avail.max(1.0);
|
||||
let per_row = (((avail + gap) / (CHIP_W.min(avail) + gap)).floor() as usize).max(1);
|
||||
n.div_ceil(per_row)
|
||||
}
|
||||
|
||||
/// The files for the next message, as chips in rows that fit the chat: a picture's thumbnail (or
|
||||
/// a file mark), the name cut to fit, and a × that takes the file off. Laid out by hand: egui
|
||||
/// wraps plain widgets, not framed groups, and one row wider than the chat widens all below it.
|
||||
|
||||
@@ -26,12 +26,12 @@ fn streamed_text_joins_and_tool_updates_change_their_call() {
|
||||
#[test]
|
||||
fn a_sign_in_request_keeps_the_question_for_after_the_login() {
|
||||
let mut c = chat();
|
||||
c.last = Some(("make a cube".into(), false, vec![attach::Attachment::imported("motor.step".into())]));
|
||||
c.last = Some(("make a cube".into(), false, None, vec![attach::Attachment::imported("motor.step".into())]));
|
||||
c.state = Some(State::Busy);
|
||||
assert!(c.on_event(Event::TurnEnded { reason: "auth".into() }).is_some());
|
||||
c.on_event(Event::AuthRequired { methods: vec![] });
|
||||
assert_eq!(c.state(), State::NeedsAuth);
|
||||
let (text, image, files) = c.queued.clone().expect("queued for after the login");
|
||||
let (text, image, _, files) = c.queued.clone().expect("queued for after the login");
|
||||
assert_eq!((text.as_str(), image), ("make a cube", false));
|
||||
assert_eq!(files.iter().map(|f| f.name.as_str()).collect::<Vec<_>>(), ["motor.step"], "its files go again too");
|
||||
}
|
||||
@@ -39,7 +39,7 @@ fn a_sign_in_request_keeps_the_question_for_after_the_login() {
|
||||
#[test]
|
||||
fn a_plan_update_replaces_the_turns_plan() {
|
||||
let mut c = chat();
|
||||
c.push(Item::User { text: "go".into(), image: false, files: Vec::new() });
|
||||
c.push(Item::User { text: "go".into(), image: false, pictures: Vec::new(), files: Vec::new() });
|
||||
c.on_event(Event::Plan(vec![("a".into(), "pending".into())]));
|
||||
c.on_event(Event::Text { thought: false, text: "working".into() });
|
||||
c.on_event(Event::Plan(vec![("a".into(), "completed".into())]));
|
||||
@@ -68,7 +68,7 @@ fn app_in_a_turn() -> (SolveApp, Sender<ControlRequest>) {
|
||||
let (tx, rx) = channel();
|
||||
app.agent_chat.tools_rx = Some(rx);
|
||||
app.agent_chat.state = Some(State::Busy);
|
||||
start_turn(&mut app, "two boxes".into(), false, Vec::new());
|
||||
start_turn(&mut app, "two boxes".into(), false, None, Vec::new());
|
||||
(app, tx)
|
||||
}
|
||||
|
||||
@@ -1380,3 +1380,164 @@ fn an_agents_markdown_is_drawn_as_blocks() {
|
||||
h.app.agent_chat.items.push(Item::Agent { thought: false, text: "*".repeat(50_000) + &"[".repeat(5_000) + &"`a".repeat(5_000) });
|
||||
h.frames(3);
|
||||
}
|
||||
|
||||
/// A long message scrolls inside its box: the buttons under the box stay in the window (#2224).
|
||||
#[test]
|
||||
fn a_long_message_keeps_the_buttons_in_the_window() {
|
||||
let mut h = crate::scenario::Harness::new();
|
||||
h.call("ui.resize", json!({"width": 1600.0, "height": 1000.0}));
|
||||
h.call("ui.agent_chat", json!({"open": true}));
|
||||
h.frames(8);
|
||||
let [_, rest] = h.widget("Send").expect("the Send button is drawn");
|
||||
let box_before = h.app.agent_chat.panel_rect.map(|r| r.bottom());
|
||||
h.app.agent_chat.input = "Please make the plate thicker and then add a fillet on every outside edge.\n".repeat(60);
|
||||
h.frames(8);
|
||||
let window = 1000.0;
|
||||
for name in ["Send", "Attach view"] {
|
||||
let [_, y] = h.widget(name).unwrap_or_else(|| panic!("{name} is drawn"));
|
||||
assert!(y < window, "{name} is at {y}, below the window's bottom at {window}");
|
||||
// The box is pinned to the bottom and grows upwards: the buttons do not move.
|
||||
assert!((y - rest).abs() < 1.0, "{name} moved from {rest} to {y}");
|
||||
}
|
||||
assert_eq!(h.app.agent_chat.panel_rect.map(|r| r.bottom()), box_before, "the panel keeps its size");
|
||||
}
|
||||
|
||||
fn test_png(colour: [u8; 4], width: u32) -> Vec<u8> {
|
||||
let mut png = Vec::new();
|
||||
let image = image::RgbaImage::from_pixel(width, 4, image::Rgba(colour));
|
||||
image::DynamicImage::ImageRgba8(image).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).expect("a PNG");
|
||||
png
|
||||
}
|
||||
|
||||
/// A message's pictures show as thumbnails; a click opens the viewer on that one, where Next, the
|
||||
/// arrow keys and Esc work (#2224). A message whose picture is not taken yet keeps a plain caption.
|
||||
#[test]
|
||||
fn a_messages_pictures_open_in_a_viewer() {
|
||||
let mut h = crate::scenario::Harness::new();
|
||||
h.call("ui.agent_chat", json!({"open": true}));
|
||||
h.app.agent_chat.items.push(Item::User { text: "waiting".into(), image: true, pictures: Vec::new(), files: Vec::new() });
|
||||
h.frames(3);
|
||||
let [x, y] = h.widget("View attached").expect("the plain caption is drawn");
|
||||
h.call("ui.click", json!({"x": x, "y": y}));
|
||||
h.frames(3);
|
||||
assert!(h.app.agent_chat.gallery.is_none(), "no picture yet: nothing opens");
|
||||
|
||||
h.app.agent_chat.items.clear();
|
||||
let pictures = vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view"), Picture::new(test_png([30, 30, 200, 255], 16), "")];
|
||||
h.app.agent_chat.items.push(Item::User { text: "look".into(), image: true, pictures, files: Vec::new() });
|
||||
h.frames(4);
|
||||
assert!(h.widget("View attached").is_none(), "thumbnails replace the caption");
|
||||
let point = |name: &str| {
|
||||
let p = crate::scenario::handle_at(name).unwrap_or_else(|| panic!("{name} is drawn"));
|
||||
json!({"x": p.x, "y": p.y})
|
||||
};
|
||||
// Hovering is not clicking.
|
||||
let second = point("agent_chat:picture:0:1");
|
||||
h.call("ui.move", second.clone());
|
||||
h.frames(3);
|
||||
assert!(h.app.agent_chat.gallery.is_none());
|
||||
h.call("ui.click", second);
|
||||
h.frames(4);
|
||||
assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(1), "the click opens the viewer on that picture");
|
||||
assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "the picture is drawn big");
|
||||
// Next wraps round; the arrow keys step too.
|
||||
let next = point("agent_chat:gallery_next");
|
||||
h.call("ui.click", next);
|
||||
h.frames(3);
|
||||
assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(0));
|
||||
h.call("ui.key", json!({"key": "ArrowRight"}));
|
||||
h.frames(3);
|
||||
assert_eq!(h.app.agent_chat.gallery.as_ref().map(|g| g.at), Some(1));
|
||||
let state = h.call("ui.agent_chat", json!({}));
|
||||
assert_eq!(state["result"]["items"][0]["pictures"], 2);
|
||||
h.call("ui.key", json!({"key": "Escape"}));
|
||||
h.frames(3);
|
||||
assert!(h.app.agent_chat.gallery.is_none(), "Esc closes the viewer");
|
||||
// New chat closes it with the conversation.
|
||||
h.app.agent_chat.gallery = Some(Gallery { pictures: vec![Picture::new(test_png([0, 0, 0, 255], 4), "")], at: 0 });
|
||||
super::new_chat(&mut h.app);
|
||||
assert!(h.app.agent_chat.gallery.is_none());
|
||||
}
|
||||
|
||||
/// A message's picture is kept on exactly the message it was taken for: not an earlier one that
|
||||
/// also wanted a picture, not one that has it already, and nothing that isn't such a message.
|
||||
#[test]
|
||||
fn the_picture_goes_to_the_message_it_was_taken_for() {
|
||||
let mut chat = AgentChat::default();
|
||||
chat.push(Item::User { text: "no picture".into(), image: false, pictures: Vec::new(), files: Vec::new() });
|
||||
chat.push(Item::User { text: "first".into(), image: true, pictures: vec![Picture::new(vec![1], "")], files: Vec::new() });
|
||||
chat.push(Item::User { text: "second".into(), image: true, pictures: Vec::new(), files: Vec::new() });
|
||||
chat.push(Item::User { text: "third".into(), image: true, pictures: Vec::new(), files: Vec::new() });
|
||||
let sizes = |chat: &AgentChat| -> Vec<usize> {
|
||||
chat.items
|
||||
.iter()
|
||||
.filter_map(|i| match i {
|
||||
Item::User { pictures, .. } => Some(pictures.iter().map(|p| p.png.len()).sum()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
// The picture of the third message goes to the third, though the second still waits for one.
|
||||
chat.keep_picture(Some(3), vec![2, 3]);
|
||||
assert_eq!(sizes(&chat), [0, 1, 0, 2]);
|
||||
// The same message again (it was sent again): nothing is added, nor moved to an older message.
|
||||
chat.keep_picture(Some(3), vec![9; 5]);
|
||||
assert_eq!(sizes(&chat), [0, 1, 0, 2]);
|
||||
// A message that did not want one, no message, or a place that is gone.
|
||||
chat.keep_picture(Some(0), vec![1]);
|
||||
chat.keep_picture(None, vec![1]);
|
||||
chat.keep_picture(Some(99), vec![1]);
|
||||
assert_eq!(sizes(&chat), [0, 1, 0, 2]);
|
||||
chat.keep_picture(Some(2), vec![0; super::MAX_PICTURE_BYTES + 1]);
|
||||
assert_eq!(sizes(&chat), [0, 1, 0, 2], "an oversized picture is not kept");
|
||||
}
|
||||
|
||||
/// A message sent again after a sign-in is still the message it was: its turn points at its own
|
||||
/// place in the conversation, so its picture is kept there, once, and never on an older message that
|
||||
/// wanted one (#2224 review). The picture itself can't be drawn without a renderer in a test, so the
|
||||
/// place a turn is started for is what is checked.
|
||||
#[test]
|
||||
fn a_message_sent_again_after_sign_in_stays_the_message_it_was() {
|
||||
let (mut app, script, ctx) = app_with_script();
|
||||
app.agent_chat.images = true;
|
||||
// An older message that wanted a picture and has none.
|
||||
app.agent_chat.push(Item::User { text: "older".into(), image: true, pictures: Vec::new(), files: Vec::new() });
|
||||
app.agent_chat.attach_view = true;
|
||||
send(&mut app, &ctx, "the new one".into());
|
||||
let place = app.agent_chat.items.len() - 1;
|
||||
assert_eq!(app.agent_chat.last, Some(("the new one".into(), true, Some(place), Vec::new())), "the turn knows its message");
|
||||
// The agent asked for a sign-in; after it the message goes again, as it is done on a sign-in.
|
||||
ended(&script, "");
|
||||
poll(&mut app, &ctx);
|
||||
app.agent_chat.queued = app.agent_chat.last.take();
|
||||
script.report(Event::Ready { agent: "fake".into(), images: true });
|
||||
poll(&mut app, &ctx);
|
||||
assert_eq!(app.agent_chat.last, Some(("the new one".into(), true, Some(place), Vec::new())), "sent again, still for the same message");
|
||||
assert_eq!(app.agent_chat.items.iter().filter(|i| matches!(i, Item::User { .. })).count(), 2, "not a new message");
|
||||
// Its picture goes there, once; the older message gets none.
|
||||
app.agent_chat.keep_picture(Some(place), test_png([1, 2, 3, 255], 4));
|
||||
app.agent_chat.keep_picture(Some(place), test_png([1, 2, 3, 255], 4));
|
||||
let counts: Vec<usize> =
|
||||
app.agent_chat.items.iter().filter_map(|i| if let Item::User { pictures, .. } = i { Some(pictures.len()) } else { None }).collect();
|
||||
assert_eq!(counts, [0, 1]);
|
||||
}
|
||||
|
||||
/// The picture viewer opens over the chat's own window too, not only over the docked panel (#1745).
|
||||
#[test]
|
||||
fn the_viewer_opens_in_the_popped_out_window() {
|
||||
let mut h = crate::scenario::Harness::new();
|
||||
h.call("ui.agent_chat", json!({"open": true}));
|
||||
h.app.preferences.agent_chat_popped = true;
|
||||
h.app.agent_chat.items.push(Item::User {
|
||||
text: "look".into(),
|
||||
image: true,
|
||||
pictures: vec![Picture::new(test_png([200, 30, 30, 255], 8), "first view")],
|
||||
files: Vec::new(),
|
||||
});
|
||||
h.frames(4);
|
||||
let at = crate::scenario::handle_at("agent_chat:picture:0:0").expect("the thumbnail is drawn in the window");
|
||||
h.call("ui.click", json!({"x": at.x, "y": at.y}));
|
||||
h.frames(4);
|
||||
assert!(h.app.agent_chat.gallery.is_some(), "the click opened the viewer");
|
||||
assert!(crate::scenario::rect_of("agent_chat:gallery_picture").is_some(), "and it is drawn");
|
||||
}
|
||||
+5
-1
@@ -195,7 +195,11 @@ The agent reaches the open design only through SolidCraft's own tools (the
|
||||
(with **Copy**), bold, italic, `code` and links (web links only; each link is a button that opens
|
||||
it in your browser). What you type, and the agent's thoughts, stay plain text.
|
||||
- **Attach view:** tick it to send a picture of the current view with your message, for agents
|
||||
that take pictures.
|
||||
that take pictures. The picture is taken when the agent starts on the message and then shows as
|
||||
a small thumbnail under it. Hold the pointer over the thumbnail for a bigger preview that stays
|
||||
while you do; click it to open the viewer (the message's pictures one at a time, with **‹** and
|
||||
**›** or the arrow keys, closed with Esc or a click outside). A long message scrolls inside its
|
||||
box, so **Send** and the buttons beside it stay in view.
|
||||
|
||||
## Permissions
|
||||
|
||||
|
||||
Reference in new issue
Block a user