Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ futures-util = "0.3.31"
image = { version = "0.25.10", default-features = false, features = ["gif", "jpeg", "png", "webp"] }
opentelemetry = "0.32.0"
pgvector = "0.4.1"
pulldown-cmark = "0.13.5"
pulldown-cmark = "0.13.6"
quick-xml = { version = "0.41.0", features = ["serialize"] }
rand = "0.10.1"
redis = "1.2.1"
Expand Down
4 changes: 3 additions & 1 deletion crates/openplotva-app/src/agent_tools.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! Dialog tools scoped to the authenticated requester and current chat.
use openplotva_dialog::{
DialogToolbox, ToolContext, ToolResult, ToolStep, ToolboxFuture, sanitize_tool_text,
verbatim_draw_caption,
};
use openplotva_memory::{CardInput, ObservationScope, RetrievalScope};
use openplotva_storage::{PostgresHistoryStore, PostgresMemoryStore};
Expand Down Expand Up @@ -56,7 +57,8 @@ impl DialogToolbox for AgentContextTools {
user_id: ctx.user_id,
user_full_name: ctx.user_full_name,
prompt: step.prompt,
caption: sanitize_tool_text(&step.caption),
caption: verbatim_draw_caption(&ctx.message_meta)
.unwrap_or_else(|| sanitize_tool_text(&step.caption)),
thread_id: ctx.thread_id,
is_forum: ctx.thread_id.is_some(),
})
Expand Down
10 changes: 9 additions & 1 deletion crates/openplotva-app/src/dialog_jobs/input.rs
Original file line number Diff line number Diff line change
Expand Up @@ -929,6 +929,10 @@ pub(crate) fn canonical_dialog_job_params(
&& !requested_tool.is_empty()
{
message.meta.requested_tool = requested_tool.to_owned();
message.meta.draw_request = scheduled.meta["draw_request"]
.as_str()
.unwrap_or_default()
.to_owned();
}
let mut params = scheduled.clone();
params.message_text = message.text;
Expand Down Expand Up @@ -1557,7 +1561,7 @@ mod context_artifact_tests {
user_full_name: "User".into(),
message_text: "draw a fish".into(),
original_text: original.into(),
meta: serde_json::json!({"requested_tool":"draw_image"}),
meta: serde_json::json!({"requested_tool":"draw_image","draw_request":"a fish"}),
max_output_tokens: 321,
thread_id: None,
};
Expand All @@ -1571,6 +1575,10 @@ mod context_artifact_tests {
);
let input = dialog_input_from_job_params_at(&canonical, OffsetDateTime::UNIX_EPOCH);
assert_eq!(input.message.meta.requested_tool, expected);
assert_eq!(
input.message.meta.draw_request,
if expected.is_empty() { "" } else { "a fish" }
);
assert_eq!(input.message.text, text);
assert_eq!(input.user.id, user_id);
}
Expand Down
35 changes: 23 additions & 12 deletions crates/openplotva-app/src/dialog_messages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1854,6 +1854,7 @@ where
Some((
"draw_api",
format!("Requested tool: draw_api. Prompt: {}", parsed.rest_text),
parsed.rest_text.trim().to_owned(),
))
} else if let Some(topic) = direct_song_shortcut_topic(
message,
Expand All @@ -1868,6 +1869,7 @@ where
"Requested tool: generate_song. Topic: {}",
resolve_song_topic(message, &topic).unwrap_or_default()
),
String::new(),
))
} else {
direct_image_shortcut(
Expand All @@ -1881,6 +1883,7 @@ where
DirectImageShortcut::Draw(prompt) => (
"draw_image",
format!("Requested tool: draw_image. Prompt: {prompt}"),
prompt,
),
DirectImageShortcut::Edit(prompt) => (
"draw_image",
Expand All @@ -1891,11 +1894,13 @@ where
reply_context.as_ref().unwrap_or(&context)
)
),
String::new(),
),
})
};
if let Some((tool, intent)) = intent {
if let Some((tool, intent, draw_request)) = intent {
context.meta.requested_tool = tool.to_owned();
context.meta.draw_request = draw_request;
context.meta.annotation.push_str(&format!("\n{intent}"));
}

Expand Down Expand Up @@ -6542,17 +6547,21 @@ mod tests {
#[tokio::test]
async fn content_shortcuts_enter_the_agent_with_explicit_intent()
-> Result<(), Box<dyn std::error::Error>> {
for (text, tool) in [
("!song neon rain", "generate_song"),
("!song", "generate_song"),
("/song@plotva_bot neon rain", "generate_song"),
("@plotva_bot песня про море", "generate_song"),
("!draw red fish", "draw_image"),
("@plotva_bot нарисуй рыбу", "draw_image"),
("@plotva_bot пожалуйста, нарисуй рыбу", "draw_image"),
("@plotva_bot можешь нарисовать рыбу", "draw_image"),
("@plotva_bot сгенерируй картинку рыбы", "draw_image"),
("@plotva_bot % red fish", "draw_api"),
for (text, tool, draw_request) in [
("!song neon rain", "generate_song", ""),
("!song", "generate_song", ""),
("/song@plotva_bot neon rain", "generate_song", ""),
("@plotva_bot песня про море", "generate_song", ""),
("!draw red fish", "draw_image", "red fish"),
("@plotva_bot нарисуй рыбу", "draw_image", "рыбу"),
("@plotva_bot пожалуйста, нарисуй рыбу", "draw_image", "рыбу"),
("@plotva_bot можешь нарисовать рыбу", "draw_image", "рыбу"),
(
"@plotva_bot сгенерируй картинку рыбы",
"draw_image",
"картинку рыбы",
),
("@plotva_bot % red fish", "draw_api", "red fish"),
] {
let scheduler = SchedulerStub::default();
let route = handle_dialog_message_update_or_else(
Expand All @@ -6572,6 +6581,7 @@ mod tests {
);
assert_eq!(scheduler.calls().len(), 1, "{text}");
assert_eq!(scheduler.metas()[0].requested_tool, tool, "{text}");
assert_eq!(scheduler.metas()[0].draw_request, draw_request, "{text}");
assert!(
scheduler.metas()[0].annotation.contains(tool),
"{text}: {:?}",
Expand Down Expand Up @@ -6601,6 +6611,7 @@ mod tests {
);
let metas = scheduler.metas();
assert_eq!(metas[0].requested_tool, "");
assert_eq!(metas[0].draw_request, "");
assert!(!metas[0].annotation.contains("draw_api"), "{metas:?}");
Ok(())
}
Expand Down
70 changes: 67 additions & 3 deletions crates/openplotva-app/src/dialog_tools.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use openplotva_dialog::{
TOOL_RESULT_STATUS_FAILED, TOOL_RESULT_STATUS_NOOP, TOOL_RESULT_STATUS_OK,
TOOL_RESULT_STATUS_QUEUED, ToolError, ToolResult, ToolSideEffect, ToolboxError, ToolboxFuture,
VisionRequest, is_internal_not_scheduled_instruction, sanitize_tool_text,
user_facing_not_scheduled_message,
user_facing_not_scheduled_message, verbatim_draw_caption,
};
use openplotva_history::{StoredSummary, SummaryContent};
use openplotva_taskman::{
Expand Down Expand Up @@ -2535,7 +2535,6 @@ where
Ok(attachments) => attachments,
Err(reason) => return Ok(ToolResult::failed("draw_image_source_invalid", reason)),
};
let original_prompt = sanitize_tool_text(&req.caption);
let mut request = DrawImageScheduleRequest {
agent_gift: req.context.message_meta.agent_gift,
chat_id: req.context.chat_id,
Expand All @@ -2551,8 +2550,13 @@ where
negative_prompt: sanitize_tool_text(&req.negative_prompt),
aspect_ratio: sanitize_tool_text(&req.aspect_ratio),
seed: sanitize_tool_text(&req.seed),
original_prompt,
original_prompt: sanitize_tool_text(&req.caption),
};
if let Some(caption) = verbatim_draw_caption(&req.context.message_meta)
&& (!request_has_image_attachment(&request) || request.original_prompt.is_empty())
{
request.original_prompt = caption;
}
if request_has_image_attachment(&request)
&& !request.agent_gift
&& scheduler
Expand Down Expand Up @@ -4544,6 +4548,66 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn short_explicit_text_to_image_request_keeps_the_users_words_as_caption()
-> Result<(), ToolboxError> {
let long_request =
"огромный рыжий кот в сапогах скачет на ракете над ночной Москвой под салютом";
for (draw_request, attachments, caption, expected) in [
("собаквака", Vec::new(), "Собака-квака", "собаквака"),
(
"кот, который съел всю сметану",
Vec::new(),
"",
"кот, который съел всю сметану",
),
(
long_request,
Vec::new(),
"Рыжий кот на ракете",
"Рыжий кот на ракете",
),
("", Vec::new(), "Собака-квака", "Собака-квака"),
(
"в стиле аниме",
vec![image_attachment("source-photo")],
"Портрет в стиле аниме",
"Портрет в стиле аниме",
),
] {
let scheduler = Arc::new(ImageSchedulerStub::successful(DrawImageScheduleResult {
status: "scheduled".to_owned(),
..Default::default()
}));
let tools = toolbox(None).with_image_scheduler(scheduler.clone());
let result = tools
.draw_image(DrawRequest {
context: ToolContext {
message_text: format!("Плотва, нарисуй {draw_request}"),
message_meta: ChatMessageMeta {
requested_tool: "draw_image".to_owned(),
draw_request: draw_request.to_owned(),
attachments,
..Default::default()
},
..context()
},
prompt: "Enriched visual instructions, cinematic lighting".into(),
caption: caption.into(),
..Default::default()
})
.await?;
assert_eq!(result.status, TOOL_RESULT_STATUS_QUEUED, "{draw_request}");
let calls = scheduler.calls();
assert_eq!(
calls[0].prompt,
"Enriched visual instructions, cinematic lighting"
);
assert_eq!(calls[0].original_prompt, expected, "{draw_request}");
}
Ok(())
}

#[tokio::test]
async fn image_style_followup_edits_the_selected_result_with_a_separate_caption()
-> Result<(), ToolboxError> {
Expand Down
2 changes: 2 additions & 0 deletions crates/openplotva-app/src/dialog_turn/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,7 @@ where
) {
tool_context.message_id = message["message_id"].as_i64().and_then(|id| i32::try_from(id).ok()).unwrap_or(tool_context.message_id);
tool_context.message_meta.requested_tool = message["meta"]["requested_tool"].as_str().unwrap_or_default().to_owned();
tool_context.message_meta.draw_request = message["meta"]["draw_request"].as_str().unwrap_or_default().to_owned();
requested_tool_call_offset = recorded_tool_calls.len();
if let Some(text) = message["text"].as_str() { tool_context.message_text = text.to_owned(); }
}
Expand All @@ -660,6 +661,7 @@ where
) {
tool_context.message_id = params.message_id;
tool_context.message_meta.requested_tool = params.meta["requested_tool"].as_str().unwrap_or_default().to_owned();
tool_context.message_meta.draw_request = params.meta["draw_request"].as_str().unwrap_or_default().to_owned();
requested_tool_call_offset = recorded_tool_calls.len();
tool_context.message_text = if params.original_text.trim().is_empty() { params.message_text.clone() } else { params.original_text.clone() };
}
Expand Down
3 changes: 3 additions & 0 deletions crates/openplotva-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,9 @@ pub struct ChatMessageMeta {
/// Explicit action recognized by the trusted message router.
#[serde(default, skip_serializing_if = "String::is_empty")]
pub requested_tool: String,
/// Verbatim request text of an explicit draw command, recognized by the trusted message router.
#[serde(default, skip_serializing_if = "String::is_empty")]
pub draw_request: String,
/// Optional annotation.
#[serde(default, skip_serializing_if = "String::is_empty")]
pub annotation: String,
Expand Down
69 changes: 69 additions & 0 deletions crates/openplotva-dialog/src/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@ use crate::{
ToolboxError, VisionRequest,
};

/// Longest explicit draw request, in words, that is shown as the caption unchanged.
pub const VERBATIM_DRAW_CAPTION_MAX_WORDS: usize = 10;

/// Caption for a short explicit draw request: the user's own words, unchanged.
///
/// Longer requests return `None`; the model writes a concise caption for them.
#[must_use]
pub fn verbatim_draw_caption(meta: &ChatMessageMeta) -> Option<String> {
let request = crate::sanitize_tool_text(&meta.draw_request);
let words = request.split_whitespace().count();
(words > 0 && words <= VERBATIM_DRAW_CAPTION_MAX_WORDS).then_some(request)
}

/// Dispatch one tool step to the toolbox.
pub async fn dispatch_dialog_tool(
toolbox: &dyn DialogToolbox,
Expand All @@ -23,6 +36,7 @@ pub async fn dispatch_dialog_tool(
) -> Result<ToolResult, ToolboxError> {
if matches!(step.step.as_str(), STEP_DRAW_IMAGE | "draw_api")
&& crate::sanitize_tool_text(&step.caption).is_empty()
&& verbatim_draw_caption(&meta.message_meta).is_none()
{
return Ok(ToolResult::failed(
"image_caption_required",
Expand Down Expand Up @@ -358,6 +372,61 @@ mod tests {
);
}

fn draw_request_meta(request: &str) -> ChatMessageMeta {
ChatMessageMeta {
draw_request: request.to_owned(),
..ChatMessageMeta::default()
}
}

#[test]
fn verbatim_draw_caption_keeps_requests_up_to_ten_words() {
for (request, expected) in [
("собаквака", Some("собаквака")),
(" кот в сапогах ", Some("кот в сапогах")),
(
"раз два три четыре пять шесть семь восемь девять десять",
Some("раз два три четыре пять шесть семь восемь девять десять"),
),
(
"раз два три четыре пять шесть семь восемь девять десять одиннадцать",
None,
),
(" ", None),
("", None),
] {
assert_eq!(
verbatim_draw_caption(&draw_request_meta(request)).as_deref(),
expected,
"{request:?}"
);
}
}

#[test]
fn short_explicit_draw_request_does_not_need_a_model_caption() {
let toolbox = ImageRecorder::default();
let meta = ToolContext {
message_text: "Плотва, нарисуй собаквака".to_owned(),
message_meta: draw_request_meta("собаквака"),
..ToolContext::default()
};
for tool in [STEP_DRAW_IMAGE, "draw_api"] {
let result = poll_ready(dispatch_dialog_tool(
&toolbox,
&meta,
&ToolStep {
step: tool.to_owned(),
prompt: "A dog-frog hybrid, cinematic lighting".to_owned(),
..ToolStep::default()
},
))
.expect("image dispatch");
assert!(result.error.is_none(), "{tool}: {:?}", result.error);
}
assert_eq!(toolbox.0.lock().expect("images").len(), 2);
}

#[derive(Default)]
struct MediaRecorder(Mutex<Vec<String>>);

Expand Down
2 changes: 1 addition & 1 deletion crates/openplotva-dialog/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ pub mod tool_telemetry;
pub mod transcript;
pub mod turn;

pub use dispatch::dispatch_dialog_tool;
pub use dispatch::{VERBATIM_DRAW_CAPTION_MAX_WORDS, dispatch_dialog_tool, verbatim_draw_caption};
pub use leak_guard::{DialogLeak, ReplyLeakGuard, ReplyLeakGuardBuilder};

pub use history::{
Expand Down
Loading