From 50eaeb7e80b660f6f5933a068cb76918313d510e Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 21 Aug 2026 19:22:50 +0000 Subject: [PATCH 1/4] test: cleanup get_smtp_rows_for_msg() It was incorrectly converting unused row ID to MsgId type and selecting already known msg_id. --- src/test_utils.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/test_utils.rs b/src/test_utils.rs index 87e96e16d6..dd38d624cf 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -685,20 +685,18 @@ ORDER BY id" .ctx .sql .query_map_vec( - "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?", + "SELECT mime, recipients FROM smtp WHERE msg_id=?", (msg_id,), |row| { - let _id: MsgId = row.get(0)?; - let msg_id: MsgId = row.get(1)?; - let mime: String = row.get(2)?; - let recipients: String = row.get(3)?; - Ok((msg_id, mime, recipients)) + let mime: String = row.get(0)?; + let recipients: String = row.get(1)?; + Ok((mime, recipients)) }, ) .await .unwrap() .into_iter() - .map(|(msg_id, mime, recipients)| SentMessage { + .map(|(mime, recipients)| SentMessage { payload: mime, sender_msg_id: msg_id, sender_context: &self.ctx, From e65d6373a7572b78dc2672f3888ed5f3d0e9f969 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 25 Aug 2026 19:05:28 +0000 Subject: [PATCH 2/4] refactor: add Encryption.is_encrypted() --- src/mimefactory.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 3d18644c9b..ff9e1a119f 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -96,6 +96,16 @@ enum Encryption { Symmetric { shared_secret: String }, } +impl Encryption { + pub fn is_encrypted(&self) -> bool { + match self { + Self::No => false, + Self::Asymmetric { .. } => true, + Self::Symmetric { .. } => true, + } + } +} + /// Helper to construct mime messages. #[derive(Debug, Clone)] pub struct MimeFactory { @@ -281,7 +291,7 @@ pub(crate) fn render_queued_mail( let mut inner_headers: Vec = Vec::new(); let mut outer_headers: Vec = Vec::new(); - let is_encrypted = !matches!(encryption, Encryption::No); + let is_encrypted = encryption.is_encrypted(); fn add_header( name: &[u8], @@ -848,8 +858,8 @@ impl MimeFactory { // We don't display avatars for address-contacts, so sending avatars w/o encryption is not // useful and causes e.g. Outlook to reject a message with a big header, see // https://support.delta.chat/t/invalid-mime-content-single-text-value-size-32822-exceeded-allowed-maximum-32768-for-the-chat-user-avatar-header/4067. - let attach_selfavatar = Self::should_attach_selfavatar(context, &msg).await - && !matches!(encryption, Encryption::No); + let attach_selfavatar = + Self::should_attach_selfavatar(context, &msg).await && encryption.is_encrypted(); ensure_and_debug_assert!( member_timestamps.is_empty() @@ -2277,10 +2287,7 @@ impl MimeFactory { } pub fn will_be_encrypted(&self) -> bool { - match self.encryption { - Encryption::No => false, - Encryption::Asymmetric { .. } | Encryption::Symmetric { .. } => true, - } + self.encryption.is_encrypted() } pub fn set_as_post_message(&mut self) { From ad5cbe517550ce991baa9f9d300528331076a5e1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 31 Aug 2026 21:18:35 +0000 Subject: [PATCH 3/4] refactor: separate QueuedEncryption This is similar to mimefactory::Encryption, but does not have email addresses for asymmetrically encrypted messages. Queued messages don't need email addresses for public keys. Addresses are only needed to render Autocrypt-Gossip headers. --- src/mimefactory.rs | 70 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index ff9e1a119f..b0207f196f 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -104,6 +104,22 @@ impl Encryption { Self::Symmetric { .. } => true, } } + + /// Converts into [`QueuedEncryption`] by dropping email addresses corresponding to the keys. + fn into_queued_encryption(self) -> QueuedEncryption { + match self { + Encryption::No => QueuedEncryption::No, + Encryption::Asymmetric { encryption_pubkeys } => QueuedEncryption::Asymmetric { + encryption_pubkeys: encryption_pubkeys + .into_iter() + .map(|(_addr, key)| key) + .collect(), + }, + Encryption::Symmetric { shared_secret } => { + QueuedEncryption::Symmetric { shared_secret } + } + } + } } /// Helper to construct mime messages. @@ -205,6 +221,34 @@ pub struct RenderedMessage { sync_ids_to_delete: Option, } +#[derive(Debug, Clone)] +pub(crate) enum QueuedEncryption { + /// Unencrypted message. + No, + + /// The message is encrypted asymmetrically to public keys. + Asymmetric { + /// OpenPGP keys to use for encryption. + /// + /// The message is always encrypted to self, + /// no need to include own key here. + encryption_pubkeys: Vec, + }, + + /// Symmetrically encrypted message with a shared secret. + Symmetric { shared_secret: String }, +} + +impl QueuedEncryption { + pub(crate) fn is_encrypted(&self) -> bool { + match self { + Self::No => false, + Self::Asymmetric { .. } => true, + Self::Symmetric { .. } => true, + } + } +} + /// Email message queued, but not sent yet. /// /// It is stored unencrypted to @@ -229,7 +273,7 @@ pub(crate) struct QueuedMail { rfc724_mid: String, /// Whether the message is encrypted and encryption keys. - encryption: Encryption, + encryption: QueuedEncryption, /// If true, Autocrypt header should be added before sending. should_attach_pubkey: bool, @@ -429,18 +473,15 @@ pub(crate) fn render_queued_mail( let sign_key = if should_sign { Some(secret_key) } else { None }; let message = match encryption { - Encryption::No => raw_message, - Encryption::Asymmetric { encryption_pubkeys } => { + QueuedEncryption::No => raw_message, + QueuedEncryption::Asymmetric { encryption_pubkeys } => { let mut full_raw_message = inner_headers.clone(); full_raw_message.extend(raw_message); // Asymmetric encryption // Use SEIPDv2 if all recipients support it. - let seipd_version = if encryption_pubkeys - .iter() - .all(|(_addr, pubkey)| pubkey_supports_seipdv2(pubkey)) - { + let seipd_version = if encryption_pubkeys.iter().all(pubkey_supports_seipdv2) { SeipdVersion::V2 } else { SeipdVersion::V1 @@ -450,7 +491,7 @@ pub(crate) fn render_queued_mail( // even for a single-device setup, // to not reveal if we have a multi-device setup to contacts. let mut encryption_keyring = vec![public_key.clone()]; - encryption_keyring.extend(encryption_pubkeys.iter().map(|(_addr, key)| (*key).clone())); + encryption_keyring.extend(encryption_pubkeys); let encrypted = crate::pgp::pk_encrypt( full_raw_message, @@ -463,7 +504,7 @@ pub(crate) fn render_queued_mail( let message = wrap_encrypted_part(encrypted); part_to_bytes(message) } - Encryption::Symmetric { shared_secret } => { + QueuedEncryption::Symmetric { shared_secret } => { let mut full_raw_message = inner_headers.clone(); full_raw_message.extend(raw_message); @@ -1605,7 +1646,7 @@ impl MimeFactory { raw_message, rfc724_mid, display_name, - encryption: self.encryption, + encryption: self.encryption.into_queued_encryption(), should_attach_pubkey, should_sign, should_compress, @@ -2511,7 +2552,7 @@ pub(crate) async fn render_symm_encrypted_securejoin_message( raw_message: part_to_bytes(message), display_name: String::new(), rfc724_mid: rfc724_mid.to_string(), - encryption: Encryption::Symmetric { + encryption: QueuedEncryption::Symmetric { shared_secret: shared_secret.to_string(), }, should_attach_pubkey, @@ -2573,11 +2614,8 @@ pub(crate) async fn render_keyupdate_message( raw_message: part_to_bytes(message), display_name: String::new(), rfc724_mid: rfc724_mid.to_string(), - encryption: Encryption::Asymmetric { - encryption_pubkeys: recipient_keys - .into_iter() - .map(|key| (String::new(), key)) - .collect(), + encryption: QueuedEncryption::Asymmetric { + encryption_pubkeys: recipient_keys, }, // Attached key with its relay list notation is the actual payload. From 46248e77107ae2bc297d6c5267bbb787cfa49d55 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 9 Sep 2026 09:32:23 +0000 Subject: [PATCH 4/4] feat: queue messages for SMTP before encryption Headers like From and Autocrypt are now added late, right before sending the message over SMTP. This way we advertise the latest list of transports and use the correct From address in the encrypted part even for messages queued while being offline. BCC-self recipients are also added late. For unencrypted messages we only want to send a copy to the sending address, but we don't know the sending address when queueing the message. Adding bcc-self recipients when dequeuing the message also makes it possible to send copies to updated list of relays. --- .../tests/test_multitransport.py | 19 + docs/schema.sql | 63 +++- src/chat.rs | 335 +++++++++--------- src/chat/chat_tests.rs | 5 +- src/config.rs | 7 - src/download.rs | 3 - src/ephemeral/ephemeral_tests.rs | 2 +- src/keyupdate.rs | 10 +- src/location.rs | 16 - src/message.rs | 2 +- src/message/message_tests.rs | 2 +- src/mimefactory.rs | 162 +++++---- src/mimefactory/mimefactory_tests.rs | 4 +- src/receive_imf.rs | 12 +- src/receive_imf/receive_imf_tests.rs | 35 +- src/securejoin.rs | 8 +- src/securejoin/bob.rs | 7 +- src/smtp.rs | 234 ++++++++++-- src/sql/migrations.rs | 31 ++ src/test_utils.rs | 96 +++-- src/tests/pre_messages/sending.rs | 4 +- 21 files changed, 698 insertions(+), 359 deletions(-) diff --git a/deltachat-rpc-client/tests/test_multitransport.py b/deltachat-rpc-client/tests/test_multitransport.py index 6a014ba18e..e0e04eb2ba 100644 --- a/deltachat-rpc-client/tests/test_multitransport.py +++ b/deltachat-rpc-client/tests/test_multitransport.py @@ -98,6 +98,25 @@ def test_change_address(acf) -> None: assert sender_addr2 == new_alice_addr +def test_remove_transport_keep_messages(acf) -> None: + """Test that deleting current sending transport keeps queued messages.""" + alice, bob = acf.get_online_accounts(2) + + qr = acf.get_account_qr() + alice.add_transport_from_qr(qr) + + alice.stop_io() + alice_chat_bob = alice.create_chat(bob) + alice_chat_bob.send_text("Hello!") + + new_alice_addr = alice.list_transports()[1]["addr"] + alice.delete_transport(alice.list_transports()[0]["addr"]) + alice.start_io() + + bob_msg = bob.wait_for_incoming_msg().get_snapshot() + assert bob_msg.sender.get_snapshot().address == new_alice_addr + + def test_download_on_demand(acf, rpcdata) -> None: alice, bob = acf.get_online_accounts(2) alice.set_config("download_limit", "1") diff --git a/docs/schema.sql b/docs/schema.sql index a9a743d1fd..ac042ec858 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -400,14 +400,51 @@ CREATE TABLE bobstate ( chat_id INTEGER NOT NULL ); -CREATE TABLE smtp ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rfc724_mid TEXT NOT NULL, -- Message-ID - mime TEXT NOT NULL, -- SMTP payload - msg_id INTEGER NOT NULL, -- ID of the message in `msgs` table - recipients TEXT NOT NULL, -- List of recipients separated by space - retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message -); +CREATE TABLE smtp2 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + display_name TEXT NOT NULL, -- Display name to put into the From field. + rfc724_mid TEXT NOT NULL, -- Message-ID + + -- Unencrypted payload with some headers. + mime BLOB NOT NULL, + + -- True if Autocrypt header should be added before sending. + should_attach_pubkey INTEGER NOT NULL, + + -- True if OpenPGP-encrypted message may use compression. + should_compress INTEGER NOT NULL, + + -- True if encrypted message should be signed. + should_sign INTEGER NOT NULL, + + -- ID of the message in `msgs` table + msg_id INTEGER NOT NULL, + + -- Space-separated recipient addresses. + recipients TEXT NOT NULL, + + -- Space-separated addresses the message was sent to. + sent_to TEXT NOT NULL DEFAULT '', + + -- If true, copy should be sent to self in addition to the recipient list. + -- + -- For encrypted messages copy is sent to all addresses. + -- For unencrypted messages, copy is sent to the From address only. + bcc_self INTEGER NOT NULL, + + -- True if the message is encrypted. + -- If true, at most one of the shared_secret or encryption_fingerprints should be non-empty. + -- If false, both must be empty. + is_encrypted INTEGER NOT NULL, + + -- Shared secret if the message is to be encrypted symmetrically. + shared_secret TEXT NOT NULL DEFAULT '', + + -- Space-separated fingerprints of the keys the message should be encrypted to. + encryption_fingerprints TEXT NOT NULL DEFAULT '', + + retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message +) STRICT; CREATE TABLE smtp_mdns ( msg_id INTEGER NOT NULL, -- id of the message in msgs table which requested MDN (DEPRECATED 2024-06-21) @@ -781,3 +818,13 @@ CREATE TABLE sending_domains( domain TEXT PRIMARY KEY, dkim_works INTEGER DEFAULT 0 ); + +-- Replaced with smtp2. +CREATE TABLE smtp ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rfc724_mid TEXT NOT NULL, -- Message-ID + mime TEXT NOT NULL, -- SMTP payload + msg_id INTEGER NOT NULL, -- ID of the message in `msgs` table + recipients TEXT NOT NULL, -- List of recipients separated by space + retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message +); diff --git a/src/chat.rs b/src/chat.rs index d570652095..6b6859369f 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -11,7 +11,6 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail, ensure}; use chrono::TimeZone; use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line}; -use humansize::{BINARY, format_size}; use mail_builder::mime::MimePart; use serde::{Deserialize, Serialize}; use strum_macros::EnumIter; @@ -27,26 +26,22 @@ use crate::constants::{ use crate::contact::{self, Contact, ContactId, Origin}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc; -use crate::download::{ - DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, PRE_MSG_SIZE_WARNING_THRESHOLD, -}; +use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD}; use crate::ensure_and_debug_assert_eq; use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers}; use crate::events::EventType; -use crate::key; -use crate::key::{Fingerprint, self_fingerprint}; -use crate::location; +use crate::key::{DcKey as _, Fingerprint, self_fingerprint}; use crate::log::{LogExt, warn}; use crate::logged_debug_assert; use crate::message::{self, Message, MessageState, MsgId, Viewtype}; use crate::mimefactory; -use crate::mimefactory::{MimeFactory, RenderedEmail}; +use crate::mimefactory::{MimeFactory, QueueSideEffects, QueuedMail, ToBeQueuedMail}; use crate::mimeparser::SystemMessage; use crate::param::{Param, Params}; use crate::pgp::addresses_from_public_key; use crate::reaction::broadcast_reactions; use crate::receive_imf::ReceivedMsg; -use crate::smtp::{self, send_msg_to_smtp}; +use crate::smtp::send_msg_to_smtp; use crate::stock_str; use crate::sync::{self, Sync::*, SyncData}; use crate::tools::{ @@ -336,16 +331,18 @@ impl ChatId { Ok(chat_id) } - async fn set_selfavatar_timestamp(self, context: &Context, timestamp: i64) -> Result<()> { - context - .sql + fn set_selfavatar_timestamp( + self, + transaction: &mut rusqlite::Transaction<'_>, + timestamp: i64, + ) -> Result<()> { + transaction .execute( "UPDATE contacts SET selfavatar_sent=? WHERE id IN(SELECT contact_id FROM chats_contacts WHERE chat_id=? AND add_timestamp >= remove_timestamp)", (timestamp, self), - ) - .await?; + ) ?; Ok(()) } @@ -2776,11 +2773,8 @@ async fn render_mime_message_and_pre_message( context: &Context, msg: &mut Message, mimefactory: MimeFactory, -) -> Result<(Option, RenderedEmail)> { - let from_addr = context.get_primary_self_addr().await?; - let public_key = key::load_self_public_key(context).await?; - let secret_key = key::load_self_secret_key(context).await?; - + bcc_self: bool, +) -> Result<(Option, ToBeQueuedMail)> { let needs_pre_message = msg.viewtype.has_file() && mimefactory.will_be_encrypted() // unencrypted is likely email, we don't want to spam by sending multiple messages && msg @@ -2797,54 +2791,121 @@ async fn render_mime_message_and_pre_message( let mut mimefactory_post_msg = mimefactory.clone(); mimefactory_post_msg.set_as_post_message(); - let (queued_msg, side_effects) = Box::pin(mimefactory_post_msg.into_queued_mail(context)) - .await - .context("Failed to render post-message")?; - - let rendered_msg = mimefactory::render_queued_mail( - queued_msg, - &public_key, - &secret_key, - from_addr.clone(), - side_effects, - )?; + let (queued_msg, side_effects) = + Box::pin(mimefactory_post_msg.into_queued_mail(context, bcc_self)) + .await + .context("Failed to render post-message")?; let mut mimefactory_pre_msg = mimefactory; - mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg); + mimefactory_pre_msg.set_as_pre_message_for(&queued_msg.rfc724_mid); let (queued_pre_msg, pre_side_effects) = - Box::pin(mimefactory_pre_msg.into_queued_mail(context)) + Box::pin(mimefactory_pre_msg.into_queued_mail(context, bcc_self)) .await .context("pre-message failed to render")?; - let rendered_pre_msg = mimefactory::render_queued_mail( - queued_pre_msg, - &public_key, - &secret_key, - from_addr, - pre_side_effects, - )?; - if rendered_pre_msg.message.len() > PRE_MSG_SIZE_WARNING_THRESHOLD { - warn!( - context, - "Pre-message for message {} is larger than expected: {}.", - msg.id, - rendered_pre_msg.message.len() - ); + Ok(( + Some((queued_pre_msg, pre_side_effects)), + (queued_msg, side_effects), + )) + } else { + let (queued_msg, side_effects) = + Box::pin(mimefactory.into_queued_mail(context, bcc_self)).await?; + + Ok((None, (queued_msg, side_effects))) + } +} + +/// Process side effects and store queued mail. +pub(crate) fn enqueue_mail( + transaction: &mut rusqlite::Transaction<'_>, + now: i64, + msg_id: MsgId, + queued_mail: &QueuedMail, + side_effects: Option<&QueueSideEffects>, +) -> Result { + if let Some(side_effects) = side_effects { + if let Some(last_added_location_timestamp) = side_effects.last_added_location_timestamp { + transaction.execute( + "UPDATE chats SET locations_last_sent=? WHERE id=?;", + (last_added_location_timestamp, side_effects.chat_id), + )?; } - Ok((Some(rendered_pre_msg), rendered_msg)) - } else { - let (queued_msg, side_effects) = Box::pin(mimefactory.into_queued_mail(context)).await?; - let rendered_msg = mimefactory::render_queued_mail( - queued_msg, - &public_key, - &secret_key, - from_addr, - side_effects, - )?; + if side_effects.avatar_is_attached { + side_effects + .chat_id + .set_selfavatar_timestamp(transaction, now) + .context("Failed to set selfavatar timestamp")?; + } - Ok((None, rendered_msg)) + if let Some(ref sync_ids) = side_effects.sync_ids_to_delete { + transaction.execute( + &format!("DELETE FROM multi_device_sync WHERE id IN ({sync_ids})"), + (), + )?; + } } + + // Store mail into queue. + let all_recipients = queued_mail.recipients.join(" "); + let is_encrypted = queued_mail.encryption.is_encrypted(); + + transaction + .execute( + " + INSERT INTO smtp2 ( + display_name, + rfc724_mid, + mime, + should_attach_pubkey, + should_compress, + should_sign, + msg_id, + recipients, + bcc_self, + is_encrypted, + shared_secret, + encryption_fingerprints + ) + VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ", + ( + &queued_mail.display_name, + &queued_mail.rfc724_mid, + &queued_mail.raw_message, + queued_mail.should_attach_pubkey, + queued_mail.should_compress, + queued_mail.should_sign, + msg_id, + &all_recipients, + queued_mail.bcc_self, + is_encrypted, + if let mimefactory::QueuedEncryption::Symmetric { ref shared_secret } = + queued_mail.encryption + { + shared_secret + } else { + "" + }, + if let mimefactory::QueuedEncryption::Asymmetric { + ref encryption_pubkeys, + } = queued_mail.encryption + { + let res: Vec = encryption_pubkeys + .iter() + .map(|pubkey| pubkey.dc_fingerprint().hex()) + .collect(); + res.join(" ") + } else { + "".to_string() + }, + ), + ) + .context("Failed to insert a row into smtp2 table")?; + let row_id = transaction.last_insert_rowid(); + Ok(row_id) } /// Constructs jobs for sending a message and inserts them into the `smtp` table. @@ -2857,6 +2918,8 @@ async fn render_mime_message_and_pre_message( /// /// The caller has to interrupt SMTP loop or otherwise process new rows. async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -> Result> { + let now = time(); + let cmd = msg.param.get_cmd(); if cmd == SystemMessage::GroupNameChanged || cmd == SystemMessage::GroupDescriptionChanged { msg.chat_id @@ -2888,12 +2951,14 @@ async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -> Result