feat(telegram): hand off audio/document files to the agent

config.toml.example

@@ -11,6 +11,11 @@ assistant_root = "~/Code/assistant"
 bot_token = "replace-with-the-token-from-BotFather"
 # Replace this once with your numeric Telegram user ID.
 allow_user_ids = [123456789]
+# Optional. Local Bot API server (default: public api.telegram.org).
+# base_url = "http://127.0.0.1:8081/bot"
+# base_file_url = "http://127.0.0.1:8081/file/bot"
+# Optional. Raise above 20 MiB when using a local Bot API for larger voice files.
+# max_audio_bytes = 104857600
 
 # Slack alternative. See docs/slack.md.
 # channel = "slack"

docs/configuration.md

@@ -225,6 +225,9 @@ requests. Review [permissions and security](security.md) before enabling jobs.
 | `telegram.bot_token` | `TELEGRAM_BOT_TOKEN` fallback | Private Bot API token; the environment value is used when this is omitted |
 | `telegram.allow_user_ids` | `[]` | Trusted numeric sender IDs |
 | `telegram.allow_chat_ids` | `[]` | Trusted numeric private-chat IDs |
+| `telegram.base_url` | `https://api.telegram.org/bot` | Bot API method base URL; point at a local Bot API server such as `http://127.0.0.1:8081/bot` |
+| `telegram.base_file_url` | `https://api.telegram.org/file/bot` | Bot API file download base URL; for a local server use `http://127.0.0.1:8081/file/bot` |
+| `telegram.max_audio_bytes` | `20971520` (20 MiB) | Maximum inbound voice download size; raise for a self-hosted Bot API that accepts larger files |
 
 ### Slack
 

docs/telegram.md

@@ -37,6 +37,27 @@ thread = "telegram:dm:123456789"
 agent = "claude"
 ```
 
+## Local Bot API Server
+
+By default Push talks to `https://api.telegram.org`. To use a self-hosted
+[Telegram Bot API server](https://github.com/tdlib/telegram-bot-api) (for
+example to download files larger than the public 20 MB limit), point Push at
+that server:
+
+```toml
+[telegram]
+bot_token = "replace-with-the-token-from-BotFather"
+allow_user_ids = [123456789]
+base_url = "http://127.0.0.1:8081/bot"
+base_file_url = "http://127.0.0.1:8081/file/bot"
+max_audio_bytes = 104857600
+```
+
+Trailing slashes on the base URLs are stripped. Keep the local Bot API process
+running before `push doctor` or `push`. Omitting these keys keeps the public
+defaults. With `telegram-bot-api --local`, `getFile` returns absolute filesystem
+paths; Push reads those files directly instead of using `base_file_url`.
+
 `push init` creates a new config with owner-only mode `0600` on Unix. An
 environment variable remains supported through `TELEGRAM_BOT_TOKEN`. Push never
 prints the token. Run:
@@ -84,10 +105,19 @@ passes the normal allowlist, transcribes it with `gpt-4o-transcribe`, sends the
 transcript through the selected coding agent, then returns the answer as both
 text and an Opus voice note generated by `gpt-4o-mini-tts`.
 
-There is no local Whisper, FFmpeg, or other audio dependency. Audio stays in
-memory and is limited to 20 MB. If neither `voice.openai_api_key` nor
-`OPENAI_API_KEY` is set, normal text messages keep working and voice notes
-receive an actionable text reply. API or speech generation errors also fall
+Telegram `audio` messages and audio `document` files (for example long phone
+recordings) take a different path: Push downloads the file into
+`$PUSH_HOME/cache/inbound-audio/`, replaces the inbound content with that path
+plus a short transcription handoff prompt, and lets the agent run its local
+pipeline/skill. Those files do not use OpenAI speech-to-text and do not trigger
+spoken replies.
+
+There is no local Whisper, FFmpeg, or other audio dependency inside Push itself.
+Voice-note audio stays in memory and is limited to 20 MB for the OpenAI path.
+File downloads honor `telegram.max_audio_bytes` (raise this when using a
+self-hosted Bot API for large recordings). If neither `voice.openai_api_key` nor
+`OPENAI_API_KEY` is set, normal text and audio-file handoffs keep working; voice
+notes receive an actionable text reply. API or speech generation errors also fall
 back to text without stopping the gateway.
 The spoken reply is AI-generated. Voice-note audio is sent to OpenAI for
 processing, so review OpenAI's data controls before enabling this feature.
@@ -100,10 +130,11 @@ channel can add those transport operations without changing the OpenAI layer.
 
 An incoming Telegram update reaches the agent only when all of these are true:
 
-- it is a normal text message or voice note in a private chat
+- it is a normal text message, voice note, audio message, or audio document in a
+  private chat
 - its numeric sender id is in `telegram.allow_user_ids`, or its numeric chat id
   is in `telegram.allow_chat_ids`
-- the message contains non-empty text or a voice attachment
+- the message contains non-empty text/caption or an accepted audio attachment
 
 Group chats, channels, group forum topics, edited messages, and other update
 types are out of scope and ignored. The private-chat thread key is

src/channel.rs

@@ -23,6 +23,9 @@ pub struct InboundVoice {
     pub filename: String,
     /// Channels that already have the bytes may provide them directly.
     pub data: Option<Vec<u8>>,
+    /// When true, Push downloads the file to disk and hands the path to the
+    /// agent instead of cloud speech-to-text (Telegram audio/document files).
+    pub agent_handoff: bool,
 }
 
 #[derive(Debug, Clone)]
@@ -165,6 +168,11 @@ impl Channel {
                     .ok_or_else(|| anyhow::anyhow!("Telegram bot token is not configured"))?,
                 cfg.telegram_allow_user_ids.clone(),
                 cfg.telegram_allow_chat_ids.clone(),
+                crate::telegram::TelegramEndpoints {
+                    base_url: cfg.telegram_base_url.clone(),
+                    base_file_url: cfg.telegram_base_file_url.clone(),
+                    max_audio_bytes: cfg.telegram_max_audio_bytes,
+                },
             ))),
             ChannelKind::Slack => Ok(Self::Slack(Slack::new(
                 cfg.slack_app_token()
@@ -755,7 +763,12 @@ mod tests {
     }
 
     fn telegram() -> Channel {
-        Channel::Telegram(Telegram::new("secret".to_string(), vec![7], vec![9]))
+        Channel::Telegram(Telegram::new(
+            "secret".to_string(),
+            vec![7],
+            vec![9],
+            crate::telegram::TelegramEndpoints::default(),
+        ))
     }
 
     fn slack() -> Channel {

src/config.rs

@@ -14,6 +14,9 @@ pub const TELEGRAM_BOT_TOKEN_ENV: &str = "TELEGRAM_BOT_TOKEN";
 pub const SLACK_APP_TOKEN_ENV: &str = "SLACK_APP_TOKEN";
 pub const SLACK_BOT_TOKEN_ENV: &str = "SLACK_BOT_TOKEN";
 pub const DEFAULT_VOICE_NAME: &str = "cedar";
+pub const DEFAULT_TELEGRAM_BASE_URL: &str = "https://api.telegram.org/bot";
+pub const DEFAULT_TELEGRAM_BASE_FILE_URL: &str = "https://api.telegram.org/file/bot";
+pub const DEFAULT_TELEGRAM_MAX_AUDIO_BYTES: usize = 20 * 1024 * 1024;
 const SUPPORTED_VOICE_NAMES: &[&str] = &[
     "alloy", "ash", "ballad", "coral", "echo", "fable", "nova", "onyx", "sage", "shimmer", "verse",
     "marin", "cedar",
@@ -62,6 +65,12 @@ pub struct Config {
     pub telegram_allow_user_ids: Vec<i64>,
     #[serde(default)]
     pub telegram_allow_chat_ids: Vec<i64>,
+    #[serde(default = "default_telegram_base_url")]
+    pub telegram_base_url: String,
+    #[serde(default = "default_telegram_base_file_url")]
+    pub telegram_base_file_url: String,
+    #[serde(default = "default_telegram_max_audio_bytes")]
+    pub telegram_max_audio_bytes: usize,
     #[serde(default)]
     pub slack_app_token: Option<String>,
     #[serde(default)]
@@ -223,6 +232,9 @@ impl Config {
                 ("bot_token", "telegram_bot_token"),
                 ("allow_user_ids", "telegram_allow_user_ids"),
                 ("allow_chat_ids", "telegram_allow_chat_ids"),
+                ("base_url", "telegram_base_url"),
+                ("base_file_url", "telegram_base_file_url"),
+                ("max_audio_bytes", "telegram_max_audio_bytes"),
             ],
         )?;
         flatten_provider_section(
@@ -288,6 +300,8 @@ impl Config {
         c.assistant_root = assistant_root.to_string_lossy().to_string();
         c.assistant_dir = c.assistant_root.clone();
         c.jobs_dir = assistant_root.join("jobs").to_string_lossy().to_string();
+        c.telegram_base_url = normalize_telegram_api_base(&c.telegram_base_url);
+        c.telegram_base_file_url = normalize_telegram_api_base(&c.telegram_base_file_url);
         validate_runtime_outside_assistant(&c)?;
         c.validate()?;
         c.config_path = config_path.to_string_lossy().to_string();
@@ -536,6 +550,15 @@ impl Config {
                     {
                         bail!("telegram.bot_token cannot be empty");
                     }
+                    if self.telegram_base_url.trim().is_empty() {
+                        bail!("telegram.base_url cannot be empty");
+                    }
+                    if self.telegram_base_file_url.trim().is_empty() {
+                        bail!("telegram.base_file_url cannot be empty");
+                    }
+                    if self.telegram_max_audio_bytes == 0 {
+                        bail!("telegram.max_audio_bytes must be positive");
+                    }
                 }
                 ChannelKind::Slack => {
                     if self.slack_allow_user_ids.is_empty()
@@ -882,6 +905,19 @@ fn default_agent() -> String {
 fn default_voice_name() -> String {
     DEFAULT_VOICE_NAME.to_string()
 }
+fn default_telegram_base_url() -> String {
+    DEFAULT_TELEGRAM_BASE_URL.to_string()
+}
+fn default_telegram_base_file_url() -> String {
+    DEFAULT_TELEGRAM_BASE_FILE_URL.to_string()
+}
+fn default_telegram_max_audio_bytes() -> usize {
+    DEFAULT_TELEGRAM_MAX_AUDIO_BYTES
+}
+
+fn normalize_telegram_api_base(value: &str) -> String {
+    value.trim().trim_end_matches('/').to_string()
+}
 fn default_jobs_dir() -> String {
     "~/.push/jobs".to_string()
 }
@@ -914,6 +950,9 @@ mod tests {
             telegram_bot_token: None,
             telegram_allow_user_ids: Vec::new(),
             telegram_allow_chat_ids: Vec::new(),
+            telegram_base_url: DEFAULT_TELEGRAM_BASE_URL.to_string(),
+            telegram_base_file_url: DEFAULT_TELEGRAM_BASE_FILE_URL.to_string(),
+            telegram_max_audio_bytes: DEFAULT_TELEGRAM_MAX_AUDIO_BYTES,
             slack_app_token: None,
             slack_bot_token: None,
             slack_allow_user_ids: Vec::new(),
@@ -1048,4 +1087,39 @@ mod tests {
         let _ = std::fs::remove_dir_all(assistant);
         let _ = std::fs::remove_dir_all(outside);
     }
+
+    #[test]
+    fn telegram_local_bot_api_settings_parse_and_normalize() {
+        let root = temp_dir("config-telegram-local-api");
+        let config_path = root.join("config.toml");
+        let assistant = root.join("assistant");
+        std::fs::create_dir_all(&assistant).unwrap();
+        std::fs::write(
+            &config_path,
+            format!(
+                r#"
+channel = "telegram"
+agent = "pi"
+assistant_root = "{assistant}"
+
+[telegram]
+bot_token = "secret"
+allow_user_ids = [1]
+base_url = "http://127.0.0.1:8081/bot/"
+base_file_url = "http://127.0.0.1:8081/file/bot/"
+max_audio_bytes = 104857600
+"#,
+                assistant = assistant.display()
+            ),
+        )
+        .unwrap();
+        let cfg = Config::load_with_paths(
+            config_path.to_str().unwrap(),
+            PushPaths::from_root(root.join("runtime")).unwrap(),
+        )
+        .unwrap();
+        assert_eq!(cfg.telegram_base_url, "http://127.0.0.1:8081/bot");
+        assert_eq!(cfg.telegram_base_file_url, "http://127.0.0.1:8081/file/bot");
+        assert_eq!(cfg.telegram_max_audio_bytes, 104857600);
+    }
 }

src/gateway/mod.rs

@@ -598,11 +598,18 @@ impl Gateway {
                 continue;
             }
             if let Some((thread, target)) = self.channel.accept(m) {
-                let reply_with_voice = m.voice.is_some();
-                let message_text = if reply_with_voice {
-                    "[Voice message]".to_string()
-                } else {
-                    m.text.trim().to_string()
+                let reply_with_voice = m.voice.as_ref().is_some_and(|voice| !voice.agent_handoff);
+                let message_text = match m.voice.as_ref() {
+                    Some(voice) if voice.agent_handoff => {
+                        let caption = m.text.trim();
+                        if caption.is_empty() {
+                            "[Audio file]".to_string()
+                        } else {
+                            caption.to_string()
+                        }
+                    }
+                    Some(_) => "[Voice message]".to_string(),
+                    None => m.text.trim().to_string(),
                 };
                 let approval_origin = self.channel.approval_origin(m, &thread);
                 let approval = if reply_with_voice {

src/gateway/tests.rs

@@ -3021,6 +3021,9 @@ fn test_config(state_path: &str, _sessions_dir: &str, assistant_dir: &str) -> Co
         telegram_bot_token: None,
         telegram_allow_user_ids: Vec::new(),
         telegram_allow_chat_ids: Vec::new(),
+        telegram_base_url: crate::config::DEFAULT_TELEGRAM_BASE_URL.to_string(),
+        telegram_base_file_url: crate::config::DEFAULT_TELEGRAM_BASE_FILE_URL.to_string(),
+        telegram_max_audio_bytes: crate::config::DEFAULT_TELEGRAM_MAX_AUDIO_BYTES,
         slack_app_token: None,
         slack_bot_token: None,
         slack_allow_user_ids: Vec::new(),
@@ -3177,6 +3180,7 @@ fn telegram_voice_message(row_id: i64, user_id: i64, chat_id: i64) -> RawMessage
         mime_type: "audio/ogg".to_string(),
         filename: "voice.ogg".to_string(),
         data: Some(vec![1, 2, 3]),
+        agent_handoff: false,
     });
     message
 }