From 150b37a46dc076ab60b255ab5cd2f56002271763 Mon Sep 17 00:00:00 2001 From: mvsrcdst <43897151+mvsrcdst@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:04:23 +0300 Subject: [PATCH 1/2] fix: reads actual on-device recognition capability for current locale directly --- .../Sources/LocalAudioTranscription.swift | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/submodules/Media/LocalAudioTranscription/Sources/LocalAudioTranscription.swift b/submodules/Media/LocalAudioTranscription/Sources/LocalAudioTranscription.swift index 9188875b84..4fbe89e844 100644 --- a/submodules/Media/LocalAudioTranscription/Sources/LocalAudioTranscription.swift +++ b/submodules/Media/LocalAudioTranscription/Sources/LocalAudioTranscription.swift @@ -47,13 +47,6 @@ private func transcribeAudio(path: String, locale: String) -> Signal Signal Date: Mon, 13 Jul 2026 17:05:30 +0300 Subject: [PATCH 2/2] feat: on-device transcription for video circle messages --- Swiftgram/SGLocalTranscription/BUILD | 22 ++++++ .../Sources/SGLocalTranscription.swift | 40 +++++++++++ .../Chat/ChatMessageInteractiveFileNode/BUILD | 5 +- .../ChatMessageInteractiveFileNode.swift | 42 ++---------- .../BUILD | 7 +- ...atMessageInteractiveInstantVideoNode.swift | 67 ++++++++++++++----- 6 files changed, 125 insertions(+), 58 deletions(-) create mode 100644 Swiftgram/SGLocalTranscription/BUILD create mode 100644 Swiftgram/SGLocalTranscription/Sources/SGLocalTranscription.swift diff --git a/Swiftgram/SGLocalTranscription/BUILD b/Swiftgram/SGLocalTranscription/BUILD new file mode 100644 index 0000000000..dac4ecbc09 --- /dev/null +++ b/Swiftgram/SGLocalTranscription/BUILD @@ -0,0 +1,22 @@ +load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") + +swift_library( + name = "SGLocalTranscription", + module_name = "SGLocalTranscription", + srcs = glob([ + "Sources/**/*.swift", + ]), + copts = [ + "-warnings-as-errors", + ], + deps = [ + "//submodules/SSignalKit/SwiftSignalKit:SwiftSignalKit", + "//submodules/AccountContext:AccountContext", + "//submodules/TelegramCore:TelegramCore", + "//submodules/Media/LocalAudioTranscription", + "//submodules/Media/ConvertOpusToAAC", + ], + visibility = [ + "//visibility:public", + ], +) diff --git a/Swiftgram/SGLocalTranscription/Sources/SGLocalTranscription.swift b/Swiftgram/SGLocalTranscription/Sources/SGLocalTranscription.swift new file mode 100644 index 0000000000..5429afedc0 --- /dev/null +++ b/Swiftgram/SGLocalTranscription/Sources/SGLocalTranscription.swift @@ -0,0 +1,40 @@ +import Foundation +import SwiftSignalKit +import TelegramCore +import AccountContext +import LocalAudioTranscription +import ConvertOpusToAAC + +public func sgLocallyTranscribeAudioMessage(context: AccountContext, messageId: EngineMessage.Id, appLocale: String) -> Signal { + return context.engine.data.get(TelegramEngine.EngineData.Item.Messages.Message(id: messageId)) + |> mapToSignal { message -> Signal in + guard let message = message else { + return .single(nil) + } + guard let file = message.media.first(where: { $0 is TelegramMediaFile }) as? TelegramMediaFile else { + return .single(nil) + } + return context.engine.resources.data(id: EngineMediaResource.Id(file.resource.id)) + |> take(1) + |> mapToSignal { data -> Signal in + if !data.isComplete { + return .single(nil) + } + return .single(data.path) + } + } + |> mapToSignal { result -> Signal in + guard let result = result else { + return .single(nil) + } + return convertOpusToAAC(sourcePath: result, allocateTempFile: { + return EngineTempBox.shared.tempFile(fileName: "audio.m4a").path + }) + } + |> mapToSignal { result -> Signal in + guard let result = result else { + return .single(nil) + } + return transcribeAudio(path: result, appLocale: appLocale) + } +} diff --git a/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/BUILD b/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/BUILD index 31df24c033..c99e21fef9 100644 --- a/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/BUILD +++ b/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/BUILD @@ -1,7 +1,8 @@ load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") sgdeps = [ - "//Swiftgram/SGSimpleSettings:SGSimpleSettings" + "//Swiftgram/SGSimpleSettings:SGSimpleSettings", + "//Swiftgram/SGLocalTranscription:SGLocalTranscription", ] @@ -36,8 +37,6 @@ swift_library( "//submodules/TelegramUI/Components/AudioTranscriptionButtonComponent", "//submodules/TelegramUI/Components/AudioWaveformComponent", "//submodules/ShimmerEffect", - "//submodules/Media/ConvertOpusToAAC", - "//submodules/Media/LocalAudioTranscription", "//submodules/TextSelectionNode", "//submodules/TelegramUI/Components/AudioTranscriptionPendingIndicatorComponent", "//submodules/UndoUI", diff --git a/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/Sources/ChatMessageInteractiveFileNode.swift b/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/Sources/ChatMessageInteractiveFileNode.swift index e7d32d76dd..447ad91b3f 100644 --- a/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/Sources/ChatMessageInteractiveFileNode.swift +++ b/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/Sources/ChatMessageInteractiveFileNode.swift @@ -21,8 +21,7 @@ import ComponentFlow import AudioTranscriptionButtonComponent import AudioWaveformComponent import ShimmerEffect -import ConvertOpusToAAC -import LocalAudioTranscription +import SGLocalTranscription import TextSelectionNode import AudioTranscriptionPendingIndicatorComponent import UndoUI @@ -423,40 +422,11 @@ public final class ChatMessageInteractiveFileNode: ASDisplayNode { if context.sharedContext.immediateExperimentalUISettings.localTranscription || !arguments.associatedData.isPremium || SGSimpleSettings.shared.transcriptionBackend == SGSimpleSettings.TranscriptionBackend.apple.rawValue { let appLocale = presentationData.strings.baseLanguageCode - - let signal: Signal = context.engine.data.get(TelegramEngine.EngineData.Item.Messages.Message(id: message.id)) - |> mapToSignal { message -> Signal in - guard let message = message else { - return .single(nil) - } - guard let file = message.media.first(where: { $0 is TelegramMediaFile }) as? TelegramMediaFile else { - return .single(nil) - } - return context.engine.resources.data(id: EngineMediaResource.Id(file.resource.id)) - |> take(1) - |> mapToSignal { data -> Signal in - if !data.isComplete { - return .single(nil) - } - return .single(data.path) - } - } - |> mapToSignal { result -> Signal in - guard let result = result else { - return .single(nil) - } - return convertOpusToAAC(sourcePath: result, allocateTempFile: { - return EngineTempBox.shared.tempFile(fileName: "audio.m4a").path - }) - } - |> mapToSignal { result -> Signal in - guard let result = result else { - return .single(nil) - } - - return transcribeAudio(path: result, appLocale: arguments.controllerInteraction.sgGetChatPredictedLang() ?? appLocale) - } - + // MARK: Swiftgram + // Extracted to SGLocalTranscription so video circle transcription + // can reuse it instead of duplicating the whole signal chain. + let signal = sgLocallyTranscribeAudioMessage(context: context, messageId: message.id, appLocale: arguments.controllerInteraction.sgGetChatPredictedLang() ?? appLocale) + self.transcribeDisposable = (signal |> deliverOnMainQueue).startStrict(next: { [weak self] result in guard let strongSelf = self, let arguments = strongSelf.arguments else { diff --git a/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/BUILD b/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/BUILD index 63595c29a2..aa90d2a9f3 100644 --- a/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/BUILD +++ b/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/BUILD @@ -1,5 +1,10 @@ load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") +sgdeps = [ + "//Swiftgram/SGSimpleSettings:SGSimpleSettings", + "//Swiftgram/SGLocalTranscription:SGLocalTranscription", +] + swift_library( name = "ChatMessageInteractiveInstantVideoNode", module_name = "ChatMessageInteractiveInstantVideoNode", @@ -10,7 +15,7 @@ swift_library( "-warnings-as-errors", #"-Xfrontend", "-debug-time-function-bodies" ], - deps = [ + deps = sgdeps + [ "//submodules/AsyncDisplayKit", "//submodules/Display", "//submodules/SSignalKit/SwiftSignalKit", diff --git a/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/Sources/ChatMessageInteractiveInstantVideoNode.swift b/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/Sources/ChatMessageInteractiveInstantVideoNode.swift index fe966ee839..6e4b6bf0f5 100644 --- a/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/Sources/ChatMessageInteractiveInstantVideoNode.swift +++ b/submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/Sources/ChatMessageInteractiveInstantVideoNode.swift @@ -17,6 +17,8 @@ import ComponentFlow import AudioTranscriptionButtonComponent import UndoUI import TelegramNotices +import SGLocalTranscription +import SGSimpleSettings import Markdown import TextFormat import ChatMessageForwardInfoNode @@ -841,7 +843,8 @@ public class ChatMessageInteractiveInstantVideoNode: ASDisplayNode { var displayTranscribe = false if item.message.id.peerId.namespace != Namespaces.Peer.SecretChat && statusDisplayType == .free && !isViewOnceMessage && !item.presentationData.isPreview { let premiumConfiguration = PremiumConfiguration.with(appConfiguration: item.context.currentAppConfiguration.with { $0 }) - if item.associatedData.isPremium || item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost { + // MARK: Swiftgram + if item.associatedData.isPremium || item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost || true { displayTranscribe = true } else if premiumConfiguration.audioTransciptionTrialCount > 0 { if incoming { @@ -1833,7 +1836,8 @@ public class ChatMessageInteractiveInstantVideoNode: ASDisplayNode { let premiumConfiguration = PremiumConfiguration.with(appConfiguration: item.context.currentAppConfiguration.with { $0 }) let transcriptionText = transcribedText(message: EngineMessage(item.message)) - if transcriptionText == nil && !item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost { + // MARK: Swiftgram + if transcriptionText == nil && !item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost && false { if premiumConfiguration.audioTransciptionTrialCount > 0 { if !item.associatedData.isPremium { if self.presentAudioTranscriptionTooltip(finished: false) { @@ -1888,26 +1892,53 @@ public class ChatMessageInteractiveInstantVideoNode: ASDisplayNode { } } - // TODO(swiftgram): Transcribe Video Messages + // MARK: Swiftgram if shouldBeginTranscription { if self.transcribeDisposable == nil { self.audioTranscriptionState = .inProgress self.requestUpdateLayout(true) - - self.transcribeDisposable = (item.context.engine.messages.transcribeAudio(messageId: item.message.id) - |> deliverOnMainQueue).startStrict(next: { [weak self] result in - guard let strongSelf = self else { - return - } - strongSelf.transcribeDisposable?.dispose() - strongSelf.transcribeDisposable = nil - - if let item = strongSelf.item, !item.associatedData.isPremium && !item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost { - Queue.mainQueue().after(0.1, { - let _ = strongSelf.presentAudioTranscriptionTooltip(finished: true) - }) - } - }) + + let context = item.context + + if context.sharedContext.immediateExperimentalUISettings.localTranscription || !item.associatedData.isPremium || SGSimpleSettings.shared.transcriptionBackend == SGSimpleSettings.TranscriptionBackend.apple.rawValue { + let appLocale = presentationData.strings.baseLanguageCode + let signal = sgLocallyTranscribeAudioMessage(context: context, messageId: item.message.id, appLocale: item.controllerInteraction.sgGetChatPredictedLang() ?? appLocale) + + self.transcribeDisposable = (signal + |> deliverOnMainQueue).startStrict(next: { [weak self] result in + guard let strongSelf = self, let item = strongSelf.item else { + return + } + + if let result = result { + let _ = item.context.engine.messages.storeLocallyTranscribedAudio(messageId: item.message.id, text: result.text, isFinal: result.isFinal, error: nil).startStandalone() + } else { + strongSelf.audioTranscriptionState = .collapsed + strongSelf.requestUpdateLayout(true) + } + }, completed: { [weak self] in + guard let strongSelf = self else { + return + } + strongSelf.transcribeDisposable?.dispose() + strongSelf.transcribeDisposable = nil + }) + } else { + self.transcribeDisposable = (context.engine.messages.transcribeAudio(messageId: item.message.id) + |> deliverOnMainQueue).startStrict(next: { [weak self] result in + guard let strongSelf = self else { + return + } + strongSelf.transcribeDisposable?.dispose() + strongSelf.transcribeDisposable = nil + + if let item = strongSelf.item, !item.associatedData.isPremium && !item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost { + Queue.mainQueue().after(0.1, { + let _ = strongSelf.presentAudioTranscriptionTooltip(finished: true) + }) + } + }) + } } }