Video circles: add on-device transcription

Swiftgram/SGLocalTranscription/BUILD

@@ -0,0 +1,22 @@
+load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
+
+swift_library(
+    name = "SGLocalTranscription",
+    module_name = "SGLocalTranscription",
+    srcs = glob([
+        "Sources/**/*.swift",
+    ]),
+    copts = [
+        "-warnings-as-errors",
+    ],
+    deps = [
+        "//submodules/SSignalKit/SwiftSignalKit:SwiftSignalKit",
+        "//submodules/AccountContext:AccountContext",
+        "//submodules/TelegramCore:TelegramCore",
+        "//submodules/Media/LocalAudioTranscription",
+        "//submodules/Media/ConvertOpusToAAC",
+    ],
+    visibility = [
+        "//visibility:public",
+    ],
+)

Swiftgram/SGLocalTranscription/Sources/SGLocalTranscription.swift

@@ -0,0 +1,40 @@
+import Foundation
+import SwiftSignalKit
+import TelegramCore
+import AccountContext
+import LocalAudioTranscription
+import ConvertOpusToAAC
+
+public func sgLocallyTranscribeAudioMessage(context: AccountContext, messageId: EngineMessage.Id, appLocale: String) -> Signal<LocallyTranscribedAudio?, NoError> {
+    return context.engine.data.get(TelegramEngine.EngineData.Item.Messages.Message(id: messageId))
+    |> mapToSignal { message -> Signal<String?, NoError> in
+        guard let message = message else {
+            return .single(nil)
+        }
+        guard let file = message.media.first(where: { $0 is TelegramMediaFile }) as? TelegramMediaFile else {
+            return .single(nil)
+        }
+        return context.engine.resources.data(id: EngineMediaResource.Id(file.resource.id))
+        |> take(1)
+        |> mapToSignal { data -> Signal<String?, NoError> in
+            if !data.isComplete {
+                return .single(nil)
+            }
+            return .single(data.path)
+        }
+    }
+    |> mapToSignal { result -> Signal<String?, NoError> in
+        guard let result = result else {
+            return .single(nil)
+        }
+        return convertOpusToAAC(sourcePath: result, allocateTempFile: {
+            return EngineTempBox.shared.tempFile(fileName: "audio.m4a").path
+        })
+    }
+    |> mapToSignal { result -> Signal<LocallyTranscribedAudio?, NoError> in
+        guard let result = result else {
+            return .single(nil)
+        }
+        return transcribeAudio(path: result, appLocale: appLocale)
+    }
+}

submodules/Media/LocalAudioTranscription/Sources/LocalAudioTranscription.swift

@@ -47,13 +47,6 @@ private func transcribeAudio(path: String, locale: String) -> Signal<Transcripti
                             speechRecognizerValue.defaultTaskHint = .dictation
                             sharedRecognizers[locale] = speechRecognizerValue
                             speechRecognizer = speechRecognizerValue
-
-                            if locale == "en-US" {
-                                speechRecognizer.supportsOnDeviceRecognition = true
-                            } else {
-                                speechRecognizer.supportsOnDeviceRecognition = false
-                            }
-                            speechRecognizer.supportsOnDeviceRecognition = true
                         }
                         speechRecognizer.defaultTaskHint = .dictation
 
@@ -64,6 +57,8 @@ private func transcribeAudio(path: String, locale: String) -> Signal<Transcripti
                         if #available(iOS 16.0, *) {
                             request.addsPunctuation = true
                         }
+                        // MARK: Swiftgram
+                        // Reads actual on-device recognition capability for this locale directly.
                         request.requiresOnDeviceRecognition = speechRecognizer.supportsOnDeviceRecognition
                         request.shouldReportPartialResults = false
 

submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/BUILD

@@ -1,7 +1,8 @@
 load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
 
 sgdeps = [
-    "//Swiftgram/SGSimpleSettings:SGSimpleSettings" 
+    "//Swiftgram/SGSimpleSettings:SGSimpleSettings",
+    "//Swiftgram/SGLocalTranscription:SGLocalTranscription",
 ]
 
 
@@ -36,8 +37,6 @@ swift_library(
         "//submodules/TelegramUI/Components/AudioTranscriptionButtonComponent",
         "//submodules/TelegramUI/Components/AudioWaveformComponent",
         "//submodules/ShimmerEffect",
-        "//submodules/Media/ConvertOpusToAAC",
-        "//submodules/Media/LocalAudioTranscription",
         "//submodules/TextSelectionNode",
         "//submodules/TelegramUI/Components/AudioTranscriptionPendingIndicatorComponent",
         "//submodules/UndoUI",

submodules/TelegramUI/Components/Chat/ChatMessageInteractiveFileNode/Sources/ChatMessageInteractiveFileNode.swift

@@ -21,8 +21,7 @@ import ComponentFlow
 import AudioTranscriptionButtonComponent
 import AudioWaveformComponent
 import ShimmerEffect
-import ConvertOpusToAAC
-import LocalAudioTranscription
+import SGLocalTranscription
 import TextSelectionNode
 import AudioTranscriptionPendingIndicatorComponent
 import UndoUI
@@ -423,40 +422,11 @@ public final class ChatMessageInteractiveFileNode: ASDisplayNode {
 
                 if context.sharedContext.immediateExperimentalUISettings.localTranscription || !arguments.associatedData.isPremium || SGSimpleSettings.shared.transcriptionBackend == SGSimpleSettings.TranscriptionBackend.apple.rawValue {
                     let appLocale = presentationData.strings.baseLanguageCode
-
-                    let signal: Signal<LocallyTranscribedAudio?, NoError> = context.engine.data.get(TelegramEngine.EngineData.Item.Messages.Message(id: message.id))
-                    |> mapToSignal { message -> Signal<String?, NoError> in
-                        guard let message = message else {
-                            return .single(nil)
-                        }
-                        guard let file = message.media.first(where: { $0 is TelegramMediaFile }) as? TelegramMediaFile else {
-                            return .single(nil)
-                        }
-                        return context.engine.resources.data(id: EngineMediaResource.Id(file.resource.id))
-                        |> take(1)
-                        |> mapToSignal { data -> Signal<String?, NoError> in
-                            if !data.isComplete {
-                                return .single(nil)
-                            }
-                            return .single(data.path)
-                        }
-                    }
-                    |> mapToSignal { result -> Signal<String?, NoError> in
-                        guard let result = result else {
-                            return .single(nil)
-                        }
-                        return convertOpusToAAC(sourcePath: result, allocateTempFile: {
-                            return EngineTempBox.shared.tempFile(fileName: "audio.m4a").path
-                        })
-                    }
-                    |> mapToSignal { result -> Signal<LocallyTranscribedAudio?, NoError> in
-                        guard let result = result else {
-                            return .single(nil)
-                        }
-
-                        return transcribeAudio(path: result, appLocale: arguments.controllerInteraction.sgGetChatPredictedLang() ?? appLocale)
-                    }
-
+                    // MARK: Swiftgram
+                    // Extracted to SGLocalTranscription so video circle transcription
+                    // can reuse it instead of duplicating the whole signal chain.
+                    let signal = sgLocallyTranscribeAudioMessage(context: context, messageId: message.id, appLocale: arguments.controllerInteraction.sgGetChatPredictedLang() ?? appLocale)
+
                     self.transcribeDisposable = (signal
                     |> deliverOnMainQueue).startStrict(next: { [weak self] result in
                         guard let strongSelf = self, let arguments = strongSelf.arguments else {

submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/BUILD

@@ -1,5 +1,10 @@
 load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
 
+sgdeps = [
+    "//Swiftgram/SGSimpleSettings:SGSimpleSettings",
+    "//Swiftgram/SGLocalTranscription:SGLocalTranscription",
+]
+
 swift_library(
     name = "ChatMessageInteractiveInstantVideoNode",
     module_name = "ChatMessageInteractiveInstantVideoNode",
@@ -10,7 +15,7 @@ swift_library(
         "-warnings-as-errors",
         #"-Xfrontend", "-debug-time-function-bodies"
     ],
-    deps = [
+    deps = sgdeps + [
         "//submodules/AsyncDisplayKit",
         "//submodules/Display",
         "//submodules/SSignalKit/SwiftSignalKit",

submodules/TelegramUI/Components/Chat/ChatMessageInteractiveInstantVideoNode/Sources/ChatMessageInteractiveInstantVideoNode.swift

@@ -17,6 +17,8 @@ import ComponentFlow
 import AudioTranscriptionButtonComponent
 import UndoUI
 import TelegramNotices
+import SGLocalTranscription
+import SGSimpleSettings
 import Markdown
 import TextFormat
 import ChatMessageForwardInfoNode
@@ -841,7 +843,8 @@ public class ChatMessageInteractiveInstantVideoNode: ASDisplayNode {
                     var displayTranscribe = false
                     if item.message.id.peerId.namespace != Namespaces.Peer.SecretChat && statusDisplayType == .free && !isViewOnceMessage && !item.presentationData.isPreview {
                         let premiumConfiguration = PremiumConfiguration.with(appConfiguration: item.context.currentAppConfiguration.with { $0 })
-                        if item.associatedData.isPremium || item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost {
+                        // MARK: Swiftgram
+                        if item.associatedData.isPremium || item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost || true {
                             displayTranscribe = true
                         } else if premiumConfiguration.audioTransciptionTrialCount > 0 {
                             if incoming {
@@ -1833,7 +1836,8 @@ public class ChatMessageInteractiveInstantVideoNode: ASDisplayNode {
         let premiumConfiguration = PremiumConfiguration.with(appConfiguration: item.context.currentAppConfiguration.with { $0 })
 
         let transcriptionText = transcribedText(message: EngineMessage(item.message))
-        if transcriptionText == nil && !item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost {
+        // MARK: Swiftgram
+        if transcriptionText == nil && !item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost && false {
             if premiumConfiguration.audioTransciptionTrialCount > 0 {
                 if !item.associatedData.isPremium {
                     if self.presentAudioTranscriptionTooltip(finished: false) {
@@ -1888,26 +1892,53 @@ public class ChatMessageInteractiveInstantVideoNode: ASDisplayNode {
             }
         }
 
-        // TODO(swiftgram): Transcribe Video Messages
+        // MARK: Swiftgram
         if shouldBeginTranscription {
             if self.transcribeDisposable == nil {
                 self.audioTranscriptionState = .inProgress
                 self.requestUpdateLayout(true)
-
-                self.transcribeDisposable = (item.context.engine.messages.transcribeAudio(messageId: item.message.id)
-                |> deliverOnMainQueue).startStrict(next: { [weak self] result in
-                    guard let strongSelf = self else {
-                        return
-                    }
-                    strongSelf.transcribeDisposable?.dispose()
-                    strongSelf.transcribeDisposable = nil
-
-                    if let item = strongSelf.item, !item.associatedData.isPremium && !item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost {
-                        Queue.mainQueue().after(0.1, {
-                            let _ = strongSelf.presentAudioTranscriptionTooltip(finished: true)
-                        })
-                    }
-                })
+
+                let context = item.context
+
+                if context.sharedContext.immediateExperimentalUISettings.localTranscription || !item.associatedData.isPremium || SGSimpleSettings.shared.transcriptionBackend == SGSimpleSettings.TranscriptionBackend.apple.rawValue {
+                    let appLocale = presentationData.strings.baseLanguageCode
+                    let signal = sgLocallyTranscribeAudioMessage(context: context, messageId: item.message.id, appLocale: item.controllerInteraction.sgGetChatPredictedLang() ?? appLocale)
+
+                    self.transcribeDisposable = (signal
+                    |> deliverOnMainQueue).startStrict(next: { [weak self] result in
+                        guard let strongSelf = self, let item = strongSelf.item else {
+                            return
+                        }
+
+                        if let result = result {
+                            let _ = item.context.engine.messages.storeLocallyTranscribedAudio(messageId: item.message.id, text: result.text, isFinal: result.isFinal, error: nil).startStandalone()
+                        } else {
+                            strongSelf.audioTranscriptionState = .collapsed
+                            strongSelf.requestUpdateLayout(true)
+                        }
+                    }, completed: { [weak self] in
+                        guard let strongSelf = self else {
+                            return
+                        }
+                        strongSelf.transcribeDisposable?.dispose()
+                        strongSelf.transcribeDisposable = nil
+                    })
+                } else {
+                    self.transcribeDisposable = (context.engine.messages.transcribeAudio(messageId: item.message.id)
+                    |> deliverOnMainQueue).startStrict(next: { [weak self] result in
+                        guard let strongSelf = self else {
+                            return
+                        }
+                        strongSelf.transcribeDisposable?.dispose()
+                        strongSelf.transcribeDisposable = nil
+
+                        if let item = strongSelf.item, !item.associatedData.isPremium && !item.associatedData.alwaysDisplayTranscribeButton.providedByGroupBoost {
+                            Queue.mainQueue().after(0.1, {
+                                let _ = strongSelf.presentAudioTranscriptionTooltip(finished: true)
+                            })
+                        }
+                    })
+                }
             }
         }