From 1c6112ee1dfc5f77593c1e713ca99731ede61837 Mon Sep 17 00:00:00 2001 From: Aleksandr Shabelnikov Date: Sat, 18 Jul 2026 17:31:41 +0200 Subject: [PATCH] refactor(isoch): make transmit queue payload opaque --- ASFW.xcodeproj/project.pbxproj | 6 +- .../Audio/Core/AudioEndpointRuntime.hpp | 6 +- .../Audio/DriverKit/ASFWAudioDevice.cpp | 60 ++- .../Audio/DriverKit/ASFWAudioDriverGraph.cpp | 2 +- .../Audio/DriverKit/ASFWAudioDriverIO.cpp | 8 +- .../DriverKit/ASFWAudioDriverLifecycle.cpp | 8 +- .../DriverKit/ASFWAudioDriverPrivate.hpp | 53 +- .../Audio/DriverKit/ASFWAudioDriverZts.cpp | 72 ++- .../Runtime/AudioTransportControlBlock.hpp | 140 +++++ .../Runtime/TxWirePayloadTelemetry.hpp | 102 ++++ ASFWDriver/Isoch/Core/IsochDmaGeometry.hpp | 5 + ASFWDriver/Isoch/Core/IsochEventGroup.hpp | 25 +- ASFWDriver/Isoch/Core/IsochTxQueue.hpp | 189 +++++++ ASFWDriver/Isoch/IsochService.cpp | 12 +- .../Isoch/Transmit/IsochTransmitContext.cpp | 107 ++-- .../Isoch/Transmit/IsochTransmitContext.hpp | 12 +- ASFWDriver/Isoch/Transmit/IsochTxDmaRing.cpp | 179 ++----- ASFWDriver/Isoch/Transmit/IsochTxDmaRing.hpp | 38 +- ASFWDriver/Isoch/Transmit/IsochTxLayout.hpp | 5 +- .../Shared/Isoch/IsochAudioTransport.hpp | 477 ------------------ .../AudioTransportControlBlockTests.cpp | 57 +++ tests/audio/CMakeLists.txt | 4 + .../audio/IsochServiceTxPreparationTests.cpp | 69 ++- tests/audio/IsochTxDmaRingTests.cpp | 193 +++---- tests/audio/TransmitBoundaryTests.cpp | 37 ++ tests/audio/TxRefillCoverageTests.cpp | 20 +- 26 files changed, 907 insertions(+), 979 deletions(-) create mode 100644 ASFWDriver/Audio/DriverKit/Runtime/TxWirePayloadTelemetry.hpp create mode 100644 ASFWDriver/Isoch/Core/IsochTxQueue.hpp delete mode 100644 ASFWDriver/Shared/Isoch/IsochAudioTransport.hpp create mode 100644 tests/audio/TransmitBoundaryTests.cpp diff --git a/ASFW.xcodeproj/project.pbxproj b/ASFW.xcodeproj/project.pbxproj index 4035ecd0..3e4a52c2 100644 --- a/ASFW.xcodeproj/project.pbxproj +++ b/ASFW.xcodeproj/project.pbxproj @@ -893,10 +893,10 @@ A6D30FC4AF96CBDC4F5B068E /* ASFWMCPMockTransport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ASFWMCPMockTransport.swift; sourceTree = ""; }; A6F6F643214F7280A18611DC /* DiceProfileRegistry.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = DiceProfileRegistry.hpp; sourceTree = ""; }; A7EFDA6CC5CB7D2153F013D3 /* FWTypes.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = FWTypes.hpp; sourceTree = ""; }; + A805D0AD31C67B2D10A9507A /* TxWirePayloadTelemetry.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = TxWirePayloadTelemetry.hpp; sourceTree = ""; }; A855F971A611AE8F859F3579 /* DICERestartSession.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = DICERestartSession.hpp; sourceTree = ""; }; A94CBEC3E242E8D452CA2EEF /* ZtsTelemetry.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = ZtsTelemetry.hpp; sourceTree = ""; }; A9EFB67D49B05DCD5C9C661D /* RomSummarizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RomSummarizer.swift; sourceTree = ""; }; - AA48E4986D21FB2A52856349 /* IsochAudioTransport.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = IsochAudioTransport.hpp; sourceTree = ""; }; AA497F39DA363DBDFD6C9CCE /* WriteCommand.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = WriteCommand.cpp; sourceTree = ""; }; AA6636F29737F75466534B0E /* GapCountOptimizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = GapCountOptimizer.cpp; sourceTree = ""; }; AA6E283C66645521931AA351 /* StatusPublisher.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = StatusPublisher.hpp; sourceTree = ""; }; @@ -1118,6 +1118,7 @@ F49607C2B6C75A5EACC56433 /* IAmdtpTxSlotProvider.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = IAmdtpTxSlotProvider.hpp; sourceTree = ""; }; F4A7F52D42A80A09468F41F1 /* PayloadWriterTelemetry.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = PayloadWriterTelemetry.hpp; sourceTree = ""; }; F50437CCB59475ED6F54FC52 /* MCPBusResetToolsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MCPBusResetToolsTests.swift; sourceTree = ""; }; + F5252BDAC15F6AA21DCD3E28 /* IsochTxQueue.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = IsochTxQueue.hpp; sourceTree = ""; }; F714F267E66F185F1C86F905 /* SBP2CommandORB.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = SBP2CommandORB.hpp; sourceTree = ""; }; F71A638FD501761865722676 /* AudioCoordinator.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = AudioCoordinator.cpp; sourceTree = ""; }; F7AAEEA6F540F467BB7E4B7D /* DeviceDiscoveryWireFormats.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = DeviceDiscoveryWireFormats.hpp; sourceTree = ""; }; @@ -1512,6 +1513,7 @@ children = ( 8548E1DA1006A1671CC85CF1 /* IsochDmaGeometry.hpp */, 3DC0A6CF3AE91F3EF65A4BAB /* IsochEventGroup.hpp */, + F5252BDAC15F6AA21DCD3E28 /* IsochTxQueue.hpp */, 72BE64F16BB205405D7F37A1 /* IsochTypes.hpp */, ); path = Core; @@ -1913,7 +1915,6 @@ 845D81D6F290A50E0EE157C1 /* AudioGeometryPolicy.hpp */, D4073B97E1CA0CE1A021785D /* AudioHalBufferProfiles.hpp */, F46044243554D4B325071905 /* AudioTimingGeometry.hpp */, - AA48E4986D21FB2A52856349 /* IsochAudioTransport.hpp */, ); path = Isoch; sourceTree = ""; @@ -2706,6 +2707,7 @@ 8A1E58C964900982CCFC790B /* DirectAudioDebugSnapshot.hpp */, F4A7F52D42A80A09468F41F1 /* PayloadWriterTelemetry.hpp */, C211D6E382A537B9689088FD /* TxSytTrace.hpp */, + A805D0AD31C67B2D10A9507A /* TxWirePayloadTelemetry.hpp */, ); path = Runtime; sourceTree = ""; diff --git a/ASFWDriver/Audio/Core/AudioEndpointRuntime.hpp b/ASFWDriver/Audio/Core/AudioEndpointRuntime.hpp index 41fcf5a6..7de3f943 100644 --- a/ASFWDriver/Audio/Core/AudioEndpointRuntime.hpp +++ b/ASFWDriver/Audio/Core/AudioEndpointRuntime.hpp @@ -5,7 +5,7 @@ #include "../DriverKit/Runtime/DirectAudioBindingSource.hpp" #include "../Model/ASFWAudioDevice.hpp" -#include "../../Shared/Isoch/IsochAudioTransport.hpp" +#include "../Config/AudioConstants.hpp" #include "../Wire/AMDTP/AmdtpRateGeometry.hpp" #include "../../Logging/Logging.hpp" @@ -430,8 +430,8 @@ class AudioEndpointRuntime final : public Runtime::IDirectAudioBindingSource { const uint32_t inputChannels = ClampAudioChannels( config_.inputChannelCount ? config_.inputChannelCount : config_.channelCount); const uint32_t sampleRateHz = config_.currentSampleRate ? config_.currentSampleRate : 48000; - const uint32_t outputFrames = IsochTransport::kAudioRingBufferFrames; - const uint32_t inputFrames = IsochTransport::kAudioRingBufferFrames; + const uint32_t outputFrames = Isoch::Config::kAudioRingBufferFrames; + const uint32_t inputFrames = Isoch::Config::kAudioRingBufferFrames; if (outputChannels == 0 || inputChannels == 0 || sampleRateHz == 0) { ASFW_LOG(DirectAudio, diff --git a/ASFWDriver/Audio/DriverKit/ASFWAudioDevice.cpp b/ASFWDriver/Audio/DriverKit/ASFWAudioDevice.cpp index 240f5b30..3dc8b3e6 100644 --- a/ASFWDriver/Audio/DriverKit/ASFWAudioDevice.cpp +++ b/ASFWDriver/Audio/DriverKit/ASFWAudioDevice.cpp @@ -12,7 +12,7 @@ #include "../Config/TimingCursorPolicy.hpp" #include "Config/AudioProfileRegistry.hpp" #include "../../Common/DriverKitOwnership.hpp" -#include "../../Shared/Isoch/IsochAudioTransport.hpp" +#include "../../Isoch/Core/IsochTxQueue.hpp" #include #include @@ -79,9 +79,10 @@ kern_return_t ASFWAudioDevice::StartIO(IOUserAudioStartStopFlags in_flags) { ivars.txControlBuffer = nullptr; ivars.runtime.txSlotProvider.payloadBase = nullptr; ivars.runtime.txSlotProvider.metadataRing = nullptr; - ivars.runtime.txSlotProvider.controlBlock = nullptr; + ivars.runtime.txSlotProvider.queueControl = nullptr; + ivars.runtime.txSlotProvider.audioControl = nullptr; ivars.runtime.txSlotProvider.numSlots = 0; - ivars.runtime.txExecutionTimeline.controlBlock = nullptr; + ivars.runtime.txExecutionTimeline.queueControl = nullptr; // Secondary playback stream resources. ivars.txPayloadMapSecondary = nullptr; @@ -92,7 +93,8 @@ kern_return_t ASFWAudioDevice::StartIO(IOUserAudioStartStopFlags in_flags) { ivars.txControlBufferSecondary = nullptr; ivars.runtime.txSlotProviderSecondary.payloadBase = nullptr; ivars.runtime.txSlotProviderSecondary.metadataRing = nullptr; - ivars.runtime.txSlotProviderSecondary.controlBlock = nullptr; + ivars.runtime.txSlotProviderSecondary.queueControl = nullptr; + ivars.runtime.txSlotProviderSecondary.audioControl = nullptr; ivars.runtime.txSlotProviderSecondary.numSlots = 0; ivars.runtime.txSecondaryActive = false; @@ -217,22 +219,23 @@ kern_return_t ASFWAudioDevice::StartIO(IOUserAudioStartStopFlags in_flags) { } uint8_t* payloadBase = reinterpret_cast(ivars.txPayloadMap->GetAddress()); - auto* metadataRing = reinterpret_cast(ivars.txMetadataMap->GetAddress()); - auto* controlBlock = reinterpret_cast(ivars.txControlMap->GetAddress()); + auto* metadataRing = reinterpret_cast(ivars.txMetadataMap->GetAddress()); + auto* queueControl = reinterpret_cast(ivars.txControlMap->GetAddress()); // Clear stale runtime cursors before prefill: the shared slab can be // reused across StartIO/StopIO probes (CoreAudio re-probes on a - // sample-rate change), and a carried-over exposeCursor fails the IT + // sample-rate change), and a carried-over committed cursor fails the IT // prime ("committed prefill > slots"). - controlBlock->ResetForStart(); + queueControl->ResetProducerForStart(); ivars.runtime.txSlotProvider.payloadBase = payloadBase; ivars.runtime.txSlotProvider.metadataRing = metadataRing; - ivars.runtime.txSlotProvider.controlBlock = controlBlock; + ivars.runtime.txSlotProvider.queueControl = queueControl; + ivars.runtime.txSlotProvider.audioControl = control; ivars.runtime.txSlotProvider.numSlots = numSlots; ivars.runtime.txSlotProvider.slotStrideBytes = maxPacketBytes; - ivars.runtime.txExecutionTimeline.controlBlock = controlBlock; + ivars.runtime.txExecutionTimeline.queueControl = queueControl; if (!ivars.runtime.txStreamEngine.Configure(*profile, txConfig)) { ASFW_LOG(Audio, "ASFWAudioDevice: txStreamEngine Configure failed"); @@ -308,14 +311,15 @@ kern_return_t ASFWAudioDevice::StartIO(IOUserAudioStartStopFlags in_flags) { if (allocKr2 != kIOReturnSuccess) { kr = failStart(allocKr2, "MapTxControl2"); return; } uint8_t* payloadBase2 = reinterpret_cast(ivars.txPayloadMapSecondary->GetAddress()); - auto* metadataRing2 = reinterpret_cast(ivars.txMetadataMapSecondary->GetAddress()); - auto* controlBlock2 = reinterpret_cast(ivars.txControlMapSecondary->GetAddress()); + auto* metadataRing2 = reinterpret_cast(ivars.txMetadataMapSecondary->GetAddress()); + auto* queueControl2 = reinterpret_cast(ivars.txControlMapSecondary->GetAddress()); - controlBlock2->ResetForStart(); + queueControl2->ResetProducerForStart(); ivars.runtime.txSlotProviderSecondary.payloadBase = payloadBase2; ivars.runtime.txSlotProviderSecondary.metadataRing = metadataRing2; - ivars.runtime.txSlotProviderSecondary.controlBlock = controlBlock2; + ivars.runtime.txSlotProviderSecondary.queueControl = queueControl2; + ivars.runtime.txSlotProviderSecondary.audioControl = control; ivars.runtime.txSlotProviderSecondary.numSlots = numSlots2; ivars.runtime.txSlotProviderSecondary.slotStrideBytes = maxPacketBytes2; @@ -337,10 +341,10 @@ kern_return_t ASFWAudioDevice::StartIO(IOUserAudioStartStopFlags in_flags) { // --- Prefill TX ring --- ASFW::Audio::DriverKit::PrefillTxRingBeforeStart(ivars); - auto* prefillControl = ivars.runtime.txSlotProvider.controlBlock; + auto* prefillControl = ivars.runtime.txSlotProvider.queueControl; const uint64_t prefillExpose = prefillControl - ? prefillControl->exposeCursor.load(std::memory_order_acquire) + ? prefillControl->committedEnd.load(std::memory_order_acquire) : 0; const uint32_t expectedPrefill = ivars.runtime.txSlotProvider.numSlots; @@ -374,18 +378,20 @@ kern_return_t ASFWAudioDevice::StartIO(IOUserAudioStartStopFlags in_flags) { // StartAudioStreaming initializes the shared transport control block. // Validate it immediately afterward; failStart stops the partially // started stream before returning any mismatch to AudioDriverKit. - auto* txControl = ivars.runtime.txSlotProvider.controlBlock; + auto* txControl = ivars.runtime.txSlotProvider.queueControl; if (!txControl || - txControl->abiVersion != ASFW::IsochTransport::kTransportAbiVersion || + txControl->abiVersion != ASFW::Isoch::kTxQueueAbiVersion || txControl->numSlots != ASFW::IsochTransport::AudioTimingGeometry::kTxSharedSlotPackets || - txControl->interruptInterval != ASFW::IsochTransport::AudioTimingGeometry::kTxPacketsPerGroup || - txControl->ztsPeriodFrames != ASFW::IsochTransport::AudioTimingGeometry::kHalZeroTimestampPeriodFrames) { + txControl->slotStrideBytes != ivars.runtime.txSlotProvider.slotStrideBytes || + txControl->maxPacketBytes != ivars.runtime.txSlotProvider.slotStrideBytes || + txControl->interruptInterval != ASFW::IsochTransport::AudioTimingGeometry::kTxPacketsPerGroup) { ASFW_LOG(Audio, - "ASFWAudioDevice: TX geometry/ABI mismatch abi=%u slots=%u group=%u zts=%u", + "ASFWAudioDevice: TX queue ABI/geometry mismatch abi=%u slots=%u stride=%u max=%u group=%u", txControl ? txControl->abiVersion : 0, txControl ? txControl->numSlots : 0, - txControl ? txControl->interruptInterval : 0, - txControl ? txControl->ztsPeriodFrames : 0); + txControl ? txControl->slotStrideBytes : 0, + txControl ? txControl->maxPacketBytes : 0, + txControl ? txControl->interruptInterval : 0); kr = failStart( kIOReturnUnsupported, "ValidateTxTransportGeometry"); return; @@ -585,9 +591,10 @@ kern_return_t ASFWAudioDevice::StopIO(IOUserAudioStartStopFlags in_flags) { ivars.txControlBuffer = nullptr; ivars.runtime.txSlotProvider.payloadBase = nullptr; ivars.runtime.txSlotProvider.metadataRing = nullptr; - ivars.runtime.txSlotProvider.controlBlock = nullptr; + ivars.runtime.txSlotProvider.queueControl = nullptr; + ivars.runtime.txSlotProvider.audioControl = nullptr; ivars.runtime.txSlotProvider.numSlots = 0; - ivars.runtime.txExecutionTimeline.controlBlock = nullptr; + ivars.runtime.txExecutionTimeline.queueControl = nullptr; // Secondary playback stream teardown. Drop txSecondaryActive first so the // RT pump/IO paths stop touching the secondary engine before its mapped @@ -601,7 +608,8 @@ kern_return_t ASFWAudioDevice::StopIO(IOUserAudioStartStopFlags in_flags) { ivars.txControlBufferSecondary = nullptr; ivars.runtime.txSlotProviderSecondary.payloadBase = nullptr; ivars.runtime.txSlotProviderSecondary.metadataRing = nullptr; - ivars.runtime.txSlotProviderSecondary.controlBlock = nullptr; + ivars.runtime.txSlotProviderSecondary.queueControl = nullptr; + ivars.runtime.txSlotProviderSecondary.audioControl = nullptr; ivars.runtime.txSlotProviderSecondary.numSlots = 0; if (ivars.device.audioNub) { diff --git a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverGraph.cpp b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverGraph.cpp index da52d287..c9743076 100644 --- a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverGraph.cpp +++ b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverGraph.cpp @@ -13,7 +13,7 @@ #include "../Config/TimingCursorPolicy.hpp" #include "../../Common/TimingUtils.hpp" #include "../../Common/DriverKitOwnership.hpp" -#include "../../Shared/Isoch/IsochAudioTransport.hpp" +#include "../../Shared/Isoch/AudioTimingGeometry.hpp" #include "../../Audio/Wire/AMDTP/AmdtpRateGeometry.hpp" #include "../../Logging/Logging.hpp" diff --git a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverIO.cpp b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverIO.cpp index 036a2435..f114b662 100644 --- a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverIO.cpp +++ b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverIO.cpp @@ -197,8 +197,8 @@ kern_return_t InstallIOOperationHandler(IOUserAudioDevice& audioDevice, hostBuffer.frameCapacity = memory.outputFrameCapacity; hostBuffer.channels = channels; - const uint64_t completionCursor = driverIvars->runtime.txSlotProvider.controlBlock - ? driverIvars->runtime.txSlotProvider.controlBlock->completionCursor.load(std::memory_order_acquire) + const uint64_t completionCursor = driverIvars->runtime.txSlotProvider.queueControl + ? driverIvars->runtime.txSlotProvider.queueControl->completionCursor.load(std::memory_order_acquire) : 0; const uint64_t writeEndFrame = sampleTime + ioBufferFrameSize; @@ -214,8 +214,8 @@ kern_return_t InstallIOOperationHandler(IOUserAudioDevice& audioDevice, // payload writer reads channels [16, 32) via sourceChannelOffset. if (driverIvars->runtime.txSecondaryActive) { const uint64_t secondaryCompletion = - driverIvars->runtime.txSlotProviderSecondary.controlBlock - ? driverIvars->runtime.txSlotProviderSecondary.controlBlock + driverIvars->runtime.txSlotProviderSecondary.queueControl + ? driverIvars->runtime.txSlotProviderSecondary.queueControl ->completionCursor.load(std::memory_order_acquire) : 0; driverIvars->runtime.txStreamEngineSecondary.WriteHostOutputFloat32( diff --git a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverLifecycle.cpp b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverLifecycle.cpp index 44bb91ba..d1dfa7f2 100644 --- a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverLifecycle.cpp +++ b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverLifecycle.cpp @@ -292,9 +292,10 @@ void PerformLoudTeardown(ASFWAudioDriver_IVars& ivars, const char* reason) noexc ivars.txControlBuffer = nullptr; ivars.runtime.txSlotProvider.payloadBase = nullptr; ivars.runtime.txSlotProvider.metadataRing = nullptr; - ivars.runtime.txSlotProvider.controlBlock = nullptr; + ivars.runtime.txSlotProvider.queueControl = nullptr; + ivars.runtime.txSlotProvider.audioControl = nullptr; ivars.runtime.txSlotProvider.numSlots = 0; - ivars.runtime.txExecutionTimeline.controlBlock = nullptr; + ivars.runtime.txExecutionTimeline.queueControl = nullptr; // Secondary playback stream teardown (mirrors the master above). ivars.runtime.txSecondaryActive = false; @@ -306,7 +307,8 @@ void PerformLoudTeardown(ASFWAudioDriver_IVars& ivars, const char* reason) noexc ivars.txControlBufferSecondary = nullptr; ivars.runtime.txSlotProviderSecondary.payloadBase = nullptr; ivars.runtime.txSlotProviderSecondary.metadataRing = nullptr; - ivars.runtime.txSlotProviderSecondary.controlBlock = nullptr; + ivars.runtime.txSlotProviderSecondary.queueControl = nullptr; + ivars.runtime.txSlotProviderSecondary.audioControl = nullptr; ivars.runtime.txSlotProviderSecondary.numSlots = 0; ivars.device.audioNub->FreeTxIsochResources(); diff --git a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverPrivate.hpp b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverPrivate.hpp index 30235828..be587471 100644 --- a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverPrivate.hpp +++ b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverPrivate.hpp @@ -10,7 +10,8 @@ #include "../Engine/Direct/FireWireAudioEngine.hpp" #include "../Config/AudioTxProfiles.hpp" #include "../Engine/Direct/Tx/DiceTxStreamEngine.hpp" -#include "../../Shared/Isoch/IsochAudioTransport.hpp" +#include "../../Isoch/Core/IsochTxQueue.hpp" +#include "../../Logging/Logging.hpp" #include "../../Common/TimingUtils.hpp" #include @@ -58,23 +59,23 @@ struct AudioDriverDeviceState { class DextTxExecutionTimeline final { public: - const ASFW::IsochTransport::TxStreamControl* controlBlock{nullptr}; + const ASFW::Isoch::IsochTxQueueControl* queueControl{nullptr}; [[nodiscard]] bool AnchorForPacket(uint64_t packetIndex, int64_t& outTicks) const noexcept { - if (!controlBlock) { + if (!queueControl) { return false; } const uint64_t count = - controlBlock->completionStampCount.load(std::memory_order_acquire); + queueControl->completionStampCount.load(std::memory_order_acquire); if (count == 0) { return false; } uint64_t completedPacketIndex = 0; uint32_t timestamp = 0; - if (!controlBlock->ReadCompletionStamp( + if (!queueControl->ReadCompletionStamp( count - 1, completedPacketIndex, timestamp) || packetIndex < completedPacketIndex) { return false; @@ -101,8 +102,9 @@ class DextTxExecutionTimeline final { class DextTxSlotProvider final : public ASFW::Protocols::Audio::AMDTP::IAmdtpTxSlotProvider { public: uint8_t* payloadBase{nullptr}; - ASFW::IsochTransport::TxPacketMeta* metadataRing{nullptr}; - ASFW::IsochTransport::TxStreamControl* controlBlock{nullptr}; + ASFW::Isoch::IsochTxPacketMeta* metadataRing{nullptr}; + ASFW::Isoch::IsochTxQueueControl* queueControl{nullptr}; + ASFW::Audio::Runtime::AudioTransportControlBlock* audioControl{nullptr}; uint32_t numSlots{0}; uint32_t slotStrideBytes{0}; @@ -123,7 +125,7 @@ class DextTxSlotProvider final : public ASFW::Protocols::Audio::AMDTP::IAmdtpTxS [[nodiscard]] bool PublishSlot( const ASFW::Protocols::Audio::AMDTP::PreparedTxPacket& packet) noexcept override { - if (!metadataRing || !controlBlock || numSlots == 0) { + if (!metadataRing || !queueControl || numSlots == 0) { return false; } const uint32_t slotIdx = packet.packetIndex % numSlots; @@ -154,12 +156,37 @@ class DextTxSlotProvider final : public ASFW::Protocols::Audio::AMDTP::IAmdtpTxS meta.immediateHeader[1] = OSSwapHostToLittleInt32( static_cast(packet.byteCount & 0xFFFF) << 16); - // Compute expectedGen and release-store commitGen - const uint64_t gen = ASFW::IsochTransport::ExpectedCommitGen(packet.packetIndex, numSlots); - meta.commitGen.store(gen, std::memory_order_release); + // Content inspection belongs to Audio and runs immediately before the + // release commit. Transport receives only opaque bytes and metadata. + if (audioControl) { + const uint32_t slotIndex = packet.packetIndex % numSlots; + const auto observation = audioControl->txWirePayloadTelemetry.Observe( + packet.packetIndex, + payloadBase + static_cast(slotIndex) * slotStrideBytes, + packet.byteCount); + if (observation.firstInfo || observation.dropout) { + ASFW_LOG_RING_ONLY_RL( + DirectAudio, + "tx-wire-payload", + observation.firstInfo ? 0u : 1000u, + ::ASFW::Logging::LogLevel::Warning, + "[TxWire] packet=%u first=%d dropout=%d infoQuads=%u maxAbs24=%u lastQuad=0x%08x", + packet.packetIndex, + observation.firstInfo ? 1 : 0, + observation.dropout ? 1 : 0, + observation.infoQuads, + observation.maxAbs24, + observation.lastInfoQuad); + } + } + + // Compute expected generation and release-store it last. + const uint64_t generation = + ASFW::Isoch::ExpectedTxCommitGeneration(packet.packetIndex, numSlots); + meta.commitGeneration.store(generation, std::memory_order_release); - // Expose cursor progress to core - controlBlock->exposeCursor.store(packet.packetIndex + 1, std::memory_order_release); + queueControl->committedEnd.store(packet.packetIndex + 1, + std::memory_order_release); return true; } diff --git a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverZts.cpp b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverZts.cpp index 5cdba430..0ceed1e0 100644 --- a/ASFWDriver/Audio/DriverKit/ASFWAudioDriverZts.cpp +++ b/ASFWDriver/Audio/DriverKit/ASFWAudioDriverZts.cpp @@ -100,12 +100,12 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, uint32_t preparedCount = 0; const auto failProducer = - [&](ASFW::IsochTransport::TxProducerStage stage, - ASFW::IsochTransport::TxProducerFailureReason producerReason, + [&](ASFW::Audio::Runtime::TxProducerFaultStage stage, + ASFW::Audio::Runtime::TxProducerFaultReason producerReason, ASFW::Audio::Runtime::FatalStreamReason runtimeReason, uint64_t packetIndex) noexcept { auto* txControl = - ivars.runtime.txSlotProvider.controlBlock; + ivars.runtime.txSlotProvider.queueControl; const uint64_t completionCursor = txControl ? txControl->completionCursor.load( @@ -113,11 +113,11 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, : 0; const uint64_t exposeCursor = txControl - ? txControl->exposeCursor.load( + ? txControl->committedEnd.load( std::memory_order_acquire) : 0; - ASFW::IsochTransport::TxProducerFailureRecord failure{ + ASFW::Audio::Runtime::TxProducerFaultRecord failure{ .stage = stage, .reason = producerReason, .packetIndex = packetIndex, @@ -125,16 +125,14 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, .rangeTarget = limitPacketIndex, .preparedCount = preparedCount, .completionCursor = completionCursor, - .exposeCursor = exposeCursor, + .committedEnd = exposeCursor, .replayProducerCursor = directControl->rxSequenceReplay.ProducerCursor(), .replayEpoch = directControl->rxSequenceReplay.Epoch(), }; const uint64_t producerGeneration = - txControl - ? txControl->producerFailure.Publish(failure) - : 0; + directControl->txProducerFault.Publish(failure); directControl->fatalReason.store( runtimeReason, std::memory_order_release); @@ -150,10 +148,10 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, "[TxProducerFatal] stage=%{public}s reason=%{public}s " "producerGen=%llu runtimeReason=%u runtimeGen=%llu " "packet=%llu range=[%llu,%llu) prepared=%u " - "completion=%llu expose=%llu replayProducer=%llu " + "completion=%llu committedEnd=%llu replayProducer=%llu " "replayEpoch=%u", - ASFW::IsochTransport::TxProducerStageName(stage), - ASFW::IsochTransport::TxProducerFailureReasonName( + ASFW::Audio::Runtime::TxProducerFaultStageName(stage), + ASFW::Audio::Runtime::TxProducerFaultReasonName( producerReason), producerGeneration, static_cast(runtimeReason), @@ -169,8 +167,7 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, if (txControl) { txControl->statusWord.store( - ASFW::IsochTransport::TxStreamStatus:: - kUnderrunFatal, + ASFW::Isoch::IsochTxQueueStatus::kProducerFault, std::memory_order_release); } ivars.runtime.txActive.store( @@ -178,10 +175,10 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, }; if (numSlots == 0 || metadataRing == nullptr || - ivars.runtime.txSlotProvider.controlBlock == nullptr) { + ivars.runtime.txSlotProvider.queueControl == nullptr) { failProducer( - ASFW::IsochTransport::TxProducerStage::kPreflight, - ASFW::IsochTransport::TxProducerFailureReason:: + ASFW::Audio::Runtime::TxProducerFaultStage::kPreflight, + ASFW::Audio::Runtime::TxProducerFaultReason:: kInvalidTransport, ASFW::Audio::Runtime::FatalStreamReason:: InvalidGeometry, @@ -215,9 +212,9 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, directControl->txReplayUnderflows.fetch_add( 1, std::memory_order_relaxed); failProducer( - ASFW::IsochTransport::TxProducerStage:: + ASFW::Audio::Runtime::TxProducerFaultStage:: kExecutionAnchor, - ASFW::IsochTransport::TxProducerFailureReason:: + ASFW::Audio::Runtime::TxProducerFaultReason:: kReplayUnavailable, ASFW::Audio::Runtime::FatalStreamReason:: TxReplayUnavailable, @@ -281,10 +278,9 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, directControl->txReplayInvalidSyt.fetch_add( 1, std::memory_order_relaxed); failProducer( - ASFW::IsochTransport::TxProducerStage:: + ASFW::Audio::Runtime::TxProducerFaultStage:: kReplaySytValidation, - ASFW::IsochTransport:: - TxProducerFailureReason:: + ASFW::Audio::Runtime::TxProducerFaultReason:: kInvalidReplaySyt, ASFW::Audio::Runtime::FatalStreamReason:: TxReplayInvalidSyt, @@ -402,10 +398,10 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, if (prepareResult != ASFW::Protocols::Audio::DICE::TxSlotPrepareResult:: kPrepared) { - ASFW::IsochTransport::TxProducerStage stage = - ASFW::IsochTransport::TxProducerStage::kSlotAcquire; - ASFW::IsochTransport::TxProducerFailureReason producerReason = - ASFW::IsochTransport::TxProducerFailureReason:: + ASFW::Audio::Runtime::TxProducerFaultStage stage = + ASFW::Audio::Runtime::TxProducerFaultStage::kSlotAcquire; + ASFW::Audio::Runtime::TxProducerFaultReason producerReason = + ASFW::Audio::Runtime::TxProducerFaultReason:: kSlotUnavailable; ASFW::Audio::Runtime::FatalStreamReason runtimeReason = ASFW::Audio::Runtime::FatalStreamReason:: @@ -415,11 +411,10 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, case ASFW::Protocols::Audio::DICE:: TxSlotPrepareResult::kPacketizerRejected: stage = - ASFW::IsochTransport::TxProducerStage:: + ASFW::Audio::Runtime::TxProducerFaultStage:: kPacketize; producerReason = - ASFW::IsochTransport:: - TxProducerFailureReason:: + ASFW::Audio::Runtime::TxProducerFaultReason:: kPacketizerRejected; runtimeReason = ASFW::Audio::Runtime::FatalStreamReason:: @@ -428,11 +423,10 @@ uint32_t PrepareTransmitSlots(ASFWAudioDriver_IVars& ivars, case ASFW::Protocols::Audio::DICE:: TxSlotPrepareResult::kSlotPublishFailed: stage = - ASFW::IsochTransport::TxProducerStage:: + ASFW::Audio::Runtime::TxProducerFaultStage:: kSlotPublish; producerReason = - ASFW::IsochTransport:: - TxProducerFailureReason:: + ASFW::Audio::Runtime::TxProducerFaultReason:: kSlotPublishFailed; break; case ASFW::Protocols::Audio::DICE:: @@ -557,14 +551,14 @@ void IMPL(ASFWAudioDriver, TxPreparationReady) return; } - auto* txControl = ivars->runtime.txSlotProvider.controlBlock; + auto* txControl = ivars->runtime.txSlotProvider.queueControl; const uint32_t numSlots = ivars->runtime.txSlotProvider.numSlots; if (!txControl || numSlots == 0) { return; } const uint64_t requested = - txControl->preparationRequestGeneration.load( + txControl->refillRequestGeneration.load( std::memory_order_acquire); if (generation > requested) { return; @@ -573,7 +567,7 @@ void IMPL(ASFWAudioDriver, TxPreparationReady) const uint64_t completionCursor = txControl->completionCursor.load(std::memory_order_acquire); const uint64_t exposeCursor = - txControl->exposeCursor.load(std::memory_order_acquire); + txControl->committedEnd.load(std::memory_order_acquire); const uint64_t packetCoverageTarget = completionCursor + ASFW::IsochTransport::AudioTimingGeometry:: @@ -682,7 +676,7 @@ void IMPL(ASFWAudioDriver, TxPreparationReady) if (directControl) { const uint64_t now = mach_absolute_time(); const uint64_t requestedAt = - txControl->preparationRequestHostTicks.load( + txControl->refillRequestHostTicks.load( std::memory_order_relaxed); const uint64_t latency = now >= requestedAt ? now - requestedAt : 0; @@ -802,13 +796,13 @@ void IMPL(ASFWAudioDriver, TxPreparationReady) directControl->txPreparationRequests.requestHostTicks.store( requestedAt, std::memory_order_relaxed); directControl->counters.txPreparationWakeRequests.store( - txControl->preparationRequestCount.load( + txControl->refillRequestCount.load( std::memory_order_relaxed), std::memory_order_relaxed); directControl->counters.txPreparationWakeDispatches.fetch_add( 1, std::memory_order_relaxed); directControl->counters.txPreparationWakeCoalesced.store( - txControl->preparationCoalescedCount.load( + txControl->refillCoalescedCount.load( std::memory_order_relaxed), std::memory_order_relaxed); directControl->counters.txPreparationDrainPasses.fetch_add( @@ -817,5 +811,5 @@ void IMPL(ASFWAudioDriver, TxPreparationReady) requested, now); } - txControl->MarkPreparationHandled(requested); + txControl->MarkRefillHandled(requested); } diff --git a/ASFWDriver/Audio/DriverKit/Runtime/AudioTransportControlBlock.hpp b/ASFWDriver/Audio/DriverKit/Runtime/AudioTransportControlBlock.hpp index aa79ec80..f0bff42f 100644 --- a/ASFWDriver/Audio/DriverKit/Runtime/AudioTransportControlBlock.hpp +++ b/ASFWDriver/Audio/DriverKit/Runtime/AudioTransportControlBlock.hpp @@ -5,6 +5,7 @@ #include "DeviceTimeline.hpp" #include "TxSytTrace.hpp" #include "PayloadWriterTelemetry.hpp" +#include "TxWirePayloadTelemetry.hpp" #include "../../Runtime/HostClockAnchor.hpp" #include "../../Wire/AMDTP/RxSequenceReplay.hpp" #include "../../Wire/AMDTP/RxSytCadence.hpp" @@ -29,6 +30,141 @@ enum class FatalStreamReason : uint32_t { TxReplayInvalidSyt, }; +enum class TxProducerFaultStage : uint32_t { + kNone = 0, + kPreflight, + kExecutionAnchor, + kReplayBegin, + kReplayRead, + kReplaySytValidation, + kSlotAcquire, + kPacketize, + kSlotPublish, +}; + +[[nodiscard]] inline const char* TxProducerFaultStageName( + TxProducerFaultStage stage) noexcept { + switch (stage) { + case TxProducerFaultStage::kNone: return "none"; + case TxProducerFaultStage::kPreflight: return "preflight"; + case TxProducerFaultStage::kExecutionAnchor: return "execution-anchor"; + case TxProducerFaultStage::kReplayBegin: return "replay-begin"; + case TxProducerFaultStage::kReplayRead: return "replay-read"; + case TxProducerFaultStage::kReplaySytValidation: return "replay-syt-validation"; + case TxProducerFaultStage::kSlotAcquire: return "slot-acquire"; + case TxProducerFaultStage::kPacketize: return "packetize"; + case TxProducerFaultStage::kSlotPublish: return "slot-publish"; + } + return "unknown"; +} + +enum class TxProducerFaultReason : uint32_t { + kNone = 0, + kInvalidTransport, + kReplayUnavailable, + kInvalidReplaySyt, + kSlotUnavailable, + kPacketizerRejected, + kSlotPublishFailed, +}; + +[[nodiscard]] inline const char* TxProducerFaultReasonName( + TxProducerFaultReason reason) noexcept { + switch (reason) { + case TxProducerFaultReason::kNone: return "none"; + case TxProducerFaultReason::kInvalidTransport: return "invalid-transport"; + case TxProducerFaultReason::kReplayUnavailable: return "replay-unavailable"; + case TxProducerFaultReason::kInvalidReplaySyt: return "invalid-replay-syt"; + case TxProducerFaultReason::kSlotUnavailable: return "slot-unavailable"; + case TxProducerFaultReason::kPacketizerRejected: return "packetizer-rejected"; + case TxProducerFaultReason::kSlotPublishFailed: return "slot-publish-failed"; + } + return "unknown"; +} + +struct TxProducerFaultRecord final { + uint64_t generation{0}; + TxProducerFaultStage stage{TxProducerFaultStage::kNone}; + TxProducerFaultReason reason{TxProducerFaultReason::kNone}; + uint64_t packetIndex{0}; + uint64_t rangeStart{0}; + uint64_t rangeTarget{0}; + uint32_t preparedCount{0}; + uint64_t completionCursor{0}; + uint64_t committedEnd{0}; + uint64_t replayProducerCursor{0}; + uint32_t replayEpoch{0}; +}; + +struct TxProducerFaultSnapshot final { + std::atomic generation{0}; + std::atomic stage{static_cast(TxProducerFaultStage::kNone)}; + std::atomic reason{static_cast(TxProducerFaultReason::kNone)}; + std::atomic packetIndex{0}; + std::atomic rangeStart{0}; + std::atomic rangeTarget{0}; + std::atomic preparedCount{0}; + std::atomic completionCursor{0}; + std::atomic committedEnd{0}; + std::atomic replayProducerCursor{0}; + std::atomic replayEpoch{0}; + + void Reset() noexcept { + stage.store(static_cast(TxProducerFaultStage::kNone), std::memory_order_relaxed); + reason.store(static_cast(TxProducerFaultReason::kNone), std::memory_order_relaxed); + packetIndex.store(0, std::memory_order_relaxed); + rangeStart.store(0, std::memory_order_relaxed); + rangeTarget.store(0, std::memory_order_relaxed); + preparedCount.store(0, std::memory_order_relaxed); + completionCursor.store(0, std::memory_order_relaxed); + committedEnd.store(0, std::memory_order_relaxed); + replayProducerCursor.store(0, std::memory_order_relaxed); + replayEpoch.store(0, std::memory_order_relaxed); + generation.store(0, std::memory_order_release); + } + + [[nodiscard]] uint64_t Publish(const TxProducerFaultRecord& record) noexcept { + const uint64_t next = generation.load(std::memory_order_relaxed) + 1; + stage.store(static_cast(record.stage), std::memory_order_relaxed); + reason.store(static_cast(record.reason), std::memory_order_relaxed); + packetIndex.store(record.packetIndex, std::memory_order_relaxed); + rangeStart.store(record.rangeStart, std::memory_order_relaxed); + rangeTarget.store(record.rangeTarget, std::memory_order_relaxed); + preparedCount.store(record.preparedCount, std::memory_order_relaxed); + completionCursor.store(record.completionCursor, std::memory_order_relaxed); + committedEnd.store(record.committedEnd, std::memory_order_relaxed); + replayProducerCursor.store(record.replayProducerCursor, std::memory_order_relaxed); + replayEpoch.store(record.replayEpoch, std::memory_order_relaxed); + generation.store(next, std::memory_order_release); + return next; + } + + [[nodiscard]] bool TryRead(TxProducerFaultRecord& out) const noexcept { + for (uint32_t attempt = 0; attempt < 4; ++attempt) { + const uint64_t before = generation.load(std::memory_order_acquire); + if (before == 0) return false; + TxProducerFaultRecord record{}; + record.generation = before; + record.stage = static_cast(stage.load(std::memory_order_relaxed)); + record.reason = static_cast(reason.load(std::memory_order_relaxed)); + record.packetIndex = packetIndex.load(std::memory_order_relaxed); + record.rangeStart = rangeStart.load(std::memory_order_relaxed); + record.rangeTarget = rangeTarget.load(std::memory_order_relaxed); + record.preparedCount = preparedCount.load(std::memory_order_relaxed); + record.completionCursor = completionCursor.load(std::memory_order_relaxed); + record.committedEnd = committedEnd.load(std::memory_order_relaxed); + record.replayProducerCursor = replayProducerCursor.load(std::memory_order_relaxed); + record.replayEpoch = replayEpoch.load(std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + if (generation.load(std::memory_order_relaxed) == before) { + out = record; + return true; + } + } + return false; + } +}; + struct TxPreparationRequestState final { std::atomic requestedGeneration{0}; std::atomic handledGeneration{0}; @@ -124,11 +260,13 @@ struct AudioTransportControlBlock final { // TX control block members PayloadWriterTelemetryRing payloadWriterTelemetry{}; + TxWirePayloadTelemetry txWirePayloadTelemetry{}; // Latest-value trace of the live replay TX SYT decision (diagnostics). TxSytTraceLatest txSytTrace{}; TxPreparationRequestState txPreparationRequests{}; TxFatalSnapshot txFatalSnapshot{}; + TxProducerFaultSnapshot txProducerFault{}; std::atomic outputConsumedEndFrame{0}; std::atomic outputUnderruns{0}; @@ -211,9 +349,11 @@ struct AudioTransportControlBlock final { // Reset TX members payloadWriterTelemetry.Reset(); + txWirePayloadTelemetry.Reset(); txSytTrace.Reset(); txPreparationRequests.Reset(); txFatalSnapshot.Reset(); + txProducerFault.Reset(); outputConsumedEndFrame.store(0, std::memory_order_relaxed); outputUnderruns.store(0, std::memory_order_relaxed); diff --git a/ASFWDriver/Audio/DriverKit/Runtime/TxWirePayloadTelemetry.hpp b/ASFWDriver/Audio/DriverKit/Runtime/TxWirePayloadTelemetry.hpp new file mode 100644 index 00000000..4bab746f --- /dev/null +++ b/ASFWDriver/Audio/DriverKit/Runtime/TxWirePayloadTelemetry.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include + +namespace ASFW::Audio::Runtime { + +/// Audio-owned, producer-side payload diagnostics. This deliberately lives +/// beside the packetizer rather than at the IT DMA refill: transport must not +/// know CIP or AM824 layout. All counters are drained through ring-only log +/// events, never unified logging on the packet hot path. +struct TxWirePayloadObservation final { + bool firstInfo{false}; + bool dropout{false}; + uint32_t infoQuads{0}; + uint32_t maxAbs24{0}; + uint32_t lastInfoQuad{0}; +}; + +struct TxWirePayloadTelemetry final { + std::atomic dataPackets{0}; + std::atomic zeroPcmPackets{0}; + std::atomic infoQuads{0}; + std::atomic pcmDropouts{0}; + std::atomic maxAbs24{0}; + std::atomic lastInfoQuad{0}; + std::atomic firstInfoPacketIndex{0}; + + void Reset() noexcept { + dataPackets.store(0, std::memory_order_relaxed); + zeroPcmPackets.store(0, std::memory_order_relaxed); + infoQuads.store(0, std::memory_order_relaxed); + pcmDropouts.store(0, std::memory_order_relaxed); + maxAbs24.store(0, std::memory_order_relaxed); + lastInfoQuad.store(0, std::memory_order_relaxed); + firstInfoPacketIndex.store(0, std::memory_order_relaxed); + lastPacketHadInfo_ = false; + firstInfoSeen_ = false; + } + + [[nodiscard]] TxWirePayloadObservation Observe( + uint64_t packetIndex, const uint8_t* packetBytes, + uint32_t payloadLength) noexcept { + constexpr uint32_t kCipHeaderBytes = 8; + constexpr uint32_t kIdleSlotWord = 0x80000000u; + TxWirePayloadObservation observation{}; + if (packetBytes == nullptr || payloadLength <= kCipHeaderBytes) { + return observation; + } + + dataPackets.fetch_add(1, std::memory_order_relaxed); + const uint8_t* quadBytes = packetBytes + kCipHeaderBytes; + const uint32_t quadCount = (payloadLength - kCipHeaderBytes) / 4; + for (uint32_t index = 0; index < quadCount; ++index, quadBytes += 4) { + const uint32_t quad = + (static_cast(quadBytes[0]) << 24) | + (static_cast(quadBytes[1]) << 16) | + (static_cast(quadBytes[2]) << 8) | + static_cast(quadBytes[3]); + if (quad == 0 || quad == kIdleSlotWord) continue; + ++observation.infoQuads; + observation.lastInfoQuad = quad; + const int32_t sample24 = static_cast(quad << 8) >> 8; + const uint32_t magnitude = static_cast( + sample24 < 0 ? -static_cast(sample24) : sample24); + observation.maxAbs24 = magnitude > observation.maxAbs24 + ? magnitude : observation.maxAbs24; + } + + if (observation.infoQuads == 0) { + zeroPcmPackets.fetch_add(1, std::memory_order_relaxed); + observation.dropout = lastPacketHadInfo_; + if (observation.dropout) { + pcmDropouts.fetch_add(1, std::memory_order_relaxed); + } + lastPacketHadInfo_ = false; + return observation; + } + + infoQuads.fetch_add(observation.infoQuads, std::memory_order_relaxed); + lastInfoQuad.store(observation.lastInfoQuad, std::memory_order_relaxed); + uint32_t previous = maxAbs24.load(std::memory_order_relaxed); + while (observation.maxAbs24 > previous && + !maxAbs24.compare_exchange_weak( + previous, observation.maxAbs24, std::memory_order_relaxed, + std::memory_order_relaxed)) { + } + observation.firstInfo = !firstInfoSeen_; + if (observation.firstInfo) { + firstInfoPacketIndex.store(packetIndex, std::memory_order_relaxed); + firstInfoSeen_ = true; + } + lastPacketHadInfo_ = true; + return observation; + } + +private: + bool lastPacketHadInfo_{false}; + bool firstInfoSeen_{false}; +}; + +} // namespace ASFW::Audio::Runtime diff --git a/ASFWDriver/Isoch/Core/IsochDmaGeometry.hpp b/ASFWDriver/Isoch/Core/IsochDmaGeometry.hpp index 83f52287..be673a1c 100644 --- a/ASFWDriver/Isoch/Core/IsochDmaGeometry.hpp +++ b/ASFWDriver/Isoch/Core/IsochDmaGeometry.hpp @@ -11,6 +11,7 @@ namespace ASFW::Isoch { struct IsochDmaGeometry final { static constexpr uint32_t kPacketsPerInterrupt = 6; static constexpr uint32_t kReceiveDescriptorPackets = 504; + static constexpr uint32_t kTransmitDescriptorPackets = 48; }; static_assert(IsochDmaGeometry::kPacketsPerInterrupt != 0); @@ -18,5 +19,9 @@ static_assert(IsochDmaGeometry::kReceiveDescriptorPackets % IsochDmaGeometry::kPacketsPerInterrupt == 0, "IR descriptor ring must contain complete interrupt groups"); +static_assert(IsochDmaGeometry::kTransmitDescriptorPackets % + IsochDmaGeometry::kPacketsPerInterrupt == + 0, + "IT descriptor ring must contain complete interrupt groups"); } // namespace ASFW::Isoch diff --git a/ASFWDriver/Isoch/Core/IsochEventGroup.hpp b/ASFWDriver/Isoch/Core/IsochEventGroup.hpp index 5278424a..eff10d87 100644 --- a/ASFWDriver/Isoch/Core/IsochEventGroup.hpp +++ b/ASFWDriver/Isoch/Core/IsochEventGroup.hpp @@ -1,6 +1,6 @@ #pragma once -#include "../../Shared/Isoch/AudioTimingGeometry.hpp" +#include "IsochDmaGeometry.hpp" #include @@ -11,28 +11,13 @@ enum class IsochEventDirection : uint8_t { kTransmit, }; -struct IsochEventGroup final { - IsochEventDirection direction{IsochEventDirection::kTransmit}; - uint64_t hostTicks{0}; - uint32_t hwPacketIndex{0}; - uint32_t completedPacketIndex{0}; - uint32_t completedPacketCount{0}; - uint32_t firstRefillPacket{0}; - uint32_t refillPacketCount{0}; - uint16_t outputLastTimestamp{0}; - uint64_t sampleFrame{0}; - - [[nodiscard]] bool HasCompletionTimestamp() const noexcept { - return outputLastTimestamp != 0; - } -}; - -[[nodiscard]] constexpr uint32_t TimingGroupPacketCount48k() noexcept { - return ASFW::IsochTransport::AudioTimingGeometry::kTimingGroupPackets; +[[nodiscard]] constexpr uint32_t InterruptGroupPacketCount() noexcept { + return IsochDmaGeometry::kPacketsPerInterrupt; } [[nodiscard]] constexpr bool IsTimingGroupBoundary(uint32_t packetIndex) noexcept { - return (packetIndex % TimingGroupPacketCount48k()) == (TimingGroupPacketCount48k() - 1); + return (packetIndex % InterruptGroupPacketCount()) == + (InterruptGroupPacketCount() - 1); } [[nodiscard]] constexpr uint32_t PreviousPacketIndex(uint32_t packetIndex, diff --git a/ASFWDriver/Isoch/Core/IsochTxQueue.hpp b/ASFWDriver/Isoch/Core/IsochTxQueue.hpp new file mode 100644 index 00000000..66a3a8d2 --- /dev/null +++ b/ASFWDriver/Isoch/Core/IsochTxQueue.hpp @@ -0,0 +1,189 @@ +// IsochTxQueue.hpp +// ASFW - Payload-opaque shared queue contract for OHCI isochronous transmit. +// +// This is the only shared ABI between an IT packet producer and the OHCI +// transport consumer. It deliberately contains no content-format, audio-clock, +// or producer-policy concepts: the immediate header and payload bytes are +// opaque to transport. Bump kTxQueueAbiVersion for every layout change. + +#pragma once + +#include +#include +#include + +namespace ASFW::Isoch { + +inline constexpr uint32_t kTxQueueAbiVersion = 5; + +/// Producer fills the plain fields, then release-stores commitGeneration. +/// Consumer acquire-loads it and accepts only ExpectedTxCommitGeneration(). +struct alignas(64) IsochTxPacketMeta final { + uint32_t immediateHeader[2]; ///< Opaque OUTPUT_MORE_IMMEDIATE quadlets. + uint32_t payloadLength; ///< Opaque payload byte count. + uint32_t reserved0; + uint64_t packetIndex; ///< Absolute packet index. + std::atomic commitGeneration{0}; + uint8_t reserved1[64 - 32]; +}; + +static_assert(sizeof(IsochTxPacketMeta) == 64); +static_assert(alignof(IsochTxPacketMeta) == 64); +static_assert(offsetof(IsochTxPacketMeta, immediateHeader) == 0); +static_assert(offsetof(IsochTxPacketMeta, payloadLength) == 8); +static_assert(offsetof(IsochTxPacketMeta, packetIndex) == 16); +static_assert(offsetof(IsochTxPacketMeta, commitGeneration) == 24); +static_assert(std::atomic::is_always_lock_free); + +[[nodiscard]] constexpr uint32_t TxQueueSlotIndexFor( + uint64_t packetIndex, uint32_t numSlots) noexcept { + return static_cast(packetIndex % numSlots); +} + +[[nodiscard]] constexpr uint64_t ExpectedTxCommitGeneration( + uint64_t packetIndex, uint32_t numSlots) noexcept { + return packetIndex / numSlots + 1; +} + +/// Raw host/cycle anchor sampled by the IT consumer. Interpretation belongs +/// to the packet producer's content/timing domain. +struct IsochTxClockPairSample final { + uint64_t hostTimeMid{0}; + uint32_t cycleTimer32{0}; +}; + +struct IsochTxClockPairSeqlock final { + std::atomic sequence{0}; + std::atomic hostTimeMid{0}; + std::atomic cycleTimer32{0}; + + void Publish(const IsochTxClockPairSample& sample) noexcept { + const uint32_t sequenceBefore = sequence.load(std::memory_order_relaxed); + sequence.store(sequenceBefore + 1, std::memory_order_release); + hostTimeMid.store(sample.hostTimeMid, std::memory_order_relaxed); + cycleTimer32.store(sample.cycleTimer32, std::memory_order_relaxed); + sequence.store(sequenceBefore + 2, std::memory_order_release); + } + + [[nodiscard]] bool TryRead(IsochTxClockPairSample& out, + uint32_t maxAttempts = 4) const noexcept { + for (uint32_t attempt = 0; attempt < maxAttempts; ++attempt) { + const uint32_t before = sequence.load(std::memory_order_acquire); + if ((before & 1u) != 0) continue; + const IsochTxClockPairSample sample{ + .hostTimeMid = hostTimeMid.load(std::memory_order_relaxed), + .cycleTimer32 = cycleTimer32.load(std::memory_order_relaxed), + }; + std::atomic_thread_fence(std::memory_order_acquire); + if (sequence.load(std::memory_order_relaxed) == before) { + out = sample; + return true; + } + } + return false; + } +}; + +inline constexpr uint32_t kIsochTxCompletionStampSlots = 32; + +struct IsochTxCompletionStamp final { + std::atomic packetIndex{0}; + std::atomic cycleTimestamp{0}; + uint32_t reserved{0}; +}; +static_assert(sizeof(IsochTxCompletionStamp) == 16); + +enum class IsochTxQueueStatus : uint32_t { + kStopped = 0, + kRunning = 1, + kProducerFault = 2, + kDeadContext = 3, +}; + +/// Fixed, neutral third shared descriptor of the three-descriptor TX RPC. +/// Geometry is initialized by transport. Producer-owned and consumer-owned +/// progress are reset independently to prevent one side from erasing the +/// other's prefill or completion state. +struct IsochTxQueueControl final { + uint32_t abiVersion{0}; + uint32_t numSlots{0}; + uint32_t slotStrideBytes{0}; + uint32_t maxPacketBytes{0}; + uint32_t interruptInterval{0}; + uint32_t reserved0{0}; + + // Consumer → producer: raw execution progress/anchors. + std::atomic streamGeneration{0}; + std::atomic statusWord{IsochTxQueueStatus::kStopped}; + IsochTxClockPairSeqlock clockPair{}; + std::atomic startCycleMatch{0}; + std::atomic startFirstPacketIndex{0}; + std::atomic completionCursor{0}; + std::atomic completionStampCount{0}; + IsochTxCompletionStamp completionStamps[kIsochTxCompletionStampSlots]{}; + std::atomic refillRequestGeneration{0}; + std::atomic refillHandledGeneration{0}; + std::atomic refillRequestHostTicks{0}; + std::atomic refillRequestCount{0}; + std::atomic refillCoalescedCount{0}; + std::atomic maxCompletionDelta{0}; + std::atomic maxCompletionDeltaEvents{0}; + + // Producer → consumer: end-exclusive committed packet cursor. + std::atomic committedEnd{0}; + + /// Runs in the producer before prefill. It never resets consumer state. + void ResetProducerForStart() noexcept { + committedEnd.store(0, std::memory_order_release); + } + + /// Runs in the transport immediately before arm. It never resets prefill. + void ResetConsumerForArm() noexcept { + streamGeneration.store(0, std::memory_order_relaxed); + statusWord.store(IsochTxQueueStatus::kStopped, std::memory_order_relaxed); + startCycleMatch.store(0, std::memory_order_relaxed); + startFirstPacketIndex.store(0, std::memory_order_relaxed); + completionCursor.store(0, std::memory_order_relaxed); + completionStampCount.store(0, std::memory_order_relaxed); + refillRequestGeneration.store(0, std::memory_order_relaxed); + refillHandledGeneration.store(0, std::memory_order_relaxed); + refillRequestHostTicks.store(0, std::memory_order_relaxed); + refillRequestCount.store(0, std::memory_order_relaxed); + refillCoalescedCount.store(0, std::memory_order_relaxed); + maxCompletionDelta.store(0, std::memory_order_relaxed); + maxCompletionDeltaEvents.store(0, std::memory_order_relaxed); + } + + void MarkRefillHandled(uint64_t generation) noexcept { + uint64_t handled = refillHandledGeneration.load(std::memory_order_relaxed); + while (handled < generation && + !refillHandledGeneration.compare_exchange_weak( + handled, generation, std::memory_order_release, + std::memory_order_relaxed)) { + } + } + + void PushCompletionStamp(uint64_t packetIndex, uint32_t cycleTimestamp) noexcept { + const uint64_t count = completionStampCount.load(std::memory_order_relaxed); + auto& slot = completionStamps[count % kIsochTxCompletionStampSlots]; + slot.packetIndex.store(packetIndex, std::memory_order_relaxed); + slot.cycleTimestamp.store(cycleTimestamp, std::memory_order_relaxed); + completionStampCount.store(count + 1, std::memory_order_release); + } + + [[nodiscard]] bool ReadCompletionStamp(uint64_t stampIndex, + uint64_t& outPacketIndex, + uint32_t& outCycleTimestamp) const noexcept { + const auto& slot = completionStamps[stampIndex % kIsochTxCompletionStampSlots]; + outPacketIndex = slot.packetIndex.load(std::memory_order_relaxed); + outCycleTimestamp = slot.cycleTimestamp.load(std::memory_order_relaxed); + const uint64_t countAfter = completionStampCount.load(std::memory_order_acquire); + return stampIndex < countAfter && + countAfter - stampIndex <= kIsochTxCompletionStampSlots; + } +}; + +static_assert(std::atomic::is_always_lock_free); +static_assert(std::atomic::is_always_lock_free); + +} // namespace ASFW::Isoch diff --git a/ASFWDriver/Isoch/IsochService.cpp b/ASFWDriver/Isoch/IsochService.cpp index 4286e6de..359a9436 100644 --- a/ASFWDriver/Isoch/IsochService.cpp +++ b/ASFWDriver/Isoch/IsochService.cpp @@ -7,7 +7,7 @@ #include #include #endif -#include "../Shared/Isoch/IsochAudioTransport.hpp" +#include "Core/IsochTxQueue.hpp" #include "Memory/IsochDMAMemoryManager.hpp" namespace ASFW::Driver { @@ -315,8 +315,7 @@ kern_return_t IsochService::PrepareTransmit(uint8_t channel, HardwareInterface& if (txPayloadSlab_[0] && txMetadataRing_[0] && txControlBlock_[0]) { const kern_return_t memKr = isochTransmitContext_->SetSharedMemoryDescriptors( txPayloadSlab_[0].get(), txMetadataRing_[0].get(), txControlBlock_[0].get(), - interruptInterval_, - ASFW::IsochTransport::AudioTimingGeometry::kHalZeroTimestampPeriodFrames); + interruptInterval_); if (memKr != kIOReturnSuccess) { ASFW_LOG(Isoch, "IsochService: IT shared-memory setup failed: 0x%08x", memKr); return memKr; @@ -376,8 +375,7 @@ kern_return_t IsochService::PrepareTransmitStream(uint32_t streamIndex, uint8_t txControlBlock_[streamIndex]) { const kern_return_t memKr = slot->SetSharedMemoryDescriptors( txPayloadSlab_[streamIndex].get(), txMetadataRing_[streamIndex].get(), - txControlBlock_[streamIndex].get(), interruptInterval_, - ASFW::IsochTransport::AudioTimingGeometry::kHalZeroTimestampPeriodFrames); + txControlBlock_[streamIndex].get(), interruptInterval_); if (memKr != kIOReturnSuccess) { ASFW_LOG(Isoch, "IsochService: secondary IT shared-memory setup failed (stream %u): 0x%08x", @@ -597,7 +595,7 @@ kern_return_t IsochService::AllocateTxIsochResources(uint32_t streamIndex, uint3 // 2. Allocate metadata ring (cacheline aligned) const size_t metadataRingBytes = - static_cast(numSlots) * sizeof(ASFW::IsochTransport::TxPacketMeta); + static_cast(numSlots) * sizeof(ASFW::Isoch::IsochTxPacketMeta); IOBufferMemoryDescriptor* metadataDescriptor = nullptr; kr = IOBufferMemoryDescriptor::Create(kIOMemoryDirectionInOut, metadataRingBytes, 64, &metadataDescriptor); @@ -610,7 +608,7 @@ kern_return_t IsochService::AllocateTxIsochResources(uint32_t streamIndex, uint3 OSSharedPtr(metadataDescriptor, OSNoRetain); // 3. Allocate control block (cacheline aligned) - const size_t controlBlockBytes = sizeof(ASFW::IsochTransport::TxStreamControl); + const size_t controlBlockBytes = sizeof(ASFW::Isoch::IsochTxQueueControl); IOBufferMemoryDescriptor* controlDescriptor = nullptr; kr = IOBufferMemoryDescriptor::Create(kIOMemoryDirectionInOut, controlBlockBytes, 64, &controlDescriptor); diff --git a/ASFWDriver/Isoch/Transmit/IsochTransmitContext.cpp b/ASFWDriver/Isoch/Transmit/IsochTransmitContext.cpp index ef736ebf..81bbd72d 100644 --- a/ASFWDriver/Isoch/Transmit/IsochTransmitContext.cpp +++ b/ASFWDriver/Isoch/Transmit/IsochTransmitContext.cpp @@ -79,8 +79,7 @@ kern_return_t IsochTransmitContext::SetSharedMemoryDescriptors( IOMemoryDescriptor* payloadSlab, IOMemoryDescriptor* metadataRing, IOMemoryDescriptor* controlBlock, - uint32_t interruptInterval, - uint32_t ztsPeriodFrames) noexcept { + uint32_t interruptInterval) noexcept { if (!payloadSlab || !metadataRing || !controlBlock) { return kIOReturnBadArgument; @@ -193,7 +192,7 @@ kern_return_t IsochTransmitContext::SetSharedMemoryDescriptors( return kr; } metadataMap_ = OSSharedPtr(mMap, OSNoRetain); - metadataRing_ = reinterpret_cast(metadataMap_->GetAddress()); + metadataRing_ = reinterpret_cast(metadataMap_->GetAddress()); // 3. Map Control Block IOMemoryMap* cMap = nullptr; @@ -203,46 +202,45 @@ kern_return_t IsochTransmitContext::SetSharedMemoryDescriptors( return kr; } controlMap_ = OSSharedPtr(cMap, OSNoRetain); - controlBlock_ = reinterpret_cast(controlMap_->GetAddress()); + controlBlock_ = reinterpret_cast(controlMap_->GetAddress()); // Populate structural fields uint64_t metadataLen = 0; metadataRing->GetLength(&metadataLen); uint64_t payloadLen = 0; payloadSlab->GetLength(&payloadLen); + uint64_t controlLen = 0; + controlBlock->GetLength(&controlLen); - const uint32_t numSlots = static_cast(metadataLen / sizeof(ASFW::IsochTransport::TxPacketMeta)); + if (metadataLen == 0 || + (metadataLen % sizeof(IsochTxPacketMeta)) != 0 || + controlLen < sizeof(IsochTxQueueControl)) { + ASFW_LOG(Isoch, + "IT: Invalid TX queue mapping metadataBytes=%llu controlBytes=%llu", + metadataLen, controlLen); + return kIOReturnBadArgument; + } + + const uint32_t numSlots = static_cast(metadataLen / sizeof(IsochTxPacketMeta)); + if (numSlots == 0 || payloadLen == 0 || payloadLen % numSlots != 0) { + ASFW_LOG(Isoch, + "IT: Invalid TX queue geometry payloadBytes=%llu slots=%u", + payloadLen, numSlots); + return kIOReturnBadArgument; + } const uint32_t maxPacketBytes = static_cast(payloadLen / numSlots); - controlBlock_->abiVersion = ASFW::IsochTransport::kTransportAbiVersion; + controlBlock_->abiVersion = kTxQueueAbiVersion; controlBlock_->numSlots = numSlots; controlBlock_->slotStrideBytes = maxPacketBytes; controlBlock_->maxPacketBytes = maxPacketBytes; controlBlock_->interruptInterval = interruptInterval; - controlBlock_->ztsPeriodFrames = ztsPeriodFrames; - - // Reset consumer-owned runtime counters / status only. exposeCursor is - // PRODUCER-owned (the audio side advances it as it commits slots) and the - // audio-side prefill has already run by the time this consumer maps the - // block — resetting it here would stomp the prefill's committed lead back to - // zero, desyncing the pump's lead math and starving the ring after the - // prefilled packets drain. The buffer is freshly allocated (zero-filled) - // each start, so exposeCursor is 0 unless the prefill legitimately set it. - controlBlock_->streamGeneration.store(0, std::memory_order_relaxed); - controlBlock_->statusWord.store(ASFW::IsochTransport::TxStreamStatus::kStopped, std::memory_order_relaxed); - controlBlock_->completionCursor.store(0, std::memory_order_relaxed); - controlBlock_->completionStampCount.store(0, std::memory_order_relaxed); - controlBlock_->preparationRequestGeneration.store( - 0, std::memory_order_relaxed); - controlBlock_->preparationHandledGeneration.store( - 0, std::memory_order_relaxed); - controlBlock_->preparationRequestHostTicks.store( - 0, std::memory_order_relaxed); - controlBlock_->preparationRequestCount.store( - 0, std::memory_order_relaxed); - controlBlock_->preparationCoalescedCount.store( - 0, std::memory_order_relaxed); - controlBlock_->producerFailure.Reset(); + controlBlock_->ResetConsumerForArm(); + if (controlBlock_->abiVersion != kTxQueueAbiVersion) { + ASFW_LOG(Isoch, "IT: TX queue ABI write validation failed abi=%u", + controlBlock_->abiVersion); + return kIOReturnInternalError; + } ASFW_LOG(Isoch, "IT: Mapped shared memory. payloadSegments=%zu metadataRing=%p controlBlock=%p slots=%u maxBytes=%u", payloadDmaMap_.SegmentCount(), metadataRing_, controlBlock_, numSlots, maxPacketBytes); @@ -290,7 +288,7 @@ kern_return_t IsochTransmitContext::Start() noexcept { ASFW_LOG(Isoch, "IT: Starting transmit context (Stage 3 - ADK Phase 2)"); - const uint64_t preFillCount = controlBlock_->exposeCursor.load(std::memory_order_relaxed); + const uint64_t preFillCount = controlBlock_->committedEnd.load(std::memory_order_relaxed); const auto primeStats = ring_.Prime(payloadDmaMap_, controlBlock_->numSlots, controlBlock_->slotStrideBytes, metadataRing_, preFillCount); if (primeStats.packetsAssembled != Tx::Layout::kNumPackets) { @@ -299,7 +297,7 @@ kern_return_t IsochTransmitContext::Start() noexcept { } packetsAssembled_ = primeStats.packetsAssembled; controlBlock_->statusWord.store( - ASFW::IsochTransport::TxStreamStatus::kRunning, + IsochTxQueueStatus::kRunning, std::memory_order_release); Register32 cmdPtrReg = static_cast(DMAContextHelpers::IsoXmitCommandPtr(contextIndex_)); @@ -391,23 +389,13 @@ kern_return_t IsochTransmitContext::Stop() noexcept { } if (controlBlock_) { - controlBlock_->statusWord.store(ASFW::IsochTransport::TxStreamStatus::kStopped, std::memory_order_release); + controlBlock_->statusWord.store(IsochTxQueueStatus::kStopped, std::memory_order_release); } state_ = State::Stopped; refillInProgress_.clear(std::memory_order_release); ASFW_LOG(Isoch, "IT: Stopped. Stats: %llu pkts IRQs=%llu", packetsAssembled_, interruptCount_.load(std::memory_order_relaxed)); - const auto& rc = ring_.RTCounters(); - ASFW_LOG(Isoch, - "IT WIRE final data=%llu zeroPcm=%llu infoQuads=%llu dropouts=%llu maxAbs24=%u lastQuad=0x%08x firstInfoAbsIdx=%llu", - rc.wireDataPackets.load(std::memory_order_relaxed), - rc.wireZeroPcmPackets.load(std::memory_order_relaxed), - rc.wireInfoQuads.load(std::memory_order_relaxed), - rc.wirePcmDropouts.load(std::memory_order_relaxed), - rc.wireMaxAbs24.load(std::memory_order_relaxed), - rc.wireLastInfoQuad.load(std::memory_order_relaxed), - rc.wireFirstInfoAbsIdx.load(std::memory_order_relaxed)); return kIOReturnSuccess; } @@ -441,31 +429,6 @@ void IsochTransmitContext::DoRefillOnce(uint64_t eventHostTicks, payloadDmaMap_); if (!outcome.ok) { const auto& counters = ring_.RTCounters(); - if (outcome.failureReason == - Tx::IsochTxDmaRing::RefillFailureReason:: - ProducerFatalStatus && - outcome.producerFailureAvailable) { - const auto& failure = outcome.producerFailure; - ASFW_LOG( - Isoch, - "IT: Producer fatal stage=%{public}s reason=%{public}s " - "generation=%llu packet=%llu range=[%llu,%llu) " - "prepared=%u completion=%llu expose=%llu " - "replayProducer=%llu replayEpoch=%u", - ASFW::IsochTransport::TxProducerStageName( - failure.stage), - ASFW::IsochTransport::TxProducerFailureReasonName( - failure.reason), - failure.generation, - failure.packetIndex, - failure.rangeStart, - failure.rangeTarget, - failure.preparedCount, - failure.completionCursor, - failure.exposeCursor, - failure.replayProducerCursor, - failure.replayEpoch); - } ASFW_LOG( Isoch, "IT: Refill failed reason=%{public}s ctrl=0x%08x streamStatus=%u " @@ -494,9 +457,9 @@ void IsochTransmitContext::DoRefillOnce(uint64_t eventHostTicks, if (outcome.packetsFilled > 0) { ring_.WakeHardwareIfIdle(*hardware_, contextIndex_); } - if (outcome.preparationRequestGeneration != 0 && + if (outcome.refillRequestGeneration != 0 && txPreparationCallback_) { - txPreparationCallback_(outcome.preparationRequestGeneration); + txPreparationCallback_(outcome.refillRequestGeneration); } } } @@ -519,8 +482,8 @@ void IsochTransmitContext::StopImmediatelyForTxFault() noexcept { } if (controlBlock_) { const auto currentStatus = controlBlock_->statusWord.load(std::memory_order_acquire); - if (currentStatus != ASFW::IsochTransport::TxStreamStatus::kUnderrunFatal) { - controlBlock_->statusWord.store(ASFW::IsochTransport::TxStreamStatus::kDeadContext, std::memory_order_release); + if (currentStatus != IsochTxQueueStatus::kProducerFault) { + controlBlock_->statusWord.store(IsochTxQueueStatus::kDeadContext, std::memory_order_release); } } state_ = State::Stopped; diff --git a/ASFWDriver/Isoch/Transmit/IsochTransmitContext.hpp b/ASFWDriver/Isoch/Transmit/IsochTransmitContext.hpp index 1fa7e098..9a68d0da 100644 --- a/ASFWDriver/Isoch/Transmit/IsochTransmitContext.hpp +++ b/ASFWDriver/Isoch/Transmit/IsochTransmitContext.hpp @@ -11,9 +11,9 @@ #include "IsochTxDmaRing.hpp" #include "IsochTxLayout.hpp" +#include "../Core/IsochTxQueue.hpp" #include "../Memory/IIsochDMAMemory.hpp" #include "../../Hardware/RegisterMap.hpp" -#include "../../Shared/Isoch/IsochAudioTransport.hpp" #include "../../Logging/Logging.hpp" #include @@ -46,7 +46,7 @@ enum class ITState { * @brief Orchestrator for an Isochronous Transmit (IT) context. * * This class owns the OHCI DMA ring (IsochTxDmaRing) and manages the lifecycle - * of the transport. It does not interpret the payload bytes (AMDTP/CIP/Audio). + * of the transport. It treats packet headers and payload bytes as opaque. */ class IsochTransmitContext final { public: @@ -77,14 +77,12 @@ class IsochTransmitContext final { * @param metadataRing Shared memory descriptor containing packet metadata. * @param controlBlock Shared memory descriptor containing stream control states. * @param interruptInterval Interrupt interval in packets. - * @param ztsPeriodFrames Verification parameter for HAL ring wrap (passed through to control block). */ kern_return_t SetSharedMemoryDescriptors( IOMemoryDescriptor* payloadSlab, IOMemoryDescriptor* metadataRing, IOMemoryDescriptor* controlBlock, - uint32_t interruptInterval, - uint32_t ztsPeriodFrames) noexcept; + uint32_t interruptInterval) noexcept; kern_return_t Start() noexcept; // Clearing RUN only prevents new descriptor fetches. The caller must not @@ -143,8 +141,8 @@ class IsochTransmitContext final { OSSharedPtr controlMap_{nullptr}; uint8_t* payloadBase_{nullptr}; - ASFW::IsochTransport::TxPacketMeta* metadataRing_{nullptr}; - ASFW::IsochTransport::TxStreamControl* controlBlock_{nullptr}; + IsochTxPacketMeta* metadataRing_{nullptr}; + IsochTxQueueControl* controlBlock_{nullptr}; Tx::TxPayloadDmaMap payloadDmaMap_{}; OSSharedPtr payloadDmaCmd_{nullptr}; diff --git a/ASFWDriver/Isoch/Transmit/IsochTxDmaRing.cpp b/ASFWDriver/Isoch/Transmit/IsochTxDmaRing.cpp index 6b0c52b7..ed7f846a 100644 --- a/ASFWDriver/Isoch/Transmit/IsochTxDmaRing.cpp +++ b/ASFWDriver/Isoch/Transmit/IsochTxDmaRing.cpp @@ -42,94 +42,6 @@ void IsochTxDmaRing::ResetForStart() noexcept { counters_.lastDmaGapPackets.store(Layout::kNumPackets, std::memory_order_relaxed); counters_.minDmaGapPackets.store(Layout::kNumPackets, std::memory_order_relaxed); - counters_.wireDataPackets.store(0, std::memory_order_relaxed); - counters_.wireZeroPcmPackets.store(0, std::memory_order_relaxed); - counters_.wireInfoQuads.store(0, std::memory_order_relaxed); - counters_.wirePcmDropouts.store(0, std::memory_order_relaxed); - counters_.wireMaxAbs24.store(0, std::memory_order_relaxed); - counters_.wireLastInfoQuad.store(0, std::memory_order_relaxed); - counters_.wireFirstInfoAbsIdx.store(0, std::memory_order_relaxed); - wireLastPacketHadInfo_ = false; - wireFirstInfoLogged_ = false; -} - -void IsochTxDmaRing::GaugeWirePayload(uint64_t fillAbsIdx, - const uint8_t* packetBytes, - uint32_t payloadLength) noexcept { - constexpr uint32_t kCipHeaderBytes = 8; - constexpr uint32_t kIdleSlotWord = 0x80000000u; // AM824 no-info / idle MIDI - if (payloadLength <= kCipHeaderBytes) { - return; // NO-DATA packet: CIP header only - } - - const uint64_t dataPackets = - counters_.wireDataPackets.fetch_add(1, std::memory_order_relaxed) + 1; - if ((dataPackets % 8192) == 0) { - ASFW_LOG(Isoch, - "IT WIRE gauge data=%llu zeroPcm=%llu infoQuads=%llu dropouts=%llu maxAbs24=%u lastQuad=0x%08x", - dataPackets, - counters_.wireZeroPcmPackets.load(std::memory_order_relaxed), - counters_.wireInfoQuads.load(std::memory_order_relaxed), - counters_.wirePcmDropouts.load(std::memory_order_relaxed), - counters_.wireMaxAbs24.load(std::memory_order_relaxed), - counters_.wireLastInfoQuad.load(std::memory_order_relaxed)); - } - - const uint32_t quadCount = (payloadLength - kCipHeaderBytes) / 4; - const uint8_t* quadBytes = packetBytes + kCipHeaderBytes; - uint32_t infoQuads = 0; - uint32_t lastInfoQuad = 0; - uint32_t maxAbs24 = 0; - for (uint32_t i = 0; i < quadCount; ++i, quadBytes += 4) { - // Payload is stored in bus (big-endian) byte order. - const uint32_t quad = (static_cast(quadBytes[0]) << 24) | - (static_cast(quadBytes[1]) << 16) | - (static_cast(quadBytes[2]) << 8) | - static_cast(quadBytes[3]); - if (quad == 0 || quad == kIdleSlotWord) { - continue; - } - ++infoQuads; - lastInfoQuad = quad; - // 24-bit two's-complement magnitude, label-agnostic (works for both - // raw sign-extended 24-in-32 and 0x40-labelled AM824 MBLA slots). - const int32_t sample24 = static_cast(quad << 8) >> 8; - const uint32_t abs24 = static_cast( - sample24 < 0 ? -static_cast(sample24) : sample24); - if (abs24 > maxAbs24) { - maxAbs24 = abs24; - } - } - - if (infoQuads == 0) { - counters_.wireZeroPcmPackets.fetch_add(1, std::memory_order_relaxed); - if (wireLastPacketHadInfo_) { - const uint64_t dropouts = - counters_.wirePcmDropouts.fetch_add(1, std::memory_order_relaxed) + 1; - if (dropouts <= 8 || (dropouts % 64) == 0) { - ASFW_LOG(Isoch, - "IT WIRE PCM dropout #%llu at absIdx=%llu (nonzero->zero)", - dropouts, fillAbsIdx); - } - } - wireLastPacketHadInfo_ = false; - return; - } - - counters_.wireInfoQuads.fetch_add(infoQuads, std::memory_order_relaxed); - counters_.wireLastInfoQuad.store(lastInfoQuad, std::memory_order_relaxed); - uint32_t previousMax = counters_.wireMaxAbs24.load(std::memory_order_relaxed); - if (maxAbs24 > previousMax) { - counters_.wireMaxAbs24.store(maxAbs24, std::memory_order_relaxed); - } - if (!wireFirstInfoLogged_) { - counters_.wireFirstInfoAbsIdx.store(fillAbsIdx, std::memory_order_relaxed); - ASFW_LOG(Isoch, - "IT WIRE first nonzero PCM absIdx=%llu infoQuads=%u/%u lastQuad=0x%08x maxAbs24=%u", - fillAbsIdx, infoQuads, quadCount, lastInfoQuad, maxAbs24); - wireFirstInfoLogged_ = true; - } - wireLastPacketHadInfo_ = true; } void IsochTxDmaRing::SeedCycleTracking(Driver::HardwareInterface& hw) noexcept { @@ -222,7 +134,7 @@ IsochTxDmaRing::PrimeStats IsochTxDmaRing::Prime( const TxPayloadDmaMap& payloadDmaMap, const uint32_t numSlots, const uint32_t slotStrideBytes, - const ASFW::IsochTransport::TxPacketMeta* metadataRing, + const IsochTxPacketMeta* metadataRing, const uint64_t preFillCount) noexcept { PrimeStats stats{}; if (!slab_.IsValid()) { @@ -257,8 +169,8 @@ IsochTxDmaRing::PrimeStats IsochTxDmaRing::Prime( const uint32_t producerSlot = pktIdx % numSlots; const auto& meta = metadataRing[producerSlot]; const uint64_t expectedGen = - ASFW::IsochTransport::ExpectedCommitGen(pktIdx, numSlots); - if (meta.commitGen.load(std::memory_order_acquire) != expectedGen) { + ExpectedTxCommitGeneration(pktIdx, numSlots); + if (meta.commitGeneration.load(std::memory_order_acquire) != expectedGen) { ASFW_LOG( Isoch, "IT: Prime failed - slot %u is not committed for packet %u", @@ -398,8 +310,8 @@ const char* IsochTxDmaRing::RefillFailureReasonName( return "invalid-shared-contract"; case RefillFailureReason::DeadContext: return "dead-context"; - case RefillFailureReason::ProducerFatalStatus: - return "producer-fatal-status"; + case RefillFailureReason::ProducerFaultStatus: + return "producer-fault-status"; case RefillFailureReason::CommandPointerDecode: return "command-pointer-decode"; case RefillFailureReason::UncommittedSlot: @@ -415,8 +327,8 @@ const char* IsochTxDmaRing::RefillFailureReasonName( IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( Driver::HardwareInterface& hw, uint8_t contextIndex, - ASFW::IsochTransport::TxPacketMeta* metadataRing, - ASFW::IsochTransport::TxStreamControl* controlBlock, + IsochTxPacketMeta* metadataRing, + IsochTxQueueControl* controlBlock, uint32_t numSlots, uint8_t* payloadBase, const TxPayloadDmaMap& payloadDmaMap) noexcept @@ -441,7 +353,7 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( { const uint64_t hostTime = mach_absolute_time(); - ASFW::IsochTransport::ClockPairSample sample{}; + IsochTxClockPairSample sample{}; sample.hostTimeMid = hostTime; sample.cycleTimer32 = refillCycleTimer; controlBlock->clockPair.Publish(sample); @@ -454,23 +366,20 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( const bool dead = (ctrl & Driver::ContextControl::kDead) != 0; if (dead) { counters_.exitDead.fetch_add(1, std::memory_order_relaxed); - controlBlock->statusWord.store(ASFW::IsochTransport::TxStreamStatus::kDeadContext, std::memory_order_release); + controlBlock->statusWord.store(IsochTxQueueStatus::kDeadContext, std::memory_order_release); controlBlock->streamGeneration.fetch_add(1, std::memory_order_release); out.dead = true; out.failureReason = RefillFailureReason::DeadContext; out.streamStatus = static_cast( - ASFW::IsochTransport::TxStreamStatus::kDeadContext); + IsochTxQueueStatus::kDeadContext); return out; } const auto streamStatus = controlBlock->statusWord.load(std::memory_order_acquire); out.streamStatus = static_cast(streamStatus); if (streamStatus == - ASFW::IsochTransport::TxStreamStatus::kUnderrunFatal) { - out.failureReason = RefillFailureReason::ProducerFatalStatus; - out.producerFailureAvailable = - controlBlock->producerFailure.TryRead( - out.producerFailure); + IsochTxQueueStatus::kProducerFault) { + out.failureReason = RefillFailureReason::ProducerFaultStatus; return out; } @@ -495,9 +404,8 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( UpdateGapCounters(gap); ResyncCycleTracking(hw, hwPacketIndex, deltaConsumed, out); - // Track the worst single coalesced completion. The committed slack - // (kTxPreparationSlackPackets) must cover this; a new high-water at or above - // the slack is one phase-slip away from an underrun, so surface it. + // Publish a neutral completion-delta high-water. Content consumers decide + // whether their own frame/lead policy can tolerate the observed cadence. { uint32_t prevMax = counters_.maxDeltaConsumed.load(std::memory_order_relaxed); @@ -506,16 +414,16 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( prevMax, deltaConsumed, std::memory_order_relaxed, std::memory_order_relaxed)) { } - constexpr uint32_t kSlack = - ASFW::IsochTransport::AudioTimingGeometry:: - kTxPreparationSlackPackets; - if (deltaConsumed > prevMax && deltaConsumed * 2 >= kSlack) { - ASFW_LOG( + if (deltaConsumed > prevMax) { + controlBlock->maxCompletionDelta.store( + deltaConsumed, std::memory_order_release); + controlBlock->maxCompletionDeltaEvents.fetch_add( + 1, std::memory_order_relaxed); + ASFW_LOG_RING_ONLY( Isoch, - "IT deltaConsumed high-water=%u (slack budget=%u) — coalesced " - "completion approaching the coverage bound", - deltaConsumed, - kSlack); + ::ASFW::Logging::LogLevel::Notice, + "IT completion delta high-water=%u", + deltaConsumed); } } @@ -547,22 +455,22 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( controlBlock->completionCursor.store(completedAbsIdx + deltaConsumed, std::memory_order_release); const uint64_t requested = - controlBlock->preparationRequestGeneration.load( + controlBlock->refillRequestGeneration.load( std::memory_order_relaxed); const uint64_t handled = - controlBlock->preparationHandledGeneration.load( + controlBlock->refillHandledGeneration.load( std::memory_order_acquire); if (requested == handled) { const uint64_t generation = requested + 1; - controlBlock->preparationRequestHostTicks.store( + controlBlock->refillRequestHostTicks.store( mach_absolute_time(), std::memory_order_relaxed); - controlBlock->preparationRequestGeneration.store( + controlBlock->refillRequestGeneration.store( generation, std::memory_order_release); - controlBlock->preparationRequestCount.fetch_add( + controlBlock->refillRequestCount.fetch_add( 1, std::memory_order_relaxed); - out.preparationRequestGeneration = generation; + out.refillRequestGeneration = generation; } else { - controlBlock->preparationCoalescedCount.fetch_add( + controlBlock->refillCoalescedCount.fetch_add( 1, std::memory_order_relaxed); } } @@ -579,18 +487,19 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( const uint32_t pktSlot = static_cast(fillAbsIdx % numSlots); auto& meta = metadataRing[pktSlot]; - const uint64_t expectedGen = ASFW::IsochTransport::ExpectedCommitGen(fillAbsIdx, numSlots); - const uint64_t commitGen = meta.commitGen.load(std::memory_order_acquire); + const uint64_t expectedGen = ExpectedTxCommitGeneration(fillAbsIdx, numSlots); + const uint64_t commitGen = + meta.commitGeneration.load(std::memory_order_acquire); if (commitGen != expectedGen) { const uint64_t requestGeneration = - controlBlock->preparationRequestGeneration.load( + controlBlock->refillRequestGeneration.load( std::memory_order_acquire); const uint64_t handledGeneration = - controlBlock->preparationHandledGeneration.load( + controlBlock->refillHandledGeneration.load( std::memory_order_acquire); const uint64_t requestHostTicks = - controlBlock->preparationRequestHostTicks.load( + controlBlock->refillRequestHostTicks.load( std::memory_order_relaxed); const uint64_t nowHostTicks = mach_absolute_time(); const uint64_t requestAgeUs = @@ -601,7 +510,7 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( : 0; counters_.txUnderruns.fetch_add(1, std::memory_order_relaxed); controlBlock->statusWord.store( - ASFW::IsochTransport::TxStreamStatus::kUnderrunFatal, + IsochTxQueueStatus::kProducerFault, std::memory_order_release); controlBlock->streamGeneration.fetch_add( 1, std::memory_order_release); @@ -616,12 +525,12 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( // prepared into its slot, or does the slot still hold a previous // lap's packet? meta.packetIndex == fillAbsIdx with a stale // commitGen => commit/writeback ordering bug; meta.packetIndex one - // lap behind => the producer's exposeCursor never reached this + // lap behind => the producer's committed cursor never reached this // packet (coverage/margin). The producer cursors localize it. ASFW_LOG( Isoch, "IT FATAL dump: fatalAbs=%llu slot=%u expectedGen=%llu commitGen=%llu " - "slotLastPacketAbs=%llu exposeCursor=%llu completionCursor=%llu " + "slotLastPacketAbs=%llu committedEnd=%llu completionCursor=%llu " "softwareFillAbs=%llu ringPacketsAhead=%u deltaConsumed=%u i=%u numSlots=%u " "prepReq=%llu prepHandled=%llu prepAgeUs=%llu prepCoalesced=%llu", fillAbsIdx, @@ -629,7 +538,7 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( expectedGen, commitGen, meta.packetIndex, - controlBlock->exposeCursor.load(std::memory_order_acquire), + controlBlock->committedEnd.load(std::memory_order_acquire), controlBlock->completionCursor.load(std::memory_order_acquire), softwareFillAbsIdx_, ringPacketsAhead_, @@ -639,7 +548,7 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( requestGeneration, handledGeneration, requestAgeUs, - controlBlock->preparationCoalescedCount.load( + controlBlock->refillCoalescedCount.load( std::memory_order_relaxed)); out.failureReason = RefillFailureReason::UncommittedSlot; out.failurePacketAbs = fillAbsIdx; @@ -671,12 +580,6 @@ IsochTxDmaRing::RefillOutcome IsochTxDmaRing::Refill( return out; } - if (payloadBase) { - GaugeWirePayload(fillAbsIdx, - payloadBase + payloadOffset, - payloadLength); - } - std::array payloadFragments{}; if (!payloadDmaMap.ResolveTwoFragments( payloadOffset, payloadLength, payloadFragments)) { diff --git a/ASFWDriver/Isoch/Transmit/IsochTxDmaRing.hpp b/ASFWDriver/Isoch/Transmit/IsochTxDmaRing.hpp index 9abd1948..5da49a17 100644 --- a/ASFWDriver/Isoch/Transmit/IsochTxDmaRing.hpp +++ b/ASFWDriver/Isoch/Transmit/IsochTxDmaRing.hpp @@ -8,12 +8,12 @@ #include "TxPayloadDmaMap.hpp" #include "../Core/IsochEventGroup.hpp" +#include "../Core/IsochTxQueue.hpp" #include "../../Hardware/HardwareInterface.hpp" #include "../../Hardware/OHCIConstants.hpp" #include "../../Hardware/RegisterMap.hpp" #include "../../Logging/Logging.hpp" #include "../../Common/BarrierUtils.hpp" -#include "../../Shared/Isoch/IsochAudioTransport.hpp" #include #include @@ -45,9 +45,8 @@ class IsochTxDmaRing final { std::atomic fatalDescriptorBounds{0}; std::atomic txUnderruns{0}; - // High-water mark of a single refill's coalesced deltaConsumed. The - // committed lead (kTxPreparationSlackPackets) must cover this or the - // refill ISR holes (IT FATAL). Use it to size the slack empirically. + // High-water mark of a single coalesced completion. Content consumers + // own the policy that decides whether this is an unsafe cadence. std::atomic maxDeltaConsumed{0}; // DMA ring gap monitoring @@ -55,18 +54,6 @@ class IsochTxDmaRing final { std::atomic minDmaGapPackets{Layout::kNumPackets}; std::atomic criticalGapEvents{0}; - // Wire-truth payload gauges, sampled at refill time — the last point - // software sees the bytes the DMA engine will ship, after the - // producer's commit. Independent of writer-side claims. An "info" - // quadlet is any AM824 payload quadlet other than 0x00000000 and the - // 0x80000000 idle MIDI/no-info slot word. - std::atomic wireDataPackets{0}; - std::atomic wireZeroPcmPackets{0}; - std::atomic wireInfoQuads{0}; - std::atomic wirePcmDropouts{0}; - std::atomic wireMaxAbs24{0}; - std::atomic wireLastInfoQuad{0}; - std::atomic wireFirstInfoAbsIdx{0}; }; struct PrimeStats { @@ -77,7 +64,7 @@ class IsochTxDmaRing final { None = 0, InvalidSharedContract, DeadContext, - ProducerFatalStatus, + ProducerFaultStatus, CommandPointerDecode, UncommittedSlot, InvalidPacketSize, @@ -107,9 +94,7 @@ class IsochTxDmaRing final { uint32_t firstRefillPacket{0}; uint32_t refillPacketCount{0}; uint64_t packetsFilled{0}; - uint64_t preparationRequestGeneration{0}; - bool producerFailureAvailable{false}; - ASFW::IsochTransport::TxProducerFailureRecord producerFailure{}; + uint64_t refillRequestGeneration{0}; }; IsochTxDmaRing() noexcept = default; @@ -132,13 +117,13 @@ class IsochTxDmaRing final { [[nodiscard]] PrimeStats Prime(const TxPayloadDmaMap& payloadDmaMap, uint32_t numSlots, uint32_t slotStrideBytes, - const ASFW::IsochTransport::TxPacketMeta* metadataRing, + const IsochTxPacketMeta* metadataRing, uint64_t preFillCount) noexcept; [[nodiscard]] RefillOutcome Refill(Driver::HardwareInterface& hw, uint8_t contextIndex, - ASFW::IsochTransport::TxPacketMeta* metadataRing, - ASFW::IsochTransport::TxStreamControl* controlBlock, + IsochTxPacketMeta* metadataRing, + IsochTxQueueControl* controlBlock, uint32_t numSlots, uint8_t* payloadBase, const TxPayloadDmaMap& payloadDmaMap) noexcept; @@ -168,9 +153,6 @@ class IsochTxDmaRing final { uint8_t contextIndex, uint32_t& outPacketIndex, uint32_t& outCmdPtr) noexcept; - void GaugeWirePayload(uint64_t fillAbsIdx, - const uint8_t* packetBytes, - uint32_t payloadLength) noexcept; uint8_t channel_{0}; IsochTxDescriptorSlab slab_{}; @@ -186,10 +168,6 @@ class IsochTxDmaRing final { bool cycleTrackingValid_{false}; uint32_t lastHwTimestamp_{0}; - // Wire-truth gauge state (refill is single-threaded) - bool wireLastPacketHadInfo_{false}; - bool wireFirstInfoLogged_{false}; - Counters counters_{}; }; diff --git a/ASFWDriver/Isoch/Transmit/IsochTxLayout.hpp b/ASFWDriver/Isoch/Transmit/IsochTxLayout.hpp index 7770f096..6de4d8a8 100644 --- a/ASFWDriver/Isoch/Transmit/IsochTxLayout.hpp +++ b/ASFWDriver/Isoch/Transmit/IsochTxLayout.hpp @@ -11,7 +11,7 @@ #include #include "../../Hardware/OHCIDescriptors.hpp" -#include "../../Shared/Isoch/AudioTimingGeometry.hpp" +#include "../Core/IsochDmaGeometry.hpp" namespace ASFW::Isoch::Tx { @@ -35,8 +35,7 @@ struct Layout final { static constexpr uint32_t kFirstPayloadBlock = 2; static constexpr uint32_t kCompletionBlock = 3; static constexpr uint32_t kNumPackets = - ASFW::IsochTransport::AudioTimingGeometry:: - kTxHardwareRingPackets; // 6 ms @ 8000 packets/sec + IsochDmaGeometry::kTransmitDescriptorPackets; static constexpr uint32_t kRingBlocks = kNumPackets * kBlocksPerPacket; static constexpr uint32_t kDescriptorStride = 16; diff --git a/ASFWDriver/Shared/Isoch/IsochAudioTransport.hpp b/ASFWDriver/Shared/Isoch/IsochAudioTransport.hpp deleted file mode 100644 index 971963e0..00000000 --- a/ASFWDriver/Shared/Isoch/IsochAudioTransport.hpp +++ /dev/null @@ -1,477 +0,0 @@ -// IsochAudioTransport.hpp -// ASFW - Shared isoch TX transport contract (audio dext ⇄ OHCI core ⇄ lab). -// -// One header, three consumers: the audio side (ASFWAudioDriver) produces -// packets into the payload slab and this metadata ring; the core (ASFWDriver) -// refill ISR consumes ring entries by blitting immediateHeader + payloadLength -// into OHCI descriptors; ADKVirtualAudioLab rehearses the same contract with a -// fake consumer. Design: ISOCH_AUDIO_ADK.md §3 (regions), §5 (timing), -// ISOCH_AUDIO_CLEANUP_PREP.md §7 (AudioStreamProfile). -// -// Rules enforced here: -// - Plain C++ only — no DriverKit, no Foundation; lock-free atomics only. -// - Every index crossing the boundary is an absolute 64-bit monotonic count. -// `% numSlots` happens only through the helpers below, so there is exactly -// one definition of every ring's modulus (the 4096-vs-512 wrap-mismatch -// class, commits 0d897ecb / 79e45e92, cannot recur between these sides). -// - Layout is locked by static_asserts; bump kTransportAbiVersion on any -// change and both sides assert it at allocation time. - -#pragma once - -#include "AudioTimingGeometry.hpp" - -#include -#include -#include - -namespace ASFW::IsochTransport { - -inline constexpr uint32_t kTransportAbiVersion = 4; - -// ============================================================================= -// Shared queue / buffer sizing (ADK §3.3 / §6.4). -// These constants are the "Iron Rule" ground truth for both sides. -// ============================================================================= - -inline constexpr uint32_t kAudioRingBufferFrames = - AudioTimingGeometry::kFrameRingFrames; -inline constexpr uint32_t kAudioIoPeriodFrames = - AudioTimingGeometry::kHalIoPeriodFrames; - -/// Target gap (writtenEnd - consumer cursor) the isoch TX consumer maintains. -inline constexpr uint32_t kOutputConsumerLeadFrames = 384; ///< ~0.75 period (~8ms @48k) - -/// Deadband: rebase the consumer cursor only when |lead - target| exceeds this. -inline constexpr uint32_t kOutputCursorResyncDeadbandFrames = 64; ///< ~0.125 period - -// ============================================================================= -// Metadata ring (ADK §3.2) — one entry per payload-slab slot. -// ============================================================================= - -/// Producer (audio pump): fill all plain fields, then release-store commitGen -/// last. Consumer (core refill ISR): acquire-load commitGen first; the entry is -/// trustworthy only if it equals ExpectedCommitGen(packetIndex, numSlots). -/// A mismatch at arm time is a DMA-side underrun — fatal by policy (ADK §6.2). -struct alignas(64) TxPacketMeta final { - uint32_t immediateHeader[2]; ///< Ready-to-blit OHCI IT header quadlets - ///< (OUTPUT_MORE_IMMEDIATE immediate data). - ///< The core never interprets these. - uint32_t payloadLength; ///< Bytes: 8 (NO-DATA) or 8 + frames·dbs·4. - uint32_t reserved0; - uint64_t packetIndex; ///< Absolute index this entry describes. - std::atomic commitGen; ///< Lap-numbered commit marker; 0 = never - ///< written. Release-stored last. - uint8_t reserved1[64 - 32]; -}; - -static_assert(sizeof(TxPacketMeta) == 64, "TxPacketMeta must be one cache line"); -static_assert(alignof(TxPacketMeta) == 64, "TxPacketMeta must be cache-line aligned"); -static_assert(offsetof(TxPacketMeta, immediateHeader) == 0); -static_assert(offsetof(TxPacketMeta, payloadLength) == 8); -static_assert(offsetof(TxPacketMeta, packetIndex) == 16); -static_assert(offsetof(TxPacketMeta, commitGen) == 24); -static_assert(std::atomic::is_always_lock_free); - -/// The single definition of the metadata ring's slot mapping and lap number. -[[nodiscard]] constexpr uint32_t SlotIndexFor(uint64_t packetIndex, uint32_t numSlots) noexcept { - return static_cast(packetIndex % numSlots); -} - -/// commitGen value a correctly committed entry for `packetIndex` must carry. -/// Lap-numbered (not a bare flag) so a stale entry from a previous lap of the -/// ring can never be mistaken for a fresh commit. -[[nodiscard]] constexpr uint64_t ExpectedCommitGen(uint64_t packetIndex, uint32_t numSlots) noexcept { - return packetIndex / numSlots + 1; -} - -// ============================================================================= -// Clock pair (ADK §5.1) — bus clock ⇄ host clock affine anchor. -// ============================================================================= - -struct ClockPairSample final { - uint64_t hostTimeMid{0}; ///< (hostA + hostB) / 2 around the register read. - uint32_t cycleTimer32{0}; ///< Raw CYCLE_TIMER register value. -}; - -/// Single-writer (core) seqlock. Reader retries while a write is in flight. -struct ClockPairSeqlock final { - std::atomic sequence{0}; // odd = write in progress - std::atomic hostTimeMid{0}; - std::atomic cycleTimer32{0}; - - void Publish(const ClockPairSample& sample) noexcept { - const uint32_t seq = sequence.load(std::memory_order_relaxed); - sequence.store(seq + 1, std::memory_order_release); // → odd - hostTimeMid.store(sample.hostTimeMid, std::memory_order_relaxed); - cycleTimer32.store(sample.cycleTimer32, std::memory_order_relaxed); - sequence.store(seq + 2, std::memory_order_release); // → even - } - - /// RT-safe bounded read; returns false if a write raced every attempt. - [[nodiscard]] bool TryRead(ClockPairSample& out, uint32_t maxAttempts = 4) const noexcept { - for (uint32_t attempt = 0; attempt < maxAttempts; ++attempt) { - const uint32_t before = sequence.load(std::memory_order_acquire); - if (before & 1u) continue; - ClockPairSample sample{}; - sample.hostTimeMid = hostTimeMid.load(std::memory_order_relaxed); - sample.cycleTimer32 = cycleTimer32.load(std::memory_order_relaxed); - std::atomic_thread_fence(std::memory_order_acquire); - if (sequence.load(std::memory_order_relaxed) == before) { - out = sample; - return true; - } - } - return false; - } -}; - -// ============================================================================= -// Completion stamps (ADK §3.3 / §5.3) — hardware TX timestamps, core → audio. -// ============================================================================= - -inline constexpr uint32_t kCompletionStampSlots = 32; - -struct CompletionStamp final { - std::atomic packetIndex{0}; - std::atomic cycleTimestamp{0}; ///< xferStatus timestamp field - ///< ([cycleSeconds:3][cycleCount:13]). - uint32_t reserved{0}; -}; -static_assert(sizeof(CompletionStamp) == 16); - -// ============================================================================= -// Stream control block v4 (ADK §3.3) — small, fixed, one writer per field. -// ============================================================================= - -enum class TxStreamStatus : uint32_t { - kStopped = 0, - kRunning = 1, - kUnderrunFatal = 2, ///< Producer aborted or refill met an uncommitted slot. - kDeadContext = 3, ///< OHCI context died / bus reset (ADK §6.3). -}; - -enum class TxProducerStage : uint32_t { - kNone = 0, - kPreflight, - kExecutionAnchor, - kReplayBegin, - kReplayRead, - kReplaySytValidation, - kSlotAcquire, - kPacketize, - kSlotPublish, -}; - -[[nodiscard]] inline const char* TxProducerStageName( - TxProducerStage stage) noexcept { - switch (stage) { - case TxProducerStage::kNone: return "none"; - case TxProducerStage::kPreflight: return "preflight"; - case TxProducerStage::kExecutionAnchor: return "execution-anchor"; - case TxProducerStage::kReplayBegin: return "replay-begin"; - case TxProducerStage::kReplayRead: return "replay-read"; - case TxProducerStage::kReplaySytValidation: return "replay-syt-validation"; - case TxProducerStage::kSlotAcquire: return "slot-acquire"; - case TxProducerStage::kPacketize: return "packetize"; - case TxProducerStage::kSlotPublish: return "slot-publish"; - } - return "unknown"; -} - -enum class TxProducerFailureReason : uint32_t { - kNone = 0, - kInvalidTransport, - kReplayUnavailable, - kInvalidReplaySyt, - kSlotUnavailable, - kPacketizerRejected, - kSlotPublishFailed, -}; - -[[nodiscard]] inline const char* TxProducerFailureReasonName( - TxProducerFailureReason reason) noexcept { - switch (reason) { - case TxProducerFailureReason::kNone: return "none"; - case TxProducerFailureReason::kInvalidTransport: return "invalid-transport"; - case TxProducerFailureReason::kReplayUnavailable: return "replay-unavailable"; - case TxProducerFailureReason::kInvalidReplaySyt: return "invalid-replay-syt"; - case TxProducerFailureReason::kSlotUnavailable: return "slot-unavailable"; - case TxProducerFailureReason::kPacketizerRejected: return "packetizer-rejected"; - case TxProducerFailureReason::kSlotPublishFailed: return "slot-publish-failed"; - } - return "unknown"; -} - -struct TxProducerFailureRecord final { - uint64_t generation{0}; - TxProducerStage stage{TxProducerStage::kNone}; - TxProducerFailureReason reason{TxProducerFailureReason::kNone}; - uint64_t packetIndex{0}; - uint64_t rangeStart{0}; - uint64_t rangeTarget{0}; - uint32_t preparedCount{0}; - uint64_t completionCursor{0}; - uint64_t exposeCursor{0}; - uint64_t replayProducerCursor{0}; - uint32_t replayEpoch{0}; -}; - -// Producer writes all detail fields and release-publishes generation last. -// The core observes statusWord with acquire ordering, then snapshots this -// record to explain a producer-originated kUnderrunFatal. -struct TxProducerFailureSnapshot final { - std::atomic generation{0}; - std::atomic stage{ - static_cast(TxProducerStage::kNone)}; - std::atomic reason{ - static_cast(TxProducerFailureReason::kNone)}; - std::atomic packetIndex{0}; - std::atomic rangeStart{0}; - std::atomic rangeTarget{0}; - std::atomic preparedCount{0}; - std::atomic completionCursor{0}; - std::atomic exposeCursor{0}; - std::atomic replayProducerCursor{0}; - std::atomic replayEpoch{0}; - - void Reset() noexcept { - stage.store( - static_cast(TxProducerStage::kNone), - std::memory_order_relaxed); - reason.store( - static_cast(TxProducerFailureReason::kNone), - std::memory_order_relaxed); - packetIndex.store(0, std::memory_order_relaxed); - rangeStart.store(0, std::memory_order_relaxed); - rangeTarget.store(0, std::memory_order_relaxed); - preparedCount.store(0, std::memory_order_relaxed); - completionCursor.store(0, std::memory_order_relaxed); - exposeCursor.store(0, std::memory_order_relaxed); - replayProducerCursor.store(0, std::memory_order_relaxed); - replayEpoch.store(0, std::memory_order_relaxed); - generation.store(0, std::memory_order_release); - } - - [[nodiscard]] uint64_t Publish( - const TxProducerFailureRecord& record) noexcept { - const uint64_t nextGeneration = - generation.load(std::memory_order_relaxed) + 1; - stage.store( - static_cast(record.stage), - std::memory_order_relaxed); - reason.store( - static_cast(record.reason), - std::memory_order_relaxed); - packetIndex.store(record.packetIndex, std::memory_order_relaxed); - rangeStart.store(record.rangeStart, std::memory_order_relaxed); - rangeTarget.store(record.rangeTarget, std::memory_order_relaxed); - preparedCount.store(record.preparedCount, std::memory_order_relaxed); - completionCursor.store( - record.completionCursor, std::memory_order_relaxed); - exposeCursor.store(record.exposeCursor, std::memory_order_relaxed); - replayProducerCursor.store( - record.replayProducerCursor, std::memory_order_relaxed); - replayEpoch.store(record.replayEpoch, std::memory_order_relaxed); - generation.store(nextGeneration, std::memory_order_release); - return nextGeneration; - } - - [[nodiscard]] bool TryRead( - TxProducerFailureRecord& out) const noexcept { - for (uint32_t attempt = 0; attempt < 4; ++attempt) { - const uint64_t before = - generation.load(std::memory_order_acquire); - if (before == 0) { - return false; - } - - TxProducerFailureRecord record{}; - record.generation = before; - record.stage = static_cast( - stage.load(std::memory_order_relaxed)); - record.reason = static_cast( - reason.load(std::memory_order_relaxed)); - record.packetIndex = - packetIndex.load(std::memory_order_relaxed); - record.rangeStart = - rangeStart.load(std::memory_order_relaxed); - record.rangeTarget = - rangeTarget.load(std::memory_order_relaxed); - record.preparedCount = - preparedCount.load(std::memory_order_relaxed); - record.completionCursor = - completionCursor.load(std::memory_order_relaxed); - record.exposeCursor = - exposeCursor.load(std::memory_order_relaxed); - record.replayProducerCursor = - replayProducerCursor.load(std::memory_order_relaxed); - record.replayEpoch = - replayEpoch.load(std::memory_order_relaxed); - std::atomic_thread_fence(std::memory_order_acquire); - if (generation.load(std::memory_order_relaxed) == before) { - out = record; - return true; - } - } - return false; - } -}; - -struct TxStreamControl final { - // --- geometry: written once by the core at allocation; the audio side - // asserts every field against its own expectations at bind time so a - // mismatch fails loudly at bring-up, never as a silence mystery. --- - uint32_t abiVersion{0}; ///< Must equal kTransportAbiVersion. - uint32_t numSlots{0}; ///< Slab slots == metadata ring entries. - uint32_t slotStrideBytes{0}; ///< Payload slab stride (cache-line rounded). - uint32_t maxPacketBytes{0}; ///< CIP header + max PCM payload. - uint32_t interruptInterval{0}; ///< Packets per refill-ISR batch. - uint32_t ztsPeriodFrames{0}; ///< HAL ring length == ZTS period (§5.5). - - // --- core → audio --- - std::atomic streamGeneration{0}; ///< Bumped on reset/teardown/death. - std::atomic statusWord{TxStreamStatus::kStopped}; - ClockPairSeqlock clockPair{}; - std::atomic startCycleMatch{0}; ///< Raw cycleMatch value armed. - std::atomic startFirstPacketIndex{0}; ///< Packet that ships at startCycleMatch. - std::atomic completionCursor{0}; ///< Count of retired packets - ///< (end-exclusive absolute index). - std::atomic completionStampCount{0}; ///< Total stamps ever written. - CompletionStamp completionStamps[kCompletionStampSlots]{}; - std::atomic preparationRequestGeneration{0}; - std::atomic preparationHandledGeneration{0}; - std::atomic preparationRequestHostTicks{0}; - std::atomic preparationRequestCount{0}; - std::atomic preparationCoalescedCount{0}; - - // --- audio → core --- - std::atomic exposeCursor{0}; ///< Count of committed packets - ///< (end-exclusive absolute index). - TxProducerFailureSnapshot producerFailure{}; - - // Clears all runtime progress cursors to a clean pre-start state while - // preserving the core-written geometry (numSlots, strides, ABI). The shared - // buffer is not guaranteed to be freshly zeroed across StartIO/StopIO cycles - // (CoreAudio re-probes a device on a sample-rate change, reusing the slab), - // so without this exposeCursor/completionCursor carry over and the IT prime - // fails ("committed prefill > slots"). The producer owns this reset; it runs - // in StartIO before the prefill and before the transport consumer arms. - void ResetForStart() noexcept { - startCycleMatch.store(0, std::memory_order_relaxed); - startFirstPacketIndex.store(0, std::memory_order_relaxed); - completionCursor.store(0, std::memory_order_relaxed); - completionStampCount.store(0, std::memory_order_relaxed); - preparationRequestGeneration.store(0, std::memory_order_relaxed); - preparationHandledGeneration.store(0, std::memory_order_relaxed); - preparationRequestHostTicks.store(0, std::memory_order_relaxed); - preparationRequestCount.store(0, std::memory_order_relaxed); - preparationCoalescedCount.store(0, std::memory_order_relaxed); - producerFailure.Reset(); - exposeCursor.store(0, std::memory_order_release); - } - - void MarkPreparationHandled(uint64_t generation) noexcept { - uint64_t handled = - preparationHandledGeneration.load(std::memory_order_relaxed); - while (handled < generation && - !preparationHandledGeneration.compare_exchange_weak( - handled, - generation, - std::memory_order_release, - std::memory_order_relaxed)) { - } - } - - // ------------------------------------------------------------------------- - // Completion-stamp ring access (writer = core ISR, reader = audio pump). - // ------------------------------------------------------------------------- - // `cycleTimestamp` is a full CYCLE_TIMER-format value. OHCI supplies the - // authoritative seconds/cycle; the core pairs it with the same refill - // sample's subcycle before publishing. - void PushCompletionStamp(uint64_t packetIndex, uint32_t cycleTimestamp) noexcept { - const uint64_t count = completionStampCount.load(std::memory_order_relaxed); - auto& slot = completionStamps[count % kCompletionStampSlots]; - slot.packetIndex.store(packetIndex, std::memory_order_relaxed); - slot.cycleTimestamp.store(cycleTimestamp, std::memory_order_relaxed); - completionStampCount.store(count + 1, std::memory_order_release); - } - - /// Reads the stamp at absolute index `stampIndex` (< completionStampCount). - /// Returns false if the slot has been overwritten by a newer lap — the - /// reader fell more than kCompletionStampSlots behind and must resync. - [[nodiscard]] bool ReadCompletionStamp(uint64_t stampIndex, - uint64_t& outPacketIndex, - uint32_t& outCycleTimestamp) const noexcept { - const auto& slot = completionStamps[stampIndex % kCompletionStampSlots]; - outPacketIndex = slot.packetIndex.load(std::memory_order_relaxed); - outCycleTimestamp = slot.cycleTimestamp.load(std::memory_order_relaxed); - const uint64_t countAfter = completionStampCount.load(std::memory_order_acquire); - return stampIndex < countAfter && - countAfter - stampIndex <= kCompletionStampSlots; - } -}; - -static_assert(std::atomic::is_always_lock_free); -static_assert(std::atomic::is_always_lock_free); - -// ============================================================================= -// Device stream profile (ISOCH_AUDIO_CLEANUP_PREP.md §7.2) — resolved by the -// core (discovered > known-device table > family defaults) and delivered to -// the audio dext via the nub. The audio side derives all transport geometry -// from this; the core never interprets the audio-facing fields. -// ============================================================================= - -enum class PcmWireFormat : uint8_t { - kAM824Labeled = 0, ///< Label 0x40 + big-endian 24-bit sample. - kRaw24In32SignExtended = 1, ///< Sign-extended 24-in-32, no label (Saffire TX). -}; - -enum class CipBlockingMode : uint8_t { - kBlocking = 0, ///< Fixed frames/packet, NO-DATA fills the cadence. - kNonBlocking = 1, -}; - -enum class TxSytMode : uint8_t { - /// RX-recovered device cadence, disciplined against completed OHCI IT - /// packet execution anchors (Saffire.kext full-duplex timing model). - kRxRecoveredDeviceClock = 0, -}; - -struct StreamDescriptor final { - PcmWireFormat wireFormat{PcmWireFormat::kAM824Labeled}; - uint8_t dbs{0}; ///< Quadlets per frame on the wire. - uint8_t pcmSlots{0}; ///< Audio slots; dbs == pcmSlots + midiSlots. - uint8_t midiSlots{0}; - CipBlockingMode blockingMode{CipBlockingMode::kBlocking}; - uint8_t fdf{0}; - uint8_t isoChannel{0xFF}; ///< 0xFF = not yet allocated. - uint8_t speed{2}; ///< IEEE 1394 speed code (2 = S400). -}; -static_assert(sizeof(StreamDescriptor) == 8); - -struct AudioStreamProfile final { - StreamDescriptor hostToDevice{}; - StreamDescriptor deviceToHost{}; - uint32_t sampleRateHz{0}; - uint8_t framesPerPacket{0}; ///< Rate ladder: 8 / 16 / 32 (ADK §6.6). - TxSytMode sytMode{TxSytMode::kRxRecoveredDeviceClock}; - uint8_t latencyMode{0}; ///< Index into the delayPackets table. - uint8_t inputDelayPackets{0}; - uint8_t outputDelayPackets{0}; - uint8_t reserved[1]{}; - uint16_t reportedLatencyBaseFrames{0}; - uint32_t profileGeneration{0}; ///< Bumped with streamGeneration (§7.4). -}; -static_assert(sizeof(AudioStreamProfile) == 32, "AudioStreamProfile layout is ABI"); -static_assert(offsetof(AudioStreamProfile, sampleRateHz) == 16); -static_assert(offsetof(AudioStreamProfile, profileGeneration) == 28); - -/// Wire bytes of a DATA packet for a descriptor at a given frames/packet: -/// CIP header (8) + frames · dbs · 4. This is the only place the formula lives. -[[nodiscard]] constexpr uint32_t DataPacketBytes(const StreamDescriptor& d, - uint32_t framesPerPacket) noexcept { - return 8u + framesPerPacket * static_cast(d.dbs) * 4u; -} - -} // namespace ASFW::IsochTransport diff --git a/tests/audio/AudioRuntime/AudioTransportControlBlockTests.cpp b/tests/audio/AudioRuntime/AudioTransportControlBlockTests.cpp index 9955527d..8dd25e5b 100644 --- a/tests/audio/AudioRuntime/AudioTransportControlBlockTests.cpp +++ b/tests/audio/AudioRuntime/AudioTransportControlBlockTests.cpp @@ -10,6 +10,9 @@ namespace { using ASFW::Audio::Runtime::AudioTransportControlBlock; using ASFW::Audio::Runtime::FatalStreamReason; using ASFW::Audio::Runtime::TxPreparationRequestState; +using ASFW::Audio::Runtime::TxProducerFaultReason; +using ASFW::Audio::Runtime::TxProducerFaultRecord; +using ASFW::Audio::Runtime::TxProducerFaultStage; TEST(AudioTransportControlBlockTests, PreparationRequestsAreMonotonicAndCoalescible) { TxPreparationRequestState requests{}; @@ -30,6 +33,60 @@ TEST(AudioTransportControlBlockTests, PreparationRequestsAreMonotonicAndCoalesci EXPECT_TRUE(requests.NeedsHandling()); } +TEST(AudioTransportControlBlockTests, ProducerFaultDetailIsAudioOwnedAndResettable) { + AudioTransportControlBlock control{}; + const TxProducerFaultRecord published{ + .stage = TxProducerFaultStage::kReplaySytValidation, + .reason = TxProducerFaultReason::kInvalidReplaySyt, + .packetIndex = 192, + .rangeStart = 180, + .rangeTarget = 216, + .preparedCount = 12, + .completionCursor = 144, + .committedEnd = 192, + .replayProducerCursor = 480, + .replayEpoch = 4, + }; + + EXPECT_EQ(control.txProducerFault.Publish(published), 1U); + TxProducerFaultRecord observed{}; + ASSERT_TRUE(control.txProducerFault.TryRead(observed)); + EXPECT_EQ(observed.packetIndex, 192U); + EXPECT_EQ(observed.committedEnd, 192U); + EXPECT_STREQ(ASFW::Audio::Runtime::TxProducerFaultStageName(observed.stage), + "replay-syt-validation"); + EXPECT_STREQ(ASFW::Audio::Runtime::TxProducerFaultReasonName(observed.reason), + "invalid-replay-syt"); + + control.ResetForStart(); + EXPECT_FALSE(control.txProducerFault.TryRead(observed)); +} + +TEST(AudioTransportControlBlockTests, WirePayloadTelemetryStaysInAudioAndFlagsDropout) { + AudioTransportControlBlock control{}; + const uint8_t dataPacket[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0x40, 0x00, 0x00, 0x01, + }; + const uint8_t zeroPacket[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, + }; + + const auto first = control.txWirePayloadTelemetry.Observe( + 12, dataPacket, sizeof(dataPacket)); + EXPECT_TRUE(first.firstInfo); + EXPECT_FALSE(first.dropout); + EXPECT_EQ(first.infoQuads, 1U); + + const auto dropout = control.txWirePayloadTelemetry.Observe( + 13, zeroPacket, sizeof(zeroPacket)); + EXPECT_FALSE(dropout.firstInfo); + EXPECT_TRUE(dropout.dropout); + EXPECT_EQ(control.txWirePayloadTelemetry.firstInfoPacketIndex.load(), 12U); + EXPECT_EQ(control.txWirePayloadTelemetry.pcmDropouts.load(), 1U); +} + TEST(AudioTransportControlBlockTests, ResetForStartClearsNestedStateAndIncrementsGeneration) { AudioTransportControlBlock control{}; diff --git a/tests/audio/CMakeLists.txt b/tests/audio/CMakeLists.txt index bb5273ea..bae8a1c1 100644 --- a/tests/audio/CMakeLists.txt +++ b/tests/audio/CMakeLists.txt @@ -106,6 +106,10 @@ add_audio_test(IsochTxDmaRingTests "${ASFW_DRIVER_DIR}/Common/BarrierUtils.cpp" ) +add_audio_test(TransmitBoundaryTests + TransmitBoundaryTests.cpp +) + add_audio_test(IsochServiceTxPreparationTests IsochServiceTxPreparationTests.cpp "${ASFW_DRIVER_DIR}/Isoch/IsochService.cpp" diff --git a/tests/audio/IsochServiceTxPreparationTests.cpp b/tests/audio/IsochServiceTxPreparationTests.cpp index c60cb9b2..484b34d5 100644 --- a/tests/audio/IsochServiceTxPreparationTests.cpp +++ b/tests/audio/IsochServiceTxPreparationTests.cpp @@ -7,7 +7,7 @@ #include "../ASFWDriver/Isoch/IsochService.hpp" #include "../ASFWDriver/Isoch/Transmit/IsochTxLayout.hpp" #include "../ASFWDriver/Shared/Isoch/AudioTimingGeometry.hpp" -#include "../ASFWDriver/Shared/Isoch/IsochAudioTransport.hpp" +#include "../ASFWDriver/Isoch/Core/IsochTxQueue.hpp" namespace { @@ -16,9 +16,9 @@ using ASFW::Driver::IsochService; using ASFW::Driver::Register32; using ASFW::Isoch::Tx::Layout; using ASFW::IsochTransport::AudioTimingGeometry; -using ASFW::IsochTransport::ExpectedCommitGen; -using ASFW::IsochTransport::TxPacketMeta; -using ASFW::IsochTransport::TxStreamControl; +using ASFW::Isoch::ExpectedTxCommitGeneration; +using ASFW::Isoch::IsochTxPacketMeta; +using ASFW::Isoch::IsochTxQueueControl; class RecordingReceiveConsumer final : public ASFW::Isoch::IIsochReceiveConsumer { public: @@ -61,22 +61,22 @@ TEST(IsochServiceTxPreparation, CallbackRegisteredBeforeContextCreationSurvivesS IOAddressSegment metadataRange{}; ASSERT_EQ(metadataDescriptor->GetAddressRange(&metadataRange), kIOReturnSuccess); std::memset(reinterpret_cast(metadataRange.address), 0, metadataRange.length); - auto* metadata = reinterpret_cast(metadataRange.address); + auto* metadata = reinterpret_cast(metadataRange.address); for (uint64_t packetIndex = 0; packetIndex < AudioTimingGeometry::kTxSharedSlotPackets; ++packetIndex) { auto& meta = metadata[packetIndex % AudioTimingGeometry::kTxSharedSlotPackets]; meta.packetIndex = packetIndex; meta.payloadLength = 8; - meta.commitGen.store( - ExpectedCommitGen(packetIndex, AudioTimingGeometry::kTxSharedSlotPackets), + meta.commitGeneration.store( + ExpectedTxCommitGeneration(packetIndex, AudioTimingGeometry::kTxSharedSlotPackets), std::memory_order_release); } IOAddressSegment controlRange{}; ASSERT_EQ(controlDescriptor->GetAddressRange(&controlRange), kIOReturnSuccess); std::memset(reinterpret_cast(controlRange.address), 0, controlRange.length); - auto* control = reinterpret_cast(controlRange.address); - control->exposeCursor.store(AudioTimingGeometry::kTxPreparationLeadPackets, + auto* control = reinterpret_cast(controlRange.address); + control->committedEnd.store(AudioTimingGeometry::kTxPreparationLeadPackets, std::memory_order_release); ASSERT_EQ(service.StartTransmit(/*channel=*/3, hardware, @@ -99,7 +99,56 @@ TEST(IsochServiceTxPreparation, CallbackRegisteredBeforeContextCreationSurvivesS EXPECT_EQ(callbackCount, 1U); EXPECT_EQ(callbackGeneration, 1U); - EXPECT_EQ(control->preparationRequestGeneration.load(std::memory_order_acquire), 1U); + EXPECT_EQ(control->refillRequestGeneration.load(std::memory_order_acquire), 1U); +} + +TEST(IsochServiceTxPreparation, ActiveTransmitStopRetainsQueueUntilHardwareQuiesces) { + IsochService service; + HardwareInterface hardware; + IOMemoryDescriptor* payloadDescriptor = nullptr; + IOMemoryDescriptor* metadataDescriptor = nullptr; + IOMemoryDescriptor* controlDescriptor = nullptr; + ASSERT_EQ(service.AllocateTxIsochResources( + 0, AudioTimingGeometry::kTxSharedSlotPackets, 512, + AudioTimingGeometry::kTxPacketsPerGroup, &payloadDescriptor, + &metadataDescriptor, &controlDescriptor), + kIOReturnSuccess); + + IOAddressSegment metadataRange{}; + ASSERT_EQ(metadataDescriptor->GetAddressRange(&metadataRange), kIOReturnSuccess); + auto* metadata = reinterpret_cast(metadataRange.address); + for (uint64_t packetIndex = 0; + packetIndex < AudioTimingGeometry::kTxSharedSlotPackets; + ++packetIndex) { + auto& meta = metadata[packetIndex % AudioTimingGeometry::kTxSharedSlotPackets]; + meta.packetIndex = packetIndex; + meta.payloadLength = 8; + meta.commitGeneration.store( + ExpectedTxCommitGeneration( + packetIndex, AudioTimingGeometry::kTxSharedSlotPackets), + std::memory_order_release); + } + IOAddressSegment controlRange{}; + ASSERT_EQ(controlDescriptor->GetAddressRange(&controlRange), kIOReturnSuccess); + auto* queue = reinterpret_cast(controlRange.address); + queue->ResetProducerForStart(); + queue->committedEnd.store(AudioTimingGeometry::kTxPreparationLeadPackets, + std::memory_order_release); + + ASSERT_EQ(service.StartTransmit(3, hardware, 0x3f), kIOReturnSuccess); + auto* context = service.TransmitContext(); + ASSERT_NE(context, nullptr); + EXPECT_EQ(context->GetState(), ASFW::Isoch::ITState::Running); + + const Register32 controlSet = static_cast( + DMAContextHelpers::IsoXmitContextControlSet(0)); + hardware.SetTestRegister(controlSet, ASFW::Driver::ContextControl::kActive); + EXPECT_EQ(service.StopAll(), kIOReturnTimeout); + EXPECT_EQ(context->GetState(), ASFW::Isoch::ITState::Running); + + hardware.SetTestRegister(controlSet, 0); + EXPECT_EQ(service.StopAll(), kIOReturnSuccess); + EXPECT_EQ(context->GetState(), ASFW::Isoch::ITState::Stopped); } // Secondary-stream container: a multi-stream DICE device (Venice F32 = 2×16) diff --git a/tests/audio/IsochTxDmaRingTests.cpp b/tests/audio/IsochTxDmaRingTests.cpp index 749d275e..251aeecb 100644 --- a/tests/audio/IsochTxDmaRingTests.cpp +++ b/tests/audio/IsochTxDmaRingTests.cpp @@ -11,7 +11,8 @@ #include "Isoch/Memory/IsochDMAMemoryManager.hpp" #include "Hardware/HardwareInterface.hpp" #include "Hardware/OHCIConstants.hpp" -#include "Shared/Isoch/IsochAudioTransport.hpp" +#include "Isoch/Core/IsochTxQueue.hpp" +#include "Shared/Isoch/AudioTimingGeometry.hpp" using ASFW::Isoch::Tx::IsochTxDmaRing; using ASFW::Isoch::Tx::Layout; @@ -19,10 +20,10 @@ using ASFW::Isoch::Tx::TxPayloadDmaMap; using ASFW::Isoch::Tx::TxPayloadDmaSegment; using ASFW::Isoch::Memory::IsochDMAMemoryManager; using ASFW::Isoch::Memory::IsochMemoryConfig; -using ASFW::IsochTransport::TxPacketMeta; -using ASFW::IsochTransport::TxStreamControl; -using ASFW::IsochTransport::TxStreamStatus; -using ASFW::IsochTransport::ExpectedCommitGen; +using ASFW::Isoch::IsochTxPacketMeta; +using ASFW::Isoch::IsochTxQueueControl; +using ASFW::Isoch::IsochTxQueueStatus; +using ASFW::Isoch::ExpectedTxCommitGeneration; using ASFW::Async::HW::OHCIDescriptor; using ASFW::Async::HW::OHCIDescriptorImmediate; using ASFW::Driver::Register32; @@ -55,15 +56,15 @@ class IsochTxDmaRingTest : public ::testing::Test { std::vector sharedPayload_ = std::vector(kSharedPayloadSlots * kSharedPayloadStride); - [[nodiscard]] static std::vector MakeMetadataRing() { - std::vector metadataRing(kSharedPayloadSlots); + [[nodiscard]] static std::vector MakeMetadataRing() { + std::vector metadataRing(kSharedPayloadSlots); for (uint32_t packetIndex = 0; packetIndex < metadataRing.size(); ++packetIndex) { auto& meta = metadataRing[packetIndex]; meta.packetIndex = packetIndex; meta.payloadLength = 8; - meta.commitGen.store(1, std::memory_order_release); + meta.commitGeneration.store(1, std::memory_order_release); } return metadataRing; } @@ -333,13 +334,13 @@ TEST_F(IsochTxDmaRingTest, RefillUsesMappedIOVAAfterPageBoundary) { TxPayloadDmaMap segmentedMap; ASSERT_TRUE(segmentedMap.Configure(segments, sharedPayload_.size())); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; for (uint32_t i = 0; i < 9; ++i) { metadataRing[i].payloadLength = 296; - metadataRing[i].commitGen.store(1, std::memory_order_release); + metadataRing[i].commitGeneration.store(1, std::memory_order_release); } const uint32_t nextPacketIOVA = @@ -385,7 +386,7 @@ TEST_F(IsochTxDmaRingTest, metadataRing[0].immediateHeader[0] = OSSwapHostToLittleInt32(kProducerHostHeader); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; @@ -433,12 +434,12 @@ TEST_F(IsochTxDmaRingTest, RefillProgramsPayloadCrossingDmaSegment) { TxPayloadDmaMap crossingMap; ASSERT_TRUE(crossingMap.Configure(segments, sharedPayload_.size())); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; metadataRing[0].payloadLength = 296; - metadataRing[0].commitGen.store(1, std::memory_order_release); + metadataRing[0].commitGeneration.store(1, std::memory_order_release); const uint32_t nextPacketIOVA = ring_.Slab().GetDescriptorIOVA(Layout::kBlocksPerPacket); @@ -480,7 +481,7 @@ TEST_F(IsochTxDmaRingTest, RefillConsumesMetadataAndPushesStamps) { ring_.SeedCycleTracking(hardware_); // Allocate host buffers for metadata ring and control block - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; const uint32_t numSlots = kSharedPayloadSlots; uint8_t* payloadBase = sharedPayload_.data(); @@ -491,7 +492,7 @@ TEST_F(IsochTxDmaRingTest, RefillConsumesMetadataAndPushesStamps) { metadataRing[i].immediateHeader[0] = 0x11110000 + i; metadataRing[i].immediateHeader[1] = 0x22220000 + i; metadataRing[i].payloadLength = 100 + i * 4; - metadataRing[i].commitGen.store(1, std::memory_order_release); // Lap 1 + metadataRing[i].commitGeneration.store(1, std::memory_order_release); // Lap 1 } // Set control block structure @@ -532,9 +533,9 @@ TEST_F(IsochTxDmaRingTest, RefillConsumesMetadataAndPushesStamps) { // Verify completed stamps EXPECT_EQ(controlBlock.completionStampCount.load(), 8); EXPECT_EQ(controlBlock.completionCursor.load(), 8); - EXPECT_EQ(outcome.preparationRequestGeneration, 1U); + EXPECT_EQ(outcome.refillRequestGeneration, 1U); EXPECT_EQ( - controlBlock.preparationRequestGeneration.load( + controlBlock.refillRequestGeneration.load( std::memory_order_acquire), 1U); for (uint32_t i = 0; i < 8; ++i) { @@ -586,13 +587,13 @@ TEST_F(IsochTxDmaRingTest, CompletionNotificationCoalescesUntilHandled) { ring_.ResetForStart(); ring_.SeedCycleTracking(hardware_); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; for (uint32_t i = 0; i < 24; ++i) { metadataRing[i].payloadLength = 8; - metadataRing[i].commitGen.store(1, std::memory_order_release); + metadataRing[i].commitGeneration.store(1, std::memory_order_release); } hardware_.SetTestRegister( static_cast( @@ -618,27 +619,27 @@ TEST_F(IsochTxDmaRingTest, CompletionNotificationCoalescesUntilHandled) { const auto first = refillTo(8); ASSERT_TRUE(first.ok); - EXPECT_EQ(first.preparationRequestGeneration, 1U); + EXPECT_EQ(first.refillRequestGeneration, 1U); const auto coalesced = refillTo(16); ASSERT_TRUE(coalesced.ok); - EXPECT_EQ(coalesced.preparationRequestGeneration, 0U); + EXPECT_EQ(coalesced.refillRequestGeneration, 0U); - controlBlock.preparationHandledGeneration.store( + controlBlock.refillHandledGeneration.store( 1, std::memory_order_release); const auto next = refillTo(24); ASSERT_TRUE(next.ok); - EXPECT_EQ(next.preparationRequestGeneration, 2U); + EXPECT_EQ(next.refillRequestGeneration, 2U); } TEST_F(IsochTxDmaRingTest, PreparationAcknowledgementNeverMovesBackward) { - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; - controlBlock.MarkPreparationHandled(2); - controlBlock.MarkPreparationHandled(1); + controlBlock.MarkRefillHandled(2); + controlBlock.MarkRefillHandled(1); EXPECT_EQ( - controlBlock.preparationHandledGeneration.load( + controlBlock.refillHandledGeneration.load( std::memory_order_acquire), 2U); } @@ -651,7 +652,7 @@ TEST_F(IsochTxDmaRingTest, RefillMapsWrappedHardwareSlotsToAbsoluteProducerSlots ring_.ResetForStart(); ring_.SeedCycleTracking(hardware_); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; @@ -664,8 +665,8 @@ TEST_F(IsochTxDmaRingTest, RefillMapsWrappedHardwareSlotsToAbsoluteProducerSlots meta.immediateHeader[0] = 0x11000000u + packetIndex; meta.immediateHeader[1] = 0x22000000u + packetIndex; meta.payloadLength = 64; - meta.commitGen.store( - ExpectedCommitGen(packetIndex, kSharedPayloadSlots), + meta.commitGeneration.store( + ExpectedTxCommitGeneration(packetIndex, kSharedPayloadSlots), std::memory_order_release); } @@ -725,7 +726,7 @@ TEST_F(IsochTxDmaRingTest, RefillRejectsStaleGenerationAtFirstSharedRingWrap) { auto& meta = metadataRing[packetIndex]; meta.packetIndex = packetIndex; meta.payloadLength = 8; - meta.commitGen.store(1, std::memory_order_release); + meta.commitGeneration.store(1, std::memory_order_release); } const auto prime = ring_.Prime( @@ -737,7 +738,7 @@ TEST_F(IsochTxDmaRingTest, RefillRejectsStaleGenerationAtFirstSharedRingWrap) { ASSERT_EQ(prime.packetsAssembled, Layout::kNumPackets); ring_.SeedCycleTracking(hardware_); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; @@ -778,7 +779,7 @@ TEST_F(IsochTxDmaRingTest, RefillRejectsStaleGenerationAtFirstSharedRingWrap) { } const auto commitBefore = - metadataRing[0].commitGen.load(std::memory_order_acquire); + metadataRing[0].commitGeneration.load(std::memory_order_acquire); const auto packetBefore = metadataRing[0].packetIndex; const std::array payloadBefore{ 0x10, 0x21, 0x32, 0x43, 0x54, 0x65, 0x76, 0x87}; @@ -795,7 +796,7 @@ TEST_F(IsochTxDmaRingTest, RefillRejectsStaleGenerationAtFirstSharedRingWrap) { Layout::kNumPackets); EXPECT_FALSE(firstSecondLapRefill.ok); EXPECT_EQ(firstSecondLapRefill.packetsFilled, 0U); - EXPECT_EQ(metadataRing[0].commitGen.load(std::memory_order_acquire), + EXPECT_EQ(metadataRing[0].commitGeneration.load(std::memory_order_acquire), commitBefore); EXPECT_EQ(metadataRing[0].packetIndex, packetBefore); EXPECT_EQ( @@ -803,7 +804,7 @@ TEST_F(IsochTxDmaRingTest, RefillRejectsStaleGenerationAtFirstSharedRingWrap) { payloadBefore.size()), 0); EXPECT_EQ(controlBlock.statusWord.load(std::memory_order_acquire), - TxStreamStatus::kUnderrunFatal); + IsochTxQueueStatus::kProducerFault); EXPECT_EQ(controlBlock.streamGeneration.load(std::memory_order_acquire), 1U); EXPECT_EQ( @@ -819,7 +820,7 @@ TEST_F(IsochTxDmaRingTest, RefillAcceptsGenerationTwoAtFirstSharedRingWrap) { auto& meta = metadataRing[packetIndex]; meta.packetIndex = packetIndex; meta.payloadLength = 8; - meta.commitGen.store(1, std::memory_order_release); + meta.commitGeneration.store(1, std::memory_order_release); } const auto prime = ring_.Prime( @@ -831,7 +832,7 @@ TEST_F(IsochTxDmaRingTest, RefillAcceptsGenerationTwoAtFirstSharedRingWrap) { ASSERT_EQ(prime.packetsAssembled, Layout::kNumPackets); ring_.SeedCycleTracking(hardware_); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; @@ -880,8 +881,8 @@ TEST_F(IsochTxDmaRingTest, RefillAcceptsGenerationTwoAtFirstSharedRingWrap) { auto& meta = metadataRing[packetIndex % kSharedPayloadSlots]; meta.packetIndex = packetIndex; meta.payloadLength = 8; - meta.commitGen.store( - ExpectedCommitGen(packetIndex, kSharedPayloadSlots), + meta.commitGeneration.store( + ExpectedTxCommitGeneration(packetIndex, kSharedPayloadSlots), std::memory_order_release); } @@ -898,7 +899,7 @@ TEST_F(IsochTxDmaRingTest, RefillAcceptsGenerationTwoAtFirstSharedRingWrap) { ASFW::IsochTransport::AudioTimingGeometry:: kTxPacketsPerGroup); EXPECT_EQ(metadataRing[0].packetIndex, kSharedPayloadSlots); - EXPECT_EQ(metadataRing[0].commitGen.load(std::memory_order_acquire), 2U); + EXPECT_EQ(metadataRing[0].commitGeneration.load(std::memory_order_acquire), 2U); } TEST_F(IsochTxDmaRingTest, @@ -916,7 +917,7 @@ TEST_F(IsochTxDmaRingTest, auto& meta = metadataRing[packetIndex]; meta.packetIndex = packetIndex; meta.payloadLength = 8; - meta.commitGen.store( + meta.commitGeneration.store( packetIndex < kHistoricalCommittedPackets ? 1U : 0U, std::memory_order_release); } @@ -930,7 +931,7 @@ TEST_F(IsochTxDmaRingTest, ASSERT_EQ(prime.packetsAssembled, Layout::kNumPackets); ring_.SeedCycleTracking(hardware_); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; @@ -963,7 +964,7 @@ TEST_F(IsochTxDmaRingTest, refillTo(3 * Geometry::kTxPacketsPerGroup); EXPECT_FALSE(third.ok); EXPECT_EQ(controlBlock.statusWord.load(std::memory_order_acquire), - TxStreamStatus::kUnderrunFatal); + IsochTxQueueStatus::kProducerFault); EXPECT_EQ( ring_.RTCounters().txUnderruns.load(std::memory_order_relaxed), 1U); @@ -977,14 +978,14 @@ TEST_F(IsochTxDmaRingTest, RefillRejectsPayloadLargerThanSharedSlot) { ring_.ResetForStart(); ring_.SeedCycleTracking(hardware_); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; metadataRing[0].packetIndex = 0; metadataRing[0].payloadLength = kSharedPayloadStride + 1; - metadataRing[0].commitGen.store(1, std::memory_order_release); + metadataRing[0].commitGeneration.store(1, std::memory_order_release); const uint32_t nextPacketIOVA = ring_.Slab().GetDescriptorIOVA(Layout::kBlocksPerPacket); @@ -1006,7 +1007,7 @@ TEST_F(IsochTxDmaRingTest, RefillRejectsPayloadLargerThanSharedSlot) { EXPECT_EQ(ring_.RTCounters().fatalPacketSize.load(std::memory_order_relaxed), 1u); } -TEST_F(IsochTxDmaRingTest, RefillHonorsProducerFatalStatusImmediately) { +TEST_F(IsochTxDmaRingTest, RefillHonorsProducerFaultStatusImmediately) { auto metadataRing = MakeMetadataRing(); (void)ring_.Prime( payloadDmaMap_, kSharedPayloadSlots, kSharedPayloadStride, @@ -1014,29 +1015,12 @@ TEST_F(IsochTxDmaRingTest, RefillHonorsProducerFatalStatusImmediately) { ring_.ResetForStart(); ring_.SeedCycleTracking(hardware_); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; controlBlock.numSlots = kSharedPayloadSlots; controlBlock.slotStrideBytes = kSharedPayloadStride; controlBlock.maxPacketBytes = kSharedPayloadStride; - const ASFW::IsochTransport::TxProducerFailureRecord producerFailure{ - .stage = - ASFW::IsochTransport::TxProducerStage::kReplayRead, - .reason = - ASFW::IsochTransport::TxProducerFailureReason:: - kReplayUnavailable, - .packetIndex = 192, - .rangeStart = 192, - .rangeTarget = 194, - .preparedCount = 0, - .completionCursor = 50, - .exposeCursor = 192, - .replayProducerCursor = 513, - .replayEpoch = 4, - }; - EXPECT_EQ( - controlBlock.producerFailure.Publish(producerFailure), 1U); controlBlock.statusWord.store( - TxStreamStatus::kUnderrunFatal, std::memory_order_release); + IsochTxQueueStatus::kProducerFault, std::memory_order_release); const uint32_t nextPacketIOVA = ring_.Slab().GetDescriptorIOVA(Layout::kBlocksPerPacket); @@ -1052,52 +1036,8 @@ TEST_F(IsochTxDmaRingTest, RefillHonorsProducerFatalStatusImmediately) { EXPECT_FALSE(outcome.ok); EXPECT_EQ( outcome.failureReason, - IsochTxDmaRing::RefillFailureReason::ProducerFatalStatus); + IsochTxDmaRing::RefillFailureReason::ProducerFaultStatus); EXPECT_EQ(outcome.packetsFilled, 0U); - ASSERT_TRUE(outcome.producerFailureAvailable); - EXPECT_EQ(outcome.producerFailure.generation, 1U); - EXPECT_EQ( - outcome.producerFailure.stage, - ASFW::IsochTransport::TxProducerStage::kReplayRead); - EXPECT_EQ( - outcome.producerFailure.reason, - ASFW::IsochTransport::TxProducerFailureReason:: - kReplayUnavailable); - EXPECT_EQ(outcome.producerFailure.packetIndex, 192U); - EXPECT_EQ(outcome.producerFailure.rangeStart, 192U); - EXPECT_EQ(outcome.producerFailure.rangeTarget, 194U); - EXPECT_EQ(outcome.producerFailure.preparedCount, 0U); - EXPECT_EQ(outcome.producerFailure.completionCursor, 50U); - EXPECT_EQ(outcome.producerFailure.exposeCursor, 192U); - EXPECT_EQ(outcome.producerFailure.replayProducerCursor, 513U); - EXPECT_EQ(outcome.producerFailure.replayEpoch, 4U); -} - -TEST(TxProducerFailureSnapshotTests, ResetClearsRecordAndNamesArePublic) { - ASFW::IsochTransport::TxProducerFailureSnapshot snapshot{}; - const ASFW::IsochTransport::TxProducerFailureRecord failure{ - .stage = - ASFW::IsochTransport::TxProducerStage:: - kReplaySytValidation, - .reason = - ASFW::IsochTransport::TxProducerFailureReason:: - kInvalidReplaySyt, - .packetIndex = 192, - }; - - EXPECT_EQ(snapshot.Publish(failure), 1U); - ASFW::IsochTransport::TxProducerFailureRecord observed{}; - ASSERT_TRUE(snapshot.TryRead(observed)); - EXPECT_STREQ( - ASFW::IsochTransport::TxProducerStageName(observed.stage), - "replay-syt-validation"); - EXPECT_STREQ( - ASFW::IsochTransport::TxProducerFailureReasonName( - observed.reason), - "invalid-replay-syt"); - - snapshot.Reset(); - EXPECT_FALSE(snapshot.TryRead(observed)); } TEST_F(IsochTxDmaRingTest, @@ -1109,12 +1049,12 @@ TEST_F(IsochTxDmaRingTest, ring_.ResetForStart(); ring_.SeedCycleTracking(hardware_); - TxStreamControl controlBlock{}; + IsochTxQueueControl controlBlock{}; const uint32_t numSlots = kSharedPayloadSlots; uint8_t* payloadBase = sharedPayload_.data(); - metadataRing[0].commitGen.store(0, std::memory_order_release); + metadataRing[0].commitGeneration.store(0, std::memory_order_release); metadataRing[0].immediateHeader[0] = 0x11110000; metadataRing[0].immediateHeader[1] = 0x22220000; std::array payloadBefore{ @@ -1138,9 +1078,9 @@ TEST_F(IsochTxDmaRingTest, EXPECT_FALSE(outcome.ok); EXPECT_EQ(outcome.packetsFilled, 0); EXPECT_EQ(controlBlock.statusWord.load(), - TxStreamStatus::kUnderrunFatal); + IsochTxQueueStatus::kProducerFault); EXPECT_EQ(controlBlock.streamGeneration.load(), 1); - EXPECT_EQ(metadataRing[0].commitGen.load(), 0); + EXPECT_EQ(metadataRing[0].commitGeneration.load(), 0); auto* desc0 = ring_.Slab().GetDescriptorPtr(0); auto* immDesc = reinterpret_cast(desc0); @@ -1151,3 +1091,28 @@ TEST_F(IsochTxDmaRingTest, payloadBefore.size()), 0); } + +TEST(IsochTxQueueControlTests, ProducerAndConsumerResetsHaveDisjointOwnership) { + IsochTxQueueControl queue{}; + queue.abiVersion = ASFW::Isoch::kTxQueueAbiVersion; + queue.committedEnd.store(408, std::memory_order_release); + queue.completionCursor.store(144, std::memory_order_release); + queue.statusWord.store(IsochTxQueueStatus::kRunning, + std::memory_order_release); + + queue.ResetConsumerForArm(); + EXPECT_EQ(queue.abiVersion, ASFW::Isoch::kTxQueueAbiVersion); + EXPECT_EQ(queue.committedEnd.load(std::memory_order_acquire), 408U); + EXPECT_EQ(queue.completionCursor.load(std::memory_order_acquire), 0U); + EXPECT_EQ(queue.statusWord.load(std::memory_order_acquire), + IsochTxQueueStatus::kStopped); + + queue.statusWord.store(IsochTxQueueStatus::kRunning, + std::memory_order_release); + queue.completionCursor.store(12, std::memory_order_release); + queue.ResetProducerForStart(); + EXPECT_EQ(queue.committedEnd.load(std::memory_order_acquire), 0U); + EXPECT_EQ(queue.completionCursor.load(std::memory_order_acquire), 12U); + EXPECT_EQ(queue.statusWord.load(std::memory_order_acquire), + IsochTxQueueStatus::kRunning); +} diff --git a/tests/audio/TransmitBoundaryTests.cpp b/tests/audio/TransmitBoundaryTests.cpp new file mode 100644 index 00000000..178f732b --- /dev/null +++ b/tests/audio/TransmitBoundaryTests.cpp @@ -0,0 +1,37 @@ +#include + +#include +#include +#include +#include + +namespace { + +[[nodiscard]] std::string ReadSource(const char* relativePath) { + const auto repositoryRoot = + std::filesystem::path(__FILE__).parent_path().parent_path().parent_path(); + std::ifstream source(repositoryRoot / relativePath); + std::ostringstream contents; + contents << source.rdbuf(); + return contents.str(); +} + +TEST(TransmitBoundaryTests, CoreSourcesDoNotDependOnAudioPacketSemantics) { + const std::string headers = + ReadSource("ASFWDriver/Isoch/Transmit/IsochTransmitContext.hpp") + + ReadSource("ASFWDriver/Isoch/Transmit/IsochTxDmaRing.hpp") + + ReadSource("ASFWDriver/Isoch/Transmit/IsochTxLayout.hpp"); + const std::string sources = + ReadSource("ASFWDriver/Isoch/Transmit/IsochTransmitContext.cpp") + + ReadSource("ASFWDriver/Isoch/Transmit/IsochTxDmaRing.cpp"); + + for (const char* forbidden : { + "Audio/", "AudioTimingGeometry", "IsochAudioTransport", + "AM824", "CipHeader", "SYT", "Replay", "ZTS", "sampleFrame", + }) { + EXPECT_EQ(headers.find(forbidden), std::string::npos) << forbidden; + EXPECT_EQ(sources.find(forbidden), std::string::npos) << forbidden; + } +} + +} // namespace diff --git a/tests/audio/TxRefillCoverageTests.cpp b/tests/audio/TxRefillCoverageTests.cpp index 63767c07..8fa99fa8 100644 --- a/tests/audio/TxRefillCoverageTests.cpp +++ b/tests/audio/TxRefillCoverageTests.cpp @@ -3,7 +3,7 @@ // // TxRefillCoverageTests.cpp // -// Decides whether the IT underrun ("slot N not committed, commitGen one lap +// Decides whether the IT underrun ("slot N not committed, commitGeneration one lap // behind") is a refill RANGE/GENERATION-coverage problem or merely a scheduling // margin race. It models the exact producer/consumer generation contract from // IsochTxDmaRing::Refill + PrepareTransmitSlots. Steady-state coverage is @@ -14,7 +14,7 @@ // full-ring prefill: // * Consumer refill checks slots [completion + ringAhead, +deltaConsumed), // where ringAhead == kTxHardwareRingPackets steady-state, and requires -// commitGen[slot] == ExpectedCommitGen(absPacket) (IsochTxDmaRing.cpp:503). +// commitGeneration[slot] == ExpectedTxCommitGeneration(absPacket) (IsochTxDmaRing.cpp:503). // * Producer tops the committed cursor to completion + lead each wake // (PrepareTransmitSlots, linear in absolute index → no modulo split). // * Coverage therefore holds iff a single coalesced deltaConsumed @@ -22,7 +22,7 @@ // the committed region regardless of latency. #include "Shared/Isoch/AudioTimingGeometry.hpp" -#include "Shared/Isoch/IsochAudioTransport.hpp" +#include "Isoch/Core/IsochTxQueue.hpp" #include @@ -32,7 +32,7 @@ namespace { using ASFW::IsochTransport::AudioTimingGeometry; -using ASFW::IsochTransport::ExpectedCommitGen; +using ASFW::Isoch::ExpectedTxCommitGeneration; constexpr uint32_t kNumSlots = AudioTimingGeometry::kTxSharedSlotPackets; // 408 constexpr uint32_t kHwRing = AudioTimingGeometry::kTxHardwareRingPackets; // 48 @@ -56,7 +56,7 @@ class RefillSim { : numSlots_(numSlots), lead_(lead), ringAhead_(ringAhead), - commitGen_(numSlots, 0) {} + commitGeneration_(numSlots, 0) {} // Producer tops the committed cursor to completion + lead (one wake). void RunProducer() { @@ -72,7 +72,7 @@ class RefillSim { void CommitThrough(uint64_t target) { for (uint64_t abs = expose_; abs < target; ++abs) { - commitGen_[abs % numSlots_] = ExpectedCommitGen(abs, numSlots_); + commitGeneration_[abs % numSlots_] = ExpectedTxCommitGeneration(abs, numSlots_); } if (target > expose_) { expose_ = target; @@ -87,8 +87,8 @@ class RefillSim { uint64_t firstMiss = kNoMiss; for (uint32_t i = 0; i < deltaConsumed; ++i) { const uint64_t fillAbs = fillBase + i; - if (commitGen_[fillAbs % numSlots_] != - ExpectedCommitGen(fillAbs, numSlots_)) { + if (commitGeneration_[fillAbs % numSlots_] != + ExpectedTxCommitGeneration(fillAbs, numSlots_)) { firstMiss = fillAbs; break; } @@ -107,7 +107,7 @@ class RefillSim { uint32_t numSlots_; uint32_t lead_; uint32_t ringAhead_; - std::vector commitGen_; + std::vector commitGeneration_; uint64_t completion_{0}; uint64_t expose_{0}; }; @@ -200,7 +200,7 @@ TEST(TxRefillCoverage, TwoGroupCoalesceCoveredAtOldLead) { // ----------------------------------------------------------------------------- // THE BUG (historical, at the pre-fix lead=60): a coalesced IT completion of // >=3 groups overruns the committed region even with zero producer latency. -// This is the field FATAL ("slot N not committed, commitGen one lap behind"). +// This is the field FATAL ("slot N not committed, commitGeneration one lap behind"). // ----------------------------------------------------------------------------- TEST(TxRefillCoverage, ThreeGroupCoalesceHolesAtOldLead) {