From 2f5618ebb6fd0997e8f7d03e00a53f8fdb045330 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Tue, 31 Mar 2026 18:37:49 +0300 Subject: [PATCH 01/27] Add Record Track (COLMAP) node to nosTrack plugin New node that records camera tracking data per frame and exports COLMAP-format files (cameras.txt + images.txt). - RecordTrackCOLMAP.cpp: Node implementation with Record/Stop/Save/Clear/ Open Folder functions. Captures position, rotation, FOV, sensor size, and lens distortion each frame. Exports OPENCV camera model intrinsics and world-to-camera extrinsics. - RecordTrackCOLMAP.nosdef: Node definition with Track input/output, output directory, image resolution, euler order, record toggle, and frame count pins. - Track.fbs: Added EulerOrder enum (ZYX, XYZ, YXZ, YZX, ZXY, XZY) for configurable euler angle rotation order in COLMAP export. - TrackMain.cpp: Registered RecordTrackCOLMAP in TrackNode enum and ExportNodeFunctions switch. - Track.noscfg: Bumped plugin version to 1.10.0, added nosdef entry. Review fixes applied: - Pin buffer size looked up by name instead of hardcoded index - Null checks on Track flatbuffer fields to prevent crashes - Euler convention matches MakeRotation (eulerAngleZYX with sign negation) - Float output precision set to 12 digits for camera parameters - macOS support added to Open Folder Co-Authored-By: Claude Opus 4.6 (1M context) # Conflicts: # Subsystems/nosTrackSubsystem/Config/Track.fbs --- Plugins/nosTrack/CHANGES.md | 57 +++ .../nosTrack/Config/RecordTrackCOLMAP.nosdef | 114 +++++ Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 467 ++++++++++++++++++ Plugins/nosTrack/Source/TrackMain.cpp | 7 +- Plugins/nosTrack/Track.noscfg | 3 +- Subsystems/nosTrackSubsystem/Config/Track.fbs | 9 + 6 files changed, 655 insertions(+), 2 deletions(-) create mode 100644 Plugins/nosTrack/CHANGES.md create mode 100644 Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef create mode 100644 Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp diff --git a/Plugins/nosTrack/CHANGES.md b/Plugins/nosTrack/CHANGES.md new file mode 100644 index 00000000..ec713c45 --- /dev/null +++ b/Plugins/nosTrack/CHANGES.md @@ -0,0 +1,57 @@ +# Record Track (COLMAP) Node + +## Summary + +A new node "Record Track (COLMAP)" added to the `nosTrack` plugin. It records incoming camera tracking data per frame and exports it in COLMAP's text format (`cameras.txt` + `images.txt`). + +## Files Changed + +### New files +- `Source/RecordTrackCOLMAP.cpp` — Node implementation +- `Config/RecordTrackCOLMAP.nosdef` — Node definition (pins, functions, metadata) + +### Modified files +- `Source/TrackMain.cpp` — Added `RecordTrackCOLMAP` to the `TrackNode` enum and `ExportNodeFunctions` switch +- `Track.noscfg` — Added `Config/RecordTrackCOLMAP.nosdef` to `node_definitions` + +## Node Design + +### Pins +| Pin | Type | Direction | Description | +|-----|------|-----------|-------------| +| Track | `nos.track.Track` | Input | Incoming tracking data | +| Track Out | `nos.track.Track` | Output (only) | Pass-through of input | +| Output Directory | `string` | Property | Folder picker for output | +| Image Resolution | `nos.fb.vec2u` | Property | Width/height (default 1920x1080) | +| Record | `bool` | Property | Mirrors Record/Stop functions | +| Frame Count | `uint` | Output (only) | Frames in buffer | + +### Functions +| Function | Behavior | +|----------|----------| +| Record | Validates folder is empty, clears buffer, starts recording. Orphaned while recording. | +| Stop | Stops recording (does NOT save). Orphaned while idle. | +| Save | Writes `cameras.txt` + `images.txt` to disk. Does not clear buffer. | +| Clear | Clears frame buffer and resets count. | +| Open Folder | Opens output directory in explorer (Windows) or xdg-open (Linux). | + +### State Management +- Record pin and functions are kept in sync bidirectionally. A `SyncingRecordPin` bool guard prevents re-entrant loops between pin changes and function calls. +- Function orphan states: Record/Stop toggle via `SetNodeOrphanState` using a `Name -> UUID` map built in constructor. +- Status messages show recording state + frame count, and persist error messages (e.g., "Target folder is not empty") via `LastError` until user changes the output directory. +- Non-empty folder check: Recording fails with a FAILURE status if the target folder already has files. + +### COLMAP Output Format +- `cameras.txt`: One OPENCV camera per frame — `fx, fy, cx, cy, k1, k2, p1, p2` derived from Track FOV, sensor size, pixel aspect ratio, lens distortion. +- `images.txt`: Per-frame pose — Euler angles converted to quaternion (world-to-camera), translation as `t = -R * C`. + +## Known Review Points +- Euler-to-quaternion convention: The Track's rotation fields (roll/tilt/pan) are passed through `glm::quat(eulerRadians)` then inverted for COLMAP's world-to-camera convention. May need validation against actual tracker output. +- One camera per frame: Each frame gets its own camera entry. This handles zoom/FOV changes but may be unusual for COLMAP workflows with constant intrinsics. +- No `points3D.txt`: COLMAP expects this file too (can be empty). Not currently written. +- `std::system()` for Open Folder: Works but is a simple shell call. Could be replaced with platform APIs if needed. + +## Build +``` +./nodos dev build -p Project13 --target nosTrack +``` diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef new file mode 100644 index 00000000..ad4288b7 --- /dev/null +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -0,0 +1,114 @@ +{ + "nodes": [ + { + "class_name": "RecordTrackCOLMAP", + "menu_info": { + "category": "nosTrack", + "display_name": "Record Track (COLMAP)", + "name_aliases": [ "colmap", "export camera", "record camera" ] + }, + "node": { + "class_name": "RecordTrackCOLMAP", + "display_name": "Record Track (COLMAP)", + "contents_type": "Job", + "description": "Records camera tracking data each frame while recording is enabled, then exports cameras.txt and images.txt in COLMAP format. Intrinsics (focal length, distortion) are derived from the Track's FOV, sensor size, and lens distortion. Extrinsics (rotation, translation) are stored per frame in world-to-camera convention.", + "pins": [ + { + "name": "Track", + "type_name": "nos.track.Track", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Incoming camera tracking data to record. Position, rotation, FOV, sensor size, and lens distortion are captured each frame." + }, + { + "name": "TrackOut", + "display_name": "Track Out", + "type_name": "nos.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "description": "Pass-through of the incoming Track data." + }, + { + "name": "OutputDirectory", + "display_name": "Output Directory", + "type_name": "string", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { "type": "FOLDER_PICKER" }, + "description": "Directory where cameras.txt and images.txt will be written when recording stops. Must be empty to start recording." + }, + { + "name": "ImageResolution", + "display_name": "Image Resolution", + "type_name": "nos.fb.vec2u", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "x": 1920, + "y": 1080 + }, + "description": "Image resolution in pixels (width, height). Used to compute focal length and principal point for COLMAP camera model." + }, + { + "name": "EulerOrder", + "display_name": "Euler Order", + "type_name": "nos.track.EulerOrder", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "ZYX", + "description": "Euler angle rotation order used when converting Track rotation to COLMAP extrinsics. Default ZYX matches the FreeD node convention." + }, + { + "name": "Record", + "type_name": "bool", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": false, + "description": "Toggle recording. Mirrors Record/Stop functions. Enabling clears previous frames and starts capturing. Will fail if the output directory is not empty." + }, + { + "name": "FrameCount", + "display_name": "Frame Count", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Number of frames recorded in the current session." + } + ], + "functions": [ + { + "class_name": "RecordTrackCOLMAP_Record", + "display_name": "Record", + "contents_type": "Job", + "pins": [] + }, + { + "class_name": "RecordTrackCOLMAP_Stop", + "display_name": "Stop", + "contents_type": "Job", + "pins": [] + }, + { + "class_name": "RecordTrackCOLMAP_Save", + "display_name": "Save", + "contents_type": "Job", + "pins": [] + }, + { + "class_name": "RecordTrackCOLMAP_Clear", + "display_name": "Clear", + "contents_type": "Job", + "pins": [] + }, + { + "class_name": "RecordTrackCOLMAP_OpenFolder", + "display_name": "Open Folder", + "contents_type": "Job", + "pins": [] + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp new file mode 100644 index 00000000..9b2751cd --- /dev/null +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -0,0 +1,467 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include "Track_generated.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace nos::track +{ + +NOS_REGISTER_NAME(OutputDirectory); +NOS_REGISTER_NAME(ImageResolution); +NOS_REGISTER_NAME(EulerOrder); +NOS_REGISTER_NAME(Record); +NOS_REGISTER_NAME(FrameCount); + +NOS_REGISTER_NAME(RecordTrackCOLMAP_Record); +NOS_REGISTER_NAME(RecordTrackCOLMAP_Stop); +NOS_REGISTER_NAME(RecordTrackCOLMAP_Save); +NOS_REGISTER_NAME(RecordTrackCOLMAP_Clear); +NOS_REGISTER_NAME(RecordTrackCOLMAP_OpenFolder); + +struct RecordedFrame +{ + glm::vec3 Location; + glm::vec3 Rotation; // Euler degrees (roll, tilt, pan) + float FOV; + glm::vec2 SensorSize; + float FocusDistance; + float PixelAspectRatio; + float K1; + float K2; +}; + +struct RecordTrackCOLMAPContext : NodeContext +{ + std::string OutputDir; + nosVec2u ImageResolution = {1920, 1080}; + track::EulerOrder EulerOrd = track::EulerOrder::ZYX; + bool Recording = false; + bool SyncingRecordPin = false; + std::string LastError; + std::vector Frames; + std::unordered_map FunctionIds; + + RecordTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) + { + if (node->functions()) + { + for (auto* func : *node->functions()) + FunctionIds[nos::Name(func->class_name()->c_str())] = *func->id(); + } + + if (node->pins()) + { + for (auto* pin : *node->pins()) + { + auto name = nos::Name(pin->name()->c_str()); + if (flatbuffers::IsFieldPresent(pin, fb::Pin::VT_DATA)) + { + nosBuffer value = {.Data = (void*)pin->data()->data(), .Size = pin->data()->size()}; + OnPinValueChanged(name, *pin->id(), value); + } + } + } + UpdateFunctionOrphanStates(); + UpdateStatus(); + } + + void SetFunctionOrphanState(nos::Name funcName, fb::NodeOrphanStateType type) + { + auto it = FunctionIds.find(funcName); + if (it != FunctionIds.end()) + NodeContext::SetNodeOrphanState(it->second, type); + } + + void UpdateFunctionOrphanStates() + { + if (Recording) + { + SetFunctionOrphanState(NSN_RecordTrackCOLMAP_Record, fb::NodeOrphanStateType::ORPHAN); + SetFunctionOrphanState(NSN_RecordTrackCOLMAP_Stop, fb::NodeOrphanStateType::ACTIVE); + } + else + { + SetFunctionOrphanState(NSN_RecordTrackCOLMAP_Record, fb::NodeOrphanStateType::ACTIVE); + SetFunctionOrphanState(NSN_RecordTrackCOLMAP_Stop, fb::NodeOrphanStateType::ORPHAN); + } + } + + void SyncRecordPin(bool value) + { + SyncingRecordPin = true; + nosEngine.SetPinValueByName(NodeId, NSN_Record, nosBuffer{.Data = &value, .Size = sizeof(value)}); + SyncingRecordPin = false; + } + + bool StartRecording() + { + std::string error; + if (!CanStartRecording(error)) + { + LastError = std::move(error); + UpdateStatus(); + return false; + } + LastError.clear(); + Frames.clear(); + Recording = true; + SyncRecordPin(true); + UpdateFrameCountPin(); + UpdateFunctionOrphanStates(); + UpdateStatus(); + nosEngine.LogI("RecordTrackCOLMAP: Recording started"); + return true; + } + + void StopRecording() + { + Recording = false; + SyncRecordPin(false); + UpdateFunctionOrphanStates(); + UpdateStatus(); + nosEngine.LogI("RecordTrackCOLMAP: Recording stopped (%zu frames in buffer)", Frames.size()); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer val) override + { + if (pinName == NSN_OutputDirectory) + { + OutputDir = InterpretPinValue(val.Data); + LastError.clear(); + UpdateStatus(); + } + else if (pinName == NSN_ImageResolution) + ImageResolution = *(nosVec2u*)val.Data; + else if (pinName == NSN_EulerOrder) + EulerOrd = *(track::EulerOrder*)val.Data; + else if (pinName == NSN_Record) + { + if (SyncingRecordPin) + return; + bool newVal = *(bool*)val.Data; + if (newVal && !Recording) + StartRecording(); + else if (!newVal && Recording) + StopRecording(); + } + } + + bool CanStartRecording(std::string& outError) + { + if (OutputDir.empty()) + { + outError = "Set output directory"; + return false; + } + + std::filesystem::path outDir = nos::Utf8ToPath(OutputDir); + try + { + if (std::filesystem::exists(outDir) && !std::filesystem::is_empty(outDir)) + { + outError = "Target folder is not empty"; + return false; + } + } + catch (std::filesystem::filesystem_error& e) + { + nosEngine.LogE("RecordTrackCOLMAP: %s", e.what()); + outError = e.what(); + return false; + } + return true; + } + + void UpdateFrameCountPin() + { + uint32_t count = (uint32_t)Frames.size(); + nosEngine.SetPinValueByName(NodeId, NSN_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); + } + + void UpdateStatus() + { + if (!LastError.empty()) + SetNodeStatusMessage(LastError, fb::NodeStatusMessageType::FAILURE); + else if (OutputDir.empty()) + SetNodeStatusMessage("Set output directory", fb::NodeStatusMessageType::WARNING); + else if (Recording) + SetNodeStatusMessage("Recording (" + std::to_string(Frames.size()) + " frames)", fb::NodeStatusMessageType::INFO); + else if (!Frames.empty()) + SetNodeStatusMessage("Idle (" + std::to_string(Frames.size()) + " frames in buffer)", fb::NodeStatusMessageType::INFO); + else + SetNodeStatusMessage("Idle", fb::NodeStatusMessageType::INFO); + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + auto pins = GetPinValues(params); + auto ids = GetPinIds(params); + + // Pass through Track input to output + auto trackPinData = pins[NOS_NAME("Track")]; + size_t trackDataSize = 0; + for (size_t i = 0; i < params->PinCount; ++i) + { + if (params->Pins[i].Name == NOS_NAME("Track")) + { + trackDataSize = params->Pins[i].Data->Size; + break; + } + } + nosEngine.SetPinValue(ids[NOS_NAME("TrackOut")], {.Data = trackPinData, .Size = trackDataSize}); + + if (!Recording) + return NOS_RESULT_SUCCESS; + + auto* trackData = flatbuffers::GetRoot(trackPinData); + if (!trackData) + return NOS_RESULT_SUCCESS; + + RecordedFrame frame{}; + if (auto* loc = trackData->location()) + frame.Location = {loc->x(), loc->y(), loc->z()}; + if (auto* rot = trackData->rotation()) + frame.Rotation = {rot->x(), rot->y(), rot->z()}; + frame.FOV = trackData->fov(); + if (auto* ss = trackData->sensor_size()) + frame.SensorSize = {ss->x(), ss->y()}; + frame.FocusDistance = trackData->focus_distance(); + frame.PixelAspectRatio = trackData->pixel_aspect_ratio(); + if (auto* ld = trackData->lens_distortion()) + { + frame.K1 = ld->k1k2().x(); + frame.K2 = ld->k1k2().y(); + } + Frames.push_back(frame); + + UpdateFrameCountPin(); + UpdateStatus(); + + return NOS_RESULT_SUCCESS; + } + + void WriteFiles() + { + if (OutputDir.empty()) + { + nosEngine.LogE("RecordTrackCOLMAP: Output directory is empty"); + return; + } + if (Frames.empty()) + { + nosEngine.LogW("RecordTrackCOLMAP: No frames recorded"); + return; + } + + std::filesystem::path outDir = nos::Utf8ToPath(OutputDir); + try + { + if (!std::filesystem::exists(outDir)) + std::filesystem::create_directories(outDir); + } + catch (std::filesystem::filesystem_error& e) + { + nosEngine.LogE("RecordTrackCOLMAP: %s", e.what()); + return; + } + + WriteCamerasTxt(outDir); + WriteImagesTxt(outDir); + nosEngine.LogI("RecordTrackCOLMAP: Saved %zu frames to %s", Frames.size(), OutputDir.c_str()); + } + + float ComputeFocalLengthPixels(const RecordedFrame& frame) const + { + if (frame.FOV <= 0.0f) + return static_cast(ImageResolution.x); + float fovRad = glm::radians(frame.FOV); + return (ImageResolution.x * 0.5f) / std::tan(fovRad * 0.5f); + } + + void WriteCamerasTxt(const std::filesystem::path& outDir) + { + auto path = outDir / "cameras.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + + file << std::setprecision(12); + file << "# Camera list with one line of data per camera:\n"; + file << "# CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n"; + file << "# Number of cameras: " << Frames.size() << "\n"; + + for (size_t i = 0; i < Frames.size(); ++i) + { + float fx = ComputeFocalLengthPixels(Frames[i]); + float fy = fx; + if (Frames[i].PixelAspectRatio > 0.0f) + fy = fx / Frames[i].PixelAspectRatio; + + float cx = ImageResolution.x * 0.5f; + float cy = ImageResolution.y * 0.5f; + + // OPENCV model: fx, fy, cx, cy, k1, k2, p1, p2 + float k1 = Frames[i].K1; + float k2 = Frames[i].K2; + + file << (i + 1) << " OPENCV " << ImageResolution.x << " " << ImageResolution.y << " " + << fx << " " << fy << " " << cx << " " << cy << " " + << k1 << " " << k2 << " 0 0\n"; + } + } + + static glm::mat3 EulerToRotationMatrix(glm::vec3 rot, track::EulerOrder order) + { + // rot is (roll, tilt, pan) = (x, y, z) in radians + // Sign convention matches MakeRotation: negate roll (x) and tilt (y) + float r = -rot.x, t = -rot.y, p = rot.z; + switch (order) + { + default: + case track::EulerOrder::ZYX: return glm::mat3(glm::eulerAngleZYX(p, t, r)); + case track::EulerOrder::XYZ: return glm::mat3(glm::eulerAngleXYZ(r, t, p)); + case track::EulerOrder::YXZ: return glm::mat3(glm::eulerAngleYXZ(t, r, p)); + case track::EulerOrder::YZX: return glm::mat3(glm::eulerAngleYZX(t, p, r)); + case track::EulerOrder::ZXY: return glm::mat3(glm::eulerAngleZXY(p, r, t)); + case track::EulerOrder::XZY: return glm::mat3(glm::eulerAngleXZY(r, p, t)); + } + } + + void WriteImagesTxt(const std::filesystem::path& outDir) + { + auto path = outDir / "images.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + + file << std::setprecision(12); + file << "# Image list with two lines of data per image:\n"; + file << "# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n"; + file << "# POINTS2D[] as (X, Y, POINT3D_ID)\n"; + file << "# Number of images: " << Frames.size() << "\n"; + + for (size_t i = 0; i < Frames.size(); ++i) + { + auto& frame = Frames[i]; + + // Convert Euler angles to rotation matrix + // Sign convention matches MakeRotation: negate roll (x) and tilt (y) + glm::vec3 rot = glm::radians(frame.Rotation); + glm::mat3 R_c2w = EulerToRotationMatrix(rot, EulerOrd); + + // COLMAP expects world-to-camera rotation + glm::mat3 R_w2c = glm::transpose(R_c2w); + glm::quat q_w2c = glm::quat_cast(R_w2c); + + // COLMAP translation: t = -R * C (camera center in world coords) + glm::vec3 t = -R_w2c * frame.Location; + + // IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME + file << (i + 1) << " " + << q_w2c.w << " " << q_w2c.x << " " << q_w2c.y << " " << q_w2c.z << " " + << t.x << " " << t.y << " " << t.z << " " + << (i + 1) << " " + << "frame_" << std::setfill('0') << std::setw(6) << i << ".png\n"; + // Empty points line (required by COLMAP format) + file << "\n"; + } + } + + // TODO: Replace std::system with platform APIs (ShellExecuteW / posix_spawnp) to avoid shell injection via crafted paths + static void OpenFolderInExplorer(const std::filesystem::path& folder) + { +#if defined(_WIN32) + std::string cmd = "explorer \"" + nos::PathToUtf8(folder) + "\""; +#elif defined(__APPLE__) + std::string cmd = "open \"" + nos::PathToUtf8(folder) + "\""; +#else + std::string cmd = "xdg-open \"" + nos::PathToUtf8(folder) + "\""; +#endif + std::system(cmd.c_str()); + } + + static nosResult GetFunctions(size_t* count, nosName* names, nosPfnNodeFunctionExecute* fns) + { + *count = 5; + if (!names || !fns) + return NOS_RESULT_SUCCESS; + + names[0] = NOS_NAME_STATIC("RecordTrackCOLMAP_Record"); + fns[0] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + if (self->Recording) + return NOS_RESULT_SUCCESS; + self->StartRecording(); + return NOS_RESULT_SUCCESS; + }; + + names[1] = NOS_NAME_STATIC("RecordTrackCOLMAP_Stop"); + fns[1] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + if (!self->Recording) + return NOS_RESULT_SUCCESS; + self->StopRecording(); + return NOS_RESULT_SUCCESS; + }; + + names[2] = NOS_NAME_STATIC("RecordTrackCOLMAP_Save"); + fns[2] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + self->WriteFiles(); + return NOS_RESULT_SUCCESS; + }; + + names[3] = NOS_NAME_STATIC("RecordTrackCOLMAP_Clear"); + fns[3] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + self->Frames.clear(); + self->UpdateFrameCountPin(); + self->UpdateStatus(); + nosEngine.LogI("RecordTrackCOLMAP: Buffer cleared"); + return NOS_RESULT_SUCCESS; + }; + + names[4] = NOS_NAME_STATIC("RecordTrackCOLMAP_OpenFolder"); + fns[4] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + if (self->OutputDir.empty()) + { + nosEngine.LogW("RecordTrackCOLMAP: Output directory not set"); + return NOS_RESULT_FAILED; + } + std::filesystem::path outDir = nos::Utf8ToPath(self->OutputDir); + if (!std::filesystem::exists(outDir)) + { + nosEngine.LogW("RecordTrackCOLMAP: Directory does not exist: %s", self->OutputDir.c_str()); + return NOS_RESULT_FAILED; + } + OpenFolderInExplorer(outDir); + return NOS_RESULT_SUCCESS; + }; + + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterRecordTrackCOLMAP(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("RecordTrackCOLMAP"), RecordTrackCOLMAPContext, fn); +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Source/TrackMain.cpp b/Plugins/nosTrack/Source/TrackMain.cpp index c330165d..628abd33 100644 --- a/Plugins/nosTrack/Source/TrackMain.cpp +++ b/Plugins/nosTrack/Source/TrackMain.cpp @@ -15,12 +15,14 @@ enum TrackNode : int FreeD, UserTrack, AddTrack, + RecordTrackCOLMAP, Count }; void RegisterFreeDNode(nosNodeFunctions* functions); void RegisterController(nosNodeFunctions* functions); void RegisterAddTrack(nosNodeFunctions*); +void RegisterRecordTrackCOLMAP(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** outList) { @@ -40,7 +42,10 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou RegisterController(node); break; case TrackNode::AddTrack: - RegisterAddTrack(node); + RegisterAddTrack(node); + break; + case TrackNode::RecordTrackCOLMAP: + RegisterRecordTrackCOLMAP(node); break; } } diff --git a/Plugins/nosTrack/Track.noscfg b/Plugins/nosTrack/Track.noscfg index 861423c0..d6310c82 100644 --- a/Plugins/nosTrack/Track.noscfg +++ b/Plugins/nosTrack/Track.noscfg @@ -16,7 +16,8 @@ "node_definitions": [ "Config/FreeD.nosdef", "Config/UserTrack.nosdef", - "Config/AddTrack.nosdef" + "Config/AddTrack.nosdef", + "Config/RecordTrackCOLMAP.nosdef" ], "defaults": [ "Config/Defaults.json" diff --git a/Subsystems/nosTrackSubsystem/Config/Track.fbs b/Subsystems/nosTrackSubsystem/Config/Track.fbs index e1dcce23..75e6e1ff 100644 --- a/Subsystems/nosTrackSubsystem/Config/Track.fbs +++ b/Subsystems/nosTrackSubsystem/Config/Track.fbs @@ -42,3 +42,12 @@ enum RotationSystem : uint { RPT = 4, PRT = 5, } + +enum EulerOrder : uint { + ZYX = 0, + XYZ = 1, + YXZ = 2, + YZX = 3, + ZXY = 4, + XZY = 5, +} From 803bab585622b3a9508e0e6b6c205e94ada16365 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Wed, 1 Apr 2026 15:38:38 +0300 Subject: [PATCH 02/27] Add Playback Track (COLMAP) node, rename pins, use NodeContext helpers New PlaybackTrackCOLMAP node loads cameras.txt + images.txt and outputs Track data. Two modes via PlaybackMode enum: - Sequential: Play/Stop auto-advance frames each execution - Manual: frame index input pin controls which frame to output Pins and functions are orphaned based on mode. Also: - Added PlaybackMode enum to Track.fbs - Renamed Record node pins to InTrack/OutTrack with "Track" display name - Renamed Playback frame pins to InFrameIndex/OutFrameIndex - Replaced nosEngine.SetPinValueByName with NodeContext::SetPinValue Co-Authored-By: Claude Opus 4.6 (1M context) --- .../Config/PlaybackTrackCOLMAP.nosdef | 109 ++++ .../nosTrack/Config/RecordTrackCOLMAP.nosdef | 7 +- .../nosTrack/Source/PlaybackTrackCOLMAP.cpp | 486 ++++++++++++++++++ Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 16 +- Plugins/nosTrack/Source/TrackMain.cpp | 5 + Plugins/nosTrack/Track.noscfg | 3 +- Subsystems/nosTrackSubsystem/Config/Track.fbs | 5 + 7 files changed, 618 insertions(+), 13 deletions(-) create mode 100644 Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef create mode 100644 Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp diff --git a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef new file mode 100644 index 00000000..d0f260c5 --- /dev/null +++ b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef @@ -0,0 +1,109 @@ +{ + "nodes": [ + { + "class_name": "PlaybackTrackCOLMAP", + "menu_info": { + "category": "nosTrack", + "display_name": "Playback Track (COLMAP)", + "name_aliases": [ "colmap", "import camera", "playback camera" ] + }, + "node": { + "class_name": "PlaybackTrackCOLMAP", + "display_name": "Playback Track (COLMAP)", + "contents_type": "Job", + "always_execute": true, + "description": "Loads camera tracking data from COLMAP text format (cameras.txt + images.txt) and outputs Track data. Sequential mode auto-advances each frame with Play/Stop control. Manual mode outputs the frame at a given index.", + "pins": [ + { + "name": "InputDirectory", + "display_name": "Input Directory", + "type_name": "string", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { "type": "FOLDER_PICKER" }, + "description": "Directory containing cameras.txt and images.txt in COLMAP text format." + }, + { + "name": "EulerOrder", + "display_name": "Euler Order", + "type_name": "nos.track.EulerOrder", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "ZYX", + "description": "Euler angle rotation order for converting COLMAP quaternion to Track rotation. Default ZYX matches the FreeD node convention." + }, + { + "name": "Mode", + "type_name": "nos.track.PlaybackMode", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "Sequential", + "description": "Sequential: auto-advances one frame per execution with Play/Stop control. Manual: outputs the frame at the given Frame Input index." + }, + { + "name": "Loop", + "type_name": "bool", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": true, + "description": "Loop playback to the beginning when the last frame is reached." + }, + { + "name": "InFrameIndex", + "display_name": "Frame Index", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "Frame index to output (Manual mode only)." + }, + { + "name": "Track", + "type_name": "nos.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "description": "Track data for the current frame." + }, + { + "name": "OutFrameIndex", + "display_name": "Frame Index", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Current playback frame index." + }, + { + "name": "FrameCount", + "display_name": "Frame Count", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Total number of frames loaded." + } + ], + "functions": [ + { + "class_name": "PlaybackTrackCOLMAP_Play", + "display_name": "Play", + "contents_type": "Job", + "pins": [] + }, + { + "class_name": "PlaybackTrackCOLMAP_Stop", + "display_name": "Stop", + "contents_type": "Job", + "pins": [] + }, + { + "class_name": "PlaybackTrackCOLMAP_OpenFolder", + "display_name": "Open Folder", + "contents_type": "Job", + "pins": [] + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index ad4288b7..5a17b344 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -14,15 +14,16 @@ "description": "Records camera tracking data each frame while recording is enabled, then exports cameras.txt and images.txt in COLMAP format. Intrinsics (focal length, distortion) are derived from the Track's FOV, sensor size, and lens distortion. Extrinsics (rotation, translation) are stored per frame in world-to-camera convention.", "pins": [ { - "name": "Track", + "name": "InTrack", + "display_name": "Track", "type_name": "nos.track.Track", "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", "description": "Incoming camera tracking data to record. Position, rotation, FOV, sensor size, and lens distortion are captured each frame." }, { - "name": "TrackOut", - "display_name": "Track Out", + "name": "OutTrack", + "display_name": "Track", "type_name": "nos.track.Track", "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp new file mode 100644 index 00000000..d57b5b0c --- /dev/null +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -0,0 +1,486 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include "Track_generated.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace nos::track +{ + +NOS_REGISTER_NAME_SPACED(Playback_InputDirectory, "InputDirectory"); +NOS_REGISTER_NAME_SPACED(Playback_EulerOrder, "EulerOrder"); +NOS_REGISTER_NAME_SPACED(Playback_Mode, "Mode"); +NOS_REGISTER_NAME_SPACED(Playback_Loop, "Loop"); +NOS_REGISTER_NAME_SPACED(Playback_InFrameIndex, "InFrameIndex"); +NOS_REGISTER_NAME_SPACED(Playback_OutFrameIndex, "OutFrameIndex"); +NOS_REGISTER_NAME_SPACED(Playback_FrameCount, "FrameCount"); + +NOS_REGISTER_NAME(PlaybackTrackCOLMAP_Play); +NOS_REGISTER_NAME(PlaybackTrackCOLMAP_Stop); +NOS_REGISTER_NAME(PlaybackTrackCOLMAP_OpenFolder); + +struct COLMAPCamera +{ + uint32_t Id = 0; + std::string Model; + uint32_t Width = 0; + uint32_t Height = 0; + float Fx = 0, Fy = 0, Cx = 0, Cy = 0; + float K1 = 0, K2 = 0, P1 = 0, P2 = 0; +}; + +struct COLMAPImage +{ + uint32_t Id = 0; + glm::quat Q{1, 0, 0, 0}; + glm::vec3 T{0}; + uint32_t CameraId = 0; +}; + +struct PlaybackTrackCOLMAPContext : NodeContext +{ + std::string InputDir; + track::EulerOrder EulerOrd = track::EulerOrder::ZYX; + track::PlaybackMode Mode = track::PlaybackMode::Sequential; + bool Loop = true; + bool Playing = false; + uint32_t ManualFrame = 0; + std::string LastError; + std::vector Frames; + uint32_t CurrentFrame = 0; + std::unordered_map FunctionIds; + std::unordered_map PinIds; + + PlaybackTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) + { + if (node->functions()) + { + for (auto* func : *node->functions()) + FunctionIds[nos::Name(func->class_name()->c_str())] = *func->id(); + } + + if (node->pins()) + { + for (auto* pin : *node->pins()) + { + auto name = nos::Name(pin->name()->c_str()); + PinIds[name] = *pin->id(); + if (flatbuffers::IsFieldPresent(pin, fb::Pin::VT_DATA)) + { + nosBuffer value = {.Data = (void*)pin->data()->data(), .Size = pin->data()->size()}; + OnPinValueChanged(name, *pin->id(), value); + } + } + } + UpdateOrphanStates(); + UpdateStatus(); + } + + void SetFunctionOrphanState(nos::Name funcName, fb::NodeOrphanStateType type) + { + auto it = FunctionIds.find(funcName); + if (it != FunctionIds.end()) + NodeContext::SetNodeOrphanState(it->second, type); + } + + void SetPinOrphanState(nos::Name pinName, fb::PinOrphanStateType type) + { + auto it = PinIds.find(pinName); + if (it != PinIds.end()) + NodeContext::SetPinOrphanState(it->second, type); + } + + void UpdateOrphanStates() + { + bool sequential = Mode == track::PlaybackMode::Sequential; + + // Sequential: Play/Stop active, Load/FrameInput orphaned + // Manual: Load/FrameInput active, Play/Stop orphaned + if (sequential) + { + SetPinOrphanState(NSN_Playback_InFrameIndex, fb::PinOrphanStateType::ORPHAN); + + if (Playing) + { + SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Play, fb::NodeOrphanStateType::ORPHAN); + SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Stop, fb::NodeOrphanStateType::ACTIVE); + } + else + { + SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Play, fb::NodeOrphanStateType::ACTIVE); + SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Stop, fb::NodeOrphanStateType::ORPHAN); + } + } + else + { + SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Play, fb::NodeOrphanStateType::ORPHAN); + SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Stop, fb::NodeOrphanStateType::ORPHAN); + SetPinOrphanState(NSN_Playback_InFrameIndex, fb::PinOrphanStateType::ACTIVE); + } + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer val) override + { + if (pinName == NSN_Playback_InputDirectory) + { + InputDir = InterpretPinValue(val.Data); + LastError.clear(); + if (Mode == track::PlaybackMode::Manual && !InputDir.empty()) + LoadFromDirectory(); + else + UpdateStatus(); + } + else if (pinName == NSN_Playback_EulerOrder) + EulerOrd = *(track::EulerOrder*)val.Data; + else if (pinName == NSN_Playback_Mode) + { + auto newMode = *(track::PlaybackMode*)val.Data; + if (newMode != Mode) + { + Mode = newMode; + Playing = false; + UpdateOrphanStates(); + UpdateStatus(); + } + } + else if (pinName == NSN_Playback_Loop) + Loop = *(bool*)val.Data; + else if (pinName == NSN_Playback_InFrameIndex) + ManualFrame = *(uint32_t*)val.Data; + } + + void UpdateFrameCountPin() + { + uint32_t count = (uint32_t)Frames.size(); + SetPinValue(NSN_Playback_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); + } + + void UpdateFrameIndexPin() + { + SetPinValue(NSN_Playback_OutFrameIndex, nosBuffer{.Data = &CurrentFrame, .Size = sizeof(CurrentFrame)}); + } + + void UpdateStatus() + { + if (!LastError.empty()) + SetNodeStatusMessage(LastError, fb::NodeStatusMessageType::FAILURE); + else if (InputDir.empty()) + SetNodeStatusMessage("Set input directory", fb::NodeStatusMessageType::WARNING); + else if (Frames.empty()) + SetNodeStatusMessage("No data loaded", fb::NodeStatusMessageType::WARNING); + else if (Mode == track::PlaybackMode::Sequential && Playing) + SetNodeStatusMessage("Playing (" + std::to_string(CurrentFrame + 1) + "/" + std::to_string(Frames.size()) + ")", fb::NodeStatusMessageType::INFO); + else + SetNodeStatusMessage("Loaded (" + std::to_string(Frames.size()) + " frames)", fb::NodeStatusMessageType::INFO); + } + + // --- Parsing --- + + bool LoadFromDirectory() + { + if (InputDir.empty()) + { + LastError = "Set input directory"; + UpdateStatus(); + return false; + } + + std::filesystem::path dir = nos::Utf8ToPath(InputDir); + auto camerasPath = dir / "cameras.txt"; + auto imagesPath = dir / "images.txt"; + + if (!std::filesystem::exists(camerasPath)) + { + LastError = "cameras.txt not found"; + UpdateStatus(); + return false; + } + if (!std::filesystem::exists(imagesPath)) + { + LastError = "images.txt not found"; + UpdateStatus(); + return false; + } + + std::unordered_map cameras; + if (!ParseCamerasTxt(camerasPath, cameras)) + return false; + + std::vector images; + if (!ParseImagesTxt(imagesPath, images)) + return false; + + if (images.empty()) + { + LastError = "No images found in images.txt"; + UpdateStatus(); + return false; + } + + Frames.clear(); + Frames.reserve(images.size()); + + for (auto& img : images) + { + track::TTrack trackData{}; + auto camIt = cameras.find(img.CameraId); + + // Convert COLMAP world-to-camera back to camera-to-world + glm::mat3 R_w2c = glm::mat3_cast(img.Q); + glm::mat3 R_c2w = glm::transpose(R_w2c); + glm::vec3 C = -R_c2w * img.T; + + glm::vec3 euler = RotationMatrixToEuler(R_c2w, EulerOrd); + trackData.location = reinterpret_cast(C); + trackData.rotation = reinterpret_cast(euler); + + if (camIt != cameras.end()) + { + auto& cam = camIt->second; + if (cam.Fx > 0) + trackData.fov = glm::degrees(2.0f * std::atan(cam.Width * 0.5f / cam.Fx)); + trackData.sensor_size = nos::fb::vec2(cam.Width, cam.Height); + if (cam.Fx > 0 && cam.Fy > 0) + trackData.pixel_aspect_ratio = cam.Fx / cam.Fy; + trackData.lens_distortion.mutable_k1k2() = nos::fb::vec2(cam.K1, cam.K2); + } + + Frames.push_back(std::move(trackData)); + } + + CurrentFrame = 0; + LastError.clear(); + UpdateFrameCountPin(); + UpdateFrameIndexPin(); + UpdateStatus(); + nosEngine.LogI("PlaybackTrackCOLMAP: Loaded %zu frames from %s", Frames.size(), InputDir.c_str()); + return true; + } + + bool ParseCamerasTxt(const std::filesystem::path& path, std::unordered_map& cameras) + { + std::ifstream file(path); + if (!file.is_open()) + { + LastError = "Cannot open cameras.txt"; + UpdateStatus(); + return false; + } + + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + COLMAPCamera cam; + ss >> cam.Id >> cam.Model >> cam.Width >> cam.Height; + if (cam.Model == "OPENCV") + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy >> cam.K1 >> cam.K2 >> cam.P1 >> cam.P2; + else if (cam.Model == "PINHOLE") + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy; + else if (cam.Model == "SIMPLE_PINHOLE") + { + float f; + ss >> f >> cam.Cx >> cam.Cy; + cam.Fx = cam.Fy = f; + } + else if (cam.Model == "SIMPLE_RADIAL") + { + float f; + ss >> f >> cam.Cx >> cam.Cy >> cam.K1; + cam.Fx = cam.Fy = f; + } + else if (cam.Model == "RADIAL") + { + float f; + ss >> f >> cam.Cx >> cam.Cy >> cam.K1 >> cam.K2; + cam.Fx = cam.Fy = f; + } + else + { + nosEngine.LogW("PlaybackTrackCOLMAP: Unsupported camera model '%s', treating as PINHOLE", cam.Model.c_str()); + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy; + } + cameras[cam.Id] = cam; + } + return true; + } + + bool ParseImagesTxt(const std::filesystem::path& path, std::vector& images) + { + std::ifstream file(path); + if (!file.is_open()) + { + LastError = "Cannot open images.txt"; + UpdateStatus(); + return false; + } + + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + COLMAPImage img; + float qw, qx, qy, qz; + std::string name; + ss >> img.Id >> qw >> qx >> qy >> qz + >> img.T.x >> img.T.y >> img.T.z + >> img.CameraId >> name; + img.Q = glm::quat(qw, qx, qy, qz); + images.push_back(img); + // Skip POINTS2D line + std::getline(file, line); + } + + std::sort(images.begin(), images.end(), [](auto& a, auto& b) { return a.Id < b.Id; }); + return true; + } + + // --- Euler extraction (inverse of EulerToRotationMatrix in RecordTrackCOLMAP) --- + + static glm::vec3 RotationMatrixToEuler(const glm::mat3& R_c2w, track::EulerOrder order) + { + float r, t, p; + switch (order) + { + default: + case track::EulerOrder::ZYX: glm::extractEulerAngleZYX(glm::mat4(R_c2w), p, t, r); break; + case track::EulerOrder::XYZ: glm::extractEulerAngleXYZ(glm::mat4(R_c2w), r, t, p); break; + case track::EulerOrder::YXZ: glm::extractEulerAngleYXZ(glm::mat4(R_c2w), t, r, p); break; + case track::EulerOrder::YZX: glm::extractEulerAngleYZX(glm::mat4(R_c2w), t, p, r); break; + case track::EulerOrder::ZXY: glm::extractEulerAngleZXY(glm::mat4(R_c2w), p, r, t); break; + case track::EulerOrder::XZY: glm::extractEulerAngleXZY(glm::mat4(R_c2w), r, p, t); break; + } + // Undo sign convention: r = -roll, t = -tilt, p = pan + return glm::degrees(glm::vec3(-r, -t, p)); + } + + // --- Execution --- + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Frames.empty()) + { + track::TTrack empty{}; + auto buf = nos::Buffer::From(empty); + SetPinValue(NOS_NAME("Track"), {.Data = buf.Data(), .Size = buf.Size()}); + return NOS_RESULT_SUCCESS; + } + + uint32_t frameIdx = 0; + if (Mode == track::PlaybackMode::Sequential) + { + if (!Playing) + { + frameIdx = CurrentFrame; + } + else + { + frameIdx = CurrentFrame; + uint32_t next = CurrentFrame + 1; + if (next >= (uint32_t)Frames.size()) + next = Loop ? 0 : (uint32_t)Frames.size() - 1; + CurrentFrame = next; + } + } + else + { + frameIdx = ManualFrame < (uint32_t)Frames.size() ? ManualFrame : (uint32_t)Frames.size() - 1; + CurrentFrame = frameIdx; + } + + auto buf = nos::Buffer::From(Frames[frameIdx]); + SetPinValue(NOS_NAME("Track"), {.Data = buf.Data(), .Size = buf.Size()}); + UpdateFrameIndexPin(); + + if (Mode == track::PlaybackMode::Sequential && Playing) + UpdateStatus(); + + return NOS_RESULT_SUCCESS; + } + + static nosResult GetFunctions(size_t* count, nosName* names, nosPfnNodeFunctionExecute* fns) + { + *count = 3; + if (!names || !fns) + return NOS_RESULT_SUCCESS; + + names[0] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_Play"); + fns[0] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + if (self->Playing) + return NOS_RESULT_SUCCESS; + if (self->Frames.empty()) + self->LoadFromDirectory(); + if (self->Frames.empty()) + return NOS_RESULT_SUCCESS; + self->Playing = true; + self->CurrentFrame = 0; + self->UpdateOrphanStates(); + self->UpdateStatus(); + nosEngine.LogI("PlaybackTrackCOLMAP: Playing (%zu frames)", self->Frames.size()); + return NOS_RESULT_SUCCESS; + }; + + names[1] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_Stop"); + fns[1] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + if (!self->Playing) + return NOS_RESULT_SUCCESS; + self->Playing = false; + self->UpdateOrphanStates(); + self->UpdateStatus(); + nosEngine.LogI("PlaybackTrackCOLMAP: Stopped at frame %u", self->CurrentFrame); + return NOS_RESULT_SUCCESS; + }; + + names[2] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_OpenFolder"); + fns[3] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + if (self->InputDir.empty()) + { + nosEngine.LogW("PlaybackTrackCOLMAP: Input directory not set"); + return NOS_RESULT_FAILED; + } + std::filesystem::path dir = nos::Utf8ToPath(self->InputDir); + if (!std::filesystem::exists(dir)) + { + nosEngine.LogW("PlaybackTrackCOLMAP: Directory does not exist: %s", self->InputDir.c_str()); + return NOS_RESULT_FAILED; + } + // TODO: Replace std::system with platform APIs (ShellExecuteW / posix_spawnp) +#if defined(_WIN32) + std::string cmd = "explorer \"" + nos::PathToUtf8(dir) + "\""; +#elif defined(__APPLE__) + std::string cmd = "open \"" + nos::PathToUtf8(dir) + "\""; +#else + std::string cmd = "xdg-open \"" + nos::PathToUtf8(dir) + "\""; +#endif + std::system(cmd.c_str()); + return NOS_RESULT_SUCCESS; + }; + + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterPlaybackTrackCOLMAP(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("PlaybackTrackCOLMAP"), PlaybackTrackCOLMAPContext, fn); +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index 9b2751cd..e7e91b55 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -99,7 +99,7 @@ struct RecordTrackCOLMAPContext : NodeContext void SyncRecordPin(bool value) { SyncingRecordPin = true; - nosEngine.SetPinValueByName(NodeId, NSN_Record, nosBuffer{.Data = &value, .Size = sizeof(value)}); + SetPinValue(NSN_Record, nosBuffer{.Data = &value, .Size = sizeof(value)}); SyncingRecordPin = false; } @@ -185,7 +185,7 @@ struct RecordTrackCOLMAPContext : NodeContext void UpdateFrameCountPin() { uint32_t count = (uint32_t)Frames.size(); - nosEngine.SetPinValueByName(NodeId, NSN_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); + SetPinValue(NSN_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); } void UpdateStatus() @@ -205,25 +205,23 @@ struct RecordTrackCOLMAPContext : NodeContext nosResult ExecuteNode(nosNodeExecuteParams* params) override { auto pins = GetPinValues(params); - auto ids = GetPinIds(params); // Pass through Track input to output - auto trackPinData = pins[NOS_NAME("Track")]; - size_t trackDataSize = 0; + nosBuffer trackBuf{}; for (size_t i = 0; i < params->PinCount; ++i) { - if (params->Pins[i].Name == NOS_NAME("Track")) + if (params->Pins[i].Name == NOS_NAME("InTrack")) { - trackDataSize = params->Pins[i].Data->Size; + trackBuf = {.Data = (void*)params->Pins[i].Data->Data, .Size = params->Pins[i].Data->Size}; break; } } - nosEngine.SetPinValue(ids[NOS_NAME("TrackOut")], {.Data = trackPinData, .Size = trackDataSize}); + SetPinValue(NOS_NAME("OutTrack"), trackBuf); if (!Recording) return NOS_RESULT_SUCCESS; - auto* trackData = flatbuffers::GetRoot(trackPinData); + auto* trackData = flatbuffers::GetRoot(trackBuf.Data); if (!trackData) return NOS_RESULT_SUCCESS; diff --git a/Plugins/nosTrack/Source/TrackMain.cpp b/Plugins/nosTrack/Source/TrackMain.cpp index 628abd33..acca3be8 100644 --- a/Plugins/nosTrack/Source/TrackMain.cpp +++ b/Plugins/nosTrack/Source/TrackMain.cpp @@ -16,6 +16,7 @@ enum TrackNode : int UserTrack, AddTrack, RecordTrackCOLMAP, + PlaybackTrackCOLMAP, Count }; @@ -23,6 +24,7 @@ void RegisterFreeDNode(nosNodeFunctions* functions); void RegisterController(nosNodeFunctions* functions); void RegisterAddTrack(nosNodeFunctions*); void RegisterRecordTrackCOLMAP(nosNodeFunctions*); +void RegisterPlaybackTrackCOLMAP(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** outList) { @@ -47,6 +49,9 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou case TrackNode::RecordTrackCOLMAP: RegisterRecordTrackCOLMAP(node); break; + case TrackNode::PlaybackTrackCOLMAP: + RegisterPlaybackTrackCOLMAP(node); + break; } } return NOS_RESULT_SUCCESS; diff --git a/Plugins/nosTrack/Track.noscfg b/Plugins/nosTrack/Track.noscfg index d6310c82..c8a13052 100644 --- a/Plugins/nosTrack/Track.noscfg +++ b/Plugins/nosTrack/Track.noscfg @@ -17,7 +17,8 @@ "Config/FreeD.nosdef", "Config/UserTrack.nosdef", "Config/AddTrack.nosdef", - "Config/RecordTrackCOLMAP.nosdef" + "Config/RecordTrackCOLMAP.nosdef", + "Config/PlaybackTrackCOLMAP.nosdef" ], "defaults": [ "Config/Defaults.json" diff --git a/Subsystems/nosTrackSubsystem/Config/Track.fbs b/Subsystems/nosTrackSubsystem/Config/Track.fbs index 75e6e1ff..2da4998e 100644 --- a/Subsystems/nosTrackSubsystem/Config/Track.fbs +++ b/Subsystems/nosTrackSubsystem/Config/Track.fbs @@ -51,3 +51,8 @@ enum EulerOrder : uint { ZXY = 4, XZY = 5, } + +enum PlaybackMode : uint { + Sequential = 0, + Manual = 1, +} From e3af9029ed9502e1a48173018a93dbbb8e30b487 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Fri, 3 Apr 2026 13:46:12 +0300 Subject: [PATCH 03/27] Backport MultiLiveOut node to 1.3 --- .../nosUtilities/Config/MultiLiveOut.nosdef | 30 +++ Plugins/nosUtilities/Source/MultiLiveOut.cpp | 189 ++++++++++++++++++ Plugins/nosUtilities/Source/UtilitiesMain.cpp | 3 + Plugins/nosUtilities/Utilities.noscfg | 5 +- 4 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 Plugins/nosUtilities/Config/MultiLiveOut.nosdef create mode 100644 Plugins/nosUtilities/Source/MultiLiveOut.cpp diff --git a/Plugins/nosUtilities/Config/MultiLiveOut.nosdef b/Plugins/nosUtilities/Config/MultiLiveOut.nosdef new file mode 100644 index 00000000..36997973 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiLiveOut.nosdef @@ -0,0 +1,30 @@ +{ + "nodes": [ + { + "class_name": "MultiLiveOut", + "menu_info": { + "category": "Scheduling", + "display_name": "Multi Live Out" + }, + "node": { + "class_name": "MultiLiveOut", + "contents_type": "Job", + "pins": [ + { + "name": "Input_0", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_0", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Source/MultiLiveOut.cpp b/Plugins/nosUtilities/Source/MultiLiveOut.cpp new file mode 100644 index 00000000..c4d08d88 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiLiveOut.cpp @@ -0,0 +1,189 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include + +namespace nos::utilities +{ + +struct MultiLiveOutNode : NodeContext +{ + MultiLiveOutNode(nosFbNodePtr node) : NodeContext(node) + { + for (auto* pin : *node->pins()) + { + SetPinOrphanState(*pin->id(), nos::fb::PinOrphanStateType::ACTIVE); + auto index = GetPinIndex(pin->name()->string_view()); + if (!index) + { + nosEngine.LogE("Failed to parse index from pin name: %s", pin->name()->c_str()); + continue; + } + if (pin->show_as() == nosFbShowAs::OUTPUT_PIN) + IndexToPairs[*index].second = uuid(*pin->id()); + else + IndexToPairs[*index].first = uuid(*pin->id()); + } + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto index = GetPinIndex(pin->name()->string_view()); + if (!index) + return; + if (pin->show_as() == nosFbShowAs::OUTPUT_PIN) + IndexToPairs[*index].second = uuid(*pin->id()); + else + IndexToPairs[*index].first = uuid(*pin->id()); + } + else if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + for (auto it = IndexToPairs.begin(); it != IndexToPairs.end(); ++it) + { + if (it->second.first == update->PinDeleted || it->second.second == update->PinDeleted) + { + IndexToPairs.erase(it); + break; + } + } + } + } + + void OnMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector> items; + if (*request->item_id() == NodeId) + items.push_back(nos::CreateContextMenuItemDirect(fbb, "Add New Pair", 1)); + else + { + auto* pin = GetPin(*request->item_id()); + if (!pin) + return; + if (pin->Name == NOS_NAME("Input_0") || pin->Name == NOS_NAME("Output_0")) + return; + items.push_back(nos::CreateContextMenuItemDirect(fbb, "Remove Pair", 1)); + } + HandleEvent(CreateAppEvent( + fbb, CreateAppContextMenuUpdate( + fbb, request->item_id(), request->pos(), request->instigator(), fbb.CreateVector(items)))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + flatbuffers::FlatBufferBuilder fbb; + if (itemID == NodeId) + { + int index = 0; + for (; index < (int)IndexToPairs.size(); index++) + { + if (!IndexToPairs.contains(index)) + break; + } + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = "Output_" + std::to_string(index); + outPin.type_name = NOS_NAME("nos.Generic"); + outPin.live = true; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = "Input_" + std::to_string(index); + inPin.type_name = NOS_NAME("nos.Generic"); + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + IndexToPairs[index] = {uuid(inPin.id), uuid(outPin.id)}; + } + else + { + auto* pin = GetPin(itemID); + if (!pin) + return; + auto index = GetPinIndex(pin->Name.AsString()); + if (!index) + { + nosEngine.LogE("Failed to parse index from pin name: %s", pin->Name.AsCStr()); + return; + } + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {IndexToPairs[*index].first, IndexToPairs[*index].second}; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + IndexToPairs.erase(*index); + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinName = nos::Name(params->InstigatorPinName).AsString(); + auto index = GetPinIndex(pinName); + if (!index.has_value()) + { + strcpy(params->OutErrorMessage, "Failed to parse pin index from pin name."); + return NOS_RESULT_FAILED; + } + auto const& [firstId, secondId] = IndexToPairs[*index]; + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pin = params->Pins[i]; + if (pin.Id == firstId || pin.Id == secondId) + pin.OutResolvedTypeName = params->IncomingTypeName; + else + pin.OutResolvedTypeName = NOS_NAME("nos.Generic"); + } + return NOS_RESULT_SUCCESS; + } + + std::optional GetPinIndex(std::string_view pinName) const + { + auto indexPos = pinName.find_last_of('_'); + if (indexPos == std::string::npos) + return std::nullopt; + try + { + return std::stoi(std::string(pinName.substr(indexPos + 1))); + } + catch (...) + { + nosEngine.LogE("Failed to parse index from pin name: %s", std::string(pinName).c_str()); + return std::nullopt; + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + for (auto const& [_, idPair] : IndexToPairs) + { + for (size_t i = 0; i < params->PinCount; ++i) + { + auto& pin = params->Pins[i]; + if (pin.Id == idPair.first && pin.Data) + { + nosEngine.SetPinValue(idPair.second, *pin.Data); + break; + } + } + } + return NOS_RESULT_SUCCESS; + } + + std::unordered_map> IndexToPairs; +}; + +nosResult RegisterMultiLiveOut(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiLiveOut"), MultiLiveOutNode, fn) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/UtilitiesMain.cpp b/Plugins/nosUtilities/Source/UtilitiesMain.cpp index c3d3e24a..b3fcc186 100644 --- a/Plugins/nosUtilities/Source/UtilitiesMain.cpp +++ b/Plugins/nosUtilities/Source/UtilitiesMain.cpp @@ -57,6 +57,7 @@ enum Utilities : int GridOutputLayout, LoadCubeLUT, RepeatingJunction, + MultiLiveOut, Count }; @@ -93,6 +94,7 @@ nosResult RegisterFreeOutputLayout(nosNodeFunctions*); nosResult RegisterGridOutputLayout(nosNodeFunctions*); nosResult RegisterLoadCubeLUT(nosNodeFunctions*); nosResult RegisterRepeatingJunction(nosNodeFunctions*); +nosResult RegisterMultiLiveOut(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** outList) { @@ -145,6 +147,7 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou GEN_CASE_NODE(GridOutputLayout) GEN_CASE_NODE(LoadCubeLUT) GEN_CASE_NODE(RepeatingJunction) + GEN_CASE_NODE(MultiLiveOut) } } return NOS_RESULT_SUCCESS; diff --git a/Plugins/nosUtilities/Utilities.noscfg b/Plugins/nosUtilities/Utilities.noscfg index 79883ae9..0320cd0d 100644 --- a/Plugins/nosUtilities/Utilities.noscfg +++ b/Plugins/nosUtilities/Utilities.noscfg @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.utilities", - "version": "3.14.8" + "version": "3.15.0" }, "description": "Various utility nodes.", "display_name": "Utilities", @@ -63,7 +63,8 @@ "Config/CalculateDispatchSize.nosdef", "Config/YADIF.nosdef", "Config/YADIFWithAutoDispatchSize.nosdef", - "Config/RepeatingJunction.nosdef" + "Config/RepeatingJunction.nosdef", + "Config/MultiLiveOut.nosdef" ], "custom_types": [ "Config/Merge.fbs", From a797fb84ecc2b0bee3b66e40554033ae24695da2 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Fri, 3 Apr 2026 13:46:58 +0300 Subject: [PATCH 04/27] Add RecordingFrame for track record node --- Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef | 11 ++++++++++- Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 10 ++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index 5a17b344..10e5c006 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -67,6 +67,15 @@ "data": false, "description": "Toggle recording. Mirrors Record/Stop functions. Enabling clears previous frames and starts capturing. Will fail if the output directory is not empty." }, + { + "name": "RecordingFrame", + "display_name": "Recording Frame", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Current recording frame index. Outputs 0 when not recording." + }, { "name": "FrameCount", "display_name": "Frame Count", @@ -74,7 +83,7 @@ "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", "data": 0, - "description": "Number of frames recorded in the current session." + "description": "Number of frames in the buffer." } ], "functions": [ diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index e7e91b55..8e269a0a 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -21,6 +21,7 @@ NOS_REGISTER_NAME(ImageResolution); NOS_REGISTER_NAME(EulerOrder); NOS_REGISTER_NAME(Record); NOS_REGISTER_NAME(FrameCount); +NOS_REGISTER_NAME(RecordingFrame); NOS_REGISTER_NAME(RecordTrackCOLMAP_Record); NOS_REGISTER_NAME(RecordTrackCOLMAP_Stop); @@ -117,6 +118,7 @@ struct RecordTrackCOLMAPContext : NodeContext Recording = true; SyncRecordPin(true); UpdateFrameCountPin(); + UpdateRecordingFramePin(); UpdateFunctionOrphanStates(); UpdateStatus(); nosEngine.LogI("RecordTrackCOLMAP: Recording started"); @@ -127,6 +129,7 @@ struct RecordTrackCOLMAPContext : NodeContext { Recording = false; SyncRecordPin(false); + UpdateRecordingFramePin(); UpdateFunctionOrphanStates(); UpdateStatus(); nosEngine.LogI("RecordTrackCOLMAP: Recording stopped (%zu frames in buffer)", Frames.size()); @@ -188,6 +191,12 @@ struct RecordTrackCOLMAPContext : NodeContext SetPinValue(NSN_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); } + void UpdateRecordingFramePin() + { + uint32_t frame = Recording ? (uint32_t)Frames.size() : 0; + SetPinValue(NSN_RecordingFrame, nosBuffer{.Data = &frame, .Size = sizeof(frame)}); + } + void UpdateStatus() { if (!LastError.empty()) @@ -243,6 +252,7 @@ struct RecordTrackCOLMAPContext : NodeContext Frames.push_back(frame); UpdateFrameCountPin(); + UpdateRecordingFramePin(); UpdateStatus(); return NOS_RESULT_SUCCESS; From 8d19bd3029b8f96d4c498f5e2c365d319d0e99da Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Fri, 3 Apr 2026 18:58:38 +0300 Subject: [PATCH 05/27] Fix crash when invoking OpenFolder function in PlaybackTrackCOLMAP node --- Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp index d57b5b0c..8a23b353 100644 --- a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -449,7 +449,7 @@ struct PlaybackTrackCOLMAPContext : NodeContext }; names[2] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_OpenFolder"); - fns[3] = [](void* ctx, nosFunctionExecuteParams*) { + fns[2] = [](void* ctx, nosFunctionExecuteParams*) { auto* self = static_cast(ctx); if (self->InputDir.empty()) { From 6d1c7d2e19045916f6b4e0a306419b5281fe830e Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Wed, 8 Apr 2026 19:08:10 +0300 Subject: [PATCH 06/27] RecordTrackCOLMAP node should always execute --- Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef | 1 + Plugins/nosTrack/Track.noscfg | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index 10e5c006..94a4f8cf 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -11,6 +11,7 @@ "class_name": "RecordTrackCOLMAP", "display_name": "Record Track (COLMAP)", "contents_type": "Job", + "always_execute": true, "description": "Records camera tracking data each frame while recording is enabled, then exports cameras.txt and images.txt in COLMAP format. Intrinsics (focal length, distortion) are derived from the Track's FOV, sensor size, and lens distortion. Extrinsics (rotation, translation) are stored per frame in world-to-camera convention.", "pins": [ { diff --git a/Plugins/nosTrack/Track.noscfg b/Plugins/nosTrack/Track.noscfg index c8a13052..464b68c7 100644 --- a/Plugins/nosTrack/Track.noscfg +++ b/Plugins/nosTrack/Track.noscfg @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.track", - "version": "1.10.0" + "version": "1.11.0" }, "display_name": "Track", "category": "Virtual Studio", From e89c723b5fbdcf6f07cc5ba64e6f9c82b0e15412 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Thu, 9 Apr 2026 13:30:04 +0300 Subject: [PATCH 07/27] Remove sequential playback mode from PlaybackTrackCOLMAP, keep manual frame index only Also fix frames not loading on node creation by always loading when InputDirectory or EulerOrder changes. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../Config/PlaybackTrackCOLMAP.nosdef | 32 +--- .../nosTrack/Source/PlaybackTrackCOLMAP.cpp | 143 ++---------------- Subsystems/nosTrackSubsystem/Config/Track.fbs | 5 - 3 files changed, 12 insertions(+), 168 deletions(-) diff --git a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef index d0f260c5..ead9359d 100644 --- a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef @@ -12,7 +12,7 @@ "display_name": "Playback Track (COLMAP)", "contents_type": "Job", "always_execute": true, - "description": "Loads camera tracking data from COLMAP text format (cameras.txt + images.txt) and outputs Track data. Sequential mode auto-advances each frame with Play/Stop control. Manual mode outputs the frame at a given index.", + "description": "Loads camera tracking data from COLMAP text format (cameras.txt + images.txt) and outputs Track data at a given frame index.", "pins": [ { "name": "InputDirectory", @@ -32,22 +32,6 @@ "data": "ZYX", "description": "Euler angle rotation order for converting COLMAP quaternion to Track rotation. Default ZYX matches the FreeD node convention." }, - { - "name": "Mode", - "type_name": "nos.track.PlaybackMode", - "show_as": "PROPERTY", - "can_show_as": "INPUT_PIN_OR_PROPERTY", - "data": "Sequential", - "description": "Sequential: auto-advances one frame per execution with Play/Stop control. Manual: outputs the frame at the given Frame Input index." - }, - { - "name": "Loop", - "type_name": "bool", - "show_as": "PROPERTY", - "can_show_as": "INPUT_PIN_OR_PROPERTY", - "data": true, - "description": "Loop playback to the beginning when the last frame is reached." - }, { "name": "InFrameIndex", "display_name": "Frame Index", @@ -55,7 +39,7 @@ "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": 0, - "description": "Frame index to output (Manual mode only)." + "description": "Frame index to output." }, { "name": "Track", @@ -84,18 +68,6 @@ } ], "functions": [ - { - "class_name": "PlaybackTrackCOLMAP_Play", - "display_name": "Play", - "contents_type": "Job", - "pins": [] - }, - { - "class_name": "PlaybackTrackCOLMAP_Stop", - "display_name": "Stop", - "contents_type": "Job", - "pins": [] - }, { "class_name": "PlaybackTrackCOLMAP_OpenFolder", "display_name": "Open Folder", diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp index 8a23b353..3a3e5641 100644 --- a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -20,14 +20,10 @@ namespace nos::track NOS_REGISTER_NAME_SPACED(Playback_InputDirectory, "InputDirectory"); NOS_REGISTER_NAME_SPACED(Playback_EulerOrder, "EulerOrder"); -NOS_REGISTER_NAME_SPACED(Playback_Mode, "Mode"); -NOS_REGISTER_NAME_SPACED(Playback_Loop, "Loop"); NOS_REGISTER_NAME_SPACED(Playback_InFrameIndex, "InFrameIndex"); NOS_REGISTER_NAME_SPACED(Playback_OutFrameIndex, "OutFrameIndex"); NOS_REGISTER_NAME_SPACED(Playback_FrameCount, "FrameCount"); -NOS_REGISTER_NAME(PlaybackTrackCOLMAP_Play); -NOS_REGISTER_NAME(PlaybackTrackCOLMAP_Stop); NOS_REGISTER_NAME(PlaybackTrackCOLMAP_OpenFolder); struct COLMAPCamera @@ -52,30 +48,17 @@ struct PlaybackTrackCOLMAPContext : NodeContext { std::string InputDir; track::EulerOrder EulerOrd = track::EulerOrder::ZYX; - track::PlaybackMode Mode = track::PlaybackMode::Sequential; - bool Loop = true; - bool Playing = false; - uint32_t ManualFrame = 0; + uint32_t FrameIndex = 0; std::string LastError; std::vector Frames; uint32_t CurrentFrame = 0; - std::unordered_map FunctionIds; - std::unordered_map PinIds; - PlaybackTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) { - if (node->functions()) - { - for (auto* func : *node->functions()) - FunctionIds[nos::Name(func->class_name()->c_str())] = *func->id(); - } - if (node->pins()) { for (auto* pin : *node->pins()) { auto name = nos::Name(pin->name()->c_str()); - PinIds[name] = *pin->id(); if (flatbuffers::IsFieldPresent(pin, fb::Pin::VT_DATA)) { nosBuffer value = {.Data = (void*)pin->data()->data(), .Size = pin->data()->size()}; @@ -83,81 +66,28 @@ struct PlaybackTrackCOLMAPContext : NodeContext } } } - UpdateOrphanStates(); UpdateStatus(); } - void SetFunctionOrphanState(nos::Name funcName, fb::NodeOrphanStateType type) - { - auto it = FunctionIds.find(funcName); - if (it != FunctionIds.end()) - NodeContext::SetNodeOrphanState(it->second, type); - } - - void SetPinOrphanState(nos::Name pinName, fb::PinOrphanStateType type) - { - auto it = PinIds.find(pinName); - if (it != PinIds.end()) - NodeContext::SetPinOrphanState(it->second, type); - } - - void UpdateOrphanStates() - { - bool sequential = Mode == track::PlaybackMode::Sequential; - - // Sequential: Play/Stop active, Load/FrameInput orphaned - // Manual: Load/FrameInput active, Play/Stop orphaned - if (sequential) - { - SetPinOrphanState(NSN_Playback_InFrameIndex, fb::PinOrphanStateType::ORPHAN); - - if (Playing) - { - SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Play, fb::NodeOrphanStateType::ORPHAN); - SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Stop, fb::NodeOrphanStateType::ACTIVE); - } - else - { - SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Play, fb::NodeOrphanStateType::ACTIVE); - SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Stop, fb::NodeOrphanStateType::ORPHAN); - } - } - else - { - SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Play, fb::NodeOrphanStateType::ORPHAN); - SetFunctionOrphanState(NSN_PlaybackTrackCOLMAP_Stop, fb::NodeOrphanStateType::ORPHAN); - SetPinOrphanState(NSN_Playback_InFrameIndex, fb::PinOrphanStateType::ACTIVE); - } - } - void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer val) override { if (pinName == NSN_Playback_InputDirectory) { InputDir = InterpretPinValue(val.Data); LastError.clear(); - if (Mode == track::PlaybackMode::Manual && !InputDir.empty()) + if (!InputDir.empty()) LoadFromDirectory(); else UpdateStatus(); } else if (pinName == NSN_Playback_EulerOrder) - EulerOrd = *(track::EulerOrder*)val.Data; - else if (pinName == NSN_Playback_Mode) { - auto newMode = *(track::PlaybackMode*)val.Data; - if (newMode != Mode) - { - Mode = newMode; - Playing = false; - UpdateOrphanStates(); - UpdateStatus(); - } + EulerOrd = *(track::EulerOrder*)val.Data; + if (!InputDir.empty()) + LoadFromDirectory(); } - else if (pinName == NSN_Playback_Loop) - Loop = *(bool*)val.Data; else if (pinName == NSN_Playback_InFrameIndex) - ManualFrame = *(uint32_t*)val.Data; + FrameIndex = *(uint32_t*)val.Data; } void UpdateFrameCountPin() @@ -179,8 +109,6 @@ struct PlaybackTrackCOLMAPContext : NodeContext SetNodeStatusMessage("Set input directory", fb::NodeStatusMessageType::WARNING); else if (Frames.empty()) SetNodeStatusMessage("No data loaded", fb::NodeStatusMessageType::WARNING); - else if (Mode == track::PlaybackMode::Sequential && Playing) - SetNodeStatusMessage("Playing (" + std::to_string(CurrentFrame + 1) + "/" + std::to_string(Frames.size()) + ")", fb::NodeStatusMessageType::INFO); else SetNodeStatusMessage("Loaded (" + std::to_string(Frames.size()) + " frames)", fb::NodeStatusMessageType::INFO); } @@ -381,75 +309,24 @@ struct PlaybackTrackCOLMAPContext : NodeContext return NOS_RESULT_SUCCESS; } - uint32_t frameIdx = 0; - if (Mode == track::PlaybackMode::Sequential) - { - if (!Playing) - { - frameIdx = CurrentFrame; - } - else - { - frameIdx = CurrentFrame; - uint32_t next = CurrentFrame + 1; - if (next >= (uint32_t)Frames.size()) - next = Loop ? 0 : (uint32_t)Frames.size() - 1; - CurrentFrame = next; - } - } - else - { - frameIdx = ManualFrame < (uint32_t)Frames.size() ? ManualFrame : (uint32_t)Frames.size() - 1; - CurrentFrame = frameIdx; - } + uint32_t frameIdx = FrameIndex < (uint32_t)Frames.size() ? FrameIndex : (uint32_t)Frames.size() - 1; + CurrentFrame = frameIdx; auto buf = nos::Buffer::From(Frames[frameIdx]); SetPinValue(NOS_NAME("Track"), {.Data = buf.Data(), .Size = buf.Size()}); UpdateFrameIndexPin(); - if (Mode == track::PlaybackMode::Sequential && Playing) - UpdateStatus(); - return NOS_RESULT_SUCCESS; } static nosResult GetFunctions(size_t* count, nosName* names, nosPfnNodeFunctionExecute* fns) { - *count = 3; + *count = 1; if (!names || !fns) return NOS_RESULT_SUCCESS; - names[0] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_Play"); + names[0] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_OpenFolder"); fns[0] = [](void* ctx, nosFunctionExecuteParams*) { - auto* self = static_cast(ctx); - if (self->Playing) - return NOS_RESULT_SUCCESS; - if (self->Frames.empty()) - self->LoadFromDirectory(); - if (self->Frames.empty()) - return NOS_RESULT_SUCCESS; - self->Playing = true; - self->CurrentFrame = 0; - self->UpdateOrphanStates(); - self->UpdateStatus(); - nosEngine.LogI("PlaybackTrackCOLMAP: Playing (%zu frames)", self->Frames.size()); - return NOS_RESULT_SUCCESS; - }; - - names[1] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_Stop"); - fns[1] = [](void* ctx, nosFunctionExecuteParams*) { - auto* self = static_cast(ctx); - if (!self->Playing) - return NOS_RESULT_SUCCESS; - self->Playing = false; - self->UpdateOrphanStates(); - self->UpdateStatus(); - nosEngine.LogI("PlaybackTrackCOLMAP: Stopped at frame %u", self->CurrentFrame); - return NOS_RESULT_SUCCESS; - }; - - names[2] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_OpenFolder"); - fns[2] = [](void* ctx, nosFunctionExecuteParams*) { auto* self = static_cast(ctx); if (self->InputDir.empty()) { diff --git a/Subsystems/nosTrackSubsystem/Config/Track.fbs b/Subsystems/nosTrackSubsystem/Config/Track.fbs index 2da4998e..75e6e1ff 100644 --- a/Subsystems/nosTrackSubsystem/Config/Track.fbs +++ b/Subsystems/nosTrackSubsystem/Config/Track.fbs @@ -51,8 +51,3 @@ enum EulerOrder : uint { ZXY = 4, XZY = 5, } - -enum PlaybackMode : uint { - Sequential = 0, - Manual = 1, -} From 6f537b0266738c47e078c2cf230024d24e3b6afb Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Thu, 9 Apr 2026 15:44:28 +0300 Subject: [PATCH 08/27] Bump nos.sys.track version --- Plugins/nosTrack/Track.noscfg | 2 +- Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Plugins/nosTrack/Track.noscfg b/Plugins/nosTrack/Track.noscfg index 464b68c7..97473fef 100644 --- a/Plugins/nosTrack/Track.noscfg +++ b/Plugins/nosTrack/Track.noscfg @@ -9,7 +9,7 @@ "dependencies": [ { "name": "nos.sys.track", - "version": "1.0" + "version": "1.1" } ] }, diff --git a/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys b/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys index d2f6b9cb..625fd3fe 100644 --- a/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys +++ b/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.sys.track", - "version": "1.0.0" + "version": "1.1.0" }, "display_name": "Track Subsystem", "dependencies": [ From 8b13957c4b20a111efc4ef0e7e758ebfd6fd9800 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Thu, 9 Apr 2026 18:00:47 +0300 Subject: [PATCH 09/27] Replace EulerOrder with CoordinateSystem from nos.sys.track Migrate COLMAP nodes from the removed plugin-local EulerOrder enum to the existing nos.sys.track.CoordinateSystem enum. Also fix nos.track namespace references to nos.sys.track after upstream merge. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../Config/PlaybackTrackCOLMAP.nosdef | 8 ++--- .../nosTrack/Config/RecordTrackCOLMAP.nosdef | 10 +++--- .../nosTrack/Source/PlaybackTrackCOLMAP.cpp | 32 +++++++++---------- Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 28 ++++++++-------- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef index ead9359d..67b8f74e 100644 --- a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef @@ -24,9 +24,9 @@ "description": "Directory containing cameras.txt and images.txt in COLMAP text format." }, { - "name": "EulerOrder", - "display_name": "Euler Order", - "type_name": "nos.track.EulerOrder", + "name": "CoordinateSystem", + "display_name": "Coordinate System", + "type_name": "nos.sys.track.CoordinateSystem", "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": "ZYX", @@ -43,7 +43,7 @@ }, { "name": "Track", - "type_name": "nos.track.Track", + "type_name": "nos.sys.track.Track", "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", "description": "Track data for the current frame." diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index 94a4f8cf..2237b720 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -17,7 +17,7 @@ { "name": "InTrack", "display_name": "Track", - "type_name": "nos.track.Track", + "type_name": "nos.sys.track.Track", "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", "description": "Incoming camera tracking data to record. Position, rotation, FOV, sensor size, and lens distortion are captured each frame." @@ -25,7 +25,7 @@ { "name": "OutTrack", "display_name": "Track", - "type_name": "nos.track.Track", + "type_name": "nos.sys.track.Track", "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", "description": "Pass-through of the incoming Track data." @@ -52,9 +52,9 @@ "description": "Image resolution in pixels (width, height). Used to compute focal length and principal point for COLMAP camera model." }, { - "name": "EulerOrder", - "display_name": "Euler Order", - "type_name": "nos.track.EulerOrder", + "name": "CoordinateSystem", + "display_name": "Coordinate System", + "type_name": "nos.sys.track.CoordinateSystem", "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": "ZYX", diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp index 3a3e5641..567f929b 100644 --- a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -1,7 +1,7 @@ // Copyright MediaZ Teknoloji A.S. All Rights Reserved. #include -#include "Track_generated.h" +#include "nosSysTrack/Track_generated.h" #include #include @@ -19,7 +19,7 @@ namespace nos::track { NOS_REGISTER_NAME_SPACED(Playback_InputDirectory, "InputDirectory"); -NOS_REGISTER_NAME_SPACED(Playback_EulerOrder, "EulerOrder"); +NOS_REGISTER_NAME_SPACED(Playback_CoordinateSystem, "CoordinateSystem"); NOS_REGISTER_NAME_SPACED(Playback_InFrameIndex, "InFrameIndex"); NOS_REGISTER_NAME_SPACED(Playback_OutFrameIndex, "OutFrameIndex"); NOS_REGISTER_NAME_SPACED(Playback_FrameCount, "FrameCount"); @@ -47,10 +47,10 @@ struct COLMAPImage struct PlaybackTrackCOLMAPContext : NodeContext { std::string InputDir; - track::EulerOrder EulerOrd = track::EulerOrder::ZYX; + sys::track::CoordinateSystem CoordSys = sys::track::CoordinateSystem::ZYX; uint32_t FrameIndex = 0; std::string LastError; - std::vector Frames; + std::vector Frames; uint32_t CurrentFrame = 0; PlaybackTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) { @@ -80,9 +80,9 @@ struct PlaybackTrackCOLMAPContext : NodeContext else UpdateStatus(); } - else if (pinName == NSN_Playback_EulerOrder) + else if (pinName == NSN_Playback_CoordinateSystem) { - EulerOrd = *(track::EulerOrder*)val.Data; + CoordSys = *(sys::track::CoordinateSystem*)val.Data; if (!InputDir.empty()) LoadFromDirectory(); } @@ -161,7 +161,7 @@ struct PlaybackTrackCOLMAPContext : NodeContext for (auto& img : images) { - track::TTrack trackData{}; + sys::track::TTrack trackData{}; auto camIt = cameras.find(img.CameraId); // Convert COLMAP world-to-camera back to camera-to-world @@ -169,7 +169,7 @@ struct PlaybackTrackCOLMAPContext : NodeContext glm::mat3 R_c2w = glm::transpose(R_w2c); glm::vec3 C = -R_c2w * img.T; - glm::vec3 euler = RotationMatrixToEuler(R_c2w, EulerOrd); + glm::vec3 euler = RotationMatrixToEuler(R_c2w, CoordSys); trackData.location = reinterpret_cast(C); trackData.rotation = reinterpret_cast(euler); @@ -280,18 +280,18 @@ struct PlaybackTrackCOLMAPContext : NodeContext // --- Euler extraction (inverse of EulerToRotationMatrix in RecordTrackCOLMAP) --- - static glm::vec3 RotationMatrixToEuler(const glm::mat3& R_c2w, track::EulerOrder order) + static glm::vec3 RotationMatrixToEuler(const glm::mat3& R_c2w, sys::track::CoordinateSystem order) { float r, t, p; switch (order) { default: - case track::EulerOrder::ZYX: glm::extractEulerAngleZYX(glm::mat4(R_c2w), p, t, r); break; - case track::EulerOrder::XYZ: glm::extractEulerAngleXYZ(glm::mat4(R_c2w), r, t, p); break; - case track::EulerOrder::YXZ: glm::extractEulerAngleYXZ(glm::mat4(R_c2w), t, r, p); break; - case track::EulerOrder::YZX: glm::extractEulerAngleYZX(glm::mat4(R_c2w), t, p, r); break; - case track::EulerOrder::ZXY: glm::extractEulerAngleZXY(glm::mat4(R_c2w), p, r, t); break; - case track::EulerOrder::XZY: glm::extractEulerAngleXZY(glm::mat4(R_c2w), r, p, t); break; + case sys::track::CoordinateSystem::ZYX: glm::extractEulerAngleZYX(glm::mat4(R_c2w), p, t, r); break; + case sys::track::CoordinateSystem::XYZ: glm::extractEulerAngleXYZ(glm::mat4(R_c2w), r, t, p); break; + case sys::track::CoordinateSystem::YXZ: glm::extractEulerAngleYXZ(glm::mat4(R_c2w), t, r, p); break; + case sys::track::CoordinateSystem::YZX: glm::extractEulerAngleYZX(glm::mat4(R_c2w), t, p, r); break; + case sys::track::CoordinateSystem::ZXY: glm::extractEulerAngleZXY(glm::mat4(R_c2w), p, r, t); break; + case sys::track::CoordinateSystem::XZY: glm::extractEulerAngleXZY(glm::mat4(R_c2w), r, p, t); break; } // Undo sign convention: r = -roll, t = -tilt, p = pan return glm::degrees(glm::vec3(-r, -t, p)); @@ -303,7 +303,7 @@ struct PlaybackTrackCOLMAPContext : NodeContext { if (Frames.empty()) { - track::TTrack empty{}; + sys::track::TTrack empty{}; auto buf = nos::Buffer::From(empty); SetPinValue(NOS_NAME("Track"), {.Data = buf.Data(), .Size = buf.Size()}); return NOS_RESULT_SUCCESS; diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index 8e269a0a..0284b2f6 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -1,7 +1,7 @@ // Copyright MediaZ Teknoloji A.S. All Rights Reserved. #include -#include "Track_generated.h" +#include "nosSysTrack/Track_generated.h" #include #include @@ -18,7 +18,7 @@ namespace nos::track NOS_REGISTER_NAME(OutputDirectory); NOS_REGISTER_NAME(ImageResolution); -NOS_REGISTER_NAME(EulerOrder); +NOS_REGISTER_NAME(CoordinateSystem); NOS_REGISTER_NAME(Record); NOS_REGISTER_NAME(FrameCount); NOS_REGISTER_NAME(RecordingFrame); @@ -45,7 +45,7 @@ struct RecordTrackCOLMAPContext : NodeContext { std::string OutputDir; nosVec2u ImageResolution = {1920, 1080}; - track::EulerOrder EulerOrd = track::EulerOrder::ZYX; + sys::track::CoordinateSystem CoordSys = sys::track::CoordinateSystem::ZYX; bool Recording = false; bool SyncingRecordPin = false; std::string LastError; @@ -145,8 +145,8 @@ struct RecordTrackCOLMAPContext : NodeContext } else if (pinName == NSN_ImageResolution) ImageResolution = *(nosVec2u*)val.Data; - else if (pinName == NSN_EulerOrder) - EulerOrd = *(track::EulerOrder*)val.Data; + else if (pinName == NSN_CoordinateSystem) + CoordSys = *(sys::track::CoordinateSystem*)val.Data; else if (pinName == NSN_Record) { if (SyncingRecordPin) @@ -230,7 +230,7 @@ struct RecordTrackCOLMAPContext : NodeContext if (!Recording) return NOS_RESULT_SUCCESS; - auto* trackData = flatbuffers::GetRoot(trackBuf.Data); + auto* trackData = flatbuffers::GetRoot(trackBuf.Data); if (!trackData) return NOS_RESULT_SUCCESS; @@ -331,7 +331,7 @@ struct RecordTrackCOLMAPContext : NodeContext } } - static glm::mat3 EulerToRotationMatrix(glm::vec3 rot, track::EulerOrder order) + static glm::mat3 EulerToRotationMatrix(glm::vec3 rot, sys::track::CoordinateSystem order) { // rot is (roll, tilt, pan) = (x, y, z) in radians // Sign convention matches MakeRotation: negate roll (x) and tilt (y) @@ -339,12 +339,12 @@ struct RecordTrackCOLMAPContext : NodeContext switch (order) { default: - case track::EulerOrder::ZYX: return glm::mat3(glm::eulerAngleZYX(p, t, r)); - case track::EulerOrder::XYZ: return glm::mat3(glm::eulerAngleXYZ(r, t, p)); - case track::EulerOrder::YXZ: return glm::mat3(glm::eulerAngleYXZ(t, r, p)); - case track::EulerOrder::YZX: return glm::mat3(glm::eulerAngleYZX(t, p, r)); - case track::EulerOrder::ZXY: return glm::mat3(glm::eulerAngleZXY(p, r, t)); - case track::EulerOrder::XZY: return glm::mat3(glm::eulerAngleXZY(r, p, t)); + case sys::track::CoordinateSystem::ZYX: return glm::mat3(glm::eulerAngleZYX(p, t, r)); + case sys::track::CoordinateSystem::XYZ: return glm::mat3(glm::eulerAngleXYZ(r, t, p)); + case sys::track::CoordinateSystem::YXZ: return glm::mat3(glm::eulerAngleYXZ(t, r, p)); + case sys::track::CoordinateSystem::YZX: return glm::mat3(glm::eulerAngleYZX(t, p, r)); + case sys::track::CoordinateSystem::ZXY: return glm::mat3(glm::eulerAngleZXY(p, r, t)); + case sys::track::CoordinateSystem::XZY: return glm::mat3(glm::eulerAngleXZY(r, p, t)); } } @@ -371,7 +371,7 @@ struct RecordTrackCOLMAPContext : NodeContext // Convert Euler angles to rotation matrix // Sign convention matches MakeRotation: negate roll (x) and tilt (y) glm::vec3 rot = glm::radians(frame.Rotation); - glm::mat3 R_c2w = EulerToRotationMatrix(rot, EulerOrd); + glm::mat3 R_c2w = EulerToRotationMatrix(rot, CoordSys); // COLMAP expects world-to-camera rotation glm::mat3 R_w2c = glm::transpose(R_c2w); From 3731d87b516b711092b89da7c9128864da628e7d Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Tue, 28 Apr 2026 14:08:42 +0300 Subject: [PATCH 10/27] Multi Ring/Queue --- .../Config/MultiBoundedQueue.nosdef | 56 ++ .../Config/MultiRingBuffer.nosdef | 74 ++ .../nosUtilities/Source/MultiBoundedQueue.cpp | 583 +++++++++++++++ .../nosUtilities/Source/MultiRingBuffer.cpp | 705 ++++++++++++++++++ Plugins/nosUtilities/Source/UtilitiesMain.cpp | 6 + Plugins/nosUtilities/Utilities.noscfg | 2 + 6 files changed, 1426 insertions(+) create mode 100644 Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef create mode 100644 Plugins/nosUtilities/Config/MultiRingBuffer.nosdef create mode 100644 Plugins/nosUtilities/Source/MultiBoundedQueue.cpp create mode 100644 Plugins/nosUtilities/Source/MultiRingBuffer.cpp diff --git a/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef b/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef new file mode 100644 index 00000000..df1ef4c8 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef @@ -0,0 +1,56 @@ +{ + "nodes": [ + { + "class_name": "MultiBoundedQueue", + "menu_info": { + "category": "Utilities", + "display_name": "Multi Bounded Queue", + "name_aliases": [ "data structure", "algorithm", "circular", "multi", "fifo" ] + }, + "node": { + "class_name": "MultiBoundedQueue", + "display_name": "Multi Bounded Queue", + "contents_type": "Job", + "description": "Bounded FIFO queue with one or more independent input/output channel pairs sharing a single bound. Right-click the node to add a channel, right-click an Input_X or Output_X pin to remove its channel.", + "pins": [ + { + "name": "Thread", + "type_name": "nos.exe", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Size", + "type_name": "uint", + "data": 2, + "max": 120, + "min": 1, + "show_as": "PROPERTY", + "can_show_as": "PROPERTY_ONLY" + }, + { + "name": "Alignment", + "description": "Used for creating memory-aligned buffers in memory", + "type_name": "uint", + "def": 0, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Input_A", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_A", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef b/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef new file mode 100644 index 00000000..3fa2c4c3 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef @@ -0,0 +1,74 @@ +{ + "nodes": [ + { + "class_name": "MultiRingBuffer", + "menu_info": { + "category": "Utilities", + "display_name": "Multi Ring Buffer", + "name_aliases": [ "data structure", "algorithm", "circular", "multi" ] + }, + "node": { + "class_name": "MultiRingBuffer", + "display_name": "Multi Ring Buffer", + "contents_type": "Job", + "description": "Ring buffer with one or more independent input/output channel pairs sharing a single ring size. Right-click the node to add a channel, right-click an Input_X or Output_X pin to remove its channel.", + "pins": [ + { + "name": "Thread", + "type_name": "nos.exe", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Size", + "type_name": "uint", + "data": 2, + "max": 120, + "min": 1, + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Spare", + "type_name": "uint", + "data": 0, + "max": 119, + "min": 0, + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Alignment", + "description": "Used for creating memory-aligned buffers in memory", + "type_name": "uint", + "def": 0, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Input_A", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_A", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + }, + { + "name": "RepeatWhenFilling", + "display_name": "Repeat When Filling", + "type_name": "bool", + "description": "Serves the last value while the buffer is being filled instead of waiting & resets the ring on restart", + "def": true, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp new file mode 100644 index 00000000..4b7175a4 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp @@ -0,0 +1,583 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#pragma once + +#include + +// External +#include +#include + +#include "Ring.h" +#include "nosUtil/Stopwatch.hpp" + +namespace nos::utilities +{ + +struct MultiBoundedQueueNodeContext : NodeContext +{ + using RingMode = RingNodeBase::RingMode; + using OnRestartType = RingNodeBase::OnRestartType; + + static constexpr std::string_view CHANNEL_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + enum MenuCommandType : uint8_t + { + ADD_CHANNEL = 0, + REMOVE_CHANNEL = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t Letter; + MenuCommand(uint32_t cmd) + { + Type = static_cast(cmd & 0xFF); + Letter = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t letter) : Type(type), Letter(letter) {} + operator uint32_t() const { return (Letter << 8) | Type; } + }; + + struct Channel + { + char Letter; + nos::Name InputName; + nos::Name OutputName; + uuid InputId{}; + uuid OutputId{}; + nos::TypeInfo TypeInfo; + std::unique_ptr Ring; + std::atomic_bool IsOutLive = false; + bool NeedsRecreation = false; + + Channel(char letter) + : Letter(letter), + InputName((std::string("Input_") + letter).c_str()), + OutputName((std::string("Output_") + letter).c_str()), + TypeInfo(NSN_Generic) + { + } + }; + + std::map> Channels; + std::unordered_map PinIdToLetter; + + std::optional RequestedRingSize = std::nullopt; + + std::string GetName() const { return "MultiBoundedQueue"; } + + static std::optional ParseLetter(std::string_view pinName) + { + auto pos = pinName.find_last_of('_'); + if (pos == std::string::npos || pos + 2 != pinName.size()) + return std::nullopt; + char c = pinName[pos + 1]; + if (c < 'A' || c > 'Z') + return std::nullopt; + return c; + } + + static bool IsInputPin(std::string_view pinName) { return pinName.starts_with("Input_"); } + static bool IsOutputPin(std::string_view pinName) { return pinName.starts_with("Output_"); } + + MultiBoundedQueueNodeContext(nosFbNodePtr node) : NodeContext(node) + { + std::vector pinsToUnorphan; + for (auto* pin : *node->pins()) + { + auto pinNameSv = pin->name()->string_view(); + if (!IsInputPin(pinNameSv) && !IsOutputPin(pinNameSv)) + continue; + auto letter = ParseLetter(pinNameSv); + if (!letter) + continue; + + auto& channel = Channels[*letter]; + if (!channel) + channel = std::make_unique(*letter); + + if (IsInputPin(pinNameSv)) + channel->InputId = uuid(*pin->id()); + else + { + channel->OutputId = uuid(*pin->id()); + channel->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + + nos::Name typeName(pin->type_name()->c_str()); + if (typeName != NSN_Generic && channel->TypeInfo->TypeName == NSN_Generic) + channel->TypeInfo = nos::TypeInfo(typeName); + + if (auto orphanState = pin->orphan_state()) + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(uuid(*pin->id())); + } + + for (auto& [_, ch] : Channels) + InitChannel(*ch); + + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); + + AddPinValueWatcher(NSN_Size, [this](nos::Buffer const& newSize, std::optional oldVal) { + uint32_t size = *newSize.As(); + if (oldVal && oldVal == newSize) + return; + RequestRingResize(size); + }); + AddPinValueWatcher(NSN_Alignment, [this](nos::Buffer const& newAlignment, std::optional oldVal) { + for (auto& [_, ch] : Channels) + { + if (!ch->Ring) + continue; + if (ch->Ring->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) + { + nosEngine.SendPathRestart(ch->InputId); + ch->Ring->Stop(); + ch->NeedsRecreation = true; + } + } + }); + } + + ~MultiBoundedQueueNodeContext() override + { + for (auto& [_, ch] : Channels) + if (ch->Ring) + ch->Ring->Stop(); + } + + void InitChannel(Channel& ch) + { + std::shared_ptr resource; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + resource = std::make_shared(); + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + resource = std::make_shared(); + else + resource = std::make_shared(); + + ch.Ring = std::make_unique(1, std::move(resource)); + ch.Ring->Stop(); + } + + Channel* GetChannelByPinId(uuid const& id) + { + auto it = PinIdToLetter.find(id); + if (it == PinIdToLetter.end()) + return nullptr; + auto chIt = Channels.find(it->second); + return chIt != Channels.end() ? chIt->second.get() : nullptr; + } + + void RequestRingResize(uint32_t size) + { + if (size == 0) + { + nosEngine.LogW((GetName() + " size cannot be 0").c_str()); + return; + } + bool changed = false; + for (auto& [_, ch] : Channels) + { + if (!ch->Ring) + continue; + if (ch->Ring->Size != size && (!RequestedRingSize.has_value() || *RequestedRingSize != size)) + { + nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; + nosEngine.SendPathCommand(ch->InputId, ringSizeChange); + ch->Ring->Stop(); + changed = true; + } + } + if (changed) + { + SendPathRestart(); + RequestedRingSize = size; + } + } + + void SendPathRestart() + { + for (auto& [_, ch] : Channels) + nosEngine.SendPathRestart(ch->InputId); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer value) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv)) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch || !ch->Ring) + return; + if (ch->Ring->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) + { + nosEngine.SendPathRestart(ch->InputId); + ch->Ring->Stop(); + ch->NeedsRecreation = true; + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinNameStr = nos::Name(params->InstigatorPinName).AsString(); + auto letter = ParseLetter(pinNameStr); + if (!letter) + return NOS_RESULT_FAILED; + auto chIt = Channels.find(*letter); + if (chIt == Channels.end()) + return NOS_RESULT_FAILED; + auto& ch = *chIt->second; + if (ch.TypeInfo->TypeName != NSN_Generic) + return NOS_RESULT_FAILED; + ch.TypeInfo = nos::TypeInfo(params->IncomingTypeName); + if (ch.Ring) + ch.Ring->Stop(); + ch.Ring.reset(); + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pinInfo = params->Pins[i]; + if (pinInfo.Id == ch.InputId || pinInfo.Id == ch.OutputId) + pinInfo.OutResolvedTypeName = ch.TypeInfo->TypeName; + } + return NOS_RESULT_SUCCESS; + } + + void OnPinUpdated(const nosPinUpdate*) override + { + for (auto& [_, ch] : Channels) + if (!ch->Ring) + InitChannel(*ch); + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + auto it = PinIdToLetter.find(update->PinDeleted); + if (it == PinIdToLetter.end()) + return; + char letter = it->second; + PinIdToLetter.erase(it); + auto chIt = Channels.find(letter); + if (chIt == Channels.end()) + return; + auto& ch = *chIt->second; + bool inputAlive = PinIdToLetter.contains(ch.InputId); + bool outputAlive = PinIdToLetter.contains(ch.OutputId); + if (!inputAlive && !outputAlive) + { + if (ch.Ring) + ch.Ring->Stop(); + Channels.erase(chIt); + } + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto sv = pin->name()->string_view(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + auto& chPtr = Channels[*letter]; + if (!chPtr) + chPtr = std::make_unique(*letter); + if (IsInputPin(sv)) + chPtr->InputId = uuid(*pin->id()); + else + { + chPtr->OutputId = uuid(*pin->id()); + chPtr->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + if (!chPtr->Ring) + InitChannel(*chPtr); + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Channels.empty()) + return NOS_RESULT_FAILED; + + NodeExecuteParams pins(params); + uint32_t requestedSize = *pins.GetPinData(NSN_Size); + + std::vector> inputs; + inputs.reserve(Channels.size()); + uint32_t maxRequired = requestedSize; + for (auto& [_, ch] : Channels) + { + if (!ch->Ring || ch->Ring->Exit || !ch->Ring->IsResourcesValid() || !ch->TypeInfo) + continue; + auto it = pins.find(ch->InputName); + if (it == pins.end()) + continue; + void* input = ch->Ring->ResInterface->GetPinInfo(it->second, false); + if (!input) + continue; + auto [required, _] = ch->Ring->ResInterface->GetRequiredRingSize(input, requestedSize); + if (required > maxRequired) + maxRequired = required; + inputs.emplace_back(ch.get(), input); + } + if (inputs.empty()) + { + SendScheduleRequest(0); + return NOS_RESULT_FAILED; + } + + bool anyResize = false; + for (auto& [ch, _] : inputs) + if (ch->Ring->Size != maxRequired) + { + anyResize = true; + break; + } + if (anyResize) + { + RequestRingResize(maxRequired); + return NOS_RESULT_FAILED; + } + + std::vector slots(inputs.size(), nullptr); + for (size_t i = 0; i < inputs.size(); ++i) + { + auto* ch = inputs[i].first; + auto* slot = ch->Ring->BeginPush(100); + if (!slot) + { + for (size_t j = 0; j < i; ++j) + inputs[j].first->Ring->CancelPush(slots[j]); + if (ch->Ring->Exit) + return NOS_RESULT_FAILED; + return NOS_RESULT_PENDING; + } + slots[i] = slot; + } + + for (size_t i = 0; i < inputs.size(); ++i) + { + auto* ch = inputs[i].first; + ch->Ring->ResInterface->Push(slots[i], inputs[i].second, params, + NOS_NAME_STATIC("MultiBoundedQueue"), false); + ch->Ring->EndPush(slots[i]); + if (!ch->IsOutLive) + { + ChangePinLiveness(ch->OutputName, true); + ch->IsOutLive = true; + } + } + + return NOS_RESULT_SUCCESS; + } + + nosResult CopyFrom(nosCopyInfo* cpy) override + { + auto* ch = GetChannelByPinId(cpy->ID); + if (!ch || !ch->Ring || ch->Ring->Exit) + return NOS_RESULT_FAILED; + if (!ch->IsOutLive) + return NOS_RESULT_SUCCESS; + + ResourceInterface::ResourceBase* slot; + { + ScopedProfilerEvent _({.Name = "Wait For Filled Slot"}); + slot = ch->Ring->BeginPop(100); + } + if (!slot) + return ch->Ring->Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + ch->Ring->ResInterface->Copy(slot, cpy, NodeId); + + cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; + cpy->FrameNumber = slot->FrameNumber; + + ch->Ring->EndPop(slot); + SendScheduleRequest(1); + return NOS_RESULT_SUCCESS; + } + + void OnEndFrame(uuid const& pinId, nosEndFrameCause cause) override + { + if (cause != NOS_END_FRAME_FAILED) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch) + return; + if (pinId == ch->OutputId) + return; + if (!ch->IsOutLive) + return; + ChangePinLiveness(ch->OutputName, false); + ch->IsOutLive = false; + } + + void SendScheduleRequest(uint32_t count, bool reset = false) const + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = count, .Reset = reset}; + nosEngine.ScheduleNode(&schedule); + } + + void OnPathCommand(const nosPathCommand* command) override + { + switch (command->Event) + { + case NOS_RING_SIZE_CHANGE: + if (command->RingSize == 0) + return; + RequestedRingSize = command->RingSize; + nosEngine.SetPinValue(*GetPinId(NSN_Size), nos::Buffer::From(command->RingSize)); + break; + default: return; + } + } + + void OnPathStop() override + { + for (auto& [_, ch] : Channels) + if (ch->Ring) + ch->Ring->Stop(); + } + + void OnPathStart() override + { + if (Channels.empty()) + return; + size_t totalSchedule = 0; + for (auto& [_, ch] : Channels) + { + if (!ch->Ring) + continue; + // FIFO restart: drop any frames left from the previous run. + ch->Ring->Reset(false); + if (RequestedRingSize) + { + ch->Ring->Resize(*RequestedRingSize); + ch->NeedsRecreation = false; + } + if (ch->NeedsRecreation) + { + ch->Ring = std::make_unique(ch->Ring->Size, ch->Ring->ResInterface); + ch->NeedsRecreation = false; + } + if (!ch->Ring->IsResourcesValid()) + { + totalSchedule = std::max(totalSchedule, 1); + continue; + } + auto emptySlotCount = ch->Ring->Write.Pool.size(); + totalSchedule = std::max(totalSchedule, emptySlotCount); + ch->Ring->Exit = false; + ch->Ring->ResInterface->OnPathStart(); + } + RequestedRingSize = std::nullopt; + if (totalSchedule > 0) + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = (uint32_t)totalSchedule}; + nosEngine.ScheduleNode(&schedule); + } + } + + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Channel", MenuCommand(ADD_CHANNEL, 0))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + if (Channels.size() <= 1) + return; + flatbuffers::FlatBufferBuilder fbb; + std::vector items = {nos::CreateContextMenuItemDirect( + fbb, "Remove Channel", MenuCommand(REMOVE_CHANNEL, static_cast(*letter)))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_CHANNEL: + { + char newLetter = 0; + for (char c : CHANNEL_LETTERS) + { + if (!Channels.contains(c)) + { + newLetter = c; + break; + } + } + if (newLetter == 0) + { + SetNodeStatusMessage("Maximum number of channels reached", fb::NodeStatusMessageType::WARNING); + return; + } + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = std::string("Input_") + newLetter; + inPin.type_name = "nos.Generic"; + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = std::string("Output_") + newLetter; + outPin.type_name = "nos.Generic"; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + outPin.live = true; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + case REMOVE_CHANNEL: + { + char letter = static_cast(command.Letter); + auto it = Channels.find(letter); + if (it == Channels.end()) + return; + auto& ch = *it->second; + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {ch.InputId, ch.OutputId}; + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + } + } +}; + +nosResult RegisterMultiBoundedQueue(nosNodeFunctions* functions) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiBoundedQueue"), MultiBoundedQueueNodeContext, functions) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/MultiRingBuffer.cpp b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp new file mode 100644 index 00000000..a13c7f09 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp @@ -0,0 +1,705 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#pragma once + +#include + +// External +#include +#include + +#include "Ring.h" +#include "nosUtil/Stopwatch.hpp" + +namespace nos::utilities +{ + +struct MultiRingBufferNodeContext : NodeContext +{ + using RingMode = RingNodeBase::RingMode; + using OnRestartType = RingNodeBase::OnRestartType; + + static constexpr std::string_view CHANNEL_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + enum MenuCommandType : uint8_t + { + ADD_CHANNEL = 0, + REMOVE_CHANNEL = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t Letter; + MenuCommand(uint32_t cmd) + { + Type = static_cast(cmd & 0xFF); + Letter = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t letter) : Type(type), Letter(letter) {} + operator uint32_t() const { return (Letter << 8) | Type; } + }; + + struct Channel + { + char Letter; + nos::Name InputName; + nos::Name OutputName; + uuid InputId{}; + uuid OutputId{}; + nos::TypeInfo TypeInfo; + std::unique_ptr Ring; + std::atomic_bool IsOutLive = false; + ResourceInterface::ResourceBase* LastPopped = nullptr; + bool NeedsRecreation = false; + std::size_t RemainingRepeatableCount = 0; + + Channel(char letter) + : Letter(letter), + InputName((std::string("Input_") + letter).c_str()), + OutputName((std::string("Output_") + letter).c_str()), + TypeInfo(NSN_Generic) + { + } + }; + + std::map> Channels; + std::unordered_map PinIdToLetter; + + OnRestartType OnRestart = OnRestartType::WAIT_UNTIL_FULL; + std::optional RequestedRingSize = std::nullopt; + std::atomic Mode = RingMode::CONSUME; + std::condition_variable ModeCV; + std::mutex ModeMutex; + std::atomic_bool RepeatWhenFilling = false; + + std::string GetName() const { return "MultiRingBuffer"; } + + static std::optional ParseLetter(std::string_view pinName) + { + auto pos = pinName.find_last_of('_'); + if (pos == std::string::npos || pos + 2 != pinName.size()) + return std::nullopt; + char c = pinName[pos + 1]; + if (c < 'A' || c > 'Z') + return std::nullopt; + return c; + } + + static bool IsInputPin(std::string_view pinName) { return pinName.starts_with("Input_"); } + static bool IsOutputPin(std::string_view pinName) { return pinName.starts_with("Output_"); } + + MultiRingBufferNodeContext(nosFbNodePtr node) : NodeContext(node) + { + std::vector pinsToUnorphan; + for (auto* pin : *node->pins()) + { + auto pinNameSv = pin->name()->string_view(); + if (!IsInputPin(pinNameSv) && !IsOutputPin(pinNameSv)) + continue; + auto letter = ParseLetter(pinNameSv); + if (!letter) + continue; + + auto& channel = Channels[*letter]; + if (!channel) + channel = std::make_unique(*letter); + + if (IsInputPin(pinNameSv)) + channel->InputId = uuid(*pin->id()); + else + { + channel->OutputId = uuid(*pin->id()); + channel->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + + nos::Name typeName(pin->type_name()->c_str()); + if (typeName != NSN_Generic && channel->TypeInfo->TypeName == NSN_Generic) + channel->TypeInfo = nos::TypeInfo(typeName); + + if (auto orphanState = pin->orphan_state()) + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(uuid(*pin->id())); + } + + for (auto& [_, ch] : Channels) + InitChannel(*ch); + + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); + + AddPinValueWatcher(NSN_Size, [this](nos::Buffer const& newSize, std::optional oldVal) { + uint32_t size = *newSize.As(); + if (oldVal && oldVal == newSize) + return; + RequestRingResize(size); + }); + AddPinValueWatcher(NSN_Alignment, [this](nos::Buffer const& newAlignment, std::optional oldVal) { + for (auto& [_, ch] : Channels) + { + if (!ch->Ring) + continue; + if (ch->Ring->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) + { + nosEngine.SendPathRestart(ch->InputId); + ch->Ring->Stop(); + ch->NeedsRecreation = true; + } + } + }); + AddPinValueWatcher(NOS_NAME_STATIC("RepeatWhenFilling"), + [this](nos::Buffer const& newVal, std::optional oldVal) { + RepeatWhenFilling = *newVal.As(); + }); + } + + ~MultiRingBufferNodeContext() override + { + for (auto& [_, ch] : Channels) + { + NOS_SOFT_CHECK(ch->LastPopped == nullptr); + if (ch->Ring) + ch->Ring->Stop(); + } + } + + void InitChannel(Channel& ch) + { + std::shared_ptr resource; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + resource = std::make_shared(); + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + resource = std::make_shared(); + else + resource = std::make_shared(); + + ch.Ring = std::make_unique(1, std::move(resource)); + ch.Ring->Stop(); + } + + Channel* GetChannelByPinId(uuid const& id) + { + auto it = PinIdToLetter.find(id); + if (it == PinIdToLetter.end()) + return nullptr; + auto chIt = Channels.find(it->second); + return chIt != Channels.end() ? chIt->second.get() : nullptr; + } + + void SeedOutputPin(Channel& ch) + { + if (!ch.Ring || !ch.Ring->IsResourcesValid()) + return; + auto* base = ch.Ring->Resources[0].get(); + if (!base) + return; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + { + if (auto* res = ResourceInterface::GetResource(base)) + nosEngine.SetPinValueByName(NodeId, ch.OutputName, res->VkRes.ToPinData()); + } + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + { + if (auto* res = ResourceInterface::GetResource(base)) + { + sys::vulkan::TTexture texDef = vkss::ConvertTextureInfo(res->VkRes); + texDef.unscaled = true; + nosEngine.SetPinValueByName(NodeId, ch.OutputName, nos::Buffer::From(texDef)); + } + } + } + + void RequestRingResize(uint32_t size) + { + if (size == 0) + { + nosEngine.LogW((GetName() + " size cannot be 0").c_str()); + return; + } + bool changed = false; + for (auto& [_, ch] : Channels) + { + if (!ch->Ring) + continue; + if (ch->Ring->Size != size && (!RequestedRingSize.has_value() || *RequestedRingSize != size)) + { + nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; + nosEngine.SendPathCommand(ch->InputId, ringSizeChange); + ch->Ring->Stop(); + changed = true; + } + } + if (changed) + { + SendPathRestart(); + RequestedRingSize = size; + } + } + + void SendPathRestart() + { + for (auto& [_, ch] : Channels) + nosEngine.SendPathRestart(ch->InputId); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer value) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv)) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch || !ch->Ring) + return; + if (ch->Ring->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) + { + nosEngine.SendPathRestart(ch->InputId); + ch->Ring->Stop(); + ch->NeedsRecreation = true; + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinNameStr = nos::Name(params->InstigatorPinName).AsString(); + auto letter = ParseLetter(pinNameStr); + if (!letter) + return NOS_RESULT_FAILED; + auto chIt = Channels.find(*letter); + if (chIt == Channels.end()) + return NOS_RESULT_FAILED; + auto& ch = *chIt->second; + if (ch.TypeInfo->TypeName != NSN_Generic) + return NOS_RESULT_FAILED; + ch.TypeInfo = nos::TypeInfo(params->IncomingTypeName); + // Drop the Generic-fallback ring so OnPinUpdated re-inits with the resolved type. + if (ch.Ring) + ch.Ring->Stop(); + ch.Ring.reset(); + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pinInfo = params->Pins[i]; + if (pinInfo.Id == ch.InputId || pinInfo.Id == ch.OutputId) + pinInfo.OutResolvedTypeName = ch.TypeInfo->TypeName; + } + return NOS_RESULT_SUCCESS; + } + + void OnPinUpdated(const nosPinUpdate*) override + { + for (auto& [_, ch] : Channels) + if (!ch->Ring) + InitChannel(*ch); + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + auto it = PinIdToLetter.find(update->PinDeleted); + if (it == PinIdToLetter.end()) + return; + char letter = it->second; + PinIdToLetter.erase(it); + auto chIt = Channels.find(letter); + if (chIt == Channels.end()) + return; + auto& ch = *chIt->second; + bool inputAlive = PinIdToLetter.contains(ch.InputId); + bool outputAlive = PinIdToLetter.contains(ch.OutputId); + if (!inputAlive && !outputAlive) + { + if (ch.Ring) + ch.Ring->Stop(); + Channels.erase(chIt); + } + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto sv = pin->name()->string_view(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + auto& chPtr = Channels[*letter]; + if (!chPtr) + chPtr = std::make_unique(*letter); + if (IsInputPin(sv)) + chPtr->InputId = uuid(*pin->id()); + else + { + chPtr->OutputId = uuid(*pin->id()); + chPtr->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + if (!chPtr->Ring) + InitChannel(*chPtr); + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Channels.empty()) + return NOS_RESULT_FAILED; + NodeExecuteParams pins(params); + uint32_t requestedSize = *pins.GetPinData(NSN_Size); + + std::vector> inputs; + inputs.reserve(Channels.size()); + uint32_t maxRequired = requestedSize; + std::string adjustMessage; + for (auto& [letter, ch] : Channels) + { + // Skip channels whose ring isn't ready yet (e.g. just added, + // awaiting OnNodeUpdated/OnPinUpdated to finish init). + if (!ch->Ring || ch->Ring->Exit || !ch->Ring->IsResourcesValid() || !ch->TypeInfo) + continue; + auto it = pins.find(ch->InputName); + if (it == pins.end()) + continue; + void* input = ch->Ring->ResInterface->GetPinInfo(it->second, true); + if (!input) + continue; + auto [required, message] = ch->Ring->ResInterface->GetRequiredRingSize(input, requestedSize); + if (required > maxRequired) + { + maxRequired = required; + adjustMessage = message; + } + inputs.emplace_back(ch.get(), input); + } + if (inputs.empty()) + { + SendScheduleRequest(0); + return NOS_RESULT_FAILED; + } + + bool effectiveSizeAdjusted = maxRequired != requestedSize; + ClearNodeStatusMessages(); + if (effectiveSizeAdjusted) + SetNodeStatusMessage(adjustMessage, fb::NodeStatusMessageType::WARNING); + + bool anyResize = false; + for (auto& [ch, _] : inputs) + if (ch->Ring->Size != maxRequired) + { + anyResize = true; + break; + } + + if (anyResize) + { + RequestRingResize(maxRequired); + if (effectiveSizeAdjusted) + nosEngine.LogW("%s", adjustMessage.c_str()); + return NOS_RESULT_FAILED; + } + + std::vector slots(inputs.size(), nullptr); + for (size_t i = 0; i < inputs.size(); ++i) + { + auto* ch = inputs[i].first; + auto* slot = ch->Ring->BeginPush(100); + if (!slot) + { + for (size_t j = 0; j < i; ++j) + inputs[j].first->Ring->CancelPush(slots[j]); + if (ch->Ring->Exit) + return NOS_RESULT_FAILED; + return NOS_RESULT_PENDING; + } + slots[i] = slot; + } + + for (size_t i = 0; i < inputs.size(); ++i) + { + auto* ch = inputs[i].first; + ch->Ring->ResInterface->Push(slots[i], inputs[i].second, params, + NOS_NAME_STATIC("MultiRingBuffer"), true); + ch->Ring->EndPush(slots[i]); + if (!ch->IsOutLive) + { + ChangePinLiveness(ch->OutputName, true); + ch->IsOutLive = true; + } + } + + if (Mode == RingMode::FILL) + { + bool isFillComplete = true; + for (auto& [ch, _] : inputs) + if (ch->Ring->Write.Pool.size() != 0) + { + isFillComplete = false; + break; + } + if (isFillComplete) + { + Mode = RingMode::CONSUME; + ModeCV.notify_all(); + } + } + + return NOS_RESULT_SUCCESS; + } + + nosResult CopyFrom(nosCopyInfo* cpy) override + { + auto* ch = GetChannelByPinId(cpy->ID); + if (!ch || !ch->Ring || ch->Ring->Exit) + return NOS_RESULT_FAILED; + if (!ch->IsOutLive) + return NOS_RESULT_SUCCESS; + + // EndPop the previous frame's slot before popping a new one. We can't + // rely on OnEndFrame: the engine only fires it on the path's primary + // source pin, so live secondary outputs (e.g. a second channel feeding + // the same consumer) never receive it. By the time the consumer asks + // for the next frame on this pin, it's done with the previous one. + if (ch->LastPopped) + { + ch->Ring->EndPop(ch->LastPopped); + ch->LastPopped = nullptr; + } + + if (OnRestart == OnRestartType::WAIT_UNTIL_FULL && RepeatWhenFilling) + { + if (ch->RemainingRepeatableCount > 0) + { + ch->Ring->ResInterface->OnRepeatPinValue(cpy); + ch->RemainingRepeatableCount--; + return NOS_RESULT_SUCCESS; + } + } + else if (Mode == RingMode::FILL) + { + std::unique_lock lock(ModeMutex); + if (!ModeCV.wait_for(lock, std::chrono::milliseconds(100), + [this] { return Mode != RingMode::FILL; })) + return NOS_RESULT_PENDING; + } + + ResourceInterface::ResourceBase* slot; + { + ScopedProfilerEvent _({.Name = "Wait For Filled Slot"}); + slot = ch->Ring->BeginPop(100); + } + if (!slot) + return ch->Ring->Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + nos::Buffer outPinVal; + bool changePinValue = ch->Ring->ResInterface->BeginCopyFrom(slot, *cpy->PinData, outPinVal); + if (changePinValue) + nosEngine.SetPinValueByName(NodeId, ch->OutputName, outPinVal); + + ch->Ring->ResInterface->WaitForDownloadToEnd(slot, "MultiRingBuffer", NodeName.AsString(), cpy); + + cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; + cpy->FrameNumber = slot->FrameNumber; + + ch->LastPopped = slot; + SendScheduleRequest(1); + return NOS_RESULT_SUCCESS; + } + + void OnEndFrame(uuid const& pinId, nosEndFrameCause cause) override + { + auto* ch = GetChannelByPinId(pinId); + if (!ch) + return; + + if (cause == NOS_END_FRAME_FAILED) + { + if (pinId == ch->OutputId) + return; + if (!ch->IsOutLive) + return; + ChangePinLiveness(ch->OutputName, false); + ch->IsOutLive = false; + } + // Note: EndPop happens at the start of the next CopyFrom for this + // channel rather than here, because OnEndFrame is unreliable for + // secondary live outputs. + } + + void SendScheduleRequest(uint32_t count, bool reset = false) const + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = count, .Reset = reset}; + nosEngine.ScheduleNode(&schedule); + } + + void OnPathCommand(const nosPathCommand* command) override + { + switch (command->Event) + { + case NOS_RING_SIZE_CHANGE: + if (command->RingSize == 0) + return; + RequestedRingSize = command->RingSize; + nosEngine.SetPinValue(*GetPinId(NSN_Size), nos::Buffer::From(command->RingSize)); + break; + default: return; + } + } + + void OnPathStop() override + { + if (OnRestart == OnRestartType::WAIT_UNTIL_FULL) + Mode = RingMode::FILL; + for (auto& [_, ch] : Channels) + { + if (ch->LastPopped && ch->Ring) + { + ch->Ring->EndPop(ch->LastPopped); + ch->LastPopped = nullptr; + } + if (ch->Ring) + ch->Ring->Stop(); + } + } + + void OnPathStart() override + { + if (Channels.empty()) + return; + size_t totalSchedule = 0; + for (auto& [_, ch] : Channels) + { + if (!ch->Ring) + continue; + if (OnRestart == OnRestartType::RESET || RepeatWhenFilling) + ch->Ring->Reset(false); + else if (ch->Ring->IsFull() && !ch->Ring->Read.Pool.empty()) + { + ch->Ring->Write.Pool.push_back(ch->Ring->Read.Pool.front()); + ch->Ring->Read.Pool.pop_front(); + } + if (RequestedRingSize) + { + ch->Ring->Resize(*RequestedRingSize); + ch->NeedsRecreation = false; + } + if (ch->NeedsRecreation) + { + ch->Ring = std::make_unique(ch->Ring->Size, ch->Ring->ResInterface); + ch->NeedsRecreation = false; + } + if (!ch->Ring->IsResourcesValid()) + { + totalSchedule = std::max(totalSchedule, 1); + continue; + } + auto emptySlotCount = ch->Ring->Write.Pool.size(); + if (RepeatWhenFilling) + ch->RemainingRepeatableCount = std::max(emptySlotCount, (size_t)1) - 1; + totalSchedule = std::max(totalSchedule, emptySlotCount); + ch->Ring->Exit = false; + ch->Ring->ResInterface->OnPathStart(); + SeedOutputPin(*ch); + } + RequestedRingSize = std::nullopt; + if (totalSchedule > 0) + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = (uint32_t)totalSchedule}; + nosEngine.ScheduleNode(&schedule); + } + } + + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Channel", MenuCommand(ADD_CHANNEL, 0))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + if (Channels.size() <= 1) + return; + flatbuffers::FlatBufferBuilder fbb; + std::vector items = {nos::CreateContextMenuItemDirect( + fbb, "Remove Channel", MenuCommand(REMOVE_CHANNEL, static_cast(*letter)))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_CHANNEL: + { + char newLetter = 0; + for (char c : CHANNEL_LETTERS) + { + if (!Channels.contains(c)) + { + newLetter = c; + break; + } + } + if (newLetter == 0) + { + SetNodeStatusMessage("Maximum number of channels reached", fb::NodeStatusMessageType::WARNING); + return; + } + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = std::string("Input_") + newLetter; + inPin.type_name = "nos.Generic"; + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = std::string("Output_") + newLetter; + outPin.type_name = "nos.Generic"; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + outPin.live = true; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + case REMOVE_CHANNEL: + { + char letter = static_cast(command.Letter); + auto it = Channels.find(letter); + if (it == Channels.end()) + return; + auto& ch = *it->second; + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {ch.InputId, ch.OutputId}; + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + } + } +}; + +nosResult RegisterMultiRingBuffer(nosNodeFunctions* functions) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiRingBuffer"), MultiRingBufferNodeContext, functions) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/UtilitiesMain.cpp b/Plugins/nosUtilities/Source/UtilitiesMain.cpp index b3fcc186..b0d8427d 100644 --- a/Plugins/nosUtilities/Source/UtilitiesMain.cpp +++ b/Plugins/nosUtilities/Source/UtilitiesMain.cpp @@ -41,7 +41,9 @@ enum Utilities : int PropagateExecution, UploadBufferProvider, BoundedQueue, + MultiBoundedQueue, RingBuffer, + MultiRingBuffer, Host, DeinterlacedBoundedTextureQueue, DeinterlacedBufferRing, @@ -76,7 +78,9 @@ nosResult RegisterSink(nosNodeFunctions*); nosResult RegisterPropagateExecution(nosNodeFunctions*); nosResult RegisterUploadBufferProvider(nosNodeFunctions*); nosResult RegisterBoundedQueue(nosNodeFunctions*); +nosResult RegisterMultiBoundedQueue(nosNodeFunctions*); nosResult RegisterRingBuffer(nosNodeFunctions*); +nosResult RegisterMultiRingBuffer(nosNodeFunctions*); nosResult RegisterHost(nosNodeFunctions*); nosResult RegisterPin2Json(nosNodeFunctions*); nosResult RegisterJson2Pin(nosNodeFunctions*); @@ -131,7 +135,9 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou GEN_CASE_NODE(PropagateExecution) GEN_CASE_NODE(UploadBufferProvider) GEN_CASE_NODE(BoundedQueue) + GEN_CASE_NODE(MultiBoundedQueue) GEN_CASE_NODE(RingBuffer) + GEN_CASE_NODE(MultiRingBuffer) GEN_CASE_NODE(Host) GEN_CASE_NODE(DeinterlacedBoundedTextureQueue) GEN_CASE_NODE(DeinterlacedBufferRing) diff --git a/Plugins/nosUtilities/Utilities.noscfg b/Plugins/nosUtilities/Utilities.noscfg index 0320cd0d..b83efd55 100644 --- a/Plugins/nosUtilities/Utilities.noscfg +++ b/Plugins/nosUtilities/Utilities.noscfg @@ -43,7 +43,9 @@ "Config/UploadBufferProvider.nosdef", "Config/TimedFunctionSignaller.nosdef", "Config/RingBuffer.nosdef", + "Config/MultiRingBuffer.nosdef", "Config/BoundedQueue.nosdef", + "Config/MultiBoundedQueue.nosdef", "Config/Host.nosdef", "Config/AutoResize.nosdef", "Config/ExecDepend.nosdef", From 3d10d10370cfa0fe3906ff3fe8819190bea62ed8 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Tue, 28 Apr 2026 14:32:29 +0300 Subject: [PATCH 11/27] Multi Ring/Queue: Use batch pushing instead of subsequent locks --- .../nosUtilities/Source/MultiBoundedQueue.cpp | 199 +++++++------- Plugins/nosUtilities/Source/MultiRing.h | 256 ++++++++++++++++++ .../nosUtilities/Source/MultiRingBuffer.cpp | 234 ++++++++-------- 3 files changed, 467 insertions(+), 222 deletions(-) create mode 100644 Plugins/nosUtilities/Source/MultiRing.h diff --git a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp index 4b7175a4..3f6eab32 100644 --- a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp +++ b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp @@ -8,6 +8,7 @@ #include #include +#include "MultiRing.h" #include "Ring.h" #include "nosUtil/Stopwatch.hpp" @@ -16,9 +17,6 @@ namespace nos::utilities struct MultiBoundedQueueNodeContext : NodeContext { - using RingMode = RingNodeBase::RingMode; - using OnRestartType = RingNodeBase::OnRestartType; - static constexpr std::string_view CHANNEL_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; enum MenuCommandType : uint8_t @@ -48,7 +46,7 @@ struct MultiBoundedQueueNodeContext : NodeContext uuid InputId{}; uuid OutputId{}; nos::TypeInfo TypeInfo; - std::unique_ptr Ring; + MultiRing::Channel* RingChannel = nullptr; std::atomic_bool IsOutLive = false; bool NeedsRecreation = false; @@ -63,6 +61,7 @@ struct MultiBoundedQueueNodeContext : NodeContext std::map> Channels; std::unordered_map PinIdToLetter; + MultiRing Ring; std::optional RequestedRingSize = std::nullopt; @@ -129,26 +128,24 @@ struct MultiBoundedQueueNodeContext : NodeContext RequestRingResize(size); }); AddPinValueWatcher(NSN_Alignment, [this](nos::Buffer const& newAlignment, std::optional oldVal) { + bool any = false; for (auto& [_, ch] : Channels) { - if (!ch->Ring) + if (!ch->RingChannel) continue; - if (ch->Ring->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) { nosEngine.SendPathRestart(ch->InputId); - ch->Ring->Stop(); ch->NeedsRecreation = true; + any = true; } } + if (any) + Ring.Stop(); }); } - ~MultiBoundedQueueNodeContext() override - { - for (auto& [_, ch] : Channels) - if (ch->Ring) - ch->Ring->Stop(); - } + ~MultiBoundedQueueNodeContext() override { Ring.Stop(); } void InitChannel(Channel& ch) { @@ -160,8 +157,7 @@ struct MultiBoundedQueueNodeContext : NodeContext else resource = std::make_shared(); - ch.Ring = std::make_unique(1, std::move(resource)); - ch.Ring->Stop(); + ch.RingChannel = &Ring.AddChannel(ch.Letter, std::move(resource), &ch); } Channel* GetChannelByPinId(uuid const& id) @@ -180,24 +176,18 @@ struct MultiBoundedQueueNodeContext : NodeContext nosEngine.LogW((GetName() + " size cannot be 0").c_str()); return; } - bool changed = false; + if (Ring.Size == size && (!RequestedRingSize.has_value() || *RequestedRingSize == size)) + return; for (auto& [_, ch] : Channels) { - if (!ch->Ring) + if (!ch->RingChannel) continue; - if (ch->Ring->Size != size && (!RequestedRingSize.has_value() || *RequestedRingSize != size)) - { - nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; - nosEngine.SendPathCommand(ch->InputId, ringSizeChange); - ch->Ring->Stop(); - changed = true; - } - } - if (changed) - { - SendPathRestart(); - RequestedRingSize = size; + nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; + nosEngine.SendPathCommand(ch->InputId, ringSizeChange); } + Ring.Stop(); + SendPathRestart(); + RequestedRingSize = size; } void SendPathRestart() @@ -212,12 +202,12 @@ struct MultiBoundedQueueNodeContext : NodeContext if (!IsInputPin(sv)) return; auto* ch = GetChannelByPinId(pinId); - if (!ch || !ch->Ring) + if (!ch || !ch->RingChannel) return; - if (ch->Ring->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) { nosEngine.SendPathRestart(ch->InputId); - ch->Ring->Stop(); + Ring.Stop(); ch->NeedsRecreation = true; } } @@ -235,9 +225,12 @@ struct MultiBoundedQueueNodeContext : NodeContext if (ch.TypeInfo->TypeName != NSN_Generic) return NOS_RESULT_FAILED; ch.TypeInfo = nos::TypeInfo(params->IncomingTypeName); - if (ch.Ring) - ch.Ring->Stop(); - ch.Ring.reset(); + if (ch.RingChannel) + { + Ring.Stop(); + Ring.RemoveChannel(*letter); + ch.RingChannel = nullptr; + } for (size_t i = 0; i < params->PinCount; i++) { auto& pinInfo = params->Pins[i]; @@ -250,7 +243,7 @@ struct MultiBoundedQueueNodeContext : NodeContext void OnPinUpdated(const nosPinUpdate*) override { for (auto& [_, ch] : Channels) - if (!ch->Ring) + if (!ch->RingChannel) InitChannel(*ch); } @@ -271,8 +264,11 @@ struct MultiBoundedQueueNodeContext : NodeContext bool outputAlive = PinIdToLetter.contains(ch.OutputId); if (!inputAlive && !outputAlive) { - if (ch.Ring) - ch.Ring->Stop(); + if (ch.RingChannel) + { + Ring.RemoveChannel(letter); + ch.RingChannel = nullptr; + } Channels.erase(chIt); } } @@ -296,92 +292,84 @@ struct MultiBoundedQueueNodeContext : NodeContext chPtr->IsOutLive = pin->live(); } PinIdToLetter[uuid(*pin->id())] = *letter; - if (!chPtr->Ring) + if (!chPtr->RingChannel) InitChannel(*chPtr); } } nosResult ExecuteNode(nosNodeExecuteParams* params) override { - if (Channels.empty()) + if (Channels.empty() || Ring.Exit) return NOS_RESULT_FAILED; NodeExecuteParams pins(params); uint32_t requestedSize = *pins.GetPinData(NSN_Size); - std::vector> inputs; - inputs.reserve(Channels.size()); + struct Gathered + { + Channel* NodeCh; + MultiRing::Channel* RingCh; + void* Input; + }; + std::vector gathered; + gathered.reserve(Channels.size()); + std::vector wantedRings; + wantedRings.reserve(Channels.size()); + uint32_t maxRequired = requestedSize; for (auto& [_, ch] : Channels) { - if (!ch->Ring || ch->Ring->Exit || !ch->Ring->IsResourcesValid() || !ch->TypeInfo) + if (!ch->RingChannel || ch->RingChannel->Resources.empty() || !ch->TypeInfo) continue; auto it = pins.find(ch->InputName); if (it == pins.end()) continue; - void* input = ch->Ring->ResInterface->GetPinInfo(it->second, false); + void* input = ch->RingChannel->ResInterface->GetPinInfo(it->second, false); if (!input) continue; - auto [required, _] = ch->Ring->ResInterface->GetRequiredRingSize(input, requestedSize); + auto [required, _] = ch->RingChannel->ResInterface->GetRequiredRingSize(input, requestedSize); if (required > maxRequired) maxRequired = required; - inputs.emplace_back(ch.get(), input); + gathered.push_back({ch.get(), ch->RingChannel, input}); + wantedRings.push_back(ch->RingChannel); } - if (inputs.empty()) + if (gathered.empty()) { SendScheduleRequest(0); return NOS_RESULT_FAILED; } - bool anyResize = false; - for (auto& [ch, _] : inputs) - if (ch->Ring->Size != maxRequired) - { - anyResize = true; - break; - } - if (anyResize) + if (Ring.Size != maxRequired) { RequestRingResize(maxRequired); return NOS_RESULT_FAILED; } - std::vector slots(inputs.size(), nullptr); - for (size_t i = 0; i < inputs.size(); ++i) - { - auto* ch = inputs[i].first; - auto* slot = ch->Ring->BeginPush(100); - if (!slot) - { - for (size_t j = 0; j < i; ++j) - inputs[j].first->Ring->CancelPush(slots[j]); - if (ch->Ring->Exit) - return NOS_RESULT_FAILED; - return NOS_RESULT_PENDING; - } - slots[i] = slot; - } + std::vector slots; + if (!Ring.BeginPushSubset(100, wantedRings, slots)) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; - for (size_t i = 0; i < inputs.size(); ++i) + for (size_t i = 0; i < gathered.size(); ++i) { - auto* ch = inputs[i].first; - ch->Ring->ResInterface->Push(slots[i], inputs[i].second, params, + auto& g = gathered[i]; + auto* slot = slots[i].second; + g.RingCh->ResInterface->Push(slot, g.Input, params, NOS_NAME_STATIC("MultiBoundedQueue"), false); - ch->Ring->EndPush(slots[i]); - if (!ch->IsOutLive) + if (!g.NodeCh->IsOutLive) { - ChangePinLiveness(ch->OutputName, true); - ch->IsOutLive = true; + ChangePinLiveness(g.NodeCh->OutputName, true); + g.NodeCh->IsOutLive = true; } } + Ring.EndPushAll(slots); return NOS_RESULT_SUCCESS; } nosResult CopyFrom(nosCopyInfo* cpy) override { auto* ch = GetChannelByPinId(cpy->ID); - if (!ch || !ch->Ring || ch->Ring->Exit) + if (!ch || !ch->RingChannel || Ring.Exit) return NOS_RESULT_FAILED; if (!ch->IsOutLive) return NOS_RESULT_SUCCESS; @@ -389,17 +377,17 @@ struct MultiBoundedQueueNodeContext : NodeContext ResourceInterface::ResourceBase* slot; { ScopedProfilerEvent _({.Name = "Wait For Filled Slot"}); - slot = ch->Ring->BeginPop(100); + slot = Ring.BeginPop(*ch->RingChannel, 100); } if (!slot) - return ch->Ring->Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; - ch->Ring->ResInterface->Copy(slot, cpy, NodeId); + ch->RingChannel->ResInterface->Copy(slot, cpy, NodeId); cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; cpy->FrameNumber = slot->FrameNumber; - ch->Ring->EndPop(slot); + Ring.EndPop(*ch->RingChannel, slot); SendScheduleRequest(1); return NOS_RESULT_SUCCESS; } @@ -439,45 +427,46 @@ struct MultiBoundedQueueNodeContext : NodeContext } } - void OnPathStop() override - { - for (auto& [_, ch] : Channels) - if (ch->Ring) - ch->Ring->Stop(); - } + void OnPathStop() override { Ring.Stop(); } void OnPathStart() override { if (Channels.empty()) return; - size_t totalSchedule = 0; - for (auto& [_, ch] : Channels) + + Ring.ResetAll(false); + + if (RequestedRingSize) { - if (!ch->Ring) - continue; - // FIFO restart: drop any frames left from the previous run. - ch->Ring->Reset(false); - if (RequestedRingSize) - { - ch->Ring->Resize(*RequestedRingSize); + Ring.ResizeAll(*RequestedRingSize); + for (auto& [_, ch] : Channels) ch->NeedsRecreation = false; - } - if (ch->NeedsRecreation) + RequestedRingSize = std::nullopt; + } + for (auto& [_, ch] : Channels) + { + if (ch->NeedsRecreation && ch->RingChannel) { - ch->Ring = std::make_unique(ch->Ring->Size, ch->Ring->ResInterface); + Ring.RecreateChannelResources(*ch->RingChannel); ch->NeedsRecreation = false; } - if (!ch->Ring->IsResourcesValid()) + } + + size_t totalSchedule = 0; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->Resources.empty()) { totalSchedule = std::max(totalSchedule, 1); continue; } - auto emptySlotCount = ch->Ring->Write.Pool.size(); + auto emptySlotCount = Ring.WritePoolSize(*ch->RingChannel); totalSchedule = std::max(totalSchedule, emptySlotCount); - ch->Ring->Exit = false; - ch->Ring->ResInterface->OnPathStart(); + ch->RingChannel->ResInterface->OnPathStart(); } - RequestedRingSize = std::nullopt; + Ring.Start(); if (totalSchedule > 0) { nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = (uint32_t)totalSchedule}; diff --git a/Plugins/nosUtilities/Source/MultiRing.h b/Plugins/nosUtilities/Source/MultiRing.h new file mode 100644 index 00000000..c2d0ba14 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiRing.h @@ -0,0 +1,256 @@ +/* + * Copyright MediaZ Teknoloji A.S. All Rights Reserved. + */ + +#pragma once + +#include "Ring.h" + +namespace nos +{ + +// Ring that holds N independent channels under a single mutex / CV pair. +// Each channel still owns its own slot pools and Resources, but every push, +// pop, resize and reset goes through the shared synchronization, so an +// N-channel batch push is a single lock acquisition, not N. +struct MultiRing +{ + struct Channel + { + std::shared_ptr ResInterface; + std::vector> Resources; + std::deque WritePool; + std::deque ReadPool; + void* UserData = nullptr; + }; + + std::map> Channels; + std::mutex Mutex; + std::condition_variable WriteCV; + std::condition_variable ReadCV; + std::atomic_bool Exit = true; + uint32_t Size = 0; + + ~MultiRing() { Stop(); } + + void Stop() + { + { + std::unique_lock lock(Mutex); + Exit = true; + } + WriteCV.notify_all(); + ReadCV.notify_all(); + } + + void Start() + { + std::unique_lock lock(Mutex); + Exit = false; + } + + void AllocateChannelResourcesUnlocked(Channel& ch) + { + ch.WritePool.clear(); + ch.ReadPool.clear(); + ch.Resources.clear(); + for (uint32_t i = 0; i < Size; ++i) + { + auto res = ch.ResInterface->CreateResource(); + if (!res) + { + nosEngine.LogE("Failed to create resource for multi ring buffer."); + ch.Resources.clear(); + ch.WritePool.clear(); + ch.ReadPool.clear(); + Exit = true; + return; + } + ch.Resources.push_back(res); + ch.WritePool.push_back(res.get()); + } + } + + Channel& AddChannel(char key, std::shared_ptr resInterface, void* userData = nullptr) + { + std::unique_lock lock(Mutex); + auto& ch = Channels[key]; + if (!ch) + ch = std::make_unique(); + ch->ResInterface = std::move(resInterface); + ch->UserData = userData; + if (Size == 0) + Size = 1; + AllocateChannelResourcesUnlocked(*ch); + return *ch; + } + + void RemoveChannel(char key) + { + std::unique_lock lock(Mutex); + Channels.erase(key); + } + + void RecreateChannelResources(Channel& ch) + { + std::unique_lock lock(Mutex); + AllocateChannelResourcesUnlocked(ch); + } + + void ResizeAll(uint32_t newSize) + { + std::unique_lock lock(Mutex); + Size = newSize; + for (auto& [_, ch] : Channels) + AllocateChannelResourcesUnlocked(*ch); + } + + bool AreAllChannelsValid() + { + std::unique_lock lock(Mutex); + if (Channels.empty()) + return false; + for (auto& [_, ch] : Channels) + if (ch->Resources.empty()) + return false; + return true; + } + + // Move slots between pools for every channel. fill=false: read→write. + void ResetAll(bool fill) + { + std::unique_lock lock(Mutex); + for (auto& [_, ch] : Channels) + { + auto& from = fill ? ch->WritePool : ch->ReadPool; + auto& to = fill ? ch->ReadPool : ch->WritePool; + while (!from.empty()) + { + auto* slot = from.front(); + from.pop_front(); + ch->ResInterface->Reset(slot); + to.push_back(slot); + } + } + } + + // If this channel is full and its read pool is non-empty, hand one slot + // back to the write pool so the producer can start pushing again. + void MoveOneReadToWriteIfFull(Channel& ch) + { + std::unique_lock lock(Mutex); + if (ch.ReadPool.size() != ch.Resources.size() || ch.ReadPool.empty()) + return; + auto* slot = ch.ReadPool.front(); + ch.ReadPool.pop_front(); + ch.WritePool.push_back(slot); + } + + bool IsFull(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.size() == ch.Resources.size(); + } + + bool IsEmpty(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.empty(); + } + + size_t WritePoolSize(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.WritePool.size(); + } + + size_t ReadPoolSize(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.size(); + } + + using SlotPair = std::pair; + + // Atomically pop one slot from each requested channel's WritePool. + // Waits until every requested channel has at least one slot, or + // timeout/exit. The caller-supplied list typically excludes channels + // that don't have valid input data this frame. + bool BeginPushSubset(uint64_t timeoutMs, + std::vector const& wanted, + std::vector& outSlots) + { + std::unique_lock lock(Mutex); + auto pred = [&] { + if (Exit) + return true; + if (wanted.empty()) + return false; + for (auto* ch : wanted) + if (ch->WritePool.empty()) + return false; + return true; + }; + if (!WriteCV.wait_for(lock, std::chrono::milliseconds(timeoutMs), pred)) + return false; + if (Exit) + return false; + outSlots.clear(); + outSlots.reserve(wanted.size()); + for (auto* ch : wanted) + { + auto* slot = ch->WritePool.front(); + ch->WritePool.pop_front(); + outSlots.emplace_back(ch, slot); + } + return true; + } + + void EndPushAll(std::vector const& slots) + { + { + std::unique_lock lock(Mutex); + for (auto& [ch, slot] : slots) + ch->ReadPool.push_back(slot); + } + ReadCV.notify_all(); + } + + void CancelPushAll(std::vector const& slots) + { + { + std::unique_lock lock(Mutex); + for (auto& [ch, slot] : slots) + { + slot->FrameNumber = 0; + ch->WritePool.push_front(slot); + } + } + WriteCV.notify_all(); + } + + ResourceInterface::ResourceBase* BeginPop(Channel& ch, uint64_t timeoutMs) + { + std::unique_lock lock(Mutex); + if (!ReadCV.wait_for(lock, std::chrono::milliseconds(timeoutMs), + [&] { return !ch.ReadPool.empty() || Exit; })) + return nullptr; + if (Exit) + return nullptr; + auto* slot = ch.ReadPool.front(); + ch.ReadPool.pop_front(); + return slot; + } + + void EndPop(Channel& ch, ResourceInterface::ResourceBase* slot) + { + { + std::unique_lock lock(Mutex); + slot->FrameNumber = 0; + ch.WritePool.push_back(slot); + } + WriteCV.notify_all(); + } +}; + +} // namespace nos diff --git a/Plugins/nosUtilities/Source/MultiRingBuffer.cpp b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp index a13c7f09..54304f41 100644 --- a/Plugins/nosUtilities/Source/MultiRingBuffer.cpp +++ b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp @@ -8,6 +8,7 @@ #include #include +#include "MultiRing.h" #include "Ring.h" #include "nosUtil/Stopwatch.hpp" @@ -48,7 +49,7 @@ struct MultiRingBufferNodeContext : NodeContext uuid InputId{}; uuid OutputId{}; nos::TypeInfo TypeInfo; - std::unique_ptr Ring; + MultiRing::Channel* RingChannel = nullptr; std::atomic_bool IsOutLive = false; ResourceInterface::ResourceBase* LastPopped = nullptr; bool NeedsRecreation = false; @@ -65,6 +66,7 @@ struct MultiRingBufferNodeContext : NodeContext std::map> Channels; std::unordered_map PinIdToLetter; + MultiRing Ring; OnRestartType OnRestart = OnRestartType::WAIT_UNTIL_FULL; std::optional RequestedRingSize = std::nullopt; @@ -136,17 +138,20 @@ struct MultiRingBufferNodeContext : NodeContext RequestRingResize(size); }); AddPinValueWatcher(NSN_Alignment, [this](nos::Buffer const& newAlignment, std::optional oldVal) { + bool any = false; for (auto& [_, ch] : Channels) { - if (!ch->Ring) + if (!ch->RingChannel) continue; - if (ch->Ring->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) { nosEngine.SendPathRestart(ch->InputId); - ch->Ring->Stop(); ch->NeedsRecreation = true; + any = true; } } + if (any) + Ring.Stop(); }); AddPinValueWatcher(NOS_NAME_STATIC("RepeatWhenFilling"), [this](nos::Buffer const& newVal, std::optional oldVal) { @@ -157,11 +162,8 @@ struct MultiRingBufferNodeContext : NodeContext ~MultiRingBufferNodeContext() override { for (auto& [_, ch] : Channels) - { NOS_SOFT_CHECK(ch->LastPopped == nullptr); - if (ch->Ring) - ch->Ring->Stop(); - } + Ring.Stop(); } void InitChannel(Channel& ch) @@ -174,8 +176,7 @@ struct MultiRingBufferNodeContext : NodeContext else resource = std::make_shared(); - ch.Ring = std::make_unique(1, std::move(resource)); - ch.Ring->Stop(); + ch.RingChannel = &Ring.AddChannel(ch.Letter, std::move(resource), &ch); } Channel* GetChannelByPinId(uuid const& id) @@ -189,9 +190,9 @@ struct MultiRingBufferNodeContext : NodeContext void SeedOutputPin(Channel& ch) { - if (!ch.Ring || !ch.Ring->IsResourcesValid()) + if (!ch.RingChannel || ch.RingChannel->Resources.empty()) return; - auto* base = ch.Ring->Resources[0].get(); + auto* base = ch.RingChannel->Resources[0].get(); if (!base) return; if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) @@ -217,24 +218,18 @@ struct MultiRingBufferNodeContext : NodeContext nosEngine.LogW((GetName() + " size cannot be 0").c_str()); return; } - bool changed = false; + if (Ring.Size == size && (!RequestedRingSize.has_value() || *RequestedRingSize == size)) + return; for (auto& [_, ch] : Channels) { - if (!ch->Ring) + if (!ch->RingChannel) continue; - if (ch->Ring->Size != size && (!RequestedRingSize.has_value() || *RequestedRingSize != size)) - { - nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; - nosEngine.SendPathCommand(ch->InputId, ringSizeChange); - ch->Ring->Stop(); - changed = true; - } - } - if (changed) - { - SendPathRestart(); - RequestedRingSize = size; + nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; + nosEngine.SendPathCommand(ch->InputId, ringSizeChange); } + Ring.Stop(); + SendPathRestart(); + RequestedRingSize = size; } void SendPathRestart() @@ -249,12 +244,12 @@ struct MultiRingBufferNodeContext : NodeContext if (!IsInputPin(sv)) return; auto* ch = GetChannelByPinId(pinId); - if (!ch || !ch->Ring) + if (!ch || !ch->RingChannel) return; - if (ch->Ring->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) { nosEngine.SendPathRestart(ch->InputId); - ch->Ring->Stop(); + Ring.Stop(); ch->NeedsRecreation = true; } } @@ -272,10 +267,13 @@ struct MultiRingBufferNodeContext : NodeContext if (ch.TypeInfo->TypeName != NSN_Generic) return NOS_RESULT_FAILED; ch.TypeInfo = nos::TypeInfo(params->IncomingTypeName); - // Drop the Generic-fallback ring so OnPinUpdated re-inits with the resolved type. - if (ch.Ring) - ch.Ring->Stop(); - ch.Ring.reset(); + // Drop the Generic-fallback ring channel so OnPinUpdated re-inits with the resolved type. + if (ch.RingChannel) + { + Ring.Stop(); + Ring.RemoveChannel(*letter); + ch.RingChannel = nullptr; + } for (size_t i = 0; i < params->PinCount; i++) { auto& pinInfo = params->Pins[i]; @@ -288,7 +286,7 @@ struct MultiRingBufferNodeContext : NodeContext void OnPinUpdated(const nosPinUpdate*) override { for (auto& [_, ch] : Channels) - if (!ch->Ring) + if (!ch->RingChannel) InitChannel(*ch); } @@ -309,8 +307,11 @@ struct MultiRingBufferNodeContext : NodeContext bool outputAlive = PinIdToLetter.contains(ch.OutputId); if (!inputAlive && !outputAlive) { - if (ch.Ring) - ch.Ring->Stop(); + if (ch.RingChannel) + { + Ring.RemoveChannel(letter); + ch.RingChannel = nullptr; + } Channels.erase(chIt); } } @@ -334,43 +335,52 @@ struct MultiRingBufferNodeContext : NodeContext chPtr->IsOutLive = pin->live(); } PinIdToLetter[uuid(*pin->id())] = *letter; - if (!chPtr->Ring) + if (!chPtr->RingChannel) InitChannel(*chPtr); } } nosResult ExecuteNode(nosNodeExecuteParams* params) override { - if (Channels.empty()) + if (Channels.empty() || Ring.Exit) return NOS_RESULT_FAILED; + NodeExecuteParams pins(params); uint32_t requestedSize = *pins.GetPinData(NSN_Size); - std::vector> inputs; - inputs.reserve(Channels.size()); + struct Gathered + { + Channel* NodeCh; + MultiRing::Channel* RingCh; + void* Input; + }; + std::vector gathered; + gathered.reserve(Channels.size()); + std::vector wantedRings; + wantedRings.reserve(Channels.size()); + uint32_t maxRequired = requestedSize; std::string adjustMessage; - for (auto& [letter, ch] : Channels) + for (auto& [_, ch] : Channels) { - // Skip channels whose ring isn't ready yet (e.g. just added, - // awaiting OnNodeUpdated/OnPinUpdated to finish init). - if (!ch->Ring || ch->Ring->Exit || !ch->Ring->IsResourcesValid() || !ch->TypeInfo) + if (!ch->RingChannel || ch->RingChannel->Resources.empty() || !ch->TypeInfo) continue; auto it = pins.find(ch->InputName); if (it == pins.end()) continue; - void* input = ch->Ring->ResInterface->GetPinInfo(it->second, true); + void* input = ch->RingChannel->ResInterface->GetPinInfo(it->second, true); if (!input) continue; - auto [required, message] = ch->Ring->ResInterface->GetRequiredRingSize(input, requestedSize); + auto [required, message] = ch->RingChannel->ResInterface->GetRequiredRingSize(input, requestedSize); if (required > maxRequired) { maxRequired = required; adjustMessage = message; } - inputs.emplace_back(ch.get(), input); + gathered.push_back({ch.get(), ch->RingChannel, input}); + wantedRings.push_back(ch->RingChannel); } - if (inputs.empty()) + if (gathered.empty()) { SendScheduleRequest(0); return NOS_RESULT_FAILED; @@ -381,15 +391,7 @@ struct MultiRingBufferNodeContext : NodeContext if (effectiveSizeAdjusted) SetNodeStatusMessage(adjustMessage, fb::NodeStatusMessageType::WARNING); - bool anyResize = false; - for (auto& [ch, _] : inputs) - if (ch->Ring->Size != maxRequired) - { - anyResize = true; - break; - } - - if (anyResize) + if (Ring.Size != maxRequired) { RequestRingResize(maxRequired); if (effectiveSizeAdjusted) @@ -397,40 +399,31 @@ struct MultiRingBufferNodeContext : NodeContext return NOS_RESULT_FAILED; } - std::vector slots(inputs.size(), nullptr); - for (size_t i = 0; i < inputs.size(); ++i) - { - auto* ch = inputs[i].first; - auto* slot = ch->Ring->BeginPush(100); - if (!slot) - { - for (size_t j = 0; j < i; ++j) - inputs[j].first->Ring->CancelPush(slots[j]); - if (ch->Ring->Exit) - return NOS_RESULT_FAILED; - return NOS_RESULT_PENDING; - } - slots[i] = slot; - } + std::vector slots; + if (!Ring.BeginPushSubset(100, wantedRings, slots)) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; - for (size_t i = 0; i < inputs.size(); ++i) + // Push outside the lock — Vulkan command recording can be slow. + for (size_t i = 0; i < gathered.size(); ++i) { - auto* ch = inputs[i].first; - ch->Ring->ResInterface->Push(slots[i], inputs[i].second, params, + auto& g = gathered[i]; + auto* slot = slots[i].second; + g.RingCh->ResInterface->Push(slot, g.Input, params, NOS_NAME_STATIC("MultiRingBuffer"), true); - ch->Ring->EndPush(slots[i]); - if (!ch->IsOutLive) + if (!g.NodeCh->IsOutLive) { - ChangePinLiveness(ch->OutputName, true); - ch->IsOutLive = true; + ChangePinLiveness(g.NodeCh->OutputName, true); + g.NodeCh->IsOutLive = true; } } + Ring.EndPushAll(slots); + if (Mode == RingMode::FILL) { bool isFillComplete = true; - for (auto& [ch, _] : inputs) - if (ch->Ring->Write.Pool.size() != 0) + for (auto* rc : wantedRings) + if (Ring.WritePoolSize(*rc) != 0) { isFillComplete = false; break; @@ -448,7 +441,7 @@ struct MultiRingBufferNodeContext : NodeContext nosResult CopyFrom(nosCopyInfo* cpy) override { auto* ch = GetChannelByPinId(cpy->ID); - if (!ch || !ch->Ring || ch->Ring->Exit) + if (!ch || !ch->RingChannel || Ring.Exit) return NOS_RESULT_FAILED; if (!ch->IsOutLive) return NOS_RESULT_SUCCESS; @@ -460,7 +453,7 @@ struct MultiRingBufferNodeContext : NodeContext // for the next frame on this pin, it's done with the previous one. if (ch->LastPopped) { - ch->Ring->EndPop(ch->LastPopped); + Ring.EndPop(*ch->RingChannel, ch->LastPopped); ch->LastPopped = nullptr; } @@ -468,7 +461,7 @@ struct MultiRingBufferNodeContext : NodeContext { if (ch->RemainingRepeatableCount > 0) { - ch->Ring->ResInterface->OnRepeatPinValue(cpy); + ch->RingChannel->ResInterface->OnRepeatPinValue(cpy); ch->RemainingRepeatableCount--; return NOS_RESULT_SUCCESS; } @@ -484,17 +477,17 @@ struct MultiRingBufferNodeContext : NodeContext ResourceInterface::ResourceBase* slot; { ScopedProfilerEvent _({.Name = "Wait For Filled Slot"}); - slot = ch->Ring->BeginPop(100); + slot = Ring.BeginPop(*ch->RingChannel, 100); } if (!slot) - return ch->Ring->Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; nos::Buffer outPinVal; - bool changePinValue = ch->Ring->ResInterface->BeginCopyFrom(slot, *cpy->PinData, outPinVal); + bool changePinValue = ch->RingChannel->ResInterface->BeginCopyFrom(slot, *cpy->PinData, outPinVal); if (changePinValue) nosEngine.SetPinValueByName(NodeId, ch->OutputName, outPinVal); - ch->Ring->ResInterface->WaitForDownloadToEnd(slot, "MultiRingBuffer", NodeName.AsString(), cpy); + ch->RingChannel->ResInterface->WaitForDownloadToEnd(slot, "MultiRingBuffer", NodeName.AsString(), cpy); cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; cpy->FrameNumber = slot->FrameNumber; @@ -519,9 +512,9 @@ struct MultiRingBufferNodeContext : NodeContext ChangePinLiveness(ch->OutputName, false); ch->IsOutLive = false; } - // Note: EndPop happens at the start of the next CopyFrom for this - // channel rather than here, because OnEndFrame is unreliable for - // secondary live outputs. + // EndPop happens at the start of the next CopyFrom for this channel + // rather than here, because OnEndFrame is unreliable for secondary + // live outputs. } void SendScheduleRequest(uint32_t count, bool reset = false) const @@ -550,56 +543,63 @@ struct MultiRingBufferNodeContext : NodeContext Mode = RingMode::FILL; for (auto& [_, ch] : Channels) { - if (ch->LastPopped && ch->Ring) + if (ch->LastPopped && ch->RingChannel) { - ch->Ring->EndPop(ch->LastPopped); + Ring.EndPop(*ch->RingChannel, ch->LastPopped); ch->LastPopped = nullptr; } - if (ch->Ring) - ch->Ring->Stop(); } + Ring.Stop(); } void OnPathStart() override { if (Channels.empty()) return; - size_t totalSchedule = 0; - for (auto& [_, ch] : Channels) + + if (OnRestart == OnRestartType::RESET || RepeatWhenFilling) + Ring.ResetAll(false); + else { - if (!ch->Ring) - continue; - if (OnRestart == OnRestartType::RESET || RepeatWhenFilling) - ch->Ring->Reset(false); - else if (ch->Ring->IsFull() && !ch->Ring->Read.Pool.empty()) - { - ch->Ring->Write.Pool.push_back(ch->Ring->Read.Pool.front()); - ch->Ring->Read.Pool.pop_front(); - } - if (RequestedRingSize) - { - ch->Ring->Resize(*RequestedRingSize); + for (auto& [_, ch] : Channels) + if (ch->RingChannel) + Ring.MoveOneReadToWriteIfFull(*ch->RingChannel); + } + + if (RequestedRingSize) + { + Ring.ResizeAll(*RequestedRingSize); + for (auto& [_, ch] : Channels) ch->NeedsRecreation = false; - } - if (ch->NeedsRecreation) + RequestedRingSize = std::nullopt; + } + for (auto& [_, ch] : Channels) + { + if (ch->NeedsRecreation && ch->RingChannel) { - ch->Ring = std::make_unique(ch->Ring->Size, ch->Ring->ResInterface); + Ring.RecreateChannelResources(*ch->RingChannel); ch->NeedsRecreation = false; } - if (!ch->Ring->IsResourcesValid()) + } + + size_t totalSchedule = 0; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->Resources.empty()) { totalSchedule = std::max(totalSchedule, 1); continue; } - auto emptySlotCount = ch->Ring->Write.Pool.size(); + auto emptySlotCount = Ring.WritePoolSize(*ch->RingChannel); if (RepeatWhenFilling) ch->RemainingRepeatableCount = std::max(emptySlotCount, (size_t)1) - 1; totalSchedule = std::max(totalSchedule, emptySlotCount); - ch->Ring->Exit = false; - ch->Ring->ResInterface->OnPathStart(); + ch->RingChannel->ResInterface->OnPathStart(); SeedOutputPin(*ch); } - RequestedRingSize = std::nullopt; + Ring.Start(); if (totalSchedule > 0) { nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = (uint32_t)totalSchedule}; From 73a98816aa328f67cb7255ae588e36f1a87fd88e Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Wed, 29 Apr 2026 14:44:11 +0300 Subject: [PATCH 12/27] Add timecode based indexing to playback track node --- Plugins/nosTrack/CMakeLists.txt | 3 + Plugins/nosTrack/Config/PlaybackMode.fbs | 9 ++ .../Config/PlaybackTrackCOLMAP.nosdef | 28 ++++- .../nosTrack/Config/RecordTrackCOLMAP.nosdef | 17 +++ .../nosTrack/Source/PlaybackTrackCOLMAP.cpp | 111 +++++++++++++++++- Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 37 +++++- Plugins/nosTrack/Track.noscfg | 3 + 7 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 Plugins/nosTrack/Config/PlaybackMode.fbs diff --git a/Plugins/nosTrack/CMakeLists.txt b/Plugins/nosTrack/CMakeLists.txt index 26c05e2b..2b30615c 100644 --- a/Plugins/nosTrack/CMakeLists.txt +++ b/Plugins/nosTrack/CMakeLists.txt @@ -13,4 +13,7 @@ endforeach() list(APPEND MODULE_DEPENDENCIES_TARGETS ${NOS_PLUGIN_SDK_TARGET}) +nos_generate_flatbuffers("${CMAKE_CURRENT_SOURCE_DIR}/Config" "${CMAKE_CURRENT_SOURCE_DIR}/Source" "cpp" "" nosTrack_generated) +list(APPEND MODULE_DEPENDENCIES_TARGETS nosTrack_generated) + nos_add_plugin("nosTrack" "${MODULE_DEPENDENCIES_TARGETS}" "${CMAKE_CURRENT_LIST_DIR}/External/asio/asio/include") diff --git a/Plugins/nosTrack/Config/PlaybackMode.fbs b/Plugins/nosTrack/Config/PlaybackMode.fbs new file mode 100644 index 00000000..c0d1d952 --- /dev/null +++ b/Plugins/nosTrack/Config/PlaybackMode.fbs @@ -0,0 +1,9 @@ +namespace nos.track; + +// Selects how PlaybackTrackCOLMAP indexes into the recorded frames. +enum PlaybackTrackMode : uint +{ + FrameIndex = 0, // Use the InFrameIndex pin as a 0-based offset. + Timecode = 1, // Look up by Timecode string from timecodes.txt. + FrameNumber = 2, // Look up by FrameNumber column from timecodes.txt. +} diff --git a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef index 67b8f74e..54cb1cdc 100644 --- a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef @@ -32,6 +32,15 @@ "data": "ZYX", "description": "Euler angle rotation order for converting COLMAP quaternion to Track rotation. Default ZYX matches the FreeD node convention." }, + { + "name": "Mode", + "display_name": "Mode", + "type_name": "nos.track.PlaybackTrackMode", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "FrameIndex", + "description": "Selects how to index into the loaded frames. FrameIndex uses InFrameIndex; Timecode/FrameNumber look up via the timecodes.txt sidecar. The unused index pin is set to PASSIVE." + }, { "name": "InFrameIndex", "display_name": "Frame Index", @@ -39,7 +48,24 @@ "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": 0, - "description": "Frame index to output." + "description": "0-based frame index. Used when Mode=FrameIndex." + }, + { + "name": "InTimecode", + "display_name": "Timecode", + "type_name": "string", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Timecode string (HH:MM:SS:FF or HH:MM:SS;FF) to look up. Used when Mode=Timecode. Requires a timecodes.txt sidecar." + }, + { + "name": "InFrameNumber", + "display_name": "Frame Number", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "Absolute frame number to look up. Used when Mode=FrameNumber. Requires a timecodes.txt sidecar." }, { "name": "Track", diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index 2237b720..45342e37 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -22,6 +22,23 @@ "can_show_as": "INPUT_PIN_OR_PROPERTY", "description": "Incoming camera tracking data to record. Position, rotation, FOV, sensor size, and lens distortion are captured each frame." }, + { + "name": "Timecode", + "display_name": "Timecode", + "type_name": "string", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Optional SMPTE timecode (HH:MM:SS:FF or HH:MM:SS;FF) for the current frame. Written to a timecodes.txt sidecar when non-empty." + }, + { + "name": "FrameNumber", + "display_name": "Frame Number", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "Optional absolute frame number that pairs with Timecode (e.g. from ExtractTimecode). Written to the timecodes.txt sidecar." + }, { "name": "OutTrack", "display_name": "Track", diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp index 567f929b..79f629f1 100644 --- a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -2,6 +2,7 @@ #include #include "nosSysTrack/Track_generated.h" +#include "PlaybackMode_generated.h" #include #include @@ -14,13 +15,17 @@ #include #include #include +#include namespace nos::track { NOS_REGISTER_NAME_SPACED(Playback_InputDirectory, "InputDirectory"); NOS_REGISTER_NAME_SPACED(Playback_CoordinateSystem, "CoordinateSystem"); +NOS_REGISTER_NAME_SPACED(Playback_Mode, "Mode"); NOS_REGISTER_NAME_SPACED(Playback_InFrameIndex, "InFrameIndex"); +NOS_REGISTER_NAME_SPACED(Playback_InTimecode, "InTimecode"); +NOS_REGISTER_NAME_SPACED(Playback_InFrameNumber, "InFrameNumber"); NOS_REGISTER_NAME_SPACED(Playback_OutFrameIndex, "OutFrameIndex"); NOS_REGISTER_NAME_SPACED(Playback_FrameCount, "FrameCount"); @@ -44,14 +49,27 @@ struct COLMAPImage uint32_t CameraId = 0; }; +struct TimecodeEntry +{ + std::string Timecode; + uint32_t FrameNumber = 0; +}; + struct PlaybackTrackCOLMAPContext : NodeContext { std::string InputDir; sys::track::CoordinateSystem CoordSys = sys::track::CoordinateSystem::ZYX; + PlaybackTrackMode Mode = PlaybackTrackMode::FrameIndex; uint32_t FrameIndex = 0; + std::string InTimecode; + uint32_t InFrameNumber = 0; std::string LastError; std::vector Frames; + std::vector Timecodes; // empty or same size as Frames + std::unordered_map TimecodeToIndex; + std::unordered_map FrameNumberToIndex; uint32_t CurrentFrame = 0; + PlaybackTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) { if (node->pins()) @@ -66,6 +84,7 @@ struct PlaybackTrackCOLMAPContext : NodeContext } } } + ApplyModeOrphanStates(); UpdateStatus(); } @@ -86,8 +105,30 @@ struct PlaybackTrackCOLMAPContext : NodeContext if (!InputDir.empty()) LoadFromDirectory(); } + else if (pinName == NSN_Playback_Mode) + { + Mode = *(PlaybackTrackMode*)val.Data; + ApplyModeOrphanStates(); + } else if (pinName == NSN_Playback_InFrameIndex) FrameIndex = *(uint32_t*)val.Data; + else if (pinName == NSN_Playback_InTimecode) + InTimecode = InterpretPinValue(val.Data); + else if (pinName == NSN_Playback_InFrameNumber) + InFrameNumber = *(uint32_t*)val.Data; + } + + void ApplyModeOrphanStates() + { + auto state = [](bool active) { + return active ? fb::PinOrphanStateType::ACTIVE : fb::PinOrphanStateType::PASSIVE; + }; + const bool useIdx = Mode == PlaybackTrackMode::FrameIndex; + const bool useTC = Mode == PlaybackTrackMode::Timecode; + const bool useFN = Mode == PlaybackTrackMode::FrameNumber; + SetPinOrphanState(NSN_Playback_InFrameIndex, state(useIdx)); + SetPinOrphanState(NSN_Playback_InTimecode, state(useTC)); + SetPinOrphanState(NSN_Playback_InFrameNumber, state(useFN)); } void UpdateFrameCountPin() @@ -158,6 +199,11 @@ struct PlaybackTrackCOLMAPContext : NodeContext Frames.clear(); Frames.reserve(images.size()); + Timecodes.clear(); + + auto timecodesPath = dir / "timecodes.txt"; + if (std::filesystem::exists(timecodesPath)) + ParseTimecodesTxt(timecodesPath, images.size()); for (auto& img : images) { @@ -278,6 +324,40 @@ struct PlaybackTrackCOLMAPContext : NodeContext return true; } + void ParseTimecodesTxt(const std::filesystem::path& path, size_t expectedCount) + { + std::ifstream file(path); + if (!file.is_open()) + return; + std::unordered_map byId; + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + uint32_t id = 0; + TimecodeEntry e; + ss >> id >> e.Timecode >> e.FrameNumber; + if (e.Timecode == "-") + e.Timecode.clear(); + byId[id] = std::move(e); + } + Timecodes.assign(expectedCount, TimecodeEntry{}); + TimecodeToIndex.clear(); + FrameNumberToIndex.clear(); + for (size_t i = 0; i < expectedCount; ++i) + { + auto it = byId.find(uint32_t(i + 1)); + if (it == byId.end()) + continue; + Timecodes[i] = it->second; + if (!Timecodes[i].Timecode.empty()) + TimecodeToIndex.emplace(Timecodes[i].Timecode, uint32_t(i)); + FrameNumberToIndex.emplace(Timecodes[i].FrameNumber, uint32_t(i)); + } + } + // --- Euler extraction (inverse of EulerToRotationMatrix in RecordTrackCOLMAP) --- static glm::vec3 RotationMatrixToEuler(const glm::mat3& R_c2w, sys::track::CoordinateSystem order) @@ -299,6 +379,33 @@ struct PlaybackTrackCOLMAPContext : NodeContext // --- Execution --- + bool ResolveFrameIndex(uint32_t& outIdx) + { + switch (Mode) + { + case PlaybackTrackMode::Timecode: + { + auto it = TimecodeToIndex.find(InTimecode); + if (it == TimecodeToIndex.end()) + return false; + outIdx = it->second; + return true; + } + case PlaybackTrackMode::FrameNumber: + { + auto it = FrameNumberToIndex.find(InFrameNumber); + if (it == FrameNumberToIndex.end()) + return false; + outIdx = it->second; + return true; + } + case PlaybackTrackMode::FrameIndex: + default: + outIdx = FrameIndex < (uint32_t)Frames.size() ? FrameIndex : (uint32_t)Frames.size() - 1; + return true; + } + } + nosResult ExecuteNode(nosNodeExecuteParams* params) override { if (Frames.empty()) @@ -309,7 +416,9 @@ struct PlaybackTrackCOLMAPContext : NodeContext return NOS_RESULT_SUCCESS; } - uint32_t frameIdx = FrameIndex < (uint32_t)Frames.size() ? FrameIndex : (uint32_t)Frames.size() - 1; + uint32_t frameIdx = 0; + if (!ResolveFrameIndex(frameIdx)) + frameIdx = CurrentFrame < (uint32_t)Frames.size() ? CurrentFrame : 0; CurrentFrame = frameIdx; auto buf = nos::Buffer::From(Frames[frameIdx]); diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index 0284b2f6..8731edf3 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -39,6 +39,8 @@ struct RecordedFrame float PixelAspectRatio; float K1; float K2; + std::string Timecode; + uint32_t FrameNumber; }; struct RecordTrackCOLMAPContext : NodeContext @@ -213,7 +215,7 @@ struct RecordTrackCOLMAPContext : NodeContext nosResult ExecuteNode(nosNodeExecuteParams* params) override { - auto pins = GetPinValues(params); + nos::NodeExecuteParams execParams(params); // Pass through Track input to output nosBuffer trackBuf{}; @@ -235,6 +237,9 @@ struct RecordTrackCOLMAPContext : NodeContext return NOS_RESULT_SUCCESS; RecordedFrame frame{}; + if (const char* tc = execParams.GetPinData(NOS_NAME_STATIC("Timecode"))) + frame.Timecode = tc; + frame.FrameNumber = *execParams.GetPinData(NOS_NAME_STATIC("FrameNumber")); if (auto* loc = trackData->location()) frame.Location = {loc->x(), loc->y(), loc->z()}; if (auto* rot = trackData->rotation()) @@ -285,9 +290,39 @@ struct RecordTrackCOLMAPContext : NodeContext WriteCamerasTxt(outDir); WriteImagesTxt(outDir); + WriteTimecodesTxt(outDir); nosEngine.LogI("RecordTrackCOLMAP: Saved %zu frames to %s", Frames.size(), OutputDir.c_str()); } + void WriteTimecodesTxt(const std::filesystem::path& outDir) + { + // Skip the sidecar entirely if no frame carried a timecode — keeps the + // output minimal when the upstream graph isn't producing TC. + bool any = false; + for (auto& f : Frames) + if (!f.Timecode.empty() || f.FrameNumber != 0) { any = true; break; } + if (!any) + return; + + auto path = outDir / "timecodes.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + file << "# Timecode sidecar paired with images.txt by IMAGE_ID.\n"; + file << "# IMAGE_ID, TIMECODE, FRAME_NUMBER\n"; + file << "# Number of entries: " << Frames.size() << "\n"; + for (size_t i = 0; i < Frames.size(); ++i) + { + const auto& f = Frames[i]; + file << (i + 1) << " " + << (f.Timecode.empty() ? "-" : f.Timecode) << " " + << f.FrameNumber << "\n"; + } + } + float ComputeFocalLengthPixels(const RecordedFrame& frame) const { if (frame.FOV <= 0.0f) diff --git a/Plugins/nosTrack/Track.noscfg b/Plugins/nosTrack/Track.noscfg index 97473fef..dc975d34 100644 --- a/Plugins/nosTrack/Track.noscfg +++ b/Plugins/nosTrack/Track.noscfg @@ -20,6 +20,9 @@ "Config/RecordTrackCOLMAP.nosdef", "Config/PlaybackTrackCOLMAP.nosdef" ], + "custom_types": [ + "Config/PlaybackMode.fbs" + ], "defaults": [ "Config/Defaults.json" ], From eb36d508bccbd0563e914ba1d9706b28032ac164 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Wed, 29 Apr 2026 18:06:11 +0300 Subject: [PATCH 13/27] Dynamically add multiple pins to Sink node --- Plugins/nosUtilities/Source/Sink.cpp | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/Plugins/nosUtilities/Source/Sink.cpp b/Plugins/nosUtilities/Source/Sink.cpp index 043982f3..28294d7e 100644 --- a/Plugins/nosUtilities/Source/Sink.cpp +++ b/Plugins/nosUtilities/Source/Sink.cpp @@ -5,6 +5,7 @@ // stl #include +#include #include #include "Sink_generated.h" @@ -18,6 +19,33 @@ constexpr uint64_t VULKAN_TIMEOUT_BEFORE_LEAK = struct SinkNode : NodeContext { + enum MenuCommandType : uint8_t + { + ADD_INPUT = 0, + REMOVE_INPUT = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t InputIndex; + MenuCommand(uint32_t cmd) { + Type = static_cast(cmd & 0xFF); + InputIndex = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t inputIndex) : Type(type), InputIndex(inputIndex) {} + operator uint32_t() const { return (InputIndex << 8) | Type; } + }; + + static const std::unordered_set& StaticPinNames() + { + static const std::unordered_set names = { + "InExe", "Sink Input", "Sink FPS", "HasGPUWork", "GPUFrameBuffering", + "AcceptsRepeat", "SinkMode", "LatencyBudget" + }; + return names; + } + std::mutex Mutex; std::atomic ShouldStop = false; std::atomic Fps = 1000.0f / 60.0f; @@ -31,9 +59,27 @@ struct SinkNode : NodeContext std::optional> GPUFrameSyncEvents = std::nullopt; size_t GPUFrameBuffering = 1; uint64_t CurrentGPUEventIndex = 0; + std::vector DynamicInputs; SinkNode(nosFbNodePtr inNode) : NodeContext(inNode) { + std::list pinsToUnorphan; + for (auto i = 0; i < inNode->pins()->size(); i++) + { + auto pin = inNode->pins()->Get(i); + if (pin->show_as() != fb::ShowAs::INPUT_PIN) + continue; + if (StaticPinNames().contains(pin->name()->string_view())) + continue; + DynamicInputs.push_back(*pin->id()); + if (auto orphanState = pin->orphan_state()) + { + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(*pin->id()); + } + } + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); AddPinValueWatcher(NOS_NAME("HasGPUWork"), [this](nosBuffer const& newVal, std::optional oldValue) { bool hasGpuWork = *static_cast(newVal.Data); @@ -255,6 +301,93 @@ struct SinkNode : NodeContext } } + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + uint32_t cmd = MenuCommand(ADD_INPUT, 0); + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Sink", cmd, nullptr) + }; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + if (StaticPinNames().contains(pinName.AsString())) + return; + auto pinId = GetPinId(pinName); + if (!pinId) + return; + auto it = std::find(DynamicInputs.begin(), DynamicInputs.end(), *pinId); + if (it == DynamicInputs.end()) + return; + auto index = std::distance(DynamicInputs.begin(), it); + uint32_t cmd = MenuCommand(REMOVE_INPUT, static_cast(index)); + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Remove Input", cmd, nullptr) + }; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_INPUT: + { + std::string pinName; + for (size_t i = 2;; i++) + { + auto candidate = "Sink Input " + std::to_string(i); + if (!GetPinId(nos::Name(candidate))) + { + pinName = std::move(candidate); + break; + } + } + flatbuffers::FlatBufferBuilder fbb; + uuid pinId = nosEngine.GenerateID(); + std::vector pins = { + fb::CreatePinDirect(fbb, &pinId, pinName.c_str(), "nos.Generic", + fb::ShowAs::INPUT_PIN, fb::CanShowAs::INPUT_PIN_ONLY) + }; + HandleEvent(CreateAppEvent(fbb, CreatePartialNodeUpdateDirect(fbb, &NodeId, ClearFlags::NONE, 0, &pins))); + break; + } + case REMOVE_INPUT: + { + if (command.InputIndex >= DynamicInputs.size()) + return; + auto pinId = DynamicInputs[command.InputIndex]; + flatbuffers::FlatBufferBuilder fbb; + std::vector pinsToRemove = { *&pinId }; + HandleEvent(CreateAppEvent(fbb, CreatePartialNodeUpdateDirect(fbb, &NodeId, ClearFlags::NONE, &pinsToRemove))); + break; + } + } + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + std::erase_if(DynamicInputs, [&](auto id) { return id == update->PinDeleted; }); + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + if (pin->show_as() != fb::ShowAs::INPUT_PIN) + return; + if (StaticPinNames().contains(pin->name()->string_view())) + return; + DynamicInputs.push_back(*pin->id()); + } + } + void GetScheduleInfo(nosScheduleInfo* info) override { info->Type = NOS_SCHEDULE_TYPE_ON_DEMAND; From ec08c0107547fb00caab233243e91131ab623520 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Wed, 29 Apr 2026 18:06:36 +0300 Subject: [PATCH 14/27] Do not pile up schedule requests in multi-ring buffer --- .../nosUtilities/Source/MultiBoundedQueue.cpp | 33 +++++++++++++++++-- .../nosUtilities/Source/MultiRingBuffer.cpp | 28 +++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp index 3f6eab32..2c69bf50 100644 --- a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp +++ b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp @@ -2,6 +2,8 @@ #pragma once +#include + #include // External @@ -62,6 +64,11 @@ struct MultiBoundedQueueNodeContext : NodeContext std::map> Channels; std::unordered_map PinIdToLetter; MultiRing Ring; + // Channels popped since the last SendScheduleRequest. One producer run + // pushes one slot per live channel, so we must only schedule again once + // every live channel has been popped — otherwise schedule requests pile + // up by a factor of N (channels) per consumer tick. + std::set PoppedSinceLastSchedule; std::optional RequestedRingSize = std::nullopt; @@ -141,7 +148,10 @@ struct MultiBoundedQueueNodeContext : NodeContext } } if (any) + { Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } }); } @@ -186,6 +196,7 @@ struct MultiBoundedQueueNodeContext : NodeContext nosEngine.SendPathCommand(ch->InputId, ringSizeChange); } Ring.Stop(); + PoppedSinceLastSchedule.clear(); SendPathRestart(); RequestedRingSize = size; } @@ -208,6 +219,7 @@ struct MultiBoundedQueueNodeContext : NodeContext { nosEngine.SendPathRestart(ch->InputId); Ring.Stop(); + PoppedSinceLastSchedule.clear(); ch->NeedsRecreation = true; } } @@ -228,6 +240,7 @@ struct MultiBoundedQueueNodeContext : NodeContext if (ch.RingChannel) { Ring.Stop(); + PoppedSinceLastSchedule.clear(); Ring.RemoveChannel(*letter); ch.RingChannel = nullptr; } @@ -388,7 +401,17 @@ struct MultiBoundedQueueNodeContext : NodeContext cpy->FrameNumber = slot->FrameNumber; Ring.EndPop(*ch->RingChannel, slot); - SendScheduleRequest(1); + + PoppedSinceLastSchedule.insert(ch->Letter); + size_t liveCount = 0; + for (auto& [_, c] : Channels) + if (c->IsOutLive) + ++liveCount; + if (PoppedSinceLastSchedule.size() >= liveCount) + { + SendScheduleRequest(1); + PoppedSinceLastSchedule.clear(); + } return NOS_RESULT_SUCCESS; } @@ -427,13 +450,19 @@ struct MultiBoundedQueueNodeContext : NodeContext } } - void OnPathStop() override { Ring.Stop(); } + void OnPathStop() override + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } void OnPathStart() override { if (Channels.empty()) return; + PoppedSinceLastSchedule.clear(); + Ring.ResetAll(false); if (RequestedRingSize) diff --git a/Plugins/nosUtilities/Source/MultiRingBuffer.cpp b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp index 54304f41..2c446299 100644 --- a/Plugins/nosUtilities/Source/MultiRingBuffer.cpp +++ b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp @@ -2,6 +2,8 @@ #pragma once +#include + #include // External @@ -67,6 +69,11 @@ struct MultiRingBufferNodeContext : NodeContext std::map> Channels; std::unordered_map PinIdToLetter; MultiRing Ring; + // Channels popped since the last SendScheduleRequest. One producer run + // pushes one slot per live channel, so we must only schedule again once + // every live channel has been popped — otherwise schedule requests pile + // up by a factor of N (channels) per consumer tick. + std::set PoppedSinceLastSchedule; OnRestartType OnRestart = OnRestartType::WAIT_UNTIL_FULL; std::optional RequestedRingSize = std::nullopt; @@ -151,7 +158,10 @@ struct MultiRingBufferNodeContext : NodeContext } } if (any) + { Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } }); AddPinValueWatcher(NOS_NAME_STATIC("RepeatWhenFilling"), [this](nos::Buffer const& newVal, std::optional oldVal) { @@ -228,6 +238,7 @@ struct MultiRingBufferNodeContext : NodeContext nosEngine.SendPathCommand(ch->InputId, ringSizeChange); } Ring.Stop(); + PoppedSinceLastSchedule.clear(); SendPathRestart(); RequestedRingSize = size; } @@ -250,6 +261,7 @@ struct MultiRingBufferNodeContext : NodeContext { nosEngine.SendPathRestart(ch->InputId); Ring.Stop(); + PoppedSinceLastSchedule.clear(); ch->NeedsRecreation = true; } } @@ -271,6 +283,7 @@ struct MultiRingBufferNodeContext : NodeContext if (ch.RingChannel) { Ring.Stop(); + PoppedSinceLastSchedule.clear(); Ring.RemoveChannel(*letter); ch.RingChannel = nullptr; } @@ -493,7 +506,17 @@ struct MultiRingBufferNodeContext : NodeContext cpy->FrameNumber = slot->FrameNumber; ch->LastPopped = slot; - SendScheduleRequest(1); + + PoppedSinceLastSchedule.insert(ch->Letter); + size_t liveCount = 0; + for (auto& [_, c] : Channels) + if (c->IsOutLive) + ++liveCount; + if (PoppedSinceLastSchedule.size() >= liveCount) + { + SendScheduleRequest(1); + PoppedSinceLastSchedule.clear(); + } return NOS_RESULT_SUCCESS; } @@ -550,6 +573,7 @@ struct MultiRingBufferNodeContext : NodeContext } } Ring.Stop(); + PoppedSinceLastSchedule.clear(); } void OnPathStart() override @@ -557,6 +581,8 @@ struct MultiRingBufferNodeContext : NodeContext if (Channels.empty()) return; + PoppedSinceLastSchedule.clear(); + if (OnRestart == OnRestartType::RESET || RepeatWhenFilling) Ring.ResetAll(false); else From 246f304b541c3dd7731811ad8a3c61a568ae6551 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Wed, 29 Apr 2026 18:47:51 +0300 Subject: [PATCH 15/27] RecordTrackCOLMAP: Disable function orphan toggling to avoid path recompile --- Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index 8731edf3..9a20292c 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -87,16 +87,12 @@ struct RecordTrackCOLMAPContext : NodeContext void UpdateFunctionOrphanStates() { - if (Recording) - { - SetFunctionOrphanState(NSN_RecordTrackCOLMAP_Record, fb::NodeOrphanStateType::ORPHAN); - SetFunctionOrphanState(NSN_RecordTrackCOLMAP_Stop, fb::NodeOrphanStateType::ACTIVE); - } - else - { - SetFunctionOrphanState(NSN_RecordTrackCOLMAP_Record, fb::NodeOrphanStateType::ACTIVE); - SetFunctionOrphanState(NSN_RecordTrackCOLMAP_Stop, fb::NodeOrphanStateType::ORPHAN); - } + // Disabled: toggling NodeOrphanState on Record/Stop function nodes invalidates the + // engine's compiled execution path, causing a recompile hitch on the first recorded + // frame. NodeOrphanStateType has no PASSIVE variant (only pins do), so we can't + // express "visual-only, no topology change" today. Re-enable once the engine adds + // PASSIVE to NodeOrphanStateType (or otherwise skips path invalidation for + // orphan-state changes on function nodes). } void SyncRecordPin(bool value) From f3e8f6ae116e631704f64044230d04fd1305aabc Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Tue, 5 May 2026 12:29:07 +0300 Subject: [PATCH 16/27] Auto-export COLMAP files when RecordTrackCOLMAP stops WriteFiles() is now invoked from StopRecording() so cameras.txt and images.txt are written the moment recording ends. The Save function remains available for re-exporting the buffered frames on demand. --- Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef | 2 +- Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index 45342e37..3efaf7ae 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -12,7 +12,7 @@ "display_name": "Record Track (COLMAP)", "contents_type": "Job", "always_execute": true, - "description": "Records camera tracking data each frame while recording is enabled, then exports cameras.txt and images.txt in COLMAP format. Intrinsics (focal length, distortion) are derived from the Track's FOV, sensor size, and lens distortion. Extrinsics (rotation, translation) are stored per frame in world-to-camera convention.", + "description": "Records camera tracking data each frame while recording is enabled, then exports cameras.txt and images.txt in COLMAP format the moment recording stops. The Save function is still available for re-exporting the buffered frames on demand. Intrinsics (focal length, distortion) are derived from the Track's FOV, sensor size, and lens distortion. Extrinsics (rotation, translation) are stored per frame in world-to-camera convention.", "pins": [ { "name": "InTrack", diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index 9a20292c..8e775553 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -129,8 +129,10 @@ struct RecordTrackCOLMAPContext : NodeContext SyncRecordPin(false); UpdateRecordingFramePin(); UpdateFunctionOrphanStates(); - UpdateStatus(); nosEngine.LogI("RecordTrackCOLMAP: Recording stopped (%zu frames in buffer)", Frames.size()); + if (!Frames.empty()) + WriteFiles(); + UpdateStatus(); } void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer val) override From 28fe3bcc29c61c84b421b3fe7396089e6dd8984d Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Tue, 5 May 2026 16:38:40 +0300 Subject: [PATCH 17/27] MultiBoundedQueue: Propagate slot descriptor to output pin in CopyFrom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CopyFrom went straight from BeginPop to Copy, so the GPU copy used the stale default-sized output pin descriptor as its destination. Mirror the single BoundedQueue path (CommonCopyFrom) and MultiRingBuffer by calling BeginCopyFrom and writing the slot resource's descriptor onto the output pin first. The ring resources themselves are still (re)allocated from the input descriptor in OnPathStart — this only fixes the output-side descriptor propagation. --- Plugins/nosUtilities/Source/MultiBoundedQueue.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp index 2c69bf50..9f433574 100644 --- a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp +++ b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp @@ -395,6 +395,13 @@ struct MultiBoundedQueueNodeContext : NodeContext if (!slot) return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + // Propagate the slot resource's descriptor onto the output pin before + // Copy reads cpy->PinData as the destination — otherwise the GPU copy + // targets the stale (default-sized) output descriptor. + nos::Buffer outPinVal; + if (ch->RingChannel->ResInterface->BeginCopyFrom(slot, *cpy->PinData, outPinVal)) + nosEngine.SetPinValueByName(NodeId, ch->OutputName, outPinVal); + ch->RingChannel->ResInterface->Copy(slot, cpy, NodeId); cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; From f76e468e40faaeb715cca498ebe0f045a9d00b2e Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Wed, 6 May 2026 00:29:01 +0300 Subject: [PATCH 18/27] Pin-driven RecordTrackCOLMAP and extended track field capture Recording is now driven by the Record pin alone; the dedicated Record/Stop/Save/Clear/OpenFolder functions are removed. Stop is debounced via a new MinOffFrames property so brief glitches in the upstream signal (e.g. SDI bit flips on a camera-derived recording flag) don't end a take prematurely. Buffer is auto-cleared on stop. RecordedFrame is widened to capture Zoom, Focus, RenderRatio, NodalOffset, CenterShift, and DistortionScale from the incoming Track. PlaybackTrackCOLMAP gains a paired ExtrasEntry sidecar so the same fields can be replayed. --- .../nosTrack/Config/RecordTrackCOLMAP.nosdef | 46 +--- .../nosTrack/Source/PlaybackTrackCOLMAP.cpp | 113 ++++++++- Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 217 +++++++----------- 3 files changed, 203 insertions(+), 173 deletions(-) diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index 3efaf7ae..34cb779f 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -12,7 +12,7 @@ "display_name": "Record Track (COLMAP)", "contents_type": "Job", "always_execute": true, - "description": "Records camera tracking data each frame while recording is enabled, then exports cameras.txt and images.txt in COLMAP format the moment recording stops. The Save function is still available for re-exporting the buffered frames on demand. Intrinsics (focal length, distortion) are derived from the Track's FOV, sensor size, and lens distortion. Extrinsics (rotation, translation) are stored per frame in world-to-camera convention.", + "description": "Records camera tracking data each frame while the Record pin is true, then exports cameras.txt and images.txt in COLMAP format the moment recording stops, and clears the buffer. The Record pin is the single driver — wire it to an upstream Recording flag (e.g. from SonyVeniceANCParser) for automatic record-following capture. The MinOffFrames pin debounces brief drops in the Record signal so SDI bit flips don't end a take prematurely. Intrinsics (focal length, distortion) are derived from the Track's FOV, sensor size, and lens distortion. Extrinsics (rotation, translation) are stored per frame in world-to-camera convention.", "pins": [ { "name": "InTrack", @@ -83,7 +83,17 @@ "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": false, - "description": "Toggle recording. Mirrors Record/Stop functions. Enabling clears previous frames and starts capturing. Will fail if the output directory is not empty." + "description": "Drives recording: rising edge clears the buffer and starts capture; falling edge stops capture (subject to MinOffFrames debouncing) and exports the COLMAP files. Will fail to start if the output directory is not empty." + }, + { + "name": "MinOffFrames", + "display_name": "Min Off Frames", + "type_name": "uint", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1, + "min": "1", + "description": "Debounce: minimum number of consecutive frames with Record=false before recording actually stops. Default 1 = stop immediately. Use higher values (e.g. 5-15) to ride out short glitches in the upstream Record signal — useful when Record is wired to a camera-derived flag that can momentarily flip due to SDI bit errors." }, { "name": "RecordingFrame", @@ -103,38 +113,6 @@ "data": 0, "description": "Number of frames in the buffer." } - ], - "functions": [ - { - "class_name": "RecordTrackCOLMAP_Record", - "display_name": "Record", - "contents_type": "Job", - "pins": [] - }, - { - "class_name": "RecordTrackCOLMAP_Stop", - "display_name": "Stop", - "contents_type": "Job", - "pins": [] - }, - { - "class_name": "RecordTrackCOLMAP_Save", - "display_name": "Save", - "contents_type": "Job", - "pins": [] - }, - { - "class_name": "RecordTrackCOLMAP_Clear", - "display_name": "Clear", - "contents_type": "Job", - "pins": [] - }, - { - "class_name": "RecordTrackCOLMAP_OpenFolder", - "display_name": "Open Folder", - "contents_type": "Job", - "pins": [] - } ] } } diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp index 79f629f1..50d405bb 100644 --- a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -55,6 +55,22 @@ struct TimecodeEntry uint32_t FrameNumber = 0; }; +struct ExtrasEntry +{ + bool Present = false; + float Zoom = 0; + float Focus = 0; + float FocusDistance = 0; + float RenderRatio = 0; + float NodalOffset = 0; + float DistortionScale = 0; + float SensorWmm = 0; + float SensorHmm = 0; + float RotX = 0; + float RotY = 0; + float RotZ = 0; +}; + struct PlaybackTrackCOLMAPContext : NodeContext { std::string InputDir; @@ -205,29 +221,81 @@ struct PlaybackTrackCOLMAPContext : NodeContext if (std::filesystem::exists(timecodesPath)) ParseTimecodesTxt(timecodesPath, images.size()); - for (auto& img : images) + std::vector extras; + auto extrasPath = dir / "extras.txt"; + if (std::filesystem::exists(extrasPath)) + ParseExtrasTxt(extrasPath, images.size(), extras); + + for (size_t i = 0; i < images.size(); ++i) { + auto& img = images[i]; sys::track::TTrack trackData{}; auto camIt = cameras.find(img.CameraId); + const ExtrasEntry* ex = (i < extras.size() && extras[i].Present) ? &extras[i] : nullptr; - // Convert COLMAP world-to-camera back to camera-to-world + // Position: invert COLMAP world-to-camera. Stable round-trip. glm::mat3 R_w2c = glm::mat3_cast(img.Q); glm::mat3 R_c2w = glm::transpose(R_w2c); glm::vec3 C = -R_c2w * img.T; - - glm::vec3 euler = RotationMatrixToEuler(R_c2w, CoordSys); trackData.location = reinterpret_cast(C); - trackData.rotation = reinterpret_cast(euler); + + // Rotation: prefer the original Euler from extras (avoids quaternion- + // to-Euler ambiguity near gimbal lock); fall back to extracting from + // the COLMAP rotation matrix when no extras sidecar exists. + if (ex) + { + glm::vec3 euler(ex->RotX, ex->RotY, ex->RotZ); + trackData.rotation = reinterpret_cast(euler); + } + else + { + glm::vec3 euler = RotationMatrixToEuler(R_c2w, CoordSys); + trackData.rotation = reinterpret_cast(euler); + } if (camIt != cameras.end()) { auto& cam = camIt->second; if (cam.Fx > 0) trackData.fov = glm::degrees(2.0f * std::atan(cam.Width * 0.5f / cam.Fx)); - trackData.sensor_size = nos::fb::vec2(cam.Width, cam.Height); if (cam.Fx > 0 && cam.Fy > 0) trackData.pixel_aspect_ratio = cam.Fx / cam.Fy; trackData.lens_distortion.mutable_k1k2() = nos::fb::vec2(cam.K1, cam.K2); + + // sensor_size: COLMAP only stores pixel dims, but Track expects mm. + // Use the extras value when present; otherwise fall back to pixels + // (matches pre-extras behaviour). + glm::vec2 sensorMm(0); + if (ex && ex->SensorWmm > 0 && ex->SensorHmm > 0) + { + sensorMm = {ex->SensorWmm, ex->SensorHmm}; + trackData.sensor_size = nos::fb::vec2(sensorMm.x, sensorMm.y); + } + else + { + trackData.sensor_size = nos::fb::vec2(cam.Width, cam.Height); + } + + // center_shift: invert the (cx, cy) encoding written by record. + // Needs sensor_size in mm to be meaningful, so only reconstructed + // when extras provided it. + if (sensorMm.x > 0 && cam.Width > 0 && sensorMm.y > 0 && cam.Height > 0) + { + glm::vec2 shift{ + (cam.Cx - cam.Width * 0.5f) * sensorMm.x / cam.Width, + (cam.Cy - cam.Height * 0.5f) * sensorMm.y / cam.Height}; + trackData.lens_distortion.mutable_center_shift() = reinterpret_cast(shift); + } + } + + if (ex) + { + trackData.zoom = ex->Zoom; + trackData.focus = ex->Focus; + trackData.focus_distance = ex->FocusDistance; + trackData.render_ratio = ex->RenderRatio; + trackData.nodal_offset = ex->NodalOffset; + trackData.lens_distortion.mutate_distortion_scale(ex->DistortionScale); } Frames.push_back(std::move(trackData)); @@ -324,6 +392,39 @@ struct PlaybackTrackCOLMAPContext : NodeContext return true; } + void ParseExtrasTxt(const std::filesystem::path& path, size_t expectedCount, std::vector& outExtras) + { + std::ifstream file(path); + if (!file.is_open()) + return; + std::unordered_map byId; + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + uint32_t id = 0; + ExtrasEntry e; + ss >> id >> e.Zoom >> e.Focus >> e.FocusDistance >> e.RenderRatio + >> e.NodalOffset >> e.DistortionScale + >> e.SensorWmm >> e.SensorHmm + >> e.RotX >> e.RotY >> e.RotZ; + if (!ss.fail()) + { + e.Present = true; + byId[id] = e; + } + } + outExtras.assign(expectedCount, ExtrasEntry{}); + for (size_t i = 0; i < expectedCount; ++i) + { + auto it = byId.find(uint32_t(i + 1)); + if (it != byId.end()) + outExtras[i] = it->second; + } + } + void ParseTimecodesTxt(const std::filesystem::path& path, size_t expectedCount) { std::ifstream file(path); diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index 8e775553..2ae9ef2d 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -20,25 +20,26 @@ NOS_REGISTER_NAME(OutputDirectory); NOS_REGISTER_NAME(ImageResolution); NOS_REGISTER_NAME(CoordinateSystem); NOS_REGISTER_NAME(Record); +NOS_REGISTER_NAME(MinOffFrames); NOS_REGISTER_NAME(FrameCount); NOS_REGISTER_NAME(RecordingFrame); -NOS_REGISTER_NAME(RecordTrackCOLMAP_Record); -NOS_REGISTER_NAME(RecordTrackCOLMAP_Stop); -NOS_REGISTER_NAME(RecordTrackCOLMAP_Save); -NOS_REGISTER_NAME(RecordTrackCOLMAP_Clear); -NOS_REGISTER_NAME(RecordTrackCOLMAP_OpenFolder); - struct RecordedFrame { glm::vec3 Location; glm::vec3 Rotation; // Euler degrees (roll, tilt, pan) float FOV; + float Zoom; + float Focus; + float RenderRatio; glm::vec2 SensorSize; - float FocusDistance; float PixelAspectRatio; + float NodalOffset; + float FocusDistance; float K1; float K2; + glm::vec2 CenterShift; + float DistortionScale; std::string Timecode; uint32_t FrameNumber; }; @@ -49,19 +50,13 @@ struct RecordTrackCOLMAPContext : NodeContext nosVec2u ImageResolution = {1920, 1080}; sys::track::CoordinateSystem CoordSys = sys::track::CoordinateSystem::ZYX; bool Recording = false; - bool SyncingRecordPin = false; + uint32_t ConsecutiveOffFrames = 0; + bool LastRequestRecord = false; std::string LastError; std::vector Frames; - std::unordered_map FunctionIds; RecordTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) { - if (node->functions()) - { - for (auto* func : *node->functions()) - FunctionIds[nos::Name(func->class_name()->c_str())] = *func->id(); - } - if (node->pins()) { for (auto* pin : *node->pins()) @@ -74,34 +69,9 @@ struct RecordTrackCOLMAPContext : NodeContext } } } - UpdateFunctionOrphanStates(); UpdateStatus(); } - void SetFunctionOrphanState(nos::Name funcName, fb::NodeOrphanStateType type) - { - auto it = FunctionIds.find(funcName); - if (it != FunctionIds.end()) - NodeContext::SetNodeOrphanState(it->second, type); - } - - void UpdateFunctionOrphanStates() - { - // Disabled: toggling NodeOrphanState on Record/Stop function nodes invalidates the - // engine's compiled execution path, causing a recompile hitch on the first recorded - // frame. NodeOrphanStateType has no PASSIVE variant (only pins do), so we can't - // express "visual-only, no topology change" today. Re-enable once the engine adds - // PASSIVE to NodeOrphanStateType (or otherwise skips path invalidation for - // orphan-state changes on function nodes). - } - - void SyncRecordPin(bool value) - { - SyncingRecordPin = true; - SetPinValue(NSN_Record, nosBuffer{.Data = &value, .Size = sizeof(value)}); - SyncingRecordPin = false; - } - bool StartRecording() { std::string error; @@ -114,10 +84,9 @@ struct RecordTrackCOLMAPContext : NodeContext LastError.clear(); Frames.clear(); Recording = true; - SyncRecordPin(true); + ConsecutiveOffFrames = 0; UpdateFrameCountPin(); UpdateRecordingFramePin(); - UpdateFunctionOrphanStates(); UpdateStatus(); nosEngine.LogI("RecordTrackCOLMAP: Recording started"); return true; @@ -126,12 +95,12 @@ struct RecordTrackCOLMAPContext : NodeContext void StopRecording() { Recording = false; - SyncRecordPin(false); - UpdateRecordingFramePin(); - UpdateFunctionOrphanStates(); nosEngine.LogI("RecordTrackCOLMAP: Recording stopped (%zu frames in buffer)", Frames.size()); if (!Frames.empty()) WriteFiles(); + Frames.clear(); + UpdateFrameCountPin(); + UpdateRecordingFramePin(); UpdateStatus(); } @@ -147,16 +116,6 @@ struct RecordTrackCOLMAPContext : NodeContext ImageResolution = *(nosVec2u*)val.Data; else if (pinName == NSN_CoordinateSystem) CoordSys = *(sys::track::CoordinateSystem*)val.Data; - else if (pinName == NSN_Record) - { - if (SyncingRecordPin) - return; - bool newVal = *(bool*)val.Data; - if (newVal && !Recording) - StartRecording(); - else if (!newVal && Recording) - StopRecording(); - } } bool CanStartRecording(std::string& outError) @@ -205,8 +164,6 @@ struct RecordTrackCOLMAPContext : NodeContext SetNodeStatusMessage("Set output directory", fb::NodeStatusMessageType::WARNING); else if (Recording) SetNodeStatusMessage("Recording (" + std::to_string(Frames.size()) + " frames)", fb::NodeStatusMessageType::INFO); - else if (!Frames.empty()) - SetNodeStatusMessage("Idle (" + std::to_string(Frames.size()) + " frames in buffer)", fb::NodeStatusMessageType::INFO); else SetNodeStatusMessage("Idle", fb::NodeStatusMessageType::INFO); } @@ -227,6 +184,27 @@ struct RecordTrackCOLMAPContext : NodeContext } SetPinValue(NOS_NAME("OutTrack"), trackBuf); + // Drive recording state from the Record pin, with off-state debouncing to + // ride out brief glitches in the upstream signal (e.g. SDI bit flips on a + // camera-derived recording flag). Start happens immediately on a rising + // edge; stop only after MinOffFrames consecutive false frames. + const bool requestRecord = *execParams.GetPinData(NSN_Record); + const uint32_t minOffFrames = *execParams.GetPinData(NSN_MinOffFrames); + + const bool risingEdge = requestRecord && !LastRequestRecord; + LastRequestRecord = requestRecord; + + if (risingEdge && !Recording) + StartRecording(); + + if (Recording) + { + if (requestRecord) + ConsecutiveOffFrames = 0; + else if (++ConsecutiveOffFrames >= std::max(1u, minOffFrames)) + StopRecording(); + } + if (!Recording) return NOS_RESULT_SUCCESS; @@ -243,14 +221,20 @@ struct RecordTrackCOLMAPContext : NodeContext if (auto* rot = trackData->rotation()) frame.Rotation = {rot->x(), rot->y(), rot->z()}; frame.FOV = trackData->fov(); + frame.Zoom = trackData->zoom(); + frame.Focus = trackData->focus(); + frame.RenderRatio = trackData->render_ratio(); if (auto* ss = trackData->sensor_size()) frame.SensorSize = {ss->x(), ss->y()}; - frame.FocusDistance = trackData->focus_distance(); frame.PixelAspectRatio = trackData->pixel_aspect_ratio(); + frame.NodalOffset = trackData->nodal_offset(); + frame.FocusDistance = trackData->focus_distance(); if (auto* ld = trackData->lens_distortion()) { frame.K1 = ld->k1k2().x(); frame.K2 = ld->k1k2().y(); + frame.CenterShift = {ld->center_shift().x(), ld->center_shift().y()}; + frame.DistortionScale = ld->distortion_scale(); } Frames.push_back(frame); @@ -289,9 +273,44 @@ struct RecordTrackCOLMAPContext : NodeContext WriteCamerasTxt(outDir); WriteImagesTxt(outDir); WriteTimecodesTxt(outDir); + WriteExtrasTxt(outDir); nosEngine.LogI("RecordTrackCOLMAP: Saved %zu frames to %s", Frames.size(), OutputDir.c_str()); } + void WriteExtrasTxt(const std::filesystem::path& outDir) + { + // Sidecar for Track fields that don't fit COLMAP's standard cameras.txt / + // images.txt format. Keyed by IMAGE_ID so it pairs 1:1 with images.txt. + auto path = outDir / "extras.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + file << std::setprecision(12); + file << "# Nodos Track sidecar paired with images.txt by IMAGE_ID.\n"; + file << "# Carries fields that don't fit COLMAP's cameras.txt/images.txt:\n"; + file << "# - sensor_size in mm (cameras.txt only stores pixel WIDTH/HEIGHT)\n"; + file << "# - original Euler rotation in degrees (avoids quaternion round-trip drift)\n"; + file << "# - nodos-only fields with no COLMAP equivalent\n"; + file << "# IMAGE_ID, ZOOM, FOCUS, FOCUS_DISTANCE, RENDER_RATIO, NODAL_OFFSET, DISTORTION_SCALE, SENSOR_W_MM, SENSOR_H_MM, ROT_X, ROT_Y, ROT_Z\n"; + file << "# Number of entries: " << Frames.size() << "\n"; + for (size_t i = 0; i < Frames.size(); ++i) + { + const auto& f = Frames[i]; + file << (i + 1) << " " + << f.Zoom << " " + << f.Focus << " " + << f.FocusDistance << " " + << f.RenderRatio << " " + << f.NodalOffset << " " + << f.DistortionScale << " " + << f.SensorSize.x << " " << f.SensorSize.y << " " + << f.Rotation.x << " " << f.Rotation.y << " " << f.Rotation.z << "\n"; + } + } + void WriteTimecodesTxt(const std::filesystem::path& outDir) { // Skip the sidecar entirely if no frame carried a timecode — keeps the @@ -351,8 +370,15 @@ struct RecordTrackCOLMAPContext : NodeContext if (Frames[i].PixelAspectRatio > 0.0f) fy = fx / Frames[i].PixelAspectRatio; + // center_shift is in the same units as sensor_size (mm); convert to + // pixel offset on the principal point. See TrackToView.cpp:30 for the + // canonical centerShift / sensorSize relationship. float cx = ImageResolution.x * 0.5f; float cy = ImageResolution.y * 0.5f; + if (Frames[i].SensorSize.x > 0.0f) + cx += Frames[i].CenterShift.x * ImageResolution.x / Frames[i].SensorSize.x; + if (Frames[i].SensorSize.y > 0.0f) + cy += Frames[i].CenterShift.y * ImageResolution.y / Frames[i].SensorSize.y; // OPENCV model: fx, fy, cx, cy, k1, k2, p1, p2 float k1 = Frames[i].K1; @@ -423,81 +449,6 @@ struct RecordTrackCOLMAPContext : NodeContext file << "\n"; } } - - // TODO: Replace std::system with platform APIs (ShellExecuteW / posix_spawnp) to avoid shell injection via crafted paths - static void OpenFolderInExplorer(const std::filesystem::path& folder) - { -#if defined(_WIN32) - std::string cmd = "explorer \"" + nos::PathToUtf8(folder) + "\""; -#elif defined(__APPLE__) - std::string cmd = "open \"" + nos::PathToUtf8(folder) + "\""; -#else - std::string cmd = "xdg-open \"" + nos::PathToUtf8(folder) + "\""; -#endif - std::system(cmd.c_str()); - } - - static nosResult GetFunctions(size_t* count, nosName* names, nosPfnNodeFunctionExecute* fns) - { - *count = 5; - if (!names || !fns) - return NOS_RESULT_SUCCESS; - - names[0] = NOS_NAME_STATIC("RecordTrackCOLMAP_Record"); - fns[0] = [](void* ctx, nosFunctionExecuteParams*) { - auto* self = static_cast(ctx); - if (self->Recording) - return NOS_RESULT_SUCCESS; - self->StartRecording(); - return NOS_RESULT_SUCCESS; - }; - - names[1] = NOS_NAME_STATIC("RecordTrackCOLMAP_Stop"); - fns[1] = [](void* ctx, nosFunctionExecuteParams*) { - auto* self = static_cast(ctx); - if (!self->Recording) - return NOS_RESULT_SUCCESS; - self->StopRecording(); - return NOS_RESULT_SUCCESS; - }; - - names[2] = NOS_NAME_STATIC("RecordTrackCOLMAP_Save"); - fns[2] = [](void* ctx, nosFunctionExecuteParams*) { - auto* self = static_cast(ctx); - self->WriteFiles(); - return NOS_RESULT_SUCCESS; - }; - - names[3] = NOS_NAME_STATIC("RecordTrackCOLMAP_Clear"); - fns[3] = [](void* ctx, nosFunctionExecuteParams*) { - auto* self = static_cast(ctx); - self->Frames.clear(); - self->UpdateFrameCountPin(); - self->UpdateStatus(); - nosEngine.LogI("RecordTrackCOLMAP: Buffer cleared"); - return NOS_RESULT_SUCCESS; - }; - - names[4] = NOS_NAME_STATIC("RecordTrackCOLMAP_OpenFolder"); - fns[4] = [](void* ctx, nosFunctionExecuteParams*) { - auto* self = static_cast(ctx); - if (self->OutputDir.empty()) - { - nosEngine.LogW("RecordTrackCOLMAP: Output directory not set"); - return NOS_RESULT_FAILED; - } - std::filesystem::path outDir = nos::Utf8ToPath(self->OutputDir); - if (!std::filesystem::exists(outDir)) - { - nosEngine.LogW("RecordTrackCOLMAP: Directory does not exist: %s", self->OutputDir.c_str()); - return NOS_RESULT_FAILED; - } - OpenFolderInExplorer(outDir); - return NOS_RESULT_SUCCESS; - }; - - return NOS_RESULT_SUCCESS; - } }; void RegisterRecordTrackCOLMAP(nosNodeFunctions* fn) From d615a659422dd82dd8a6c0b734005847d4a2d7fe Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Thu, 7 May 2026 10:27:02 +0300 Subject: [PATCH 19/27] ChannelViewer: Use nos.mediaio.ColorSpace instead of own enum Drop the duplicated ChannelViewerFormats enum and retype the Format pin to nos.mediaio.ColorSpace, matching how the rest of the codebase (GetLumaCoeffs, ColorSpaceMatrix, AJA/DeckLink IO, CasWithColorSpace) already names this concept. The two enums disagreed on index ordering (Rec_601=0 vs REC709=0), so the luma coeffs array is reordered to match the new mapping. A MigrateNode hook rewrites legacy pin type_name and remaps the saved enum literal Rec_601/Rec_709/Rec_2020 -> REC601/REC709/REC2020 so existing graphs keep their selected color space. Co-Authored-By: Claude Opus 4.7 (1M context) --- Plugins/nosUtilities/Config/ChannelViewer.fbs | 6 --- .../nosUtilities/Config/ChannelViewer.nosdef | 4 +- Plugins/nosUtilities/Source/ChannelViewer.cpp | 39 ++++++++++++++++++- Plugins/nosUtilities/Source/UtilitiesMain.cpp | 2 +- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/Plugins/nosUtilities/Config/ChannelViewer.fbs b/Plugins/nosUtilities/Config/ChannelViewer.fbs index b5de7eb1..7042da21 100644 --- a/Plugins/nosUtilities/Config/ChannelViewer.fbs +++ b/Plugins/nosUtilities/Config/ChannelViewer.fbs @@ -9,9 +9,3 @@ enum ChannelViewerChannels : uint { Cb = 5, Cr = 6 } - -enum ChannelViewerFormats : uint { - Rec_601 = 0, - Rec_709 = 1, - Rec_2020 = 2 -} diff --git a/Plugins/nosUtilities/Config/ChannelViewer.nosdef b/Plugins/nosUtilities/Config/ChannelViewer.nosdef index 4a002d10..2722cba5 100644 --- a/Plugins/nosUtilities/Config/ChannelViewer.nosdef +++ b/Plugins/nosUtilities/Config/ChannelViewer.nosdef @@ -31,10 +31,10 @@ }, { "name": "Format", - "type_name": "nos.utilities.ChannelViewerFormats", + "type_name": "nos.mediaio.ColorSpace", "show_as": "PROPERTY", "can_show_as": "PROPERTY_ONLY", - "data": "Rec_709", + "data": "REC709", "description": "Sets the input texture color space,\nRequired for correct YCbCr conversion" }, { diff --git a/Plugins/nosUtilities/Source/ChannelViewer.cpp b/Plugins/nosUtilities/Source/ChannelViewer.cpp index a0d88f67..4fcc2e96 100644 --- a/Plugins/nosUtilities/Source/ChannelViewer.cpp +++ b/Plugins/nosUtilities/Source/ChannelViewer.cpp @@ -13,6 +13,41 @@ NOS_REGISTER_NAME_SPACED(Nos_Utilities_ChannelViewer, "nos.utilities.ChannelView namespace nos::utilities { +static nosResult MigrateNode(nosFbNodePtr nodePtr, nosBuffer* outBuffer) +{ + fb::TNode tNode; + nodePtr->UnPackTo(&tNode); + bool migrated = false; + for (auto& pin : tNode.pins) + { + if (!pin || pin->name != "Format") + continue; + bool legacyType = pin->type_name == "nos.utilities.ChannelViewerFormats" || + pin->type_name == "nos.fb.ChannelViewerFormats"; + const char* newValue = nullptr; + if (!pin->data.empty()) + { + std::string_view oldValue(reinterpret_cast(pin->data.data()), pin->data.size() - 1); + if (oldValue == "Rec_601") newValue = "REC601"; + else if (oldValue == "Rec_709") newValue = "REC709"; + else if (oldValue == "Rec_2020") newValue = "REC2020"; + } + if (!legacyType && !newValue) + continue; + pin->type_name = "nos.mediaio.ColorSpace"; + if (newValue) + { + std::string s = newValue; + pin->data = std::vector(s.c_str(), s.c_str() + s.size() + 1); + } + migrated = true; + } + if (!migrated) + return NOS_RESULT_SUCCESS; + *outBuffer = EngineBuffer::CopyFrom(tNode).Release(); + return NOS_RESULT_SUCCESS; +} + static nosResult ExecuteNode(void* ctx, nosNodeExecuteParams* pins) { auto values = GetPinValues(pins); @@ -25,7 +60,8 @@ static nosResult ExecuteNode(void* ctx, nosNodeExecuteParams* pins) glm::vec4 val{}; val[channel & 3] = 1; - constexpr glm::vec3 coeffs[3] = {{.299f, .587f, .114f}, {.2126f, .7152f, .0722f}, {.2627f, .678f, .0593f}}; + // Indexed by nos.mediaio.ColorSpace: REC709=0, REC601=1, REC2020=2 + constexpr glm::vec3 coeffs[3] = {{.2126f, .7152f, .0722f}, {.299f, .587f, .114f}, {.2627f, .678f, .0593f}}; glm::vec4 multipliers = glm::vec4(coeffs[format], channel > 3); std::vector bindings = { @@ -51,6 +87,7 @@ nosResult RegisterChannelViewer(nosNodeFunctions* out) { out->ClassName = NSN_Nos_Utilities_ChannelViewer; out->ExecuteNode = ExecuteNode; + out->MigrateNode = MigrateNode; fs::path root = nosEngine.Module->RootFolderPath; auto chViewerPath = (root / "Shaders" / "ChannelViewer.frag").generic_string(); diff --git a/Plugins/nosUtilities/Source/UtilitiesMain.cpp b/Plugins/nosUtilities/Source/UtilitiesMain.cpp index b0d8427d..826732ca 100644 --- a/Plugins/nosUtilities/Source/UtilitiesMain.cpp +++ b/Plugins/nosUtilities/Source/UtilitiesMain.cpp @@ -172,7 +172,7 @@ NOSAPI_ATTR nosResult NOSAPI_CALL nosExportPlugin(nosPluginFunctions* out) } // clang-format off outRenamedFrom[0] = NOS_NAME("nos.fb.ChannelViewerChannels"); outRenamedTo[0] = NOS_NAME("nos.utilities.ChannelViewerChannels"); - outRenamedFrom[1] = NOS_NAME("nos.fb.ChannelViewerFormats"); outRenamedTo[1] = NOS_NAME("nos.utilities.ChannelViewerFormats"); + outRenamedFrom[1] = NOS_NAME("nos.fb.ChannelViewerFormats"); outRenamedTo[1] = NOS_NAME("nos.mediaio.ColorSpace"); outRenamedFrom[2] = NOS_NAME("nos.fb.GradientKind"); outRenamedTo[2] = NOS_NAME("nos.utilities.GradientKind"); outRenamedFrom[3] = NOS_NAME("nos.fb.BlendMode"); outRenamedTo[3] = NOS_NAME("nos.utilities.BlendMode"); outRenamedFrom[4] = NOS_NAME("nos.fb.ResizeMethod"); outRenamedTo[4] = NOS_NAME("nos.utilities.ResizeMethod"); From e644526e9f556b350d4cbd4aef1a28f24fc30423 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Thu, 7 May 2026 18:59:37 +0300 Subject: [PATCH 20/27] nosMath: Add EulerToQuaternion and QuaternionToEuler nodes Introduce nosMath's first flatbuffer custom type (Math.fbs) and wire the plugin through nos_generate_flatbuffers so future nodes can share strongly-typed inputs/outputs. Adds two conversion nodes between Euler angles and quaternions, registered alongside the existing math nodes. --- Plugins/nosMath/CMakeLists.txt | 8 +- .../nosMath/Config/EulerToQuaternion.nosdef | 38 ++++++++ Plugins/nosMath/Config/Math.fbs | 10 ++ .../nosMath/Config/QuaternionToEuler.nosdef | 38 ++++++++ Plugins/nosMath/Math.noscfg | 8 +- Plugins/nosMath/Source/EulerToQuaternion.cpp | 97 +++++++++++++++++++ Plugins/nosMath/Source/Math.cpp | 12 +++ 7 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 Plugins/nosMath/Config/EulerToQuaternion.nosdef create mode 100644 Plugins/nosMath/Config/Math.fbs create mode 100644 Plugins/nosMath/Config/QuaternionToEuler.nosdef create mode 100644 Plugins/nosMath/Source/EulerToQuaternion.cpp diff --git a/Plugins/nosMath/CMakeLists.txt b/Plugins/nosMath/CMakeLists.txt index ee9ef718..bcbf2b2f 100644 --- a/Plugins/nosMath/CMakeLists.txt +++ b/Plugins/nosMath/CMakeLists.txt @@ -6,6 +6,10 @@ add_library(tinyexpr_cpp STATIC ${TINYEXPR_SOURCES}) target_include_directories(tinyexpr_cpp PUBLIC External/tinyexpr-cpp) nos_group_targets("tinyexpr_cpp" "External") -set(DEPENDENCIES ${NOS_PLUGIN_SDK_TARGET} tinyexpr_cpp) +set(GENERATED_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/Generated") +nos_generate_flatbuffers("${CMAKE_CURRENT_SOURCE_DIR}/Config" "${GENERATED_OUTPUT_DIR}" "cpp" "${NOS_SDK_DIR}/Types" nosMath_generated) -nos_add_plugin("nosMath" "${DEPENDENCIES}" "") +set(DEPENDENCIES ${NOS_PLUGIN_SDK_TARGET} tinyexpr_cpp nosMath_generated) +set(INCLUDE_FOLDERS "${GENERATED_OUTPUT_DIR}") + +nos_add_plugin("nosMath" "${DEPENDENCIES}" "${INCLUDE_FOLDERS}") diff --git a/Plugins/nosMath/Config/EulerToQuaternion.nosdef b/Plugins/nosMath/Config/EulerToQuaternion.nosdef new file mode 100644 index 00000000..fa40e5d7 --- /dev/null +++ b/Plugins/nosMath/Config/EulerToQuaternion.nosdef @@ -0,0 +1,38 @@ +{ + "nodes": [ + { + "class_name": "EulerToQuaternion", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Euler To Quaternion" + }, + "node": { + "class_name": "EulerToQuaternion", + "contents_type": "Job", + "description": "Converts an Euler-angle rotation (degrees) to a unit quaternion (x, y, z, w). The Order pin selects the intrinsic rotation order applied to the components of the input vec3 (e.g. ZYX means R = Rz(rot.z) * Ry(rot.y) * Rx(rot.x)). Default ZYX matches the FreeD/Track convention.", + "pins": [ + { + "name": "Euler", + "type_name": "nos.fb.vec3d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Order", + "type_name": "nos.math.EulerOrder", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "ZYX", + "description": "Euler intrinsic rotation order applied to the (rot.x, rot.y, rot.z) components." + }, + { + "name": "Quaternion", + "type_name": "nos.fb.vec4d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Config/Math.fbs b/Plugins/nosMath/Config/Math.fbs new file mode 100644 index 00000000..84c79eeb --- /dev/null +++ b/Plugins/nosMath/Config/Math.fbs @@ -0,0 +1,10 @@ +namespace nos.math; + +enum EulerOrder : ubyte { + ZYX = 0, + XYZ = 1, + YXZ = 2, + YZX = 3, + ZXY = 4, + XZY = 5, +} diff --git a/Plugins/nosMath/Config/QuaternionToEuler.nosdef b/Plugins/nosMath/Config/QuaternionToEuler.nosdef new file mode 100644 index 00000000..d08ffc7a --- /dev/null +++ b/Plugins/nosMath/Config/QuaternionToEuler.nosdef @@ -0,0 +1,38 @@ +{ + "nodes": [ + { + "class_name": "QuaternionToEuler", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Quaternion To Euler" + }, + "node": { + "class_name": "QuaternionToEuler", + "contents_type": "Job", + "description": "Converts a unit quaternion (x, y, z, w) to Euler angles (degrees). The Order pin selects the intrinsic rotation order extracted (e.g. ZYX yields rot.z = first rotation about Z, rot.y about Y, rot.x about X). Inverse of EulerToQuaternion when the same Order is used on both ends.", + "pins": [ + { + "name": "Quaternion", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Order", + "type_name": "nos.math.EulerOrder", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "ZYX", + "description": "Euler intrinsic rotation order extracted into the (rot.x, rot.y, rot.z) output components." + }, + { + "name": "Euler", + "type_name": "nos.fb.vec3d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Math.noscfg b/Plugins/nosMath/Math.noscfg index de4791ee..55046e44 100644 --- a/Plugins/nosMath/Math.noscfg +++ b/Plugins/nosMath/Math.noscfg @@ -14,7 +14,9 @@ } ] }, - "custom_types": [], + "custom_types": [ + "Config/Math.fbs" + ], "node_definitions": [ "Config/Math.nosdef", "Config/Eval.nosdef", @@ -25,7 +27,9 @@ "Config/Random.nosdef", "Config/Lerp.nosdef", "Config/Vec3ToVec4.nosdef", - "Config/EmbedMat3ToMat4.nosdef" + "Config/EmbedMat3ToMat4.nosdef", + "Config/EulerToQuaternion.nosdef", + "Config/QuaternionToEuler.nosdef" ], "binary_path": "Binaries/nosMath", "third_party_software": [ diff --git a/Plugins/nosMath/Source/EulerToQuaternion.cpp b/Plugins/nosMath/Source/EulerToQuaternion.cpp new file mode 100644 index 00000000..94440a60 --- /dev/null +++ b/Plugins/nosMath/Source/EulerToQuaternion.cpp @@ -0,0 +1,97 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include +#include +#include + +namespace nos::math +{ + +// Build a rotation matrix for the given intrinsic Euler order. +// In all cases, rot.x is the angle about X, rot.y about Y, rot.z about Z (radians). +// Order ZYX means R = Rz(rot.z) * Ry(rot.y) * Rx(rot.x), applied right-to-left to a point. +static glm::dmat4 EulerToMat(EulerOrder order, glm::dvec3 const& r) +{ + switch (order) + { + case EulerOrder::ZYX: return glm::eulerAngleZYX(r.z, r.y, r.x); + case EulerOrder::XYZ: return glm::eulerAngleXYZ(r.x, r.y, r.z); + case EulerOrder::YXZ: return glm::eulerAngleYXZ(r.y, r.x, r.z); + case EulerOrder::YZX: return glm::eulerAngleYZX(r.y, r.z, r.x); + case EulerOrder::ZXY: return glm::eulerAngleZXY(r.z, r.x, r.y); + case EulerOrder::XZY: return glm::eulerAngleXZY(r.x, r.z, r.y); + } + return glm::dmat4(1.0); +} + +static void MatToEuler(EulerOrder order, glm::dmat4 const& m, glm::dvec3& r) +{ + switch (order) + { + case EulerOrder::ZYX: glm::extractEulerAngleZYX(m, r.z, r.y, r.x); break; + case EulerOrder::XYZ: glm::extractEulerAngleXYZ(m, r.x, r.y, r.z); break; + case EulerOrder::YXZ: glm::extractEulerAngleYXZ(m, r.y, r.x, r.z); break; + case EulerOrder::YZX: glm::extractEulerAngleYZX(m, r.y, r.z, r.x); break; + case EulerOrder::ZXY: glm::extractEulerAngleZXY(m, r.z, r.x, r.y); break; + case EulerOrder::XZY: glm::extractEulerAngleXZY(m, r.x, r.z, r.y); break; + } +} + +struct EulerToQuaternionNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* in = params.GetPinData(NOS_NAME("Euler")); + auto* order = params.GetPinData(NOS_NAME("Order")); + auto* out = params.GetPinData(NOS_NAME("Quaternion")); + + glm::dvec3 r = glm::radians(glm::dvec3(in->x(), in->y(), in->z())); + glm::dquat q(EulerToMat(*order, r)); + + out->mutate_x(q.x); + out->mutate_y(q.y); + out->mutate_z(q.z); + out->mutate_w(q.w); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterEulerToQuaternion(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.EulerToQuaternion"), EulerToQuaternionNodeContext, fn) +} + +struct QuaternionToEulerNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* in = params.GetPinData(NOS_NAME("Quaternion")); + auto* order = params.GetPinData(NOS_NAME("Order")); + auto* out = params.GetPinData(NOS_NAME("Euler")); + + glm::dquat q(in->w(), in->x(), in->y(), in->z()); + glm::dmat4 m = glm::mat4_cast(q); + glm::dvec3 r(0.0); + MatToEuler(*order, m, r); + + out->mutate_x(glm::degrees(r.x)); + out->mutate_y(glm::degrees(r.y)); + out->mutate_z(glm::degrees(r.z)); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterQuaternionToEuler(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.QuaternionToEuler"), QuaternionToEulerNodeContext, fn) +} + +} // namespace nos::math diff --git a/Plugins/nosMath/Source/Math.cpp b/Plugins/nosMath/Source/Math.cpp index c307f2cf..a4bc6454 100644 --- a/Plugins/nosMath/Source/Math.cpp +++ b/Plugins/nosMath/Source/Math.cpp @@ -104,6 +104,8 @@ enum class MathNodeTypes : int { Or, Not, Random, + EulerToQuaternion, + QuaternionToEuler, Count }; @@ -168,6 +170,8 @@ void RegisterAnd(nosNodeFunctions*); void RegisterOr(nosNodeFunctions*); void RegisterNot(nosNodeFunctions*); void RegisterRandom(nosNodeFunctions*); +void RegisterEulerToQuaternion(nosNodeFunctions*); +void RegisterQuaternionToEuler(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outCount, nosNodeFunctions** outList) { @@ -281,6 +285,14 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outCount, nosNodeFunctions** o RegisterRandom(node); break; } + case MathNodeTypes::EulerToQuaternion: { + RegisterEulerToQuaternion(node); + break; + } + case MathNodeTypes::QuaternionToEuler: { + RegisterQuaternionToEuler(node); + break; + } default: break; } From bfe76da809bc4e3f5919ceab8e85e485555b0600 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Thu, 7 May 2026 18:59:43 +0300 Subject: [PATCH 21/27] nosTrack: Record fixed-step delta seconds in COLMAP timecode sidecar Capture TimingInfo.FixedStepTiming.DeltaSeconds during ExecuteNode and write it as the first non-comment line of the timecode sidecar (0 if the recording wasn't taken in fixed-step mode). Lets downstream tooling know the per-frame interval without having to derive it from timecodes. --- Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index 2ae9ef2d..4a4a4e48 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -54,6 +54,7 @@ struct RecordTrackCOLMAPContext : NodeContext bool LastRequestRecord = false; std::string LastError; std::vector Frames; + nosVec2u DeltaSeconds{}; // {numerator, denominator}; 0/0 if not in fixed-step mode RecordTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) { @@ -172,6 +173,9 @@ struct RecordTrackCOLMAPContext : NodeContext { nos::NodeExecuteParams execParams(params); + if (params->TimingInfo.TimingMode == NOS_EXECUTION_TIMING_MODE_FIXED_STEP) + DeltaSeconds = params->TimingInfo.FixedStepTiming.DeltaSeconds; + // Pass through Track input to output nosBuffer trackBuf{}; for (size_t i = 0; i < params->PinCount; ++i) @@ -328,9 +332,12 @@ struct RecordTrackCOLMAPContext : NodeContext nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); return; } + double dt = (DeltaSeconds.y != 0) ? (double)DeltaSeconds.x / (double)DeltaSeconds.y : 0.0; file << "# Timecode sidecar paired with images.txt by IMAGE_ID.\n"; + file << "# First non-comment line: per-frame delta seconds (0 if recording wasn't in fixed-step timing).\n"; file << "# IMAGE_ID, TIMECODE, FRAME_NUMBER\n"; file << "# Number of entries: " << Frames.size() << "\n"; + file << std::setprecision(12) << dt << "\n"; for (size_t i = 0; i < Frames.size(); ++i) { const auto& f = Frames[i]; From cf5b65c0ad1a204aa035a01fd313ab0b4301fae5 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Fri, 8 May 2026 20:25:36 +0300 Subject: [PATCH 22/27] Add QuaternionMultiply node --- .../nosMath/Config/QuaternionMultiply.nosdef | 36 +++++++++++++++++ Plugins/nosMath/Math.noscfg | 3 +- Plugins/nosMath/Source/Math.cpp | 6 +++ Plugins/nosMath/Source/QuaternionMultiply.cpp | 39 +++++++++++++++++++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 Plugins/nosMath/Config/QuaternionMultiply.nosdef create mode 100644 Plugins/nosMath/Source/QuaternionMultiply.cpp diff --git a/Plugins/nosMath/Config/QuaternionMultiply.nosdef b/Plugins/nosMath/Config/QuaternionMultiply.nosdef new file mode 100644 index 00000000..fc162b41 --- /dev/null +++ b/Plugins/nosMath/Config/QuaternionMultiply.nosdef @@ -0,0 +1,36 @@ +{ + "nodes": [ + { + "class_name": "QuaternionMultiply", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Quaternion Multiply" + }, + "node": { + "class_name": "QuaternionMultiply", + "contents_type": "Job", + "description": "Hamilton product of two unit quaternions: Result = A * B (each as (x, y, z, w)). Composing rotations: A * B applies B first, then A. To rotate (conjugate) a quaternion Q by R, compute R * Q * conjugate(R).", + "pins": [ + { + "name": "A", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "B", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Result", + "type_name": "nos.fb.vec4d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Math.noscfg b/Plugins/nosMath/Math.noscfg index 55046e44..5417aca6 100644 --- a/Plugins/nosMath/Math.noscfg +++ b/Plugins/nosMath/Math.noscfg @@ -29,7 +29,8 @@ "Config/Vec3ToVec4.nosdef", "Config/EmbedMat3ToMat4.nosdef", "Config/EulerToQuaternion.nosdef", - "Config/QuaternionToEuler.nosdef" + "Config/QuaternionToEuler.nosdef", + "Config/QuaternionMultiply.nosdef" ], "binary_path": "Binaries/nosMath", "third_party_software": [ diff --git a/Plugins/nosMath/Source/Math.cpp b/Plugins/nosMath/Source/Math.cpp index a4bc6454..1893cc8e 100644 --- a/Plugins/nosMath/Source/Math.cpp +++ b/Plugins/nosMath/Source/Math.cpp @@ -106,6 +106,7 @@ enum class MathNodeTypes : int { Random, EulerToQuaternion, QuaternionToEuler, + QuaternionMultiply, Count }; @@ -172,6 +173,7 @@ void RegisterNot(nosNodeFunctions*); void RegisterRandom(nosNodeFunctions*); void RegisterEulerToQuaternion(nosNodeFunctions*); void RegisterQuaternionToEuler(nosNodeFunctions*); +void RegisterQuaternionMultiply(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outCount, nosNodeFunctions** outList) { @@ -293,6 +295,10 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outCount, nosNodeFunctions** o RegisterQuaternionToEuler(node); break; } + case MathNodeTypes::QuaternionMultiply: { + RegisterQuaternionMultiply(node); + break; + } default: break; } diff --git a/Plugins/nosMath/Source/QuaternionMultiply.cpp b/Plugins/nosMath/Source/QuaternionMultiply.cpp new file mode 100644 index 00000000..395b4843 --- /dev/null +++ b/Plugins/nosMath/Source/QuaternionMultiply.cpp @@ -0,0 +1,39 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include +#include + +namespace nos::math +{ + +struct QuaternionMultiplyNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* a = params.GetPinData(NOS_NAME("A")); + auto* b = params.GetPinData(NOS_NAME("B")); + auto* out = params.GetPinData(NOS_NAME("Result")); + + glm::dquat qa(a->w(), a->x(), a->y(), a->z()); + glm::dquat qb(b->w(), b->x(), b->y(), b->z()); + glm::dquat qr = qa * qb; + + out->mutate_x(qr.x); + out->mutate_y(qr.y); + out->mutate_z(qr.z); + out->mutate_w(qr.w); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterQuaternionMultiply(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.QuaternionMultiply"), QuaternionMultiplyNodeContext, fn) +} + +} // namespace nos::math From 7439287183505cfcc9fee0cb903b8add43b0b1bc Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Fri, 8 May 2026 20:26:10 +0300 Subject: [PATCH 23/27] Add Track modifier node and make recorder nodes respect COLMAP quat format --- Plugins/nosTrack/CMakeLists.txt | 2 +- .../Config/PlaybackTrackCOLMAP.nosdef | 24 ++--- .../nosTrack/Config/RecordTrackCOLMAP.nosdef | 32 +++--- Plugins/nosTrack/Config/TrackTransform.nosdef | 54 ++++++++++ Plugins/nosTrack/Source/CoordinateFrameConv.h | 100 ++++++++++++++++++ .../nosTrack/Source/PlaybackTrackCOLMAP.cpp | 61 +++++------ Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp | 69 ++++++------ Plugins/nosTrack/Source/TrackMain.cpp | 5 + Plugins/nosTrack/Source/TrackTransform.cpp | 53 ++++++++++ Plugins/nosTrack/Track.noscfg | 3 +- Subsystems/nosTrackSubsystem/Config/Track.fbs | 20 ++-- 11 files changed, 317 insertions(+), 106 deletions(-) create mode 100644 Plugins/nosTrack/Config/TrackTransform.nosdef create mode 100644 Plugins/nosTrack/Source/CoordinateFrameConv.h create mode 100644 Plugins/nosTrack/Source/TrackTransform.cpp diff --git a/Plugins/nosTrack/CMakeLists.txt b/Plugins/nosTrack/CMakeLists.txt index 2b30615c..af2df47c 100644 --- a/Plugins/nosTrack/CMakeLists.txt +++ b/Plugins/nosTrack/CMakeLists.txt @@ -1,6 +1,6 @@ # Copyright MediaZ Teknoloji A.S. All Rights Reserved. -set(MODULE_DEPENDENCIES "nos.sys.track-1.0") +set(MODULE_DEPENDENCIES "nos.sys.track-1.1") set(dep_idx 0) foreach(module_name_version ${MODULE_DEPENDENCIES}) # module_name_version: - diff --git a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef index 54cb1cdc..460f02dc 100644 --- a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef @@ -12,7 +12,7 @@ "display_name": "Playback Track (COLMAP)", "contents_type": "Job", "always_execute": true, - "description": "Loads camera tracking data from COLMAP text format (cameras.txt + images.txt) and outputs Track data at a given frame index.", + "description": "Loads camera track from COLMAP-spec cameras.txt + images.txt.\nReads world-to-camera poses in the COLMAP frame (RH, +X right, +Y down, +Z forward) and converts to the chosen TargetFrame.\nWhen an extras.txt sidecar is present, original Euler/FOV/sensor metadata is restored verbatim (no quaternion round-trip drift).", "pins": [ { "name": "InputDirectory", @@ -21,16 +21,16 @@ "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", "visualizer": { "type": "FOLDER_PICKER" }, - "description": "Directory containing cameras.txt and images.txt in COLMAP text format." + "description": "Directory with cameras.txt + images.txt (and optional timecodes.txt / extras.txt sidecars)." }, { - "name": "CoordinateSystem", - "display_name": "Coordinate System", - "type_name": "nos.sys.track.CoordinateSystem", + "name": "TargetFrame", + "display_name": "Target Frame", + "type_name": "nos.sys.track.CoordinateFrame", "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", - "data": "ZYX", - "description": "Euler angle rotation order for converting COLMAP quaternion to Track rotation. Default ZYX matches the FreeD node convention." + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate frame of the produced Track.\nCOLMAP poses are converted into this frame.\nDefault matches FreeD / UE convention." }, { "name": "Mode", @@ -39,7 +39,7 @@ "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": "FrameIndex", - "description": "Selects how to index into the loaded frames. FrameIndex uses InFrameIndex; Timecode/FrameNumber look up via the timecodes.txt sidecar. The unused index pin is set to PASSIVE." + "description": "Selects how to index frames.\nFrameIndex uses InFrameIndex.\nTimecode / FrameNumber look up via timecodes.txt sidecar.\nThe unused index pin becomes PASSIVE." }, { "name": "InFrameIndex", @@ -56,7 +56,7 @@ "type_name": "string", "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", - "description": "Timecode string (HH:MM:SS:FF or HH:MM:SS;FF) to look up. Used when Mode=Timecode. Requires a timecodes.txt sidecar." + "description": "Timecode string (HH:MM:SS:FF) to look up. Used when Mode=Timecode. Requires timecodes.txt." }, { "name": "InFrameNumber", @@ -65,14 +65,14 @@ "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": 0, - "description": "Absolute frame number to look up. Used when Mode=FrameNumber. Requires a timecodes.txt sidecar." + "description": "Absolute frame number to look up. Used when Mode=FrameNumber. Requires timecodes.txt." }, { "name": "Track", "type_name": "nos.sys.track.Track", "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", - "description": "Track data for the current frame." + "description": "Track for the current frame, expressed in the TargetFrame convention." }, { "name": "OutFrameIndex", @@ -90,7 +90,7 @@ "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", "data": 0, - "description": "Total number of frames loaded." + "description": "Total frames loaded." } ], "functions": [ diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index 34cb779f..25d70356 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -12,7 +12,7 @@ "display_name": "Record Track (COLMAP)", "contents_type": "Job", "always_execute": true, - "description": "Records camera tracking data each frame while the Record pin is true, then exports cameras.txt and images.txt in COLMAP format the moment recording stops, and clears the buffer. The Record pin is the single driver — wire it to an upstream Recording flag (e.g. from SonyVeniceANCParser) for automatic record-following capture. The MinOffFrames pin debounces brief drops in the Record signal so SDI bit flips don't end a take prematurely. Intrinsics (focal length, distortion) are derived from the Track's FOV, sensor size, and lens distortion. Extrinsics (rotation, translation) are stored per frame in world-to-camera convention.", + "description": "Records camera track data each frame while Record is true.\nOn falling edge (after MinOffFrames debounce) writes COLMAP-spec cameras.txt + images.txt and clears the buffer.\nIntrinsics come from FOV/sensor/distortion. Extrinsics are written as world-to-camera in the COLMAP frame (RH, +X right, +Y down, +Z forward).\nSet SourceFrame to match the convention of the connected Track.", "pins": [ { "name": "InTrack", @@ -20,7 +20,7 @@ "type_name": "nos.sys.track.Track", "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", - "description": "Incoming camera tracking data to record. Position, rotation, FOV, sensor size, and lens distortion are captured each frame." + "description": "Camera track to record. Interpreted in the SourceFrame convention (location, rotation Euler, FOV, sensor, lens distortion)." }, { "name": "Timecode", @@ -28,7 +28,7 @@ "type_name": "string", "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", - "description": "Optional SMPTE timecode (HH:MM:SS:FF or HH:MM:SS;FF) for the current frame. Written to a timecodes.txt sidecar when non-empty." + "description": "Optional SMPTE timecode (HH:MM:SS:FF). Written to timecodes.txt sidecar when non-empty." }, { "name": "FrameNumber", @@ -37,7 +37,7 @@ "show_as": "INPUT_PIN", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": 0, - "description": "Optional absolute frame number that pairs with Timecode (e.g. from ExtractTimecode). Written to the timecodes.txt sidecar." + "description": "Optional absolute frame number paired with Timecode. Written to timecodes.txt sidecar." }, { "name": "OutTrack", @@ -45,7 +45,7 @@ "type_name": "nos.sys.track.Track", "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", - "description": "Pass-through of the incoming Track data." + "description": "Pass-through of InTrack." }, { "name": "OutputDirectory", @@ -54,7 +54,7 @@ "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", "visualizer": { "type": "FOLDER_PICKER" }, - "description": "Directory where cameras.txt and images.txt will be written when recording stops. Must be empty to start recording." + "description": "Where cameras.txt and images.txt are written when recording stops. Must be empty to start recording." }, { "name": "ImageResolution", @@ -66,16 +66,16 @@ "x": 1920, "y": 1080 }, - "description": "Image resolution in pixels (width, height). Used to compute focal length and principal point for COLMAP camera model." + "description": "Image WIDTH/HEIGHT in pixels. Used to compute focal length and principal point for cameras.txt." }, { - "name": "CoordinateSystem", - "display_name": "Coordinate System", - "type_name": "nos.sys.track.CoordinateSystem", + "name": "SourceFrame", + "display_name": "Source Frame", + "type_name": "nos.sys.track.CoordinateFrame", "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", - "data": "ZYX", - "description": "Euler angle rotation order used when converting Track rotation to COLMAP extrinsics. Default ZYX matches the FreeD node convention." + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate frame of the connected Track.\nUsed to convert location and rotation into the COLMAP frame before writing.\nDefault matches FreeD / UE convention." }, { "name": "Record", @@ -83,7 +83,7 @@ "show_as": "PROPERTY", "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": false, - "description": "Drives recording: rising edge clears the buffer and starts capture; falling edge stops capture (subject to MinOffFrames debouncing) and exports the COLMAP files. Will fail to start if the output directory is not empty." + "description": "Drives recording.\nRising edge: clear buffer and start.\nFalling edge (after MinOffFrames): stop and write files.\nFails to start if OutputDirectory is non-empty." }, { "name": "MinOffFrames", @@ -93,7 +93,7 @@ "can_show_as": "INPUT_PIN_OR_PROPERTY", "data": 1, "min": "1", - "description": "Debounce: minimum number of consecutive frames with Record=false before recording actually stops. Default 1 = stop immediately. Use higher values (e.g. 5-15) to ride out short glitches in the upstream Record signal — useful when Record is wired to a camera-derived flag that can momentarily flip due to SDI bit errors." + "description": "Debounce: minimum consecutive Record=false frames before stopping. Default 1 = stop immediately. Use 5-15 to ride out short upstream glitches (e.g. SDI bit flips on a camera-derived flag)." }, { "name": "RecordingFrame", @@ -102,7 +102,7 @@ "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", "data": 0, - "description": "Current recording frame index. Outputs 0 when not recording." + "description": "Current recording frame index. 0 when not recording." }, { "name": "FrameCount", @@ -111,7 +111,7 @@ "show_as": "OUTPUT_PIN", "can_show_as": "OUTPUT_PIN_ONLY", "data": 0, - "description": "Number of frames in the buffer." + "description": "Frames in the buffer." } ] } diff --git a/Plugins/nosTrack/Config/TrackTransform.nosdef b/Plugins/nosTrack/Config/TrackTransform.nosdef new file mode 100644 index 00000000..cb198734 --- /dev/null +++ b/Plugins/nosTrack/Config/TrackTransform.nosdef @@ -0,0 +1,54 @@ +{ + "nodes": [ + { + "class_name": "TrackTransform", + "menu_info": { + "category": "Track|Coordinate System", + "display_name": "Track Transform" + }, + "node": { + "class_name": "TrackTransform", + "contents_type": "Job", + "description": "Transforms a Track between coordinate frames.\nThe Source and Target enums select axis assignments, handedness, and the Euler convention used for the rotation field.\nLocation: basis-changed (Source -> Target), then multiplied by WorldScale (e.g. 0.01 for cm -> m, 100 for m -> cm).\nRotation: built in the source Euler convention, conjugated by the basis-change matrix, re-extracted in the target convention.\nOther Track fields (fov, focus, sensor_size, lens_distortion, ...) pass through unchanged.", + "pins": [ + { + "name": "In", + "type_name": "nos.sys.track.Track", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Source", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate system convention of the input Track." + }, + { + "name": "Target", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "RH_YUp_FwdNegZ_RightX", + "description": "Coordinate system convention of the output Track." + }, + { + "name": "WorldScale", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "description": "Uniform scale applied only to the output location after the basis change. Use to convert linear units (e.g. 0.01 for cm -> m, 100 for m -> cm). Does not affect rotation, fov, sensor size, focus, or lens distortion." + }, + { + "name": "Out", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Source/CoordinateFrameConv.h b/Plugins/nosTrack/Source/CoordinateFrameConv.h new file mode 100644 index 00000000..777fea34 --- /dev/null +++ b/Plugins/nosTrack/Source/CoordinateFrameConv.h @@ -0,0 +1,100 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Frame-conversion helpers shared by TrackTransform / RecordTrackCOLMAP / +// PlaybackTrackCOLMAP. Encodes per-frame Euler conventions and basis-change +// matrices to the COLMAP camera/world frame. +#pragma once + +#include "nosSysTrack/Track_generated.h" +#include +#include + +namespace nos::track::convention +{ + +using Frame = sys::track::CoordinateFrame; + +// Basis matrix S for a CoordinateFrame: maps semantic (forward, right, up) +// to engine coords (vx, vy, vz). v_engine = S * (forward, right, up). +// det(S) > 0 for left-handed frames, < 0 for right-handed (with this ordering). +inline glm::dmat3 BasisMatrix(Frame frame) +{ + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + // vx = forward, vy = right, vz = up. + return glm::dmat3(1.0); + case Frame::RH_YUp_FwdNegZ_RightX: + // vx = right, vy = up, vz = -forward. + return glm::dmat3( + glm::dvec3( 0.0, 0.0, -1.0), // M * (1,0,0) = forward column + glm::dvec3( 1.0, 0.0, 0.0), // M * (0,1,0) = right column + glm::dvec3( 0.0, 1.0, 0.0)); // M * (0,0,1) = up column + } + return glm::dmat3(1.0); +} + +// COLMAP camera/world frame: X right, Y down, Z forward (RH). +// Provided as a basis matrix in the same (forward, right, up) convention so +// it can be combined with BasisMatrix to build cross-frame conversions. +inline glm::dmat3 ColmapBasisMatrix() +{ + return glm::dmat3( + glm::dvec3( 0.0, 0.0, 1.0), // forward -> +Z + glm::dvec3( 1.0, 0.0, 0.0), // right -> +X + glm::dvec3( 0.0, -1.0, 0.0)); // up -> -Y (Y is down) +} + +// Build R_c2w in `frame` from Track.rotation Euler degrees. +inline glm::dmat3 EulerToMat(Frame frame, glm::dvec3 const& degRot) +{ + glm::dvec3 r = glm::radians(degRot); + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + // FRotator: rot.x = roll, rot.y = pitch, rot.z = yaw, intrinsic ZYX. + // UE sign convention has +pitch = look up and +roll = bank right via + // LH-rule rotations, equivalent to standard-RH Rz(yaw) * Ry(-pitch) * Rx(-roll). + return glm::dmat3(glm::eulerAngleZYX(r.z, -r.y, -r.x)); + case Frame::RH_YUp_FwdNegZ_RightX: + // rot.x = pitch, rot.y = yaw, rot.z = roll, intrinsic YXZ: + // R = Ry(yaw) * Rx(pitch) * Rz(roll), all standard-RH formulas. + return glm::dmat3(glm::eulerAngleYXZ(r.y, r.x, r.z)); + } + return glm::dmat3(1.0); +} + +// Inverse of EulerToMat: extract Euler degrees in `frame`'s convention. +// Output is packed into the (rot.x, rot.y, rot.z) Track layout for that frame. +inline glm::dvec3 MatToEuler(Frame frame, glm::dmat3 const& R) +{ + glm::dmat4 M(R); + double a = 0.0, b = 0.0, c = 0.0; + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + glm::extractEulerAngleZYX(M, a, b, c); // a=yaw, b=pitch, c=roll + // Negate pitch and roll back to UE sign convention; pack as (roll, pitch, yaw). + return glm::degrees(glm::dvec3(-c, -b, a)); + case Frame::RH_YUp_FwdNegZ_RightX: + glm::extractEulerAngleYXZ(M, a, b, c); // a=yaw, b=pitch, c=roll + // Pack as (pitch, yaw, roll). + return glm::degrees(glm::dvec3(b, a, c)); + } + return glm::dvec3(0.0); +} + +// Basis-change M from `frame` to COLMAP frame: M = S_colmap * S_frame^-1. +// For a vector: v_colmap = M * v_frame. +// For a rotation matrix: R_colmap = M * R_frame * M^-1. +inline glm::dmat3 BasisChangeToColmap(Frame frame) +{ + return ColmapBasisMatrix() * glm::inverse(BasisMatrix(frame)); +} + +// Inverse of BasisChangeToColmap. +inline glm::dmat3 BasisChangeFromColmap(Frame frame) +{ + return BasisMatrix(frame) * glm::inverse(ColmapBasisMatrix()); +} + +} // namespace nos::track::convention diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp index 50d405bb..56271a01 100644 --- a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include @@ -17,11 +16,13 @@ #include #include +#include "CoordinateFrameConv.h" + namespace nos::track { NOS_REGISTER_NAME_SPACED(Playback_InputDirectory, "InputDirectory"); -NOS_REGISTER_NAME_SPACED(Playback_CoordinateSystem, "CoordinateSystem"); +NOS_REGISTER_NAME_SPACED(Playback_TargetFrame, "TargetFrame"); NOS_REGISTER_NAME_SPACED(Playback_Mode, "Mode"); NOS_REGISTER_NAME_SPACED(Playback_InFrameIndex, "InFrameIndex"); NOS_REGISTER_NAME_SPACED(Playback_InTimecode, "InTimecode"); @@ -44,8 +45,8 @@ struct COLMAPCamera struct COLMAPImage { uint32_t Id = 0; - glm::quat Q{1, 0, 0, 0}; - glm::vec3 T{0}; + glm::quat Q{1, 0, 0, 0}; // R_w2c in COLMAP camera frame. + glm::vec3 T{0}; // t = -R_w2c * camera_world_position (COLMAP world frame). uint32_t CameraId = 0; }; @@ -74,7 +75,7 @@ struct ExtrasEntry struct PlaybackTrackCOLMAPContext : NodeContext { std::string InputDir; - sys::track::CoordinateSystem CoordSys = sys::track::CoordinateSystem::ZYX; + convention::Frame TargetFrame = convention::Frame::LH_ZUp_FwdX_RightY; PlaybackTrackMode Mode = PlaybackTrackMode::FrameIndex; uint32_t FrameIndex = 0; std::string InTimecode; @@ -115,9 +116,9 @@ struct PlaybackTrackCOLMAPContext : NodeContext else UpdateStatus(); } - else if (pinName == NSN_Playback_CoordinateSystem) + else if (pinName == NSN_Playback_TargetFrame) { - CoordSys = *(sys::track::CoordinateSystem*)val.Data; + TargetFrame = *(convention::Frame*)val.Data; if (!InputDir.empty()) LoadFromDirectory(); } @@ -226,6 +227,15 @@ struct PlaybackTrackCOLMAPContext : NodeContext if (std::filesystem::exists(extrasPath)) ParseExtrasTxt(extrasPath, images.size(), extras); + // Inverse of RecordTrackCOLMAP::WriteImagesTxt: + // images.txt holds R_w2c in COLMAP frame, t = -R_w2c * pos_colmap. + // pos_colmap = -R_c2w_colmap * t (R_c2w_colmap = R_w2c^T) + // pos_target = M^-1 * pos_colmap + // R_c2w_target = M^-1 * R_c2w_colmap * M + // Track.rotation = MatToEuler(TargetFrame, R_c2w_target) + const glm::dmat3 Minv = convention::BasisChangeFromColmap(TargetFrame); + const glm::dmat3 M = glm::inverse(Minv); + for (size_t i = 0; i < images.size(); ++i) { auto& img = images[i]; @@ -233,11 +243,13 @@ struct PlaybackTrackCOLMAPContext : NodeContext auto camIt = cameras.find(img.CameraId); const ExtrasEntry* ex = (i < extras.size() && extras[i].Present) ? &extras[i] : nullptr; - // Position: invert COLMAP world-to-camera. Stable round-trip. - glm::mat3 R_w2c = glm::mat3_cast(img.Q); - glm::mat3 R_c2w = glm::transpose(R_w2c); - glm::vec3 C = -R_c2w * img.T; - trackData.location = reinterpret_cast(C); + glm::dmat3 R_w2c = glm::dmat3(glm::mat3_cast(img.Q)); + glm::dmat3 R_c2w_colmap = glm::transpose(R_w2c); + glm::dvec3 pos_colmap = -R_c2w_colmap * glm::dvec3(img.T); + + glm::dvec3 pos_target = Minv * pos_colmap; + glm::vec3 locF((float)pos_target.x, (float)pos_target.y, (float)pos_target.z); + trackData.location = reinterpret_cast(locF); // Rotation: prefer the original Euler from extras (avoids quaternion- // to-Euler ambiguity near gimbal lock); fall back to extracting from @@ -249,8 +261,10 @@ struct PlaybackTrackCOLMAPContext : NodeContext } else { - glm::vec3 euler = RotationMatrixToEuler(R_c2w, CoordSys); - trackData.rotation = reinterpret_cast(euler); + glm::dmat3 R_c2w_target = Minv * R_c2w_colmap * M; + glm::dvec3 eulerD = convention::MatToEuler(TargetFrame, R_c2w_target); + glm::vec3 eulerF((float)eulerD.x, (float)eulerD.y, (float)eulerD.z); + trackData.rotation = reinterpret_cast(eulerF); } if (camIt != cameras.end()) @@ -459,25 +473,6 @@ struct PlaybackTrackCOLMAPContext : NodeContext } } - // --- Euler extraction (inverse of EulerToRotationMatrix in RecordTrackCOLMAP) --- - - static glm::vec3 RotationMatrixToEuler(const glm::mat3& R_c2w, sys::track::CoordinateSystem order) - { - float r, t, p; - switch (order) - { - default: - case sys::track::CoordinateSystem::ZYX: glm::extractEulerAngleZYX(glm::mat4(R_c2w), p, t, r); break; - case sys::track::CoordinateSystem::XYZ: glm::extractEulerAngleXYZ(glm::mat4(R_c2w), r, t, p); break; - case sys::track::CoordinateSystem::YXZ: glm::extractEulerAngleYXZ(glm::mat4(R_c2w), t, r, p); break; - case sys::track::CoordinateSystem::YZX: glm::extractEulerAngleYZX(glm::mat4(R_c2w), t, p, r); break; - case sys::track::CoordinateSystem::ZXY: glm::extractEulerAngleZXY(glm::mat4(R_c2w), p, r, t); break; - case sys::track::CoordinateSystem::XZY: glm::extractEulerAngleXZY(glm::mat4(R_c2w), r, p, t); break; - } - // Undo sign convention: r = -roll, t = -tilt, p = pan - return glm::degrees(glm::vec3(-r, -t, p)); - } - // --- Execution --- bool ResolveFrameIndex(uint32_t& outIdx) diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp index 4a4a4e48..661d06e3 100644 --- a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include @@ -13,12 +12,14 @@ #include #include +#include "CoordinateFrameConv.h" + namespace nos::track { NOS_REGISTER_NAME(OutputDirectory); NOS_REGISTER_NAME(ImageResolution); -NOS_REGISTER_NAME(CoordinateSystem); +NOS_REGISTER_NAME(SourceFrame); NOS_REGISTER_NAME(Record); NOS_REGISTER_NAME(MinOffFrames); NOS_REGISTER_NAME(FrameCount); @@ -27,7 +28,7 @@ NOS_REGISTER_NAME(RecordingFrame); struct RecordedFrame { glm::vec3 Location; - glm::vec3 Rotation; // Euler degrees (roll, tilt, pan) + glm::vec3 Rotation; // Euler degrees in the SourceFrame's convention. float FOV; float Zoom; float Focus; @@ -48,7 +49,7 @@ struct RecordTrackCOLMAPContext : NodeContext { std::string OutputDir; nosVec2u ImageResolution = {1920, 1080}; - sys::track::CoordinateSystem CoordSys = sys::track::CoordinateSystem::ZYX; + convention::Frame SourceFrame = convention::Frame::LH_ZUp_FwdX_RightY; bool Recording = false; uint32_t ConsecutiveOffFrames = 0; bool LastRequestRecord = false; @@ -115,8 +116,8 @@ struct RecordTrackCOLMAPContext : NodeContext } else if (pinName == NSN_ImageResolution) ImageResolution = *(nosVec2u*)val.Data; - else if (pinName == NSN_CoordinateSystem) - CoordSys = *(sys::track::CoordinateSystem*)val.Data; + else if (pinName == NSN_SourceFrame) + SourceFrame = *(convention::Frame*)val.Data; } bool CanStartRecording(std::string& outError) @@ -292,12 +293,17 @@ struct RecordTrackCOLMAPContext : NodeContext nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); return; } + const char* frameName = + SourceFrame == convention::Frame::LH_ZUp_FwdX_RightY ? "LH_ZUp_FwdX_RightY" + : SourceFrame == convention::Frame::RH_YUp_FwdNegZ_RightX ? "RH_YUp_FwdNegZ_RightX" + : "Unknown"; file << std::setprecision(12); file << "# Nodos Track sidecar paired with images.txt by IMAGE_ID.\n"; file << "# Carries fields that don't fit COLMAP's cameras.txt/images.txt:\n"; file << "# - sensor_size in mm (cameras.txt only stores pixel WIDTH/HEIGHT)\n"; file << "# - original Euler rotation in degrees (avoids quaternion round-trip drift)\n"; file << "# - nodos-only fields with no COLMAP equivalent\n"; + file << "# SourceFrame: " << frameName << " (Euler convention used for ROT_X, ROT_Y, ROT_Z below).\n"; file << "# IMAGE_ID, ZOOM, FOCUS, FOCUS_DISTANCE, RENDER_RATIO, NODAL_OFFSET, DISTORTION_SCALE, SENSOR_W_MM, SENSOR_H_MM, ROT_X, ROT_Y, ROT_Z\n"; file << "# Number of entries: " << Frames.size() << "\n"; for (size_t i = 0; i < Frames.size(); ++i) @@ -317,7 +323,7 @@ struct RecordTrackCOLMAPContext : NodeContext void WriteTimecodesTxt(const std::filesystem::path& outDir) { - // Skip the sidecar entirely if no frame carried a timecode — keeps the + // Skip the sidecar entirely if no frame carried a timecode -- keeps the // output minimal when the upstream graph isn't producing TC. bool any = false; for (auto& f : Frames) @@ -366,6 +372,8 @@ struct RecordTrackCOLMAPContext : NodeContext } file << std::setprecision(12); + file << "# COLMAP camera intrinsics. Standard format (colmap.github.io/format.html).\n"; + file << "# OPENCV model: PARAMS = fx, fy, cx, cy, k1, k2, p1, p2 (pixels).\n"; file << "# Camera list with one line of data per camera:\n"; file << "# CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n"; file << "# Number of cameras: " << Frames.size() << "\n"; @@ -387,7 +395,6 @@ struct RecordTrackCOLMAPContext : NodeContext if (Frames[i].SensorSize.y > 0.0f) cy += Frames[i].CenterShift.y * ImageResolution.y / Frames[i].SensorSize.y; - // OPENCV model: fx, fy, cx, cy, k1, k2, p1, p2 float k1 = Frames[i].K1; float k2 = Frames[i].K2; @@ -397,23 +404,6 @@ struct RecordTrackCOLMAPContext : NodeContext } } - static glm::mat3 EulerToRotationMatrix(glm::vec3 rot, sys::track::CoordinateSystem order) - { - // rot is (roll, tilt, pan) = (x, y, z) in radians - // Sign convention matches MakeRotation: negate roll (x) and tilt (y) - float r = -rot.x, t = -rot.y, p = rot.z; - switch (order) - { - default: - case sys::track::CoordinateSystem::ZYX: return glm::mat3(glm::eulerAngleZYX(p, t, r)); - case sys::track::CoordinateSystem::XYZ: return glm::mat3(glm::eulerAngleXYZ(r, t, p)); - case sys::track::CoordinateSystem::YXZ: return glm::mat3(glm::eulerAngleYXZ(t, r, p)); - case sys::track::CoordinateSystem::YZX: return glm::mat3(glm::eulerAngleYZX(t, p, r)); - case sys::track::CoordinateSystem::ZXY: return glm::mat3(glm::eulerAngleZXY(p, r, t)); - case sys::track::CoordinateSystem::XZY: return glm::mat3(glm::eulerAngleXZY(r, p, t)); - } - } - void WriteImagesTxt(const std::filesystem::path& outDir) { auto path = outDir / "images.txt"; @@ -425,28 +415,35 @@ struct RecordTrackCOLMAPContext : NodeContext } file << std::setprecision(12); + file << "# COLMAP poses. Standard format (colmap.github.io/format.html).\n"; + file << "# Frame: RH, +X right, +Y down, +Z forward (camera looks along +Z).\n"; + file << "# (QW, QX, QY, QZ) is the world-to-camera rotation R_w2c.\n"; + file << "# (TX, TY, TZ) is the world-to-camera translation: t = -R_w2c * camera_world_position.\n"; + file << "# Recover camera position in the COLMAP world frame as: C = -R_w2c^T * t.\n"; file << "# Image list with two lines of data per image:\n"; file << "# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n"; file << "# POINTS2D[] as (X, Y, POINT3D_ID)\n"; file << "# Number of images: " << Frames.size() << "\n"; + // M maps the SourceFrame to the COLMAP frame. Used to convert both the + // source-frame R_c2w and the source-frame camera position into COLMAP. + const glm::dmat3 M = convention::BasisChangeToColmap(SourceFrame); + const glm::dmat3 Minv = glm::inverse(M); + for (size_t i = 0; i < Frames.size(); ++i) { auto& frame = Frames[i]; - // Convert Euler angles to rotation matrix - // Sign convention matches MakeRotation: negate roll (x) and tilt (y) - glm::vec3 rot = glm::radians(frame.Rotation); - glm::mat3 R_c2w = EulerToRotationMatrix(rot, CoordSys); - - // COLMAP expects world-to-camera rotation - glm::mat3 R_w2c = glm::transpose(R_c2w); - glm::quat q_w2c = glm::quat_cast(R_w2c); + // Build R_c2w in the source frame, then conjugate by M to land in + // the COLMAP frame. Likewise frame the position. + glm::dmat3 R_c2w_src = convention::EulerToMat(SourceFrame, glm::dvec3(frame.Rotation)); + glm::dmat3 R_c2w_colmap = M * R_c2w_src * Minv; + glm::dvec3 pos_colmap = M * glm::dvec3(frame.Location); - // COLMAP translation: t = -R * C (camera center in world coords) - glm::vec3 t = -R_w2c * frame.Location; + glm::dmat3 R_w2c = glm::transpose(R_c2w_colmap); + glm::dquat q_w2c = glm::quat_cast(R_w2c); + glm::dvec3 t = -R_w2c * pos_colmap; - // IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME file << (i + 1) << " " << q_w2c.w << " " << q_w2c.x << " " << q_w2c.y << " " << q_w2c.z << " " << t.x << " " << t.y << " " << t.z << " " diff --git a/Plugins/nosTrack/Source/TrackMain.cpp b/Plugins/nosTrack/Source/TrackMain.cpp index acca3be8..48b43fcc 100644 --- a/Plugins/nosTrack/Source/TrackMain.cpp +++ b/Plugins/nosTrack/Source/TrackMain.cpp @@ -17,6 +17,7 @@ enum TrackNode : int AddTrack, RecordTrackCOLMAP, PlaybackTrackCOLMAP, + TrackTransform, Count }; @@ -25,6 +26,7 @@ void RegisterController(nosNodeFunctions* functions); void RegisterAddTrack(nosNodeFunctions*); void RegisterRecordTrackCOLMAP(nosNodeFunctions*); void RegisterPlaybackTrackCOLMAP(nosNodeFunctions*); +void RegisterTrackTransform(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** outList) { @@ -52,6 +54,9 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou case TrackNode::PlaybackTrackCOLMAP: RegisterPlaybackTrackCOLMAP(node); break; + case TrackNode::TrackTransform: + RegisterTrackTransform(node); + break; } } return NOS_RESULT_SUCCESS; diff --git a/Plugins/nosTrack/Source/TrackTransform.cpp b/Plugins/nosTrack/Source/TrackTransform.cpp new file mode 100644 index 00000000..0dabb1f1 --- /dev/null +++ b/Plugins/nosTrack/Source/TrackTransform.cpp @@ -0,0 +1,53 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include + +#include "CoordinateFrameConv.h" + +namespace nos::track +{ + +void RegisterTrackTransform(nosNodeFunctions* funcs) +{ + funcs->ClassName = NOS_NAME("TrackTransform"); + funcs->ExecuteNode = [](void*, nosNodeExecuteParams* params) { + auto pins = GetPinValues(params); + auto ids = GetPinIds(params); + + auto* inTrack = flatbuffers::GetMutableRoot(pins[NOS_NAME("In")]); + auto source = *static_cast(pins[NOS_NAME("Source")]); + auto target = *static_cast(pins[NOS_NAME("Target")]); + float worldScale = *static_cast(pins[NOS_NAME("WorldScale")]); + + nos::sys::track::TTrack out; + inTrack->UnPackTo(&out); + + const glm::dmat3 S_src = convention::BasisMatrix(source); + const glm::dmat3 S_tgt = convention::BasisMatrix(target); + const glm::dmat3 M = S_tgt * glm::inverse(S_src); + + // Location: basis change, then uniform world-scale. Other Track fields + // (rotation, fov, focus, sensor_size, lens_distortion, ...) are unaffected. + const auto& inLoc = *inTrack->location(); + glm::dvec3 loc(inLoc.x(), inLoc.y(), inLoc.z()); + glm::dvec3 outLoc = M * loc * static_cast(worldScale); + out.location.mutate_x(static_cast(outLoc.x)); + out.location.mutate_y(static_cast(outLoc.y)); + out.location.mutate_z(static_cast(outLoc.z)); + + // Rotation: build in source frame, conjugate by M, extract in target frame. + const auto& inRot = *inTrack->rotation(); + glm::dmat3 R_src = convention::EulerToMat(source, glm::dvec3(inRot.x(), inRot.y(), inRot.z())); + glm::dmat3 R_tgt = M * R_src * glm::transpose(M); + glm::dvec3 outRotDeg = convention::MatToEuler(target, R_tgt); + out.rotation.mutate_x(static_cast(outRotDeg.x)); + out.rotation.mutate_y(static_cast(outRotDeg.y)); + out.rotation.mutate_z(static_cast(outRotDeg.z)); + + return nosEngine.SetPinValue(ids[NOS_NAME("Out")], nos::Buffer::From(out)); + }; +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Track.noscfg b/Plugins/nosTrack/Track.noscfg index dc975d34..3810af05 100644 --- a/Plugins/nosTrack/Track.noscfg +++ b/Plugins/nosTrack/Track.noscfg @@ -18,7 +18,8 @@ "Config/UserTrack.nosdef", "Config/AddTrack.nosdef", "Config/RecordTrackCOLMAP.nosdef", - "Config/PlaybackTrackCOLMAP.nosdef" + "Config/PlaybackTrackCOLMAP.nosdef", + "Config/TrackTransform.nosdef" ], "custom_types": [ "Config/PlaybackMode.fbs" diff --git a/Subsystems/nosTrackSubsystem/Config/Track.fbs b/Subsystems/nosTrackSubsystem/Config/Track.fbs index 75e6e1ff..f36f2818 100644 --- a/Subsystems/nosTrackSubsystem/Config/Track.fbs +++ b/Subsystems/nosTrackSubsystem/Config/Track.fbs @@ -43,11 +43,17 @@ enum RotationSystem : uint { PRT = 5, } -enum EulerOrder : uint { - ZYX = 0, - XYZ = 1, - YXZ = 2, - YZX = 3, - ZXY = 4, - XZY = 5, +// World coordinate frame convention used by a Track endpoint. Encodes axis +// assignments to world-semantic directions (forward, right, up), the implied +// handedness, and the Euler convention for the Track.rotation field. +enum CoordinateFrame : ubyte { + // Left-handed, Z-up. +X forward, +Y right, +Z up. + // Rotation: rot.x = roll (X), rot.y = pitch (Y), rot.z = yaw (Z), + // intrinsic ZYX => R = Rz(yaw) * Ry(pitch) * Rx(roll). + LH_ZUp_FwdX_RightY = 0, + + // Right-handed, Y-up. +X right, +Y up, -Z forward. + // Rotation: rot.x = pitch (X), rot.y = yaw (Y), rot.z = roll (Z), + // intrinsic YXZ => R = Ry(yaw) * Rx(pitch) * Rz(roll). + RH_YUp_FwdNegZ_RightX = 1, } From 2bea82248767d8cf4dac788fd5f2472c505af63b Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Tue, 12 May 2026 10:21:23 +0300 Subject: [PATCH 24/27] Reorder pins in Track playback/record nodes --- .../Config/PlaybackTrackCOLMAP.nosdef | 14 ++++---- .../nosTrack/Config/RecordTrackCOLMAP.nosdef | 32 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef index 460f02dc..7f1604f2 100644 --- a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef @@ -67,13 +67,6 @@ "data": 0, "description": "Absolute frame number to look up. Used when Mode=FrameNumber. Requires timecodes.txt." }, - { - "name": "Track", - "type_name": "nos.sys.track.Track", - "show_as": "OUTPUT_PIN", - "can_show_as": "OUTPUT_PIN_ONLY", - "description": "Track for the current frame, expressed in the TargetFrame convention." - }, { "name": "OutFrameIndex", "display_name": "Frame Index", @@ -91,6 +84,13 @@ "can_show_as": "OUTPUT_PIN_ONLY", "data": 0, "description": "Total frames loaded." + }, + { + "name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "description": "Track for the current frame, expressed in the TargetFrame convention." } ], "functions": [ diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef index 25d70356..c26f52cb 100644 --- a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -14,14 +14,6 @@ "always_execute": true, "description": "Records camera track data each frame while Record is true.\nOn falling edge (after MinOffFrames debounce) writes COLMAP-spec cameras.txt + images.txt and clears the buffer.\nIntrinsics come from FOV/sensor/distortion. Extrinsics are written as world-to-camera in the COLMAP frame (RH, +X right, +Y down, +Z forward).\nSet SourceFrame to match the convention of the connected Track.", "pins": [ - { - "name": "InTrack", - "display_name": "Track", - "type_name": "nos.sys.track.Track", - "show_as": "INPUT_PIN", - "can_show_as": "INPUT_PIN_OR_PROPERTY", - "description": "Camera track to record. Interpreted in the SourceFrame convention (location, rotation Euler, FOV, sensor, lens distortion)." - }, { "name": "Timecode", "display_name": "Timecode", @@ -39,14 +31,6 @@ "data": 0, "description": "Optional absolute frame number paired with Timecode. Written to timecodes.txt sidecar." }, - { - "name": "OutTrack", - "display_name": "Track", - "type_name": "nos.sys.track.Track", - "show_as": "OUTPUT_PIN", - "can_show_as": "OUTPUT_PIN_ONLY", - "description": "Pass-through of InTrack." - }, { "name": "OutputDirectory", "display_name": "Output Directory", @@ -112,6 +96,22 @@ "can_show_as": "OUTPUT_PIN_ONLY", "data": 0, "description": "Frames in the buffer." + }, + { + "name": "InTrack", + "display_name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Camera track to record. Interpreted in the SourceFrame convention (location, rotation Euler, FOV, sensor, lens distortion)." + }, + { + "name": "OutTrack", + "display_name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "description": "Pass-through of InTrack." } ] } From 2e21bbbcf6f494aa087de0d4cd7bcb625bc7add1 Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Mon, 18 May 2026 15:47:20 +0300 Subject: [PATCH 25/27] Add depth-of-field filter nodes --- Plugins/nosFilters/Config/DepthOfField.nosdef | 990 ++++++++++++++++++ .../nosFilters/Config/DirectionalDof.nosdef | 121 +++ Plugins/nosFilters/Filters.noscfg | 2 + .../nosFilters/Shaders/DirectionalDof.frag | 95 ++ 4 files changed, 1208 insertions(+) create mode 100644 Plugins/nosFilters/Config/DepthOfField.nosdef create mode 100644 Plugins/nosFilters/Config/DirectionalDof.nosdef create mode 100644 Plugins/nosFilters/Shaders/DirectionalDof.frag diff --git a/Plugins/nosFilters/Config/DepthOfField.nosdef b/Plugins/nosFilters/Config/DepthOfField.nosdef new file mode 100644 index 00000000..1db5a70f --- /dev/null +++ b/Plugins/nosFilters/Config/DepthOfField.nosdef @@ -0,0 +1,990 @@ +{ "nodes": [ + { + "class_name": "nos.filters.DepthOfField", + "node": { + "id": "5899940c-437e-4f71-b119-bb80fb5d1e1a", + "name": "DepthOfField", + "class_name": "nos.filters.DepthOfField", + "pins": [ + { + "id": "1950c2e6-a0f6-485b-8a02-bded8a2f6ed5", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "74a1bfd0-4f2d-447b-945c-8d0cb67a2120" } + }, + { + "id": "e0b8f433-212f-48f6-ba4f-c8a194e1a707", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "PortalPin", + "contents": { "source_id": "e709c7b4-9a59-4546-be53-0dc51abc5605" } + }, + { + "id": "68187c92-92f3-40d0-8b24-df6f33f9f649", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 2.0, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "PortalPin", + "contents": { "source_id": "534a26e9-1ebd-4ed2-89fb-bdf5d34b6ec1" } + }, + { + "id": "42554a0a-2d70-4ec4-a2ea-594ad71559f3", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 16.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "PortalPin", + "contents": { "source_id": "63f77504-73aa-4b89-8849-65e27649b272" } + }, + { + "id": "312c4450-a4ad-4690-ba3d-afcbc93da6eb", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.5, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "PortalPin", + "contents": { "source_id": "97561978-6da1-4a33-a6bc-c654008a8261" } + }, + { + "id": "ce6c0d45-8ce1-47ef-bd73-addda06d826e", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "c278680b-43b5-40ce-b1af-a4551c2e58f0" } + }, + { + "id": "2be9d3ba-9386-43b0-ae1c-58168be2a289", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "479248dc-200d-4a4d-87d2-f2c7c77f667f" } + } + ], + "pos": { "x": 0.0, "y": 0.0 }, + "contents_type": "Graph", + "contents": { "nodes": [ + { + "id": "393281e0-2cb8-4b90-a98e-a8e708719229", + "name": "Output", + "class_name": "nos.internal.GraphOutput", + "pins": [ + { + "id": "2e4ec877-e014-49ae-ae1d-881a0e4d1ac5", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "c278680b-43b5-40ce-b1af-a4551c2e58f0", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "referred_by": [ + "ce6c0d45-8ce1-47ef-bd73-addda06d826e" + ], + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 1329.0, "y": 1025.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "6a261add-ff1c-49ba-b9b7-a3bbad8e1fb3", + "name": "Directional DoF (1)", + "class_name": "nos.filters.DirectionalDof", + "pins": [ + { + "id": "b1b03fce-6863-42e6-a78a-260743b5441d", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "14aab6c6-10ce-4a39-9c6f-8c5633fe59e2", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "d6a91b7a-b576-487b-bd2c-89fee90a37d1", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "55bffdc3-e0fa-4c0f-9ead-5a3b96c232bf", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "bdc5ae5a-10cc-4c3c-b013-573a64bd8ec6", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "d77d3716-69f5-4c5d-a342-414dc11597fb", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ad25df82-1942-4f9f-a062-c072261a2d92", + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 1.0, + "min": 0.0, + "max": 1.0, + "def": 1.0, + "step": 0.01, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "aaff92e1-63fe-4253-8edb-1f34a76019c9", + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { "x": 0.0, "y": 1.0 }, + "min": { "x": -1.0, "y": -1.0 }, + "max": { "x": 1.0, "y": 1.0 }, + "def": { "x": 1.0, "y": 0.0 }, + "step": 0.02, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ad6603e0-2b1d-4bf6-a1d1-af0fc05978a2", + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 12.0, + "min": 1.0, + "max": 64.0, + "def": 12.0, + "step": 0.63, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "0ef0f439-9766-4957-8931-a02ce1019bd1", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 1129.0, "y": 1073.0 }, + "contents_type": "Job", + "contents": { "type": "nos.sys.vulkan.GPUNode", "options": { "shader": "Shaders/DirectionalDof.frag" } }, + "function_category": "Default Node", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "plugin_version": { "major": 1, "minor": 7, "patch": 0 } + }, + { + "id": "deac982f-b51b-4ae0-b6c6-9b2998d3e5a9", + "name": "MaxRadius", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "63f77504-73aa-4b89-8849-65e27649b272", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 16.0, + "referred_by": [ + "42554a0a-2d70-4ec4-a2ea-594ad71559f3" + ], + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1250.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "af576b2d-dde0-4d7b-86fc-37cb9f97b49e", + "name": "Directional DoF", + "class_name": "nos.filters.DirectionalDof", + "pins": [ + { + "id": "9e368dde-bb31-44d8-aaad-782e92fe2366", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "7e88a91a-1eca-4cc5-8dce-7c4aca61368d", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "c1b814c4-e424-40a0-99d6-0437d948d1d7", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "78893474-3dfc-4a36-b897-77760ba19c8c", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "4f471215-bebf-49be-a6e4-909c394d1f1a", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "05132381-cf95-4253-9fc2-e87f84b70dd8", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "2b52da00-b45d-41ae-a1ec-c88566879043", + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 1.0, + "min": 0.0, + "max": 1.0, + "def": 1.0, + "step": 0.01, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ac684e31-1a90-462f-8b64-2b368a93b563", + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { "x": 1.0, "y": 0.0 }, + "min": { "x": -1.0, "y": -1.0 }, + "max": { "x": 1.0, "y": 1.0 }, + "def": { "x": 1.0, "y": 0.0 }, + "step": 0.02, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "b8527f03-5c5c-4a41-b485-fa05e0f50cb1", + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 12.0, + "min": 1.0, + "max": 64.0, + "def": 12.0, + "step": 0.63, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "8f4d23a7-3b94-4a1c-ba14-d1ce47e92acd", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 855.0, "y": 977.0 }, + "contents_type": "Job", + "contents": { "type": "nos.sys.vulkan.GPUNode", "options": { "shader": "Shaders/DirectionalDof.frag" } }, + "function_category": "Default Node", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "plugin_version": { "major": 1, "minor": 7, "patch": 0 } + }, + { + "id": "8b497dab-5466-4d32-a440-125976e3a3ee", + "name": "Depth", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "9587b7b1-8fc7-437b-9459-ee73f90de097", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "74a1bfd0-4f2d-447b-945c-8d0cb67a2120", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "referred_by": [ + "1950c2e6-a0f6-485b-8a02-bded8a2f6ed5" + ], + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1025.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "9813ee9d-1f75-4554-9f9c-b9ecafc2e9fe", + "name": "FocusDistance", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "7c60934b-ba19-4faf-9923-411511649cd0", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "e709c7b4-9a59-4546-be53-0dc51abc5605", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "referred_by": [ + "e0b8f433-212f-48f6-ba4f-c8a194e1a707" + ], + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1100.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "2c0861b9-e416-4741-b56d-8dfa81c49516", + "name": "FocusRange", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "6a933bab-7bf6-4388-b990-abd1b9729e64", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "534a26e9-1ebd-4ed2-89fb-bdf5d34b6ec1", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 2.0, + "referred_by": [ + "68187c92-92f3-40d0-8b24-df6f33f9f649" + ], + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1175.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "3951aaae-16df-4b07-b1a9-b8b2a01b19c7", + "name": "MinRadius", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "f0feee29-3782-49fe-a834-94e2b57916a8", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "97561978-6da1-4a33-a6bc-c654008a8261", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.5, + "referred_by": [ + "312c4450-a4ad-4690-ba3d-afcbc93da6eb" + ], + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1325.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "a1d16ddd-0144-4daa-97b2-e9b3b019c8c1", + "name": "Input", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "c271ac23-2923-45c2-b262-b654455a93c3", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "479248dc-200d-4a4d-87d2-f2c7c77f667f", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "referred_by": [ + "2be9d3ba-9386-43b0-ae1c-58168be2a289" + ], + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1400.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + } + ], "connections": [ + { "from": "8f4d23a7-3b94-4a1c-ba14-d1ce47e92acd", "to": "b1b03fce-6863-42e6-a78a-260743b5441d", "id": "83839676-0760-4699-ae80-c0a789e273d8" }, + { "from": "f0feee29-3782-49fe-a834-94e2b57916a8", "to": "d77d3716-69f5-4c5d-a342-414dc11597fb", "id": "4c05135f-6001-4679-b39c-b248559ae56d" }, + { "from": "9587b7b1-8fc7-437b-9459-ee73f90de097", "to": "14aab6c6-10ce-4a39-9c6f-8c5633fe59e2", "id": "231cdfe5-7ac7-4013-9d20-68d5af8509b7" }, + { "from": "7c60934b-ba19-4faf-9923-411511649cd0", "to": "d6a91b7a-b576-487b-bd2c-89fee90a37d1", "id": "1cdaef73-876c-472a-97ff-04bf1f01348e" }, + { "from": "c271ac23-2923-45c2-b262-b654455a93c3", "to": "9e368dde-bb31-44d8-aaad-782e92fe2366", "id": "231fc88c-a52e-48d0-a6ee-8c2fdfe3ef0d" }, + { "from": "6a933bab-7bf6-4388-b990-abd1b9729e64", "to": "55bffdc3-e0fa-4c0f-9ead-5a3b96c232bf", "id": "7c1cae59-5834-420e-9d3d-e4767f6c3273" }, + { "from": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", "to": "bdc5ae5a-10cc-4c3c-b013-573a64bd8ec6", "id": "d74bdb3a-8c8c-4f82-8038-01a237e27a89" }, + { "from": "0ef0f439-9766-4957-8931-a02ce1019bd1", "to": "2e4ec877-e014-49ae-ae1d-881a0e4d1ac5", "id": "353cc954-d098-417a-8331-357b879ba654" }, + { "from": "9587b7b1-8fc7-437b-9459-ee73f90de097", "to": "7e88a91a-1eca-4cc5-8dce-7c4aca61368d", "id": "b126f4c4-d748-46f2-be51-ce1c778c0c4b" }, + { "from": "7c60934b-ba19-4faf-9923-411511649cd0", "to": "c1b814c4-e424-40a0-99d6-0437d948d1d7", "id": "fc25a2f4-0af4-49ae-9052-133a76cfc044" }, + { "from": "6a933bab-7bf6-4388-b990-abd1b9729e64", "to": "78893474-3dfc-4a36-b897-77760ba19c8c", "id": "f6ba18f8-0ef1-42db-a774-c4b02aa78fac" }, + { "from": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", "to": "4f471215-bebf-49be-a6e4-909c394d1f1a", "id": "afd9d7ff-f9e2-4a67-b874-2cfb2f870447" }, + { "from": "f0feee29-3782-49fe-a834-94e2b57916a8", "to": "05132381-cf95-4253-9fc2-e87f84b70dd8", "id": "82919455-4a51-490a-8ab2-201952d2e126" } + ] }, + "function_category": "Default Node", + "display_name": "Depth of Field", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + } + } + ] } diff --git a/Plugins/nosFilters/Config/DirectionalDof.nosdef b/Plugins/nosFilters/Config/DirectionalDof.nosdef new file mode 100644 index 00000000..427e4385 --- /dev/null +++ b/Plugins/nosFilters/Config/DirectionalDof.nosdef @@ -0,0 +1,121 @@ +{ + "nodes": [ + { + "class_name": "DirectionalDof", + "menu_info": { + "category": "Filters", + "display_name": "Directional DoF" + }, + "node": { + "class_name": "DirectionalDof", + "name": "Directional DoF", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/DirectionalDof.frag" + } + }, + "pins": [ + { + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "NEAREST" + } + }, + { + "name": "FocusDistance", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 5.0, + "min": 0.0, + "max": 1000.0 + }, + { + "name": "FocusRange", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 2.0, + "min": 0.01, + "max": 1000.0 + }, + { + "name": "MaxRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 16.0, + "min": 0.0, + "max": 128.0 + }, + { + "name": "MinRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.5, + "min": 0.0, + "max": 8.0 + }, + { + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "min": 0.0, + "max": 1.0 + }, + { + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "x": 1.0, + "y": 0.0 + }, + "min": { + "x": -1.0, + "y": -1.0 + }, + "max": { + "x": 1.0, + "y": 1.0 + } + }, + { + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 12.0, + "min": 1.0, + "max": 64.0 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Filters.noscfg b/Plugins/nosFilters/Filters.noscfg index 072c6ce0..18bac701 100644 --- a/Plugins/nosFilters/Filters.noscfg +++ b/Plugins/nosFilters/Filters.noscfg @@ -27,6 +27,8 @@ "Config/Diff.nosdef", "Config/GaussianBlur.nosdef", "Config/DirectionalBlur.nosdef", + "Config/DirectionalDof.nosdef", + "Config/DepthOfField.nosdef", "Config/KawaseLightStreak.nosdef", "Config/Kuwahara.nosdef", "Config/PremultiplyAlpha.nosdef", diff --git a/Plugins/nosFilters/Shaders/DirectionalDof.frag b/Plugins/nosFilters/Shaders/DirectionalDof.frag new file mode 100644 index 00000000..308cfcc7 --- /dev/null +++ b/Plugins/nosFilters/Shaders/DirectionalDof.frag @@ -0,0 +1,95 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Directional depth-of-field pass. +// Computes circle-of-confusion (CoC) per pixel from a linear view-space Z input, +// then does a 1D weighted gather along Direction. Chain two instances +// (Direction = (1,0) and Direction = (0,1)) for a separable approximation of +// disc bokeh; visually close to a gaussian bokeh and cheap. + +#version 450 + +#define MASK_THRESHOLD 0.001 + +layout(binding = 0) uniform sampler2D Input; +layout(binding = 1) uniform sampler2D Depth; +layout(binding = 2) uniform DirectionalDofParams +{ + // Focus distance in the same units as the Depth input (linear view-space Z). + float FocusDistance; + // Distance from focus where CoC reaches MaxRadius. + // Smaller value = sharper focus falloff; larger = gentler. + float FocusRange; + // Maximum CoC radius in pixels. + float MaxRadius; + // 0 = treat zero depth as "no info, keep sharp"; 1 = treat zero depth as far. + float BackgroundIsFar; + vec2 Direction; + // Optional: clamp CoC near the focus plane to avoid noise; raise to skip tiny blurs. + float MinRadius; + // Sample count along the direction (one side; total taps = 2*N+1). Higher = smoother. + float SampleCount; +} +Params; + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +float CocFromDepth(float Z) +{ + // Treat Z<=0 (no depth signal) as either "near focus" (BackgroundIsFar=0) + // or as far plane (BackgroundIsFar=1). Picking far avoids halos around empty regions. + if (Z <= 0.0) + Z = mix(Params.FocusDistance, Params.FocusDistance + Params.FocusRange * 4.0, Params.BackgroundIsFar); + + float D = abs(Z - Params.FocusDistance); + float Coc = D / max(Params.FocusRange, 1e-4); + Coc = clamp(Coc * Params.MaxRadius, 0.0, Params.MaxRadius); + return Coc; +} + +void main() +{ + vec2 TextureSize = textureSize(Input, 0); + vec2 TexelSize = 1.0 / TextureSize; + + vec4 CenterColor = texture(Input, uv); + float CenterZ = texture(Depth, uv).r; + float CenterCoC = CocFromDepth(CenterZ); + + if (CenterCoC <= Params.MinRadius || Params.MaxRadius < MASK_THRESHOLD) + { + rt = CenterColor; + return; + } + + vec2 Dir = normalize(Params.Direction); + + int N = int(max(1.0, Params.SampleCount)); + float RadiusPx = CenterCoC; + float Step = RadiusPx / float(N); + + // Box-weighted average; for separable-2D this gives a soft disc. + // CoC-clamping per sample prevents fragments in focus from bleeding outward. + vec4 Accum = CenterColor; + float Weight = 1.0; + + for (int i = 1; i <= N; ++i) + { + float T = float(i) * Step; + vec2 Ofs = Dir * T * TexelSize; + + vec4 SPos = texture(Input, uv + Ofs); + float ZPos = texture(Depth, uv + Ofs).r; + float CocPos = CocFromDepth(ZPos); + float WPos = Step <= CocPos ? 1.0 : 0.0; + + vec4 SNeg = texture(Input, uv - Ofs); + float ZNeg = texture(Depth, uv - Ofs).r; + float CocNeg = CocFromDepth(ZNeg); + float WNeg = Step <= CocNeg ? 1.0 : 0.0; + + Accum += SPos * WPos + SNeg * WNeg; + Weight += WPos + WNeg; + } + + rt = Accum / Weight; +} From 187675d8c82d30e3f7bee1c1ef2698aa9ed18d4a Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Mon, 18 May 2026 15:58:29 +0300 Subject: [PATCH 26/27] Bump nos.filters to 1.8.0 --- Plugins/nosFilters/Filters.noscfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Plugins/nosFilters/Filters.noscfg b/Plugins/nosFilters/Filters.noscfg index 18bac701..c9ebb2ce 100644 --- a/Plugins/nosFilters/Filters.noscfg +++ b/Plugins/nosFilters/Filters.noscfg @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.filters", - "version": "1.7.0" + "version": "1.8.0" }, "display_name": "Filters", "description": "Collection of image filters.", From 17a5edc1e38b38c21beefef381d8b9dd3cbc593c Mon Sep 17 00:00:00 2001 From: "M. Samil Atesoglu" Date: Mon, 18 May 2026 18:13:23 +0300 Subject: [PATCH 27/27] Add BokehDof and BokehShape nodes --- Plugins/nosFilters/Config/BokehDof.nosdef | 121 ++++++++++++++++++++ Plugins/nosFilters/Config/BokehShape.nosdef | 93 +++++++++++++++ Plugins/nosFilters/Filters.noscfg | 2 + Plugins/nosFilters/Shaders/BokehDof.frag | 105 +++++++++++++++++ Plugins/nosFilters/Shaders/BokehShape.frag | 77 +++++++++++++ 5 files changed, 398 insertions(+) create mode 100644 Plugins/nosFilters/Config/BokehDof.nosdef create mode 100644 Plugins/nosFilters/Config/BokehShape.nosdef create mode 100644 Plugins/nosFilters/Shaders/BokehDof.frag create mode 100644 Plugins/nosFilters/Shaders/BokehShape.frag diff --git a/Plugins/nosFilters/Config/BokehDof.nosdef b/Plugins/nosFilters/Config/BokehDof.nosdef new file mode 100644 index 00000000..df5b08fb --- /dev/null +++ b/Plugins/nosFilters/Config/BokehDof.nosdef @@ -0,0 +1,121 @@ +{ + "nodes": [ + { + "class_name": "BokehDof", + "menu_info": { + "category": "Filters", + "display_name": "Bokeh DoF" + }, + "node": { + "class_name": "BokehDof", + "name": "Bokeh DoF", + "description": "Single-pass 2D depth-of-field. CoC is computed from a linear view-space Z input; samples are gathered on a Vogel disc weighted by the BokehShape kernel texture, so bokeh takes the shape painted into BokehShape.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/BokehDof.frag" + } + }, + "pins": [ + { + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "NEAREST" + } + }, + { + "name": "BokehShape", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "FocusDistance", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 5.0, + "min": 0.0, + "max": 1000.0 + }, + { + "name": "FocusRange", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 2.0, + "min": 0.01, + "max": 1000.0 + }, + { + "name": "MaxRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 16.0, + "min": 0.0, + "max": 128.0 + }, + { + "name": "MinRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.5, + "min": 0.0, + "max": 8.0 + }, + { + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "min": 0.0, + "max": 1.0 + }, + { + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 48.0, + "min": 4.0, + "max": 256.0 + }, + { + "name": "KernelRotation", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": -6.2832, + "max": 6.2832 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Config/BokehShape.nosdef b/Plugins/nosFilters/Config/BokehShape.nosdef new file mode 100644 index 00000000..3a466b9a --- /dev/null +++ b/Plugins/nosFilters/Config/BokehShape.nosdef @@ -0,0 +1,93 @@ +{ + "nodes": [ + { + "class_name": "BokehShape", + "menu_info": { + "category": "Filters", + "display_name": "Bokeh Shape" + }, + "node": { + "class_name": "BokehShape", + "name": "Bokeh Shape", + "description": "Procedural bokeh kernel generator. Produces a unit-disc grayscale mask shaped like a regular polygon aperture (blade count, roundness, rotation), with soft edge and optional rim brightening. Feed the Output into a Bokeh DoF node's BokehShape pin.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/BokehShape.frag" + } + }, + "pins": [ + { + "name": "BladeCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 6.0, + "min": 0.0, + "max": 16.0 + }, + { + "name": "Roundness", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.3, + "min": 0.0, + "max": 1.0 + }, + { + "name": "Rotation", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": -6.2832, + "max": 6.2832 + }, + { + "name": "EdgeSoftness", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.04, + "min": 0.0, + "max": 0.5 + }, + { + "name": "RimBoost", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": 0.0, + "max": 4.0 + }, + { + "name": "RimWidth", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.08, + "min": 0.005, + "max": 0.5 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "data": { + "resolution": "CUSTOM", + "width": 128, + "height": 128, + "format": "R16_UNORM", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET", + "filtering": "LINEAR" + } + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Filters.noscfg b/Plugins/nosFilters/Filters.noscfg index c9ebb2ce..3660bb3f 100644 --- a/Plugins/nosFilters/Filters.noscfg +++ b/Plugins/nosFilters/Filters.noscfg @@ -29,6 +29,8 @@ "Config/DirectionalBlur.nosdef", "Config/DirectionalDof.nosdef", "Config/DepthOfField.nosdef", + "Config/BokehDof.nosdef", + "Config/BokehShape.nosdef", "Config/KawaseLightStreak.nosdef", "Config/Kuwahara.nosdef", "Config/PremultiplyAlpha.nosdef", diff --git a/Plugins/nosFilters/Shaders/BokehDof.frag b/Plugins/nosFilters/Shaders/BokehDof.frag new file mode 100644 index 00000000..b365ddf2 --- /dev/null +++ b/Plugins/nosFilters/Shaders/BokehDof.frag @@ -0,0 +1,105 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Single-pass 2D bokeh depth-of-field with a kernel-texture shaping the bokeh. +// +// Computes a per-pixel circle of confusion (CoC) from a linear view-space Z +// input, then gathers samples on a Vogel (golden-angle) disc within that CoC. +// Each sample's contribution is weighted by BokehShape sampled at the same +// unit-disc position, so the bokeh takes on the shape painted into BokehShape +// (regular polygon, ring, custom artwork, etc.). + +#version 450 + +#define MASK_THRESHOLD 0.001 +#define GOLDEN_ANGLE 2.39996322972865332 + +layout(binding = 0) uniform sampler2D Input; +layout(binding = 1) uniform sampler2D Depth; +layout(binding = 2) uniform sampler2D BokehShape; +layout(binding = 3) uniform BokehDofParams +{ + // Focus distance in the same units as the Depth input (linear view-space Z). + float FocusDistance; + // Distance from focus where CoC reaches MaxRadius. + float FocusRange; + // Maximum CoC radius in pixels. + float MaxRadius; + // Skip the gather when CoC <= MinRadius (keeps focused regions crisp & cheap). + float MinRadius; + // 0 = treat zero depth as "near focus" (stays sharp); 1 = treat as far plane. + float BackgroundIsFar; + // Total Vogel-disc sample count. ~32 = soft, ~64 = clean, ~128 = no banding. + float SampleCount; + // Rotate the kernel lookup (radians). Useful for animated highlights. + float KernelRotation; +} +Params; + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +float CocFromDepth(float Z) +{ + if (Z <= 0.0) + Z = mix(Params.FocusDistance, Params.FocusDistance + Params.FocusRange * 4.0, Params.BackgroundIsFar); + + float D = abs(Z - Params.FocusDistance); + float Coc = D / max(Params.FocusRange, 1e-4); + return clamp(Coc * Params.MaxRadius, 0.0, Params.MaxRadius); +} + +void main() +{ + vec2 TextureSize = textureSize(Input, 0); + vec2 TexelSize = 1.0 / TextureSize; + + vec4 CenterColor = texture(Input, uv); + float CenterZ = texture(Depth, uv).r; + float CenterCoC = CocFromDepth(CenterZ); + + if (CenterCoC <= Params.MinRadius || Params.MaxRadius < MASK_THRESHOLD) + { + rt = CenterColor; + return; + } + + int N = int(max(1.0, Params.SampleCount)); + float CosR = cos(Params.KernelRotation); + float SinR = sin(Params.KernelRotation); + + // Vogel disc: golden-angle spiral with sqrt radius for uniform area density. + // Sample 0 is the center; included implicitly via CenterColor initialization. + vec4 Accum = CenterColor; + float Weight = texture(BokehShape, vec2(0.5)).r; + Accum *= Weight; + + for (int i = 1; i < N; ++i) + { + float Frac = float(i) / float(N); + float R = sqrt(Frac); // unit-disc radius + float Th = float(i) * GOLDEN_ANGLE; + vec2 Unit = vec2(cos(Th) * R, sin(Th) * R); // unit disc position + + // Rotated lookup into the bokeh kernel. + vec2 ShapeUv = vec2(Unit.x * CosR - Unit.y * SinR, + Unit.x * SinR + Unit.y * CosR) * 0.5 + 0.5; + float WShape = texture(BokehShape, ShapeUv).r; + if (WShape <= MASK_THRESHOLD) + continue; + + vec2 Ofs = Unit * CenterCoC * TexelSize; + vec4 Sample = texture(Input, uv + Ofs); + float ZSamp = texture(Depth, uv + Ofs).r; + float CocSmp = CocFromDepth(ZSamp); + + // Per-sample CoC rejection prevents in-focus pixels bleeding outward. + // A sample contributes only if its own CoC is at least its distance from center. + float Dist = R * CenterCoC; + float WCoc = Dist <= CocSmp ? 1.0 : 0.0; + + float W = WShape * WCoc; + Accum += Sample * W; + Weight += W; + } + + rt = Accum / max(Weight, 1e-4); +} diff --git a/Plugins/nosFilters/Shaders/BokehShape.frag b/Plugins/nosFilters/Shaders/BokehShape.frag new file mode 100644 index 00000000..cb963629 --- /dev/null +++ b/Plugins/nosFilters/Shaders/BokehShape.frag @@ -0,0 +1,77 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Procedural bokeh kernel generator. +// +// Produces a grayscale unit-disc mask shaped like a regular polygon aperture +// (number of blades configurable) with optional roundness, rotation, soft edge +// and brightened rim. Intended as input to a kernel-weighted DoF gather. +// +// Convention: image is treated as the [-1, 1] unit square; pixels outside the +// kernel shape return 0; pixels inside return ~1, with a smooth edge falloff +// over EdgeSoftness. The mask is normalized so that center stays at 1. + +#version 450 + +#define PI 3.14159265358979323846 + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +layout(binding = 1) uniform BokehShapeParams +{ + // Aperture blade count. 0 or 1 = perfect circle. + float BladeCount; + // 0 = sharp polygon, 1 = perfect circle. Interpolates polygon edge toward disc. + float Roundness; + // Rotation of the polygon (radians). + float Rotation; + // Soft falloff width at the edge, in [0, 1] of unit-disc radius. + float EdgeSoftness; + // Extra brightness boost near the rim, [0, 1]. Mimics cat's-eye / specular bokeh. + float RimBoost; + // Width of the rim brightening band, in [0, 1] of radius. + float RimWidth; +} +Params; + +void main() +{ + // Map uv [0,1] to centered coords [-1,1] + vec2 Pos = uv * 2.0 - 1.0; + float R = length(Pos); + + if (R > 1.0) + { + rt = vec4(0.0); + return; + } + + float Blades = max(Params.BladeCount, 1.0); + + // Polygon edge radius along this angular direction. + // sectorAngle = 2*pi / N; angle from sector center is a; edge distance = cos(pi/N) / cos(a). + float PolygonR = 1.0; + if (Blades >= 3.0) + { + float Theta = atan(Pos.y, Pos.x) - Params.Rotation; + float SectorAngle = 2.0 * PI / Blades; + float HalfSector = SectorAngle * 0.5; + // Angle measured from the nearest sector centerline, in [-HalfSector, +HalfSector]. + float A = mod(Theta + HalfSector, SectorAngle) - HalfSector; + PolygonR = cos(HalfSector) / max(cos(A), 1e-4); + } + + // Roundness mixes polygon edge toward the circumscribed circle (radius 1). + float EdgeR = mix(PolygonR, 1.0, clamp(Params.Roundness, 0.0, 1.0)); + + // Soft edge: 1 inside, 0 past the edge, smooth across EdgeSoftness. + float Soft = max(Params.EdgeSoftness, 1e-4); + float Mask = 1.0 - smoothstep(EdgeR - Soft, EdgeR, R); + + // Rim brightening: a soft band just inside the edge. + float RimW = max(Params.RimWidth, 1e-4); + float RimPos = (R - (EdgeR - RimW)) / RimW; // 0 at inner edge of rim, 1 at outer + float Rim = clamp(1.0 - abs(RimPos * 2.0 - 1.0), 0.0, 1.0); + Mask += Rim * Params.RimBoost * Mask; + + rt = vec4(Mask, Mask, Mask, 1.0); +}