diff --git a/Plugins/nosFilters/Config/BokehDof.nosdef b/Plugins/nosFilters/Config/BokehDof.nosdef new file mode 100644 index 00000000..df5b08fb --- /dev/null +++ b/Plugins/nosFilters/Config/BokehDof.nosdef @@ -0,0 +1,121 @@ +{ + "nodes": [ + { + "class_name": "BokehDof", + "menu_info": { + "category": "Filters", + "display_name": "Bokeh DoF" + }, + "node": { + "class_name": "BokehDof", + "name": "Bokeh DoF", + "description": "Single-pass 2D depth-of-field. CoC is computed from a linear view-space Z input; samples are gathered on a Vogel disc weighted by the BokehShape kernel texture, so bokeh takes the shape painted into BokehShape.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/BokehDof.frag" + } + }, + "pins": [ + { + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "NEAREST" + } + }, + { + "name": "BokehShape", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "FocusDistance", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 5.0, + "min": 0.0, + "max": 1000.0 + }, + { + "name": "FocusRange", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 2.0, + "min": 0.01, + "max": 1000.0 + }, + { + "name": "MaxRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 16.0, + "min": 0.0, + "max": 128.0 + }, + { + "name": "MinRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.5, + "min": 0.0, + "max": 8.0 + }, + { + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "min": 0.0, + "max": 1.0 + }, + { + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 48.0, + "min": 4.0, + "max": 256.0 + }, + { + "name": "KernelRotation", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": -6.2832, + "max": 6.2832 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Config/BokehShape.nosdef b/Plugins/nosFilters/Config/BokehShape.nosdef new file mode 100644 index 00000000..3a466b9a --- /dev/null +++ b/Plugins/nosFilters/Config/BokehShape.nosdef @@ -0,0 +1,93 @@ +{ + "nodes": [ + { + "class_name": "BokehShape", + "menu_info": { + "category": "Filters", + "display_name": "Bokeh Shape" + }, + "node": { + "class_name": "BokehShape", + "name": "Bokeh Shape", + "description": "Procedural bokeh kernel generator. Produces a unit-disc grayscale mask shaped like a regular polygon aperture (blade count, roundness, rotation), with soft edge and optional rim brightening. Feed the Output into a Bokeh DoF node's BokehShape pin.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/BokehShape.frag" + } + }, + "pins": [ + { + "name": "BladeCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 6.0, + "min": 0.0, + "max": 16.0 + }, + { + "name": "Roundness", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.3, + "min": 0.0, + "max": 1.0 + }, + { + "name": "Rotation", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": -6.2832, + "max": 6.2832 + }, + { + "name": "EdgeSoftness", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.04, + "min": 0.0, + "max": 0.5 + }, + { + "name": "RimBoost", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": 0.0, + "max": 4.0 + }, + { + "name": "RimWidth", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.08, + "min": 0.005, + "max": 0.5 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "data": { + "resolution": "CUSTOM", + "width": 128, + "height": 128, + "format": "R16_UNORM", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET", + "filtering": "LINEAR" + } + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Config/DepthOfField.nosdef b/Plugins/nosFilters/Config/DepthOfField.nosdef new file mode 100644 index 00000000..1db5a70f --- /dev/null +++ b/Plugins/nosFilters/Config/DepthOfField.nosdef @@ -0,0 +1,990 @@ +{ "nodes": [ + { + "class_name": "nos.filters.DepthOfField", + "node": { + "id": "5899940c-437e-4f71-b119-bb80fb5d1e1a", + "name": "DepthOfField", + "class_name": "nos.filters.DepthOfField", + "pins": [ + { + "id": "1950c2e6-a0f6-485b-8a02-bded8a2f6ed5", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "74a1bfd0-4f2d-447b-945c-8d0cb67a2120" } + }, + { + "id": "e0b8f433-212f-48f6-ba4f-c8a194e1a707", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "PortalPin", + "contents": { "source_id": "e709c7b4-9a59-4546-be53-0dc51abc5605" } + }, + { + "id": "68187c92-92f3-40d0-8b24-df6f33f9f649", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 2.0, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "PortalPin", + "contents": { "source_id": "534a26e9-1ebd-4ed2-89fb-bdf5d34b6ec1" } + }, + { + "id": "42554a0a-2d70-4ec4-a2ea-594ad71559f3", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 16.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "PortalPin", + "contents": { "source_id": "63f77504-73aa-4b89-8849-65e27649b272" } + }, + { + "id": "312c4450-a4ad-4690-ba3d-afcbc93da6eb", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.5, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "PortalPin", + "contents": { "source_id": "97561978-6da1-4a33-a6bc-c654008a8261" } + }, + { + "id": "ce6c0d45-8ce1-47ef-bd73-addda06d826e", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "c278680b-43b5-40ce-b1af-a4551c2e58f0" } + }, + { + "id": "2be9d3ba-9386-43b0-ae1c-58168be2a289", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "479248dc-200d-4a4d-87d2-f2c7c77f667f" } + } + ], + "pos": { "x": 0.0, "y": 0.0 }, + "contents_type": "Graph", + "contents": { "nodes": [ + { + "id": "393281e0-2cb8-4b90-a98e-a8e708719229", + "name": "Output", + "class_name": "nos.internal.GraphOutput", + "pins": [ + { + "id": "2e4ec877-e014-49ae-ae1d-881a0e4d1ac5", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "c278680b-43b5-40ce-b1af-a4551c2e58f0", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "referred_by": [ + "ce6c0d45-8ce1-47ef-bd73-addda06d826e" + ], + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 1329.0, "y": 1025.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "6a261add-ff1c-49ba-b9b7-a3bbad8e1fb3", + "name": "Directional DoF (1)", + "class_name": "nos.filters.DirectionalDof", + "pins": [ + { + "id": "b1b03fce-6863-42e6-a78a-260743b5441d", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "14aab6c6-10ce-4a39-9c6f-8c5633fe59e2", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "d6a91b7a-b576-487b-bd2c-89fee90a37d1", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "55bffdc3-e0fa-4c0f-9ead-5a3b96c232bf", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "bdc5ae5a-10cc-4c3c-b013-573a64bd8ec6", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "d77d3716-69f5-4c5d-a342-414dc11597fb", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ad25df82-1942-4f9f-a062-c072261a2d92", + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 1.0, + "min": 0.0, + "max": 1.0, + "def": 1.0, + "step": 0.01, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "aaff92e1-63fe-4253-8edb-1f34a76019c9", + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { "x": 0.0, "y": 1.0 }, + "min": { "x": -1.0, "y": -1.0 }, + "max": { "x": 1.0, "y": 1.0 }, + "def": { "x": 1.0, "y": 0.0 }, + "step": 0.02, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ad6603e0-2b1d-4bf6-a1d1-af0fc05978a2", + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 12.0, + "min": 1.0, + "max": 64.0, + "def": 12.0, + "step": 0.63, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "0ef0f439-9766-4957-8931-a02ce1019bd1", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 1129.0, "y": 1073.0 }, + "contents_type": "Job", + "contents": { "type": "nos.sys.vulkan.GPUNode", "options": { "shader": "Shaders/DirectionalDof.frag" } }, + "function_category": "Default Node", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "plugin_version": { "major": 1, "minor": 7, "patch": 0 } + }, + { + "id": "deac982f-b51b-4ae0-b6c6-9b2998d3e5a9", + "name": "MaxRadius", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "63f77504-73aa-4b89-8849-65e27649b272", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 16.0, + "referred_by": [ + "42554a0a-2d70-4ec4-a2ea-594ad71559f3" + ], + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1250.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "af576b2d-dde0-4d7b-86fc-37cb9f97b49e", + "name": "Directional DoF", + "class_name": "nos.filters.DirectionalDof", + "pins": [ + { + "id": "9e368dde-bb31-44d8-aaad-782e92fe2366", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "7e88a91a-1eca-4cc5-8dce-7c4aca61368d", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "c1b814c4-e424-40a0-99d6-0437d948d1d7", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "78893474-3dfc-4a36-b897-77760ba19c8c", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "4f471215-bebf-49be-a6e4-909c394d1f1a", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "05132381-cf95-4253-9fc2-e87f84b70dd8", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "2b52da00-b45d-41ae-a1ec-c88566879043", + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 1.0, + "min": 0.0, + "max": 1.0, + "def": 1.0, + "step": 0.01, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ac684e31-1a90-462f-8b64-2b368a93b563", + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { "x": 1.0, "y": 0.0 }, + "min": { "x": -1.0, "y": -1.0 }, + "max": { "x": 1.0, "y": 1.0 }, + "def": { "x": 1.0, "y": 0.0 }, + "step": 0.02, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "b8527f03-5c5c-4a41-b485-fa05e0f50cb1", + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 12.0, + "min": 1.0, + "max": 64.0, + "def": 12.0, + "step": 0.63, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "8f4d23a7-3b94-4a1c-ba14-d1ce47e92acd", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 855.0, "y": 977.0 }, + "contents_type": "Job", + "contents": { "type": "nos.sys.vulkan.GPUNode", "options": { "shader": "Shaders/DirectionalDof.frag" } }, + "function_category": "Default Node", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "plugin_version": { "major": 1, "minor": 7, "patch": 0 } + }, + { + "id": "8b497dab-5466-4d32-a440-125976e3a3ee", + "name": "Depth", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "9587b7b1-8fc7-437b-9459-ee73f90de097", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "74a1bfd0-4f2d-447b-945c-8d0cb67a2120", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "referred_by": [ + "1950c2e6-a0f6-485b-8a02-bded8a2f6ed5" + ], + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1025.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "9813ee9d-1f75-4554-9f9c-b9ecafc2e9fe", + "name": "FocusDistance", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "7c60934b-ba19-4faf-9923-411511649cd0", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "e709c7b4-9a59-4546-be53-0dc51abc5605", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "referred_by": [ + "e0b8f433-212f-48f6-ba4f-c8a194e1a707" + ], + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1100.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "2c0861b9-e416-4741-b56d-8dfa81c49516", + "name": "FocusRange", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "6a933bab-7bf6-4388-b990-abd1b9729e64", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "534a26e9-1ebd-4ed2-89fb-bdf5d34b6ec1", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 2.0, + "referred_by": [ + "68187c92-92f3-40d0-8b24-df6f33f9f649" + ], + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1175.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "3951aaae-16df-4b07-b1a9-b8b2a01b19c7", + "name": "MinRadius", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "f0feee29-3782-49fe-a834-94e2b57916a8", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "97561978-6da1-4a33-a6bc-c654008a8261", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.5, + "referred_by": [ + "312c4450-a4ad-4690-ba3d-afcbc93da6eb" + ], + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1325.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "a1d16ddd-0144-4daa-97b2-e9b3b019c8c1", + "name": "Input", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "c271ac23-2923-45c2-b262-b654455a93c3", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "479248dc-200d-4a4d-87d2-f2c7c77f667f", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "referred_by": [ + "2be9d3ba-9386-43b0-ae1c-58168be2a289" + ], + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1400.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + } + ], "connections": [ + { "from": "8f4d23a7-3b94-4a1c-ba14-d1ce47e92acd", "to": "b1b03fce-6863-42e6-a78a-260743b5441d", "id": "83839676-0760-4699-ae80-c0a789e273d8" }, + { "from": "f0feee29-3782-49fe-a834-94e2b57916a8", "to": "d77d3716-69f5-4c5d-a342-414dc11597fb", "id": "4c05135f-6001-4679-b39c-b248559ae56d" }, + { "from": "9587b7b1-8fc7-437b-9459-ee73f90de097", "to": "14aab6c6-10ce-4a39-9c6f-8c5633fe59e2", "id": "231cdfe5-7ac7-4013-9d20-68d5af8509b7" }, + { "from": "7c60934b-ba19-4faf-9923-411511649cd0", "to": "d6a91b7a-b576-487b-bd2c-89fee90a37d1", "id": "1cdaef73-876c-472a-97ff-04bf1f01348e" }, + { "from": "c271ac23-2923-45c2-b262-b654455a93c3", "to": "9e368dde-bb31-44d8-aaad-782e92fe2366", "id": "231fc88c-a52e-48d0-a6ee-8c2fdfe3ef0d" }, + { "from": "6a933bab-7bf6-4388-b990-abd1b9729e64", "to": "55bffdc3-e0fa-4c0f-9ead-5a3b96c232bf", "id": "7c1cae59-5834-420e-9d3d-e4767f6c3273" }, + { "from": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", "to": "bdc5ae5a-10cc-4c3c-b013-573a64bd8ec6", "id": "d74bdb3a-8c8c-4f82-8038-01a237e27a89" }, + { "from": "0ef0f439-9766-4957-8931-a02ce1019bd1", "to": "2e4ec877-e014-49ae-ae1d-881a0e4d1ac5", "id": "353cc954-d098-417a-8331-357b879ba654" }, + { "from": "9587b7b1-8fc7-437b-9459-ee73f90de097", "to": "7e88a91a-1eca-4cc5-8dce-7c4aca61368d", "id": "b126f4c4-d748-46f2-be51-ce1c778c0c4b" }, + { "from": "7c60934b-ba19-4faf-9923-411511649cd0", "to": "c1b814c4-e424-40a0-99d6-0437d948d1d7", "id": "fc25a2f4-0af4-49ae-9052-133a76cfc044" }, + { "from": "6a933bab-7bf6-4388-b990-abd1b9729e64", "to": "78893474-3dfc-4a36-b897-77760ba19c8c", "id": "f6ba18f8-0ef1-42db-a774-c4b02aa78fac" }, + { "from": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", "to": "4f471215-bebf-49be-a6e4-909c394d1f1a", "id": "afd9d7ff-f9e2-4a67-b874-2cfb2f870447" }, + { "from": "f0feee29-3782-49fe-a834-94e2b57916a8", "to": "05132381-cf95-4253-9fc2-e87f84b70dd8", "id": "82919455-4a51-490a-8ab2-201952d2e126" } + ] }, + "function_category": "Default Node", + "display_name": "Depth of Field", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + } + } + ] } diff --git a/Plugins/nosFilters/Config/DirectionalDof.nosdef b/Plugins/nosFilters/Config/DirectionalDof.nosdef new file mode 100644 index 00000000..427e4385 --- /dev/null +++ b/Plugins/nosFilters/Config/DirectionalDof.nosdef @@ -0,0 +1,121 @@ +{ + "nodes": [ + { + "class_name": "DirectionalDof", + "menu_info": { + "category": "Filters", + "display_name": "Directional DoF" + }, + "node": { + "class_name": "DirectionalDof", + "name": "Directional DoF", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/DirectionalDof.frag" + } + }, + "pins": [ + { + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "NEAREST" + } + }, + { + "name": "FocusDistance", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 5.0, + "min": 0.0, + "max": 1000.0 + }, + { + "name": "FocusRange", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 2.0, + "min": 0.01, + "max": 1000.0 + }, + { + "name": "MaxRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 16.0, + "min": 0.0, + "max": 128.0 + }, + { + "name": "MinRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.5, + "min": 0.0, + "max": 8.0 + }, + { + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "min": 0.0, + "max": 1.0 + }, + { + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "x": 1.0, + "y": 0.0 + }, + "min": { + "x": -1.0, + "y": -1.0 + }, + "max": { + "x": 1.0, + "y": 1.0 + } + }, + { + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 12.0, + "min": 1.0, + "max": 64.0 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Filters.noscfg b/Plugins/nosFilters/Filters.noscfg index 072c6ce0..3660bb3f 100644 --- a/Plugins/nosFilters/Filters.noscfg +++ b/Plugins/nosFilters/Filters.noscfg @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.filters", - "version": "1.7.0" + "version": "1.8.0" }, "display_name": "Filters", "description": "Collection of image filters.", @@ -27,6 +27,10 @@ "Config/Diff.nosdef", "Config/GaussianBlur.nosdef", "Config/DirectionalBlur.nosdef", + "Config/DirectionalDof.nosdef", + "Config/DepthOfField.nosdef", + "Config/BokehDof.nosdef", + "Config/BokehShape.nosdef", "Config/KawaseLightStreak.nosdef", "Config/Kuwahara.nosdef", "Config/PremultiplyAlpha.nosdef", diff --git a/Plugins/nosFilters/Shaders/BokehDof.frag b/Plugins/nosFilters/Shaders/BokehDof.frag new file mode 100644 index 00000000..b365ddf2 --- /dev/null +++ b/Plugins/nosFilters/Shaders/BokehDof.frag @@ -0,0 +1,105 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Single-pass 2D bokeh depth-of-field with a kernel-texture shaping the bokeh. +// +// Computes a per-pixel circle of confusion (CoC) from a linear view-space Z +// input, then gathers samples on a Vogel (golden-angle) disc within that CoC. +// Each sample's contribution is weighted by BokehShape sampled at the same +// unit-disc position, so the bokeh takes on the shape painted into BokehShape +// (regular polygon, ring, custom artwork, etc.). + +#version 450 + +#define MASK_THRESHOLD 0.001 +#define GOLDEN_ANGLE 2.39996322972865332 + +layout(binding = 0) uniform sampler2D Input; +layout(binding = 1) uniform sampler2D Depth; +layout(binding = 2) uniform sampler2D BokehShape; +layout(binding = 3) uniform BokehDofParams +{ + // Focus distance in the same units as the Depth input (linear view-space Z). + float FocusDistance; + // Distance from focus where CoC reaches MaxRadius. + float FocusRange; + // Maximum CoC radius in pixels. + float MaxRadius; + // Skip the gather when CoC <= MinRadius (keeps focused regions crisp & cheap). + float MinRadius; + // 0 = treat zero depth as "near focus" (stays sharp); 1 = treat as far plane. + float BackgroundIsFar; + // Total Vogel-disc sample count. ~32 = soft, ~64 = clean, ~128 = no banding. + float SampleCount; + // Rotate the kernel lookup (radians). Useful for animated highlights. + float KernelRotation; +} +Params; + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +float CocFromDepth(float Z) +{ + if (Z <= 0.0) + Z = mix(Params.FocusDistance, Params.FocusDistance + Params.FocusRange * 4.0, Params.BackgroundIsFar); + + float D = abs(Z - Params.FocusDistance); + float Coc = D / max(Params.FocusRange, 1e-4); + return clamp(Coc * Params.MaxRadius, 0.0, Params.MaxRadius); +} + +void main() +{ + vec2 TextureSize = textureSize(Input, 0); + vec2 TexelSize = 1.0 / TextureSize; + + vec4 CenterColor = texture(Input, uv); + float CenterZ = texture(Depth, uv).r; + float CenterCoC = CocFromDepth(CenterZ); + + if (CenterCoC <= Params.MinRadius || Params.MaxRadius < MASK_THRESHOLD) + { + rt = CenterColor; + return; + } + + int N = int(max(1.0, Params.SampleCount)); + float CosR = cos(Params.KernelRotation); + float SinR = sin(Params.KernelRotation); + + // Vogel disc: golden-angle spiral with sqrt radius for uniform area density. + // Sample 0 is the center; included implicitly via CenterColor initialization. + vec4 Accum = CenterColor; + float Weight = texture(BokehShape, vec2(0.5)).r; + Accum *= Weight; + + for (int i = 1; i < N; ++i) + { + float Frac = float(i) / float(N); + float R = sqrt(Frac); // unit-disc radius + float Th = float(i) * GOLDEN_ANGLE; + vec2 Unit = vec2(cos(Th) * R, sin(Th) * R); // unit disc position + + // Rotated lookup into the bokeh kernel. + vec2 ShapeUv = vec2(Unit.x * CosR - Unit.y * SinR, + Unit.x * SinR + Unit.y * CosR) * 0.5 + 0.5; + float WShape = texture(BokehShape, ShapeUv).r; + if (WShape <= MASK_THRESHOLD) + continue; + + vec2 Ofs = Unit * CenterCoC * TexelSize; + vec4 Sample = texture(Input, uv + Ofs); + float ZSamp = texture(Depth, uv + Ofs).r; + float CocSmp = CocFromDepth(ZSamp); + + // Per-sample CoC rejection prevents in-focus pixels bleeding outward. + // A sample contributes only if its own CoC is at least its distance from center. + float Dist = R * CenterCoC; + float WCoc = Dist <= CocSmp ? 1.0 : 0.0; + + float W = WShape * WCoc; + Accum += Sample * W; + Weight += W; + } + + rt = Accum / max(Weight, 1e-4); +} diff --git a/Plugins/nosFilters/Shaders/BokehShape.frag b/Plugins/nosFilters/Shaders/BokehShape.frag new file mode 100644 index 00000000..cb963629 --- /dev/null +++ b/Plugins/nosFilters/Shaders/BokehShape.frag @@ -0,0 +1,77 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Procedural bokeh kernel generator. +// +// Produces a grayscale unit-disc mask shaped like a regular polygon aperture +// (number of blades configurable) with optional roundness, rotation, soft edge +// and brightened rim. Intended as input to a kernel-weighted DoF gather. +// +// Convention: image is treated as the [-1, 1] unit square; pixels outside the +// kernel shape return 0; pixels inside return ~1, with a smooth edge falloff +// over EdgeSoftness. The mask is normalized so that center stays at 1. + +#version 450 + +#define PI 3.14159265358979323846 + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +layout(binding = 1) uniform BokehShapeParams +{ + // Aperture blade count. 0 or 1 = perfect circle. + float BladeCount; + // 0 = sharp polygon, 1 = perfect circle. Interpolates polygon edge toward disc. + float Roundness; + // Rotation of the polygon (radians). + float Rotation; + // Soft falloff width at the edge, in [0, 1] of unit-disc radius. + float EdgeSoftness; + // Extra brightness boost near the rim, [0, 1]. Mimics cat's-eye / specular bokeh. + float RimBoost; + // Width of the rim brightening band, in [0, 1] of radius. + float RimWidth; +} +Params; + +void main() +{ + // Map uv [0,1] to centered coords [-1,1] + vec2 Pos = uv * 2.0 - 1.0; + float R = length(Pos); + + if (R > 1.0) + { + rt = vec4(0.0); + return; + } + + float Blades = max(Params.BladeCount, 1.0); + + // Polygon edge radius along this angular direction. + // sectorAngle = 2*pi / N; angle from sector center is a; edge distance = cos(pi/N) / cos(a). + float PolygonR = 1.0; + if (Blades >= 3.0) + { + float Theta = atan(Pos.y, Pos.x) - Params.Rotation; + float SectorAngle = 2.0 * PI / Blades; + float HalfSector = SectorAngle * 0.5; + // Angle measured from the nearest sector centerline, in [-HalfSector, +HalfSector]. + float A = mod(Theta + HalfSector, SectorAngle) - HalfSector; + PolygonR = cos(HalfSector) / max(cos(A), 1e-4); + } + + // Roundness mixes polygon edge toward the circumscribed circle (radius 1). + float EdgeR = mix(PolygonR, 1.0, clamp(Params.Roundness, 0.0, 1.0)); + + // Soft edge: 1 inside, 0 past the edge, smooth across EdgeSoftness. + float Soft = max(Params.EdgeSoftness, 1e-4); + float Mask = 1.0 - smoothstep(EdgeR - Soft, EdgeR, R); + + // Rim brightening: a soft band just inside the edge. + float RimW = max(Params.RimWidth, 1e-4); + float RimPos = (R - (EdgeR - RimW)) / RimW; // 0 at inner edge of rim, 1 at outer + float Rim = clamp(1.0 - abs(RimPos * 2.0 - 1.0), 0.0, 1.0); + Mask += Rim * Params.RimBoost * Mask; + + rt = vec4(Mask, Mask, Mask, 1.0); +} diff --git a/Plugins/nosFilters/Shaders/DirectionalDof.frag b/Plugins/nosFilters/Shaders/DirectionalDof.frag new file mode 100644 index 00000000..308cfcc7 --- /dev/null +++ b/Plugins/nosFilters/Shaders/DirectionalDof.frag @@ -0,0 +1,95 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Directional depth-of-field pass. +// Computes circle-of-confusion (CoC) per pixel from a linear view-space Z input, +// then does a 1D weighted gather along Direction. Chain two instances +// (Direction = (1,0) and Direction = (0,1)) for a separable approximation of +// disc bokeh; visually close to a gaussian bokeh and cheap. + +#version 450 + +#define MASK_THRESHOLD 0.001 + +layout(binding = 0) uniform sampler2D Input; +layout(binding = 1) uniform sampler2D Depth; +layout(binding = 2) uniform DirectionalDofParams +{ + // Focus distance in the same units as the Depth input (linear view-space Z). + float FocusDistance; + // Distance from focus where CoC reaches MaxRadius. + // Smaller value = sharper focus falloff; larger = gentler. + float FocusRange; + // Maximum CoC radius in pixels. + float MaxRadius; + // 0 = treat zero depth as "no info, keep sharp"; 1 = treat zero depth as far. + float BackgroundIsFar; + vec2 Direction; + // Optional: clamp CoC near the focus plane to avoid noise; raise to skip tiny blurs. + float MinRadius; + // Sample count along the direction (one side; total taps = 2*N+1). Higher = smoother. + float SampleCount; +} +Params; + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +float CocFromDepth(float Z) +{ + // Treat Z<=0 (no depth signal) as either "near focus" (BackgroundIsFar=0) + // or as far plane (BackgroundIsFar=1). Picking far avoids halos around empty regions. + if (Z <= 0.0) + Z = mix(Params.FocusDistance, Params.FocusDistance + Params.FocusRange * 4.0, Params.BackgroundIsFar); + + float D = abs(Z - Params.FocusDistance); + float Coc = D / max(Params.FocusRange, 1e-4); + Coc = clamp(Coc * Params.MaxRadius, 0.0, Params.MaxRadius); + return Coc; +} + +void main() +{ + vec2 TextureSize = textureSize(Input, 0); + vec2 TexelSize = 1.0 / TextureSize; + + vec4 CenterColor = texture(Input, uv); + float CenterZ = texture(Depth, uv).r; + float CenterCoC = CocFromDepth(CenterZ); + + if (CenterCoC <= Params.MinRadius || Params.MaxRadius < MASK_THRESHOLD) + { + rt = CenterColor; + return; + } + + vec2 Dir = normalize(Params.Direction); + + int N = int(max(1.0, Params.SampleCount)); + float RadiusPx = CenterCoC; + float Step = RadiusPx / float(N); + + // Box-weighted average; for separable-2D this gives a soft disc. + // CoC-clamping per sample prevents fragments in focus from bleeding outward. + vec4 Accum = CenterColor; + float Weight = 1.0; + + for (int i = 1; i <= N; ++i) + { + float T = float(i) * Step; + vec2 Ofs = Dir * T * TexelSize; + + vec4 SPos = texture(Input, uv + Ofs); + float ZPos = texture(Depth, uv + Ofs).r; + float CocPos = CocFromDepth(ZPos); + float WPos = Step <= CocPos ? 1.0 : 0.0; + + vec4 SNeg = texture(Input, uv - Ofs); + float ZNeg = texture(Depth, uv - Ofs).r; + float CocNeg = CocFromDepth(ZNeg); + float WNeg = Step <= CocNeg ? 1.0 : 0.0; + + Accum += SPos * WPos + SNeg * WNeg; + Weight += WPos + WNeg; + } + + rt = Accum / Weight; +} diff --git a/Plugins/nosMath/CMakeLists.txt b/Plugins/nosMath/CMakeLists.txt index ee9ef718..bcbf2b2f 100644 --- a/Plugins/nosMath/CMakeLists.txt +++ b/Plugins/nosMath/CMakeLists.txt @@ -6,6 +6,10 @@ add_library(tinyexpr_cpp STATIC ${TINYEXPR_SOURCES}) target_include_directories(tinyexpr_cpp PUBLIC External/tinyexpr-cpp) nos_group_targets("tinyexpr_cpp" "External") -set(DEPENDENCIES ${NOS_PLUGIN_SDK_TARGET} tinyexpr_cpp) +set(GENERATED_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/Generated") +nos_generate_flatbuffers("${CMAKE_CURRENT_SOURCE_DIR}/Config" "${GENERATED_OUTPUT_DIR}" "cpp" "${NOS_SDK_DIR}/Types" nosMath_generated) -nos_add_plugin("nosMath" "${DEPENDENCIES}" "") +set(DEPENDENCIES ${NOS_PLUGIN_SDK_TARGET} tinyexpr_cpp nosMath_generated) +set(INCLUDE_FOLDERS "${GENERATED_OUTPUT_DIR}") + +nos_add_plugin("nosMath" "${DEPENDENCIES}" "${INCLUDE_FOLDERS}") diff --git a/Plugins/nosMath/Config/EulerToQuaternion.nosdef b/Plugins/nosMath/Config/EulerToQuaternion.nosdef new file mode 100644 index 00000000..fa40e5d7 --- /dev/null +++ b/Plugins/nosMath/Config/EulerToQuaternion.nosdef @@ -0,0 +1,38 @@ +{ + "nodes": [ + { + "class_name": "EulerToQuaternion", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Euler To Quaternion" + }, + "node": { + "class_name": "EulerToQuaternion", + "contents_type": "Job", + "description": "Converts an Euler-angle rotation (degrees) to a unit quaternion (x, y, z, w). The Order pin selects the intrinsic rotation order applied to the components of the input vec3 (e.g. ZYX means R = Rz(rot.z) * Ry(rot.y) * Rx(rot.x)). Default ZYX matches the FreeD/Track convention.", + "pins": [ + { + "name": "Euler", + "type_name": "nos.fb.vec3d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Order", + "type_name": "nos.math.EulerOrder", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "ZYX", + "description": "Euler intrinsic rotation order applied to the (rot.x, rot.y, rot.z) components." + }, + { + "name": "Quaternion", + "type_name": "nos.fb.vec4d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Config/Math.fbs b/Plugins/nosMath/Config/Math.fbs new file mode 100644 index 00000000..84c79eeb --- /dev/null +++ b/Plugins/nosMath/Config/Math.fbs @@ -0,0 +1,10 @@ +namespace nos.math; + +enum EulerOrder : ubyte { + ZYX = 0, + XYZ = 1, + YXZ = 2, + YZX = 3, + ZXY = 4, + XZY = 5, +} diff --git a/Plugins/nosMath/Config/QuaternionMultiply.nosdef b/Plugins/nosMath/Config/QuaternionMultiply.nosdef new file mode 100644 index 00000000..fc162b41 --- /dev/null +++ b/Plugins/nosMath/Config/QuaternionMultiply.nosdef @@ -0,0 +1,36 @@ +{ + "nodes": [ + { + "class_name": "QuaternionMultiply", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Quaternion Multiply" + }, + "node": { + "class_name": "QuaternionMultiply", + "contents_type": "Job", + "description": "Hamilton product of two unit quaternions: Result = A * B (each as (x, y, z, w)). Composing rotations: A * B applies B first, then A. To rotate (conjugate) a quaternion Q by R, compute R * Q * conjugate(R).", + "pins": [ + { + "name": "A", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "B", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Result", + "type_name": "nos.fb.vec4d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Config/QuaternionToEuler.nosdef b/Plugins/nosMath/Config/QuaternionToEuler.nosdef new file mode 100644 index 00000000..d08ffc7a --- /dev/null +++ b/Plugins/nosMath/Config/QuaternionToEuler.nosdef @@ -0,0 +1,38 @@ +{ + "nodes": [ + { + "class_name": "QuaternionToEuler", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Quaternion To Euler" + }, + "node": { + "class_name": "QuaternionToEuler", + "contents_type": "Job", + "description": "Converts a unit quaternion (x, y, z, w) to Euler angles (degrees). The Order pin selects the intrinsic rotation order extracted (e.g. ZYX yields rot.z = first rotation about Z, rot.y about Y, rot.x about X). Inverse of EulerToQuaternion when the same Order is used on both ends.", + "pins": [ + { + "name": "Quaternion", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Order", + "type_name": "nos.math.EulerOrder", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "ZYX", + "description": "Euler intrinsic rotation order extracted into the (rot.x, rot.y, rot.z) output components." + }, + { + "name": "Euler", + "type_name": "nos.fb.vec3d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Math.noscfg b/Plugins/nosMath/Math.noscfg index de4791ee..5417aca6 100644 --- a/Plugins/nosMath/Math.noscfg +++ b/Plugins/nosMath/Math.noscfg @@ -14,7 +14,9 @@ } ] }, - "custom_types": [], + "custom_types": [ + "Config/Math.fbs" + ], "node_definitions": [ "Config/Math.nosdef", "Config/Eval.nosdef", @@ -25,7 +27,10 @@ "Config/Random.nosdef", "Config/Lerp.nosdef", "Config/Vec3ToVec4.nosdef", - "Config/EmbedMat3ToMat4.nosdef" + "Config/EmbedMat3ToMat4.nosdef", + "Config/EulerToQuaternion.nosdef", + "Config/QuaternionToEuler.nosdef", + "Config/QuaternionMultiply.nosdef" ], "binary_path": "Binaries/nosMath", "third_party_software": [ diff --git a/Plugins/nosMath/Source/EulerToQuaternion.cpp b/Plugins/nosMath/Source/EulerToQuaternion.cpp new file mode 100644 index 00000000..94440a60 --- /dev/null +++ b/Plugins/nosMath/Source/EulerToQuaternion.cpp @@ -0,0 +1,97 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include +#include +#include + +namespace nos::math +{ + +// Build a rotation matrix for the given intrinsic Euler order. +// In all cases, rot.x is the angle about X, rot.y about Y, rot.z about Z (radians). +// Order ZYX means R = Rz(rot.z) * Ry(rot.y) * Rx(rot.x), applied right-to-left to a point. +static glm::dmat4 EulerToMat(EulerOrder order, glm::dvec3 const& r) +{ + switch (order) + { + case EulerOrder::ZYX: return glm::eulerAngleZYX(r.z, r.y, r.x); + case EulerOrder::XYZ: return glm::eulerAngleXYZ(r.x, r.y, r.z); + case EulerOrder::YXZ: return glm::eulerAngleYXZ(r.y, r.x, r.z); + case EulerOrder::YZX: return glm::eulerAngleYZX(r.y, r.z, r.x); + case EulerOrder::ZXY: return glm::eulerAngleZXY(r.z, r.x, r.y); + case EulerOrder::XZY: return glm::eulerAngleXZY(r.x, r.z, r.y); + } + return glm::dmat4(1.0); +} + +static void MatToEuler(EulerOrder order, glm::dmat4 const& m, glm::dvec3& r) +{ + switch (order) + { + case EulerOrder::ZYX: glm::extractEulerAngleZYX(m, r.z, r.y, r.x); break; + case EulerOrder::XYZ: glm::extractEulerAngleXYZ(m, r.x, r.y, r.z); break; + case EulerOrder::YXZ: glm::extractEulerAngleYXZ(m, r.y, r.x, r.z); break; + case EulerOrder::YZX: glm::extractEulerAngleYZX(m, r.y, r.z, r.x); break; + case EulerOrder::ZXY: glm::extractEulerAngleZXY(m, r.z, r.x, r.y); break; + case EulerOrder::XZY: glm::extractEulerAngleXZY(m, r.x, r.z, r.y); break; + } +} + +struct EulerToQuaternionNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* in = params.GetPinData(NOS_NAME("Euler")); + auto* order = params.GetPinData(NOS_NAME("Order")); + auto* out = params.GetPinData(NOS_NAME("Quaternion")); + + glm::dvec3 r = glm::radians(glm::dvec3(in->x(), in->y(), in->z())); + glm::dquat q(EulerToMat(*order, r)); + + out->mutate_x(q.x); + out->mutate_y(q.y); + out->mutate_z(q.z); + out->mutate_w(q.w); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterEulerToQuaternion(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.EulerToQuaternion"), EulerToQuaternionNodeContext, fn) +} + +struct QuaternionToEulerNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* in = params.GetPinData(NOS_NAME("Quaternion")); + auto* order = params.GetPinData(NOS_NAME("Order")); + auto* out = params.GetPinData(NOS_NAME("Euler")); + + glm::dquat q(in->w(), in->x(), in->y(), in->z()); + glm::dmat4 m = glm::mat4_cast(q); + glm::dvec3 r(0.0); + MatToEuler(*order, m, r); + + out->mutate_x(glm::degrees(r.x)); + out->mutate_y(glm::degrees(r.y)); + out->mutate_z(glm::degrees(r.z)); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterQuaternionToEuler(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.QuaternionToEuler"), QuaternionToEulerNodeContext, fn) +} + +} // namespace nos::math diff --git a/Plugins/nosMath/Source/Math.cpp b/Plugins/nosMath/Source/Math.cpp index c307f2cf..1893cc8e 100644 --- a/Plugins/nosMath/Source/Math.cpp +++ b/Plugins/nosMath/Source/Math.cpp @@ -104,6 +104,9 @@ enum class MathNodeTypes : int { Or, Not, Random, + EulerToQuaternion, + QuaternionToEuler, + QuaternionMultiply, Count }; @@ -168,6 +171,9 @@ void RegisterAnd(nosNodeFunctions*); void RegisterOr(nosNodeFunctions*); void RegisterNot(nosNodeFunctions*); void RegisterRandom(nosNodeFunctions*); +void RegisterEulerToQuaternion(nosNodeFunctions*); +void RegisterQuaternionToEuler(nosNodeFunctions*); +void RegisterQuaternionMultiply(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outCount, nosNodeFunctions** outList) { @@ -281,6 +287,18 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outCount, nosNodeFunctions** o RegisterRandom(node); break; } + case MathNodeTypes::EulerToQuaternion: { + RegisterEulerToQuaternion(node); + break; + } + case MathNodeTypes::QuaternionToEuler: { + RegisterQuaternionToEuler(node); + break; + } + case MathNodeTypes::QuaternionMultiply: { + RegisterQuaternionMultiply(node); + break; + } default: break; } diff --git a/Plugins/nosMath/Source/QuaternionMultiply.cpp b/Plugins/nosMath/Source/QuaternionMultiply.cpp new file mode 100644 index 00000000..395b4843 --- /dev/null +++ b/Plugins/nosMath/Source/QuaternionMultiply.cpp @@ -0,0 +1,39 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include +#include + +namespace nos::math +{ + +struct QuaternionMultiplyNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* a = params.GetPinData(NOS_NAME("A")); + auto* b = params.GetPinData(NOS_NAME("B")); + auto* out = params.GetPinData(NOS_NAME("Result")); + + glm::dquat qa(a->w(), a->x(), a->y(), a->z()); + glm::dquat qb(b->w(), b->x(), b->y(), b->z()); + glm::dquat qr = qa * qb; + + out->mutate_x(qr.x); + out->mutate_y(qr.y); + out->mutate_z(qr.z); + out->mutate_w(qr.w); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterQuaternionMultiply(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.QuaternionMultiply"), QuaternionMultiplyNodeContext, fn) +} + +} // namespace nos::math diff --git a/Plugins/nosTrack/CHANGES.md b/Plugins/nosTrack/CHANGES.md new file mode 100644 index 00000000..ec713c45 --- /dev/null +++ b/Plugins/nosTrack/CHANGES.md @@ -0,0 +1,57 @@ +# Record Track (COLMAP) Node + +## Summary + +A new node "Record Track (COLMAP)" added to the `nosTrack` plugin. It records incoming camera tracking data per frame and exports it in COLMAP's text format (`cameras.txt` + `images.txt`). + +## Files Changed + +### New files +- `Source/RecordTrackCOLMAP.cpp` — Node implementation +- `Config/RecordTrackCOLMAP.nosdef` — Node definition (pins, functions, metadata) + +### Modified files +- `Source/TrackMain.cpp` — Added `RecordTrackCOLMAP` to the `TrackNode` enum and `ExportNodeFunctions` switch +- `Track.noscfg` — Added `Config/RecordTrackCOLMAP.nosdef` to `node_definitions` + +## Node Design + +### Pins +| Pin | Type | Direction | Description | +|-----|------|-----------|-------------| +| Track | `nos.track.Track` | Input | Incoming tracking data | +| Track Out | `nos.track.Track` | Output (only) | Pass-through of input | +| Output Directory | `string` | Property | Folder picker for output | +| Image Resolution | `nos.fb.vec2u` | Property | Width/height (default 1920x1080) | +| Record | `bool` | Property | Mirrors Record/Stop functions | +| Frame Count | `uint` | Output (only) | Frames in buffer | + +### Functions +| Function | Behavior | +|----------|----------| +| Record | Validates folder is empty, clears buffer, starts recording. Orphaned while recording. | +| Stop | Stops recording (does NOT save). Orphaned while idle. | +| Save | Writes `cameras.txt` + `images.txt` to disk. Does not clear buffer. | +| Clear | Clears frame buffer and resets count. | +| Open Folder | Opens output directory in explorer (Windows) or xdg-open (Linux). | + +### State Management +- Record pin and functions are kept in sync bidirectionally. A `SyncingRecordPin` bool guard prevents re-entrant loops between pin changes and function calls. +- Function orphan states: Record/Stop toggle via `SetNodeOrphanState` using a `Name -> UUID` map built in constructor. +- Status messages show recording state + frame count, and persist error messages (e.g., "Target folder is not empty") via `LastError` until user changes the output directory. +- Non-empty folder check: Recording fails with a FAILURE status if the target folder already has files. + +### COLMAP Output Format +- `cameras.txt`: One OPENCV camera per frame — `fx, fy, cx, cy, k1, k2, p1, p2` derived from Track FOV, sensor size, pixel aspect ratio, lens distortion. +- `images.txt`: Per-frame pose — Euler angles converted to quaternion (world-to-camera), translation as `t = -R * C`. + +## Known Review Points +- Euler-to-quaternion convention: The Track's rotation fields (roll/tilt/pan) are passed through `glm::quat(eulerRadians)` then inverted for COLMAP's world-to-camera convention. May need validation against actual tracker output. +- One camera per frame: Each frame gets its own camera entry. This handles zoom/FOV changes but may be unusual for COLMAP workflows with constant intrinsics. +- No `points3D.txt`: COLMAP expects this file too (can be empty). Not currently written. +- `std::system()` for Open Folder: Works but is a simple shell call. Could be replaced with platform APIs if needed. + +## Build +``` +./nodos dev build -p Project13 --target nosTrack +``` diff --git a/Plugins/nosTrack/CMakeLists.txt b/Plugins/nosTrack/CMakeLists.txt index 26c05e2b..af2df47c 100644 --- a/Plugins/nosTrack/CMakeLists.txt +++ b/Plugins/nosTrack/CMakeLists.txt @@ -1,6 +1,6 @@ # Copyright MediaZ Teknoloji A.S. All Rights Reserved. -set(MODULE_DEPENDENCIES "nos.sys.track-1.0") +set(MODULE_DEPENDENCIES "nos.sys.track-1.1") set(dep_idx 0) foreach(module_name_version ${MODULE_DEPENDENCIES}) # module_name_version: - @@ -13,4 +13,7 @@ endforeach() list(APPEND MODULE_DEPENDENCIES_TARGETS ${NOS_PLUGIN_SDK_TARGET}) +nos_generate_flatbuffers("${CMAKE_CURRENT_SOURCE_DIR}/Config" "${CMAKE_CURRENT_SOURCE_DIR}/Source" "cpp" "" nosTrack_generated) +list(APPEND MODULE_DEPENDENCIES_TARGETS nosTrack_generated) + nos_add_plugin("nosTrack" "${MODULE_DEPENDENCIES_TARGETS}" "${CMAKE_CURRENT_LIST_DIR}/External/asio/asio/include") diff --git a/Plugins/nosTrack/Config/PlaybackMode.fbs b/Plugins/nosTrack/Config/PlaybackMode.fbs new file mode 100644 index 00000000..c0d1d952 --- /dev/null +++ b/Plugins/nosTrack/Config/PlaybackMode.fbs @@ -0,0 +1,9 @@ +namespace nos.track; + +// Selects how PlaybackTrackCOLMAP indexes into the recorded frames. +enum PlaybackTrackMode : uint +{ + FrameIndex = 0, // Use the InFrameIndex pin as a 0-based offset. + Timecode = 1, // Look up by Timecode string from timecodes.txt. + FrameNumber = 2, // Look up by FrameNumber column from timecodes.txt. +} diff --git a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef new file mode 100644 index 00000000..7f1604f2 --- /dev/null +++ b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef @@ -0,0 +1,107 @@ +{ + "nodes": [ + { + "class_name": "PlaybackTrackCOLMAP", + "menu_info": { + "category": "nosTrack", + "display_name": "Playback Track (COLMAP)", + "name_aliases": [ "colmap", "import camera", "playback camera" ] + }, + "node": { + "class_name": "PlaybackTrackCOLMAP", + "display_name": "Playback Track (COLMAP)", + "contents_type": "Job", + "always_execute": true, + "description": "Loads camera track from COLMAP-spec cameras.txt + images.txt.\nReads world-to-camera poses in the COLMAP frame (RH, +X right, +Y down, +Z forward) and converts to the chosen TargetFrame.\nWhen an extras.txt sidecar is present, original Euler/FOV/sensor metadata is restored verbatim (no quaternion round-trip drift).", + "pins": [ + { + "name": "InputDirectory", + "display_name": "Input Directory", + "type_name": "string", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { "type": "FOLDER_PICKER" }, + "description": "Directory with cameras.txt + images.txt (and optional timecodes.txt / extras.txt sidecars)." + }, + { + "name": "TargetFrame", + "display_name": "Target Frame", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate frame of the produced Track.\nCOLMAP poses are converted into this frame.\nDefault matches FreeD / UE convention." + }, + { + "name": "Mode", + "display_name": "Mode", + "type_name": "nos.track.PlaybackTrackMode", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "FrameIndex", + "description": "Selects how to index frames.\nFrameIndex uses InFrameIndex.\nTimecode / FrameNumber look up via timecodes.txt sidecar.\nThe unused index pin becomes PASSIVE." + }, + { + "name": "InFrameIndex", + "display_name": "Frame Index", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "0-based frame index. Used when Mode=FrameIndex." + }, + { + "name": "InTimecode", + "display_name": "Timecode", + "type_name": "string", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Timecode string (HH:MM:SS:FF) to look up. Used when Mode=Timecode. Requires timecodes.txt." + }, + { + "name": "InFrameNumber", + "display_name": "Frame Number", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "Absolute frame number to look up. Used when Mode=FrameNumber. Requires timecodes.txt." + }, + { + "name": "OutFrameIndex", + "display_name": "Frame Index", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Current playback frame index." + }, + { + "name": "FrameCount", + "display_name": "Frame Count", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Total frames loaded." + }, + { + "name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "description": "Track for the current frame, expressed in the TargetFrame convention." + } + ], + "functions": [ + { + "class_name": "PlaybackTrackCOLMAP_OpenFolder", + "display_name": "Open Folder", + "contents_type": "Job", + "pins": [] + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef new file mode 100644 index 00000000..c26f52cb --- /dev/null +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -0,0 +1,120 @@ +{ + "nodes": [ + { + "class_name": "RecordTrackCOLMAP", + "menu_info": { + "category": "nosTrack", + "display_name": "Record Track (COLMAP)", + "name_aliases": [ "colmap", "export camera", "record camera" ] + }, + "node": { + "class_name": "RecordTrackCOLMAP", + "display_name": "Record Track (COLMAP)", + "contents_type": "Job", + "always_execute": true, + "description": "Records camera track data each frame while Record is true.\nOn falling edge (after MinOffFrames debounce) writes COLMAP-spec cameras.txt + images.txt and clears the buffer.\nIntrinsics come from FOV/sensor/distortion. Extrinsics are written as world-to-camera in the COLMAP frame (RH, +X right, +Y down, +Z forward).\nSet SourceFrame to match the convention of the connected Track.", + "pins": [ + { + "name": "Timecode", + "display_name": "Timecode", + "type_name": "string", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Optional SMPTE timecode (HH:MM:SS:FF). Written to timecodes.txt sidecar when non-empty." + }, + { + "name": "FrameNumber", + "display_name": "Frame Number", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "Optional absolute frame number paired with Timecode. Written to timecodes.txt sidecar." + }, + { + "name": "OutputDirectory", + "display_name": "Output Directory", + "type_name": "string", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { "type": "FOLDER_PICKER" }, + "description": "Where cameras.txt and images.txt are written when recording stops. Must be empty to start recording." + }, + { + "name": "ImageResolution", + "display_name": "Image Resolution", + "type_name": "nos.fb.vec2u", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "x": 1920, + "y": 1080 + }, + "description": "Image WIDTH/HEIGHT in pixels. Used to compute focal length and principal point for cameras.txt." + }, + { + "name": "SourceFrame", + "display_name": "Source Frame", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate frame of the connected Track.\nUsed to convert location and rotation into the COLMAP frame before writing.\nDefault matches FreeD / UE convention." + }, + { + "name": "Record", + "type_name": "bool", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": false, + "description": "Drives recording.\nRising edge: clear buffer and start.\nFalling edge (after MinOffFrames): stop and write files.\nFails to start if OutputDirectory is non-empty." + }, + { + "name": "MinOffFrames", + "display_name": "Min Off Frames", + "type_name": "uint", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1, + "min": "1", + "description": "Debounce: minimum consecutive Record=false frames before stopping. Default 1 = stop immediately. Use 5-15 to ride out short upstream glitches (e.g. SDI bit flips on a camera-derived flag)." + }, + { + "name": "RecordingFrame", + "display_name": "Recording Frame", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Current recording frame index. 0 when not recording." + }, + { + "name": "FrameCount", + "display_name": "Frame Count", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Frames in the buffer." + }, + { + "name": "InTrack", + "display_name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Camera track to record. Interpreted in the SourceFrame convention (location, rotation Euler, FOV, sensor, lens distortion)." + }, + { + "name": "OutTrack", + "display_name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "description": "Pass-through of InTrack." + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Config/TrackTransform.nosdef b/Plugins/nosTrack/Config/TrackTransform.nosdef new file mode 100644 index 00000000..cb198734 --- /dev/null +++ b/Plugins/nosTrack/Config/TrackTransform.nosdef @@ -0,0 +1,54 @@ +{ + "nodes": [ + { + "class_name": "TrackTransform", + "menu_info": { + "category": "Track|Coordinate System", + "display_name": "Track Transform" + }, + "node": { + "class_name": "TrackTransform", + "contents_type": "Job", + "description": "Transforms a Track between coordinate frames.\nThe Source and Target enums select axis assignments, handedness, and the Euler convention used for the rotation field.\nLocation: basis-changed (Source -> Target), then multiplied by WorldScale (e.g. 0.01 for cm -> m, 100 for m -> cm).\nRotation: built in the source Euler convention, conjugated by the basis-change matrix, re-extracted in the target convention.\nOther Track fields (fov, focus, sensor_size, lens_distortion, ...) pass through unchanged.", + "pins": [ + { + "name": "In", + "type_name": "nos.sys.track.Track", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Source", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate system convention of the input Track." + }, + { + "name": "Target", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "RH_YUp_FwdNegZ_RightX", + "description": "Coordinate system convention of the output Track." + }, + { + "name": "WorldScale", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "description": "Uniform scale applied only to the output location after the basis change. Use to convert linear units (e.g. 0.01 for cm -> m, 100 for m -> cm). Does not affect rotation, fov, sensor size, focus, or lens distortion." + }, + { + "name": "Out", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Source/CoordinateFrameConv.h b/Plugins/nosTrack/Source/CoordinateFrameConv.h new file mode 100644 index 00000000..777fea34 --- /dev/null +++ b/Plugins/nosTrack/Source/CoordinateFrameConv.h @@ -0,0 +1,100 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Frame-conversion helpers shared by TrackTransform / RecordTrackCOLMAP / +// PlaybackTrackCOLMAP. Encodes per-frame Euler conventions and basis-change +// matrices to the COLMAP camera/world frame. +#pragma once + +#include "nosSysTrack/Track_generated.h" +#include +#include + +namespace nos::track::convention +{ + +using Frame = sys::track::CoordinateFrame; + +// Basis matrix S for a CoordinateFrame: maps semantic (forward, right, up) +// to engine coords (vx, vy, vz). v_engine = S * (forward, right, up). +// det(S) > 0 for left-handed frames, < 0 for right-handed (with this ordering). +inline glm::dmat3 BasisMatrix(Frame frame) +{ + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + // vx = forward, vy = right, vz = up. + return glm::dmat3(1.0); + case Frame::RH_YUp_FwdNegZ_RightX: + // vx = right, vy = up, vz = -forward. + return glm::dmat3( + glm::dvec3( 0.0, 0.0, -1.0), // M * (1,0,0) = forward column + glm::dvec3( 1.0, 0.0, 0.0), // M * (0,1,0) = right column + glm::dvec3( 0.0, 1.0, 0.0)); // M * (0,0,1) = up column + } + return glm::dmat3(1.0); +} + +// COLMAP camera/world frame: X right, Y down, Z forward (RH). +// Provided as a basis matrix in the same (forward, right, up) convention so +// it can be combined with BasisMatrix to build cross-frame conversions. +inline glm::dmat3 ColmapBasisMatrix() +{ + return glm::dmat3( + glm::dvec3( 0.0, 0.0, 1.0), // forward -> +Z + glm::dvec3( 1.0, 0.0, 0.0), // right -> +X + glm::dvec3( 0.0, -1.0, 0.0)); // up -> -Y (Y is down) +} + +// Build R_c2w in `frame` from Track.rotation Euler degrees. +inline glm::dmat3 EulerToMat(Frame frame, glm::dvec3 const& degRot) +{ + glm::dvec3 r = glm::radians(degRot); + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + // FRotator: rot.x = roll, rot.y = pitch, rot.z = yaw, intrinsic ZYX. + // UE sign convention has +pitch = look up and +roll = bank right via + // LH-rule rotations, equivalent to standard-RH Rz(yaw) * Ry(-pitch) * Rx(-roll). + return glm::dmat3(glm::eulerAngleZYX(r.z, -r.y, -r.x)); + case Frame::RH_YUp_FwdNegZ_RightX: + // rot.x = pitch, rot.y = yaw, rot.z = roll, intrinsic YXZ: + // R = Ry(yaw) * Rx(pitch) * Rz(roll), all standard-RH formulas. + return glm::dmat3(glm::eulerAngleYXZ(r.y, r.x, r.z)); + } + return glm::dmat3(1.0); +} + +// Inverse of EulerToMat: extract Euler degrees in `frame`'s convention. +// Output is packed into the (rot.x, rot.y, rot.z) Track layout for that frame. +inline glm::dvec3 MatToEuler(Frame frame, glm::dmat3 const& R) +{ + glm::dmat4 M(R); + double a = 0.0, b = 0.0, c = 0.0; + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + glm::extractEulerAngleZYX(M, a, b, c); // a=yaw, b=pitch, c=roll + // Negate pitch and roll back to UE sign convention; pack as (roll, pitch, yaw). + return glm::degrees(glm::dvec3(-c, -b, a)); + case Frame::RH_YUp_FwdNegZ_RightX: + glm::extractEulerAngleYXZ(M, a, b, c); // a=yaw, b=pitch, c=roll + // Pack as (pitch, yaw, roll). + return glm::degrees(glm::dvec3(b, a, c)); + } + return glm::dvec3(0.0); +} + +// Basis-change M from `frame` to COLMAP frame: M = S_colmap * S_frame^-1. +// For a vector: v_colmap = M * v_frame. +// For a rotation matrix: R_colmap = M * R_frame * M^-1. +inline glm::dmat3 BasisChangeToColmap(Frame frame) +{ + return ColmapBasisMatrix() * glm::inverse(BasisMatrix(frame)); +} + +// Inverse of BasisChangeToColmap. +inline glm::dmat3 BasisChangeFromColmap(Frame frame) +{ + return BasisMatrix(frame) * glm::inverse(ColmapBasisMatrix()); +} + +} // namespace nos::track::convention diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp new file mode 100644 index 00000000..56271a01 --- /dev/null +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -0,0 +1,568 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include "nosSysTrack/Track_generated.h" +#include "PlaybackMode_generated.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "CoordinateFrameConv.h" + +namespace nos::track +{ + +NOS_REGISTER_NAME_SPACED(Playback_InputDirectory, "InputDirectory"); +NOS_REGISTER_NAME_SPACED(Playback_TargetFrame, "TargetFrame"); +NOS_REGISTER_NAME_SPACED(Playback_Mode, "Mode"); +NOS_REGISTER_NAME_SPACED(Playback_InFrameIndex, "InFrameIndex"); +NOS_REGISTER_NAME_SPACED(Playback_InTimecode, "InTimecode"); +NOS_REGISTER_NAME_SPACED(Playback_InFrameNumber, "InFrameNumber"); +NOS_REGISTER_NAME_SPACED(Playback_OutFrameIndex, "OutFrameIndex"); +NOS_REGISTER_NAME_SPACED(Playback_FrameCount, "FrameCount"); + +NOS_REGISTER_NAME(PlaybackTrackCOLMAP_OpenFolder); + +struct COLMAPCamera +{ + uint32_t Id = 0; + std::string Model; + uint32_t Width = 0; + uint32_t Height = 0; + float Fx = 0, Fy = 0, Cx = 0, Cy = 0; + float K1 = 0, K2 = 0, P1 = 0, P2 = 0; +}; + +struct COLMAPImage +{ + uint32_t Id = 0; + glm::quat Q{1, 0, 0, 0}; // R_w2c in COLMAP camera frame. + glm::vec3 T{0}; // t = -R_w2c * camera_world_position (COLMAP world frame). + uint32_t CameraId = 0; +}; + +struct TimecodeEntry +{ + std::string Timecode; + uint32_t FrameNumber = 0; +}; + +struct ExtrasEntry +{ + bool Present = false; + float Zoom = 0; + float Focus = 0; + float FocusDistance = 0; + float RenderRatio = 0; + float NodalOffset = 0; + float DistortionScale = 0; + float SensorWmm = 0; + float SensorHmm = 0; + float RotX = 0; + float RotY = 0; + float RotZ = 0; +}; + +struct PlaybackTrackCOLMAPContext : NodeContext +{ + std::string InputDir; + convention::Frame TargetFrame = convention::Frame::LH_ZUp_FwdX_RightY; + PlaybackTrackMode Mode = PlaybackTrackMode::FrameIndex; + uint32_t FrameIndex = 0; + std::string InTimecode; + uint32_t InFrameNumber = 0; + std::string LastError; + std::vector Frames; + std::vector Timecodes; // empty or same size as Frames + std::unordered_map TimecodeToIndex; + std::unordered_map FrameNumberToIndex; + uint32_t CurrentFrame = 0; + + PlaybackTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) + { + if (node->pins()) + { + for (auto* pin : *node->pins()) + { + auto name = nos::Name(pin->name()->c_str()); + if (flatbuffers::IsFieldPresent(pin, fb::Pin::VT_DATA)) + { + nosBuffer value = {.Data = (void*)pin->data()->data(), .Size = pin->data()->size()}; + OnPinValueChanged(name, *pin->id(), value); + } + } + } + ApplyModeOrphanStates(); + UpdateStatus(); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer val) override + { + if (pinName == NSN_Playback_InputDirectory) + { + InputDir = InterpretPinValue(val.Data); + LastError.clear(); + if (!InputDir.empty()) + LoadFromDirectory(); + else + UpdateStatus(); + } + else if (pinName == NSN_Playback_TargetFrame) + { + TargetFrame = *(convention::Frame*)val.Data; + if (!InputDir.empty()) + LoadFromDirectory(); + } + else if (pinName == NSN_Playback_Mode) + { + Mode = *(PlaybackTrackMode*)val.Data; + ApplyModeOrphanStates(); + } + else if (pinName == NSN_Playback_InFrameIndex) + FrameIndex = *(uint32_t*)val.Data; + else if (pinName == NSN_Playback_InTimecode) + InTimecode = InterpretPinValue(val.Data); + else if (pinName == NSN_Playback_InFrameNumber) + InFrameNumber = *(uint32_t*)val.Data; + } + + void ApplyModeOrphanStates() + { + auto state = [](bool active) { + return active ? fb::PinOrphanStateType::ACTIVE : fb::PinOrphanStateType::PASSIVE; + }; + const bool useIdx = Mode == PlaybackTrackMode::FrameIndex; + const bool useTC = Mode == PlaybackTrackMode::Timecode; + const bool useFN = Mode == PlaybackTrackMode::FrameNumber; + SetPinOrphanState(NSN_Playback_InFrameIndex, state(useIdx)); + SetPinOrphanState(NSN_Playback_InTimecode, state(useTC)); + SetPinOrphanState(NSN_Playback_InFrameNumber, state(useFN)); + } + + void UpdateFrameCountPin() + { + uint32_t count = (uint32_t)Frames.size(); + SetPinValue(NSN_Playback_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); + } + + void UpdateFrameIndexPin() + { + SetPinValue(NSN_Playback_OutFrameIndex, nosBuffer{.Data = &CurrentFrame, .Size = sizeof(CurrentFrame)}); + } + + void UpdateStatus() + { + if (!LastError.empty()) + SetNodeStatusMessage(LastError, fb::NodeStatusMessageType::FAILURE); + else if (InputDir.empty()) + SetNodeStatusMessage("Set input directory", fb::NodeStatusMessageType::WARNING); + else if (Frames.empty()) + SetNodeStatusMessage("No data loaded", fb::NodeStatusMessageType::WARNING); + else + SetNodeStatusMessage("Loaded (" + std::to_string(Frames.size()) + " frames)", fb::NodeStatusMessageType::INFO); + } + + // --- Parsing --- + + bool LoadFromDirectory() + { + if (InputDir.empty()) + { + LastError = "Set input directory"; + UpdateStatus(); + return false; + } + + std::filesystem::path dir = nos::Utf8ToPath(InputDir); + auto camerasPath = dir / "cameras.txt"; + auto imagesPath = dir / "images.txt"; + + if (!std::filesystem::exists(camerasPath)) + { + LastError = "cameras.txt not found"; + UpdateStatus(); + return false; + } + if (!std::filesystem::exists(imagesPath)) + { + LastError = "images.txt not found"; + UpdateStatus(); + return false; + } + + std::unordered_map cameras; + if (!ParseCamerasTxt(camerasPath, cameras)) + return false; + + std::vector images; + if (!ParseImagesTxt(imagesPath, images)) + return false; + + if (images.empty()) + { + LastError = "No images found in images.txt"; + UpdateStatus(); + return false; + } + + Frames.clear(); + Frames.reserve(images.size()); + Timecodes.clear(); + + auto timecodesPath = dir / "timecodes.txt"; + if (std::filesystem::exists(timecodesPath)) + ParseTimecodesTxt(timecodesPath, images.size()); + + std::vector extras; + auto extrasPath = dir / "extras.txt"; + if (std::filesystem::exists(extrasPath)) + ParseExtrasTxt(extrasPath, images.size(), extras); + + // Inverse of RecordTrackCOLMAP::WriteImagesTxt: + // images.txt holds R_w2c in COLMAP frame, t = -R_w2c * pos_colmap. + // pos_colmap = -R_c2w_colmap * t (R_c2w_colmap = R_w2c^T) + // pos_target = M^-1 * pos_colmap + // R_c2w_target = M^-1 * R_c2w_colmap * M + // Track.rotation = MatToEuler(TargetFrame, R_c2w_target) + const glm::dmat3 Minv = convention::BasisChangeFromColmap(TargetFrame); + const glm::dmat3 M = glm::inverse(Minv); + + for (size_t i = 0; i < images.size(); ++i) + { + auto& img = images[i]; + sys::track::TTrack trackData{}; + auto camIt = cameras.find(img.CameraId); + const ExtrasEntry* ex = (i < extras.size() && extras[i].Present) ? &extras[i] : nullptr; + + glm::dmat3 R_w2c = glm::dmat3(glm::mat3_cast(img.Q)); + glm::dmat3 R_c2w_colmap = glm::transpose(R_w2c); + glm::dvec3 pos_colmap = -R_c2w_colmap * glm::dvec3(img.T); + + glm::dvec3 pos_target = Minv * pos_colmap; + glm::vec3 locF((float)pos_target.x, (float)pos_target.y, (float)pos_target.z); + trackData.location = reinterpret_cast(locF); + + // Rotation: prefer the original Euler from extras (avoids quaternion- + // to-Euler ambiguity near gimbal lock); fall back to extracting from + // the COLMAP rotation matrix when no extras sidecar exists. + if (ex) + { + glm::vec3 euler(ex->RotX, ex->RotY, ex->RotZ); + trackData.rotation = reinterpret_cast(euler); + } + else + { + glm::dmat3 R_c2w_target = Minv * R_c2w_colmap * M; + glm::dvec3 eulerD = convention::MatToEuler(TargetFrame, R_c2w_target); + glm::vec3 eulerF((float)eulerD.x, (float)eulerD.y, (float)eulerD.z); + trackData.rotation = reinterpret_cast(eulerF); + } + + if (camIt != cameras.end()) + { + auto& cam = camIt->second; + if (cam.Fx > 0) + trackData.fov = glm::degrees(2.0f * std::atan(cam.Width * 0.5f / cam.Fx)); + if (cam.Fx > 0 && cam.Fy > 0) + trackData.pixel_aspect_ratio = cam.Fx / cam.Fy; + trackData.lens_distortion.mutable_k1k2() = nos::fb::vec2(cam.K1, cam.K2); + + // sensor_size: COLMAP only stores pixel dims, but Track expects mm. + // Use the extras value when present; otherwise fall back to pixels + // (matches pre-extras behaviour). + glm::vec2 sensorMm(0); + if (ex && ex->SensorWmm > 0 && ex->SensorHmm > 0) + { + sensorMm = {ex->SensorWmm, ex->SensorHmm}; + trackData.sensor_size = nos::fb::vec2(sensorMm.x, sensorMm.y); + } + else + { + trackData.sensor_size = nos::fb::vec2(cam.Width, cam.Height); + } + + // center_shift: invert the (cx, cy) encoding written by record. + // Needs sensor_size in mm to be meaningful, so only reconstructed + // when extras provided it. + if (sensorMm.x > 0 && cam.Width > 0 && sensorMm.y > 0 && cam.Height > 0) + { + glm::vec2 shift{ + (cam.Cx - cam.Width * 0.5f) * sensorMm.x / cam.Width, + (cam.Cy - cam.Height * 0.5f) * sensorMm.y / cam.Height}; + trackData.lens_distortion.mutable_center_shift() = reinterpret_cast(shift); + } + } + + if (ex) + { + trackData.zoom = ex->Zoom; + trackData.focus = ex->Focus; + trackData.focus_distance = ex->FocusDistance; + trackData.render_ratio = ex->RenderRatio; + trackData.nodal_offset = ex->NodalOffset; + trackData.lens_distortion.mutate_distortion_scale(ex->DistortionScale); + } + + Frames.push_back(std::move(trackData)); + } + + CurrentFrame = 0; + LastError.clear(); + UpdateFrameCountPin(); + UpdateFrameIndexPin(); + UpdateStatus(); + nosEngine.LogI("PlaybackTrackCOLMAP: Loaded %zu frames from %s", Frames.size(), InputDir.c_str()); + return true; + } + + bool ParseCamerasTxt(const std::filesystem::path& path, std::unordered_map& cameras) + { + std::ifstream file(path); + if (!file.is_open()) + { + LastError = "Cannot open cameras.txt"; + UpdateStatus(); + return false; + } + + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + COLMAPCamera cam; + ss >> cam.Id >> cam.Model >> cam.Width >> cam.Height; + if (cam.Model == "OPENCV") + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy >> cam.K1 >> cam.K2 >> cam.P1 >> cam.P2; + else if (cam.Model == "PINHOLE") + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy; + else if (cam.Model == "SIMPLE_PINHOLE") + { + float f; + ss >> f >> cam.Cx >> cam.Cy; + cam.Fx = cam.Fy = f; + } + else if (cam.Model == "SIMPLE_RADIAL") + { + float f; + ss >> f >> cam.Cx >> cam.Cy >> cam.K1; + cam.Fx = cam.Fy = f; + } + else if (cam.Model == "RADIAL") + { + float f; + ss >> f >> cam.Cx >> cam.Cy >> cam.K1 >> cam.K2; + cam.Fx = cam.Fy = f; + } + else + { + nosEngine.LogW("PlaybackTrackCOLMAP: Unsupported camera model '%s', treating as PINHOLE", cam.Model.c_str()); + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy; + } + cameras[cam.Id] = cam; + } + return true; + } + + bool ParseImagesTxt(const std::filesystem::path& path, std::vector& images) + { + std::ifstream file(path); + if (!file.is_open()) + { + LastError = "Cannot open images.txt"; + UpdateStatus(); + return false; + } + + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + COLMAPImage img; + float qw, qx, qy, qz; + std::string name; + ss >> img.Id >> qw >> qx >> qy >> qz + >> img.T.x >> img.T.y >> img.T.z + >> img.CameraId >> name; + img.Q = glm::quat(qw, qx, qy, qz); + images.push_back(img); + // Skip POINTS2D line + std::getline(file, line); + } + + std::sort(images.begin(), images.end(), [](auto& a, auto& b) { return a.Id < b.Id; }); + return true; + } + + void ParseExtrasTxt(const std::filesystem::path& path, size_t expectedCount, std::vector& outExtras) + { + std::ifstream file(path); + if (!file.is_open()) + return; + std::unordered_map byId; + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + uint32_t id = 0; + ExtrasEntry e; + ss >> id >> e.Zoom >> e.Focus >> e.FocusDistance >> e.RenderRatio + >> e.NodalOffset >> e.DistortionScale + >> e.SensorWmm >> e.SensorHmm + >> e.RotX >> e.RotY >> e.RotZ; + if (!ss.fail()) + { + e.Present = true; + byId[id] = e; + } + } + outExtras.assign(expectedCount, ExtrasEntry{}); + for (size_t i = 0; i < expectedCount; ++i) + { + auto it = byId.find(uint32_t(i + 1)); + if (it != byId.end()) + outExtras[i] = it->second; + } + } + + void ParseTimecodesTxt(const std::filesystem::path& path, size_t expectedCount) + { + std::ifstream file(path); + if (!file.is_open()) + return; + std::unordered_map byId; + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + uint32_t id = 0; + TimecodeEntry e; + ss >> id >> e.Timecode >> e.FrameNumber; + if (e.Timecode == "-") + e.Timecode.clear(); + byId[id] = std::move(e); + } + Timecodes.assign(expectedCount, TimecodeEntry{}); + TimecodeToIndex.clear(); + FrameNumberToIndex.clear(); + for (size_t i = 0; i < expectedCount; ++i) + { + auto it = byId.find(uint32_t(i + 1)); + if (it == byId.end()) + continue; + Timecodes[i] = it->second; + if (!Timecodes[i].Timecode.empty()) + TimecodeToIndex.emplace(Timecodes[i].Timecode, uint32_t(i)); + FrameNumberToIndex.emplace(Timecodes[i].FrameNumber, uint32_t(i)); + } + } + + // --- Execution --- + + bool ResolveFrameIndex(uint32_t& outIdx) + { + switch (Mode) + { + case PlaybackTrackMode::Timecode: + { + auto it = TimecodeToIndex.find(InTimecode); + if (it == TimecodeToIndex.end()) + return false; + outIdx = it->second; + return true; + } + case PlaybackTrackMode::FrameNumber: + { + auto it = FrameNumberToIndex.find(InFrameNumber); + if (it == FrameNumberToIndex.end()) + return false; + outIdx = it->second; + return true; + } + case PlaybackTrackMode::FrameIndex: + default: + outIdx = FrameIndex < (uint32_t)Frames.size() ? FrameIndex : (uint32_t)Frames.size() - 1; + return true; + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Frames.empty()) + { + sys::track::TTrack empty{}; + auto buf = nos::Buffer::From(empty); + SetPinValue(NOS_NAME("Track"), {.Data = buf.Data(), .Size = buf.Size()}); + return NOS_RESULT_SUCCESS; + } + + uint32_t frameIdx = 0; + if (!ResolveFrameIndex(frameIdx)) + frameIdx = CurrentFrame < (uint32_t)Frames.size() ? CurrentFrame : 0; + CurrentFrame = frameIdx; + + auto buf = nos::Buffer::From(Frames[frameIdx]); + SetPinValue(NOS_NAME("Track"), {.Data = buf.Data(), .Size = buf.Size()}); + UpdateFrameIndexPin(); + + return NOS_RESULT_SUCCESS; + } + + static nosResult GetFunctions(size_t* count, nosName* names, nosPfnNodeFunctionExecute* fns) + { + *count = 1; + if (!names || !fns) + return NOS_RESULT_SUCCESS; + + names[0] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_OpenFolder"); + fns[0] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + if (self->InputDir.empty()) + { + nosEngine.LogW("PlaybackTrackCOLMAP: Input directory not set"); + return NOS_RESULT_FAILED; + } + std::filesystem::path dir = nos::Utf8ToPath(self->InputDir); + if (!std::filesystem::exists(dir)) + { + nosEngine.LogW("PlaybackTrackCOLMAP: Directory does not exist: %s", self->InputDir.c_str()); + return NOS_RESULT_FAILED; + } + // TODO: Replace std::system with platform APIs (ShellExecuteW / posix_spawnp) +#if defined(_WIN32) + std::string cmd = "explorer \"" + nos::PathToUtf8(dir) + "\""; +#elif defined(__APPLE__) + std::string cmd = "open \"" + nos::PathToUtf8(dir) + "\""; +#else + std::string cmd = "xdg-open \"" + nos::PathToUtf8(dir) + "\""; +#endif + std::system(cmd.c_str()); + return NOS_RESULT_SUCCESS; + }; + + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterPlaybackTrackCOLMAP(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("PlaybackTrackCOLMAP"), PlaybackTrackCOLMAPContext, fn); +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp new file mode 100644 index 00000000..661d06e3 --- /dev/null +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -0,0 +1,463 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include "nosSysTrack/Track_generated.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "CoordinateFrameConv.h" + +namespace nos::track +{ + +NOS_REGISTER_NAME(OutputDirectory); +NOS_REGISTER_NAME(ImageResolution); +NOS_REGISTER_NAME(SourceFrame); +NOS_REGISTER_NAME(Record); +NOS_REGISTER_NAME(MinOffFrames); +NOS_REGISTER_NAME(FrameCount); +NOS_REGISTER_NAME(RecordingFrame); + +struct RecordedFrame +{ + glm::vec3 Location; + glm::vec3 Rotation; // Euler degrees in the SourceFrame's convention. + float FOV; + float Zoom; + float Focus; + float RenderRatio; + glm::vec2 SensorSize; + float PixelAspectRatio; + float NodalOffset; + float FocusDistance; + float K1; + float K2; + glm::vec2 CenterShift; + float DistortionScale; + std::string Timecode; + uint32_t FrameNumber; +}; + +struct RecordTrackCOLMAPContext : NodeContext +{ + std::string OutputDir; + nosVec2u ImageResolution = {1920, 1080}; + convention::Frame SourceFrame = convention::Frame::LH_ZUp_FwdX_RightY; + bool Recording = false; + uint32_t ConsecutiveOffFrames = 0; + bool LastRequestRecord = false; + std::string LastError; + std::vector Frames; + nosVec2u DeltaSeconds{}; // {numerator, denominator}; 0/0 if not in fixed-step mode + + RecordTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) + { + if (node->pins()) + { + for (auto* pin : *node->pins()) + { + auto name = nos::Name(pin->name()->c_str()); + if (flatbuffers::IsFieldPresent(pin, fb::Pin::VT_DATA)) + { + nosBuffer value = {.Data = (void*)pin->data()->data(), .Size = pin->data()->size()}; + OnPinValueChanged(name, *pin->id(), value); + } + } + } + UpdateStatus(); + } + + bool StartRecording() + { + std::string error; + if (!CanStartRecording(error)) + { + LastError = std::move(error); + UpdateStatus(); + return false; + } + LastError.clear(); + Frames.clear(); + Recording = true; + ConsecutiveOffFrames = 0; + UpdateFrameCountPin(); + UpdateRecordingFramePin(); + UpdateStatus(); + nosEngine.LogI("RecordTrackCOLMAP: Recording started"); + return true; + } + + void StopRecording() + { + Recording = false; + nosEngine.LogI("RecordTrackCOLMAP: Recording stopped (%zu frames in buffer)", Frames.size()); + if (!Frames.empty()) + WriteFiles(); + Frames.clear(); + UpdateFrameCountPin(); + UpdateRecordingFramePin(); + UpdateStatus(); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer val) override + { + if (pinName == NSN_OutputDirectory) + { + OutputDir = InterpretPinValue(val.Data); + LastError.clear(); + UpdateStatus(); + } + else if (pinName == NSN_ImageResolution) + ImageResolution = *(nosVec2u*)val.Data; + else if (pinName == NSN_SourceFrame) + SourceFrame = *(convention::Frame*)val.Data; + } + + bool CanStartRecording(std::string& outError) + { + if (OutputDir.empty()) + { + outError = "Set output directory"; + return false; + } + + std::filesystem::path outDir = nos::Utf8ToPath(OutputDir); + try + { + if (std::filesystem::exists(outDir) && !std::filesystem::is_empty(outDir)) + { + outError = "Target folder is not empty"; + return false; + } + } + catch (std::filesystem::filesystem_error& e) + { + nosEngine.LogE("RecordTrackCOLMAP: %s", e.what()); + outError = e.what(); + return false; + } + return true; + } + + void UpdateFrameCountPin() + { + uint32_t count = (uint32_t)Frames.size(); + SetPinValue(NSN_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); + } + + void UpdateRecordingFramePin() + { + uint32_t frame = Recording ? (uint32_t)Frames.size() : 0; + SetPinValue(NSN_RecordingFrame, nosBuffer{.Data = &frame, .Size = sizeof(frame)}); + } + + void UpdateStatus() + { + if (!LastError.empty()) + SetNodeStatusMessage(LastError, fb::NodeStatusMessageType::FAILURE); + else if (OutputDir.empty()) + SetNodeStatusMessage("Set output directory", fb::NodeStatusMessageType::WARNING); + else if (Recording) + SetNodeStatusMessage("Recording (" + std::to_string(Frames.size()) + " frames)", fb::NodeStatusMessageType::INFO); + else + SetNodeStatusMessage("Idle", fb::NodeStatusMessageType::INFO); + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + nos::NodeExecuteParams execParams(params); + + if (params->TimingInfo.TimingMode == NOS_EXECUTION_TIMING_MODE_FIXED_STEP) + DeltaSeconds = params->TimingInfo.FixedStepTiming.DeltaSeconds; + + // Pass through Track input to output + nosBuffer trackBuf{}; + for (size_t i = 0; i < params->PinCount; ++i) + { + if (params->Pins[i].Name == NOS_NAME("InTrack")) + { + trackBuf = {.Data = (void*)params->Pins[i].Data->Data, .Size = params->Pins[i].Data->Size}; + break; + } + } + SetPinValue(NOS_NAME("OutTrack"), trackBuf); + + // Drive recording state from the Record pin, with off-state debouncing to + // ride out brief glitches in the upstream signal (e.g. SDI bit flips on a + // camera-derived recording flag). Start happens immediately on a rising + // edge; stop only after MinOffFrames consecutive false frames. + const bool requestRecord = *execParams.GetPinData(NSN_Record); + const uint32_t minOffFrames = *execParams.GetPinData(NSN_MinOffFrames); + + const bool risingEdge = requestRecord && !LastRequestRecord; + LastRequestRecord = requestRecord; + + if (risingEdge && !Recording) + StartRecording(); + + if (Recording) + { + if (requestRecord) + ConsecutiveOffFrames = 0; + else if (++ConsecutiveOffFrames >= std::max(1u, minOffFrames)) + StopRecording(); + } + + if (!Recording) + return NOS_RESULT_SUCCESS; + + auto* trackData = flatbuffers::GetRoot(trackBuf.Data); + if (!trackData) + return NOS_RESULT_SUCCESS; + + RecordedFrame frame{}; + if (const char* tc = execParams.GetPinData(NOS_NAME_STATIC("Timecode"))) + frame.Timecode = tc; + frame.FrameNumber = *execParams.GetPinData(NOS_NAME_STATIC("FrameNumber")); + if (auto* loc = trackData->location()) + frame.Location = {loc->x(), loc->y(), loc->z()}; + if (auto* rot = trackData->rotation()) + frame.Rotation = {rot->x(), rot->y(), rot->z()}; + frame.FOV = trackData->fov(); + frame.Zoom = trackData->zoom(); + frame.Focus = trackData->focus(); + frame.RenderRatio = trackData->render_ratio(); + if (auto* ss = trackData->sensor_size()) + frame.SensorSize = {ss->x(), ss->y()}; + frame.PixelAspectRatio = trackData->pixel_aspect_ratio(); + frame.NodalOffset = trackData->nodal_offset(); + frame.FocusDistance = trackData->focus_distance(); + if (auto* ld = trackData->lens_distortion()) + { + frame.K1 = ld->k1k2().x(); + frame.K2 = ld->k1k2().y(); + frame.CenterShift = {ld->center_shift().x(), ld->center_shift().y()}; + frame.DistortionScale = ld->distortion_scale(); + } + Frames.push_back(frame); + + UpdateFrameCountPin(); + UpdateRecordingFramePin(); + UpdateStatus(); + + return NOS_RESULT_SUCCESS; + } + + void WriteFiles() + { + if (OutputDir.empty()) + { + nosEngine.LogE("RecordTrackCOLMAP: Output directory is empty"); + return; + } + if (Frames.empty()) + { + nosEngine.LogW("RecordTrackCOLMAP: No frames recorded"); + return; + } + + std::filesystem::path outDir = nos::Utf8ToPath(OutputDir); + try + { + if (!std::filesystem::exists(outDir)) + std::filesystem::create_directories(outDir); + } + catch (std::filesystem::filesystem_error& e) + { + nosEngine.LogE("RecordTrackCOLMAP: %s", e.what()); + return; + } + + WriteCamerasTxt(outDir); + WriteImagesTxt(outDir); + WriteTimecodesTxt(outDir); + WriteExtrasTxt(outDir); + nosEngine.LogI("RecordTrackCOLMAP: Saved %zu frames to %s", Frames.size(), OutputDir.c_str()); + } + + void WriteExtrasTxt(const std::filesystem::path& outDir) + { + // Sidecar for Track fields that don't fit COLMAP's standard cameras.txt / + // images.txt format. Keyed by IMAGE_ID so it pairs 1:1 with images.txt. + auto path = outDir / "extras.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + const char* frameName = + SourceFrame == convention::Frame::LH_ZUp_FwdX_RightY ? "LH_ZUp_FwdX_RightY" + : SourceFrame == convention::Frame::RH_YUp_FwdNegZ_RightX ? "RH_YUp_FwdNegZ_RightX" + : "Unknown"; + file << std::setprecision(12); + file << "# Nodos Track sidecar paired with images.txt by IMAGE_ID.\n"; + file << "# Carries fields that don't fit COLMAP's cameras.txt/images.txt:\n"; + file << "# - sensor_size in mm (cameras.txt only stores pixel WIDTH/HEIGHT)\n"; + file << "# - original Euler rotation in degrees (avoids quaternion round-trip drift)\n"; + file << "# - nodos-only fields with no COLMAP equivalent\n"; + file << "# SourceFrame: " << frameName << " (Euler convention used for ROT_X, ROT_Y, ROT_Z below).\n"; + file << "# IMAGE_ID, ZOOM, FOCUS, FOCUS_DISTANCE, RENDER_RATIO, NODAL_OFFSET, DISTORTION_SCALE, SENSOR_W_MM, SENSOR_H_MM, ROT_X, ROT_Y, ROT_Z\n"; + file << "# Number of entries: " << Frames.size() << "\n"; + for (size_t i = 0; i < Frames.size(); ++i) + { + const auto& f = Frames[i]; + file << (i + 1) << " " + << f.Zoom << " " + << f.Focus << " " + << f.FocusDistance << " " + << f.RenderRatio << " " + << f.NodalOffset << " " + << f.DistortionScale << " " + << f.SensorSize.x << " " << f.SensorSize.y << " " + << f.Rotation.x << " " << f.Rotation.y << " " << f.Rotation.z << "\n"; + } + } + + void WriteTimecodesTxt(const std::filesystem::path& outDir) + { + // Skip the sidecar entirely if no frame carried a timecode -- keeps the + // output minimal when the upstream graph isn't producing TC. + bool any = false; + for (auto& f : Frames) + if (!f.Timecode.empty() || f.FrameNumber != 0) { any = true; break; } + if (!any) + return; + + auto path = outDir / "timecodes.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + double dt = (DeltaSeconds.y != 0) ? (double)DeltaSeconds.x / (double)DeltaSeconds.y : 0.0; + file << "# Timecode sidecar paired with images.txt by IMAGE_ID.\n"; + file << "# First non-comment line: per-frame delta seconds (0 if recording wasn't in fixed-step timing).\n"; + file << "# IMAGE_ID, TIMECODE, FRAME_NUMBER\n"; + file << "# Number of entries: " << Frames.size() << "\n"; + file << std::setprecision(12) << dt << "\n"; + for (size_t i = 0; i < Frames.size(); ++i) + { + const auto& f = Frames[i]; + file << (i + 1) << " " + << (f.Timecode.empty() ? "-" : f.Timecode) << " " + << f.FrameNumber << "\n"; + } + } + + float ComputeFocalLengthPixels(const RecordedFrame& frame) const + { + if (frame.FOV <= 0.0f) + return static_cast(ImageResolution.x); + float fovRad = glm::radians(frame.FOV); + return (ImageResolution.x * 0.5f) / std::tan(fovRad * 0.5f); + } + + void WriteCamerasTxt(const std::filesystem::path& outDir) + { + auto path = outDir / "cameras.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + + file << std::setprecision(12); + file << "# COLMAP camera intrinsics. Standard format (colmap.github.io/format.html).\n"; + file << "# OPENCV model: PARAMS = fx, fy, cx, cy, k1, k2, p1, p2 (pixels).\n"; + file << "# Camera list with one line of data per camera:\n"; + file << "# CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n"; + file << "# Number of cameras: " << Frames.size() << "\n"; + + for (size_t i = 0; i < Frames.size(); ++i) + { + float fx = ComputeFocalLengthPixels(Frames[i]); + float fy = fx; + if (Frames[i].PixelAspectRatio > 0.0f) + fy = fx / Frames[i].PixelAspectRatio; + + // center_shift is in the same units as sensor_size (mm); convert to + // pixel offset on the principal point. See TrackToView.cpp:30 for the + // canonical centerShift / sensorSize relationship. + float cx = ImageResolution.x * 0.5f; + float cy = ImageResolution.y * 0.5f; + if (Frames[i].SensorSize.x > 0.0f) + cx += Frames[i].CenterShift.x * ImageResolution.x / Frames[i].SensorSize.x; + if (Frames[i].SensorSize.y > 0.0f) + cy += Frames[i].CenterShift.y * ImageResolution.y / Frames[i].SensorSize.y; + + float k1 = Frames[i].K1; + float k2 = Frames[i].K2; + + file << (i + 1) << " OPENCV " << ImageResolution.x << " " << ImageResolution.y << " " + << fx << " " << fy << " " << cx << " " << cy << " " + << k1 << " " << k2 << " 0 0\n"; + } + } + + void WriteImagesTxt(const std::filesystem::path& outDir) + { + auto path = outDir / "images.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + + file << std::setprecision(12); + file << "# COLMAP poses. Standard format (colmap.github.io/format.html).\n"; + file << "# Frame: RH, +X right, +Y down, +Z forward (camera looks along +Z).\n"; + file << "# (QW, QX, QY, QZ) is the world-to-camera rotation R_w2c.\n"; + file << "# (TX, TY, TZ) is the world-to-camera translation: t = -R_w2c * camera_world_position.\n"; + file << "# Recover camera position in the COLMAP world frame as: C = -R_w2c^T * t.\n"; + file << "# Image list with two lines of data per image:\n"; + file << "# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n"; + file << "# POINTS2D[] as (X, Y, POINT3D_ID)\n"; + file << "# Number of images: " << Frames.size() << "\n"; + + // M maps the SourceFrame to the COLMAP frame. Used to convert both the + // source-frame R_c2w and the source-frame camera position into COLMAP. + const glm::dmat3 M = convention::BasisChangeToColmap(SourceFrame); + const glm::dmat3 Minv = glm::inverse(M); + + for (size_t i = 0; i < Frames.size(); ++i) + { + auto& frame = Frames[i]; + + // Build R_c2w in the source frame, then conjugate by M to land in + // the COLMAP frame. Likewise frame the position. + glm::dmat3 R_c2w_src = convention::EulerToMat(SourceFrame, glm::dvec3(frame.Rotation)); + glm::dmat3 R_c2w_colmap = M * R_c2w_src * Minv; + glm::dvec3 pos_colmap = M * glm::dvec3(frame.Location); + + glm::dmat3 R_w2c = glm::transpose(R_c2w_colmap); + glm::dquat q_w2c = glm::quat_cast(R_w2c); + glm::dvec3 t = -R_w2c * pos_colmap; + + file << (i + 1) << " " + << q_w2c.w << " " << q_w2c.x << " " << q_w2c.y << " " << q_w2c.z << " " + << t.x << " " << t.y << " " << t.z << " " + << (i + 1) << " " + << "frame_" << std::setfill('0') << std::setw(6) << i << ".png\n"; + // Empty points line (required by COLMAP format) + file << "\n"; + } + } +}; + +void RegisterRecordTrackCOLMAP(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("RecordTrackCOLMAP"), RecordTrackCOLMAPContext, fn); +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Source/TrackMain.cpp b/Plugins/nosTrack/Source/TrackMain.cpp index c330165d..48b43fcc 100644 --- a/Plugins/nosTrack/Source/TrackMain.cpp +++ b/Plugins/nosTrack/Source/TrackMain.cpp @@ -15,12 +15,18 @@ enum TrackNode : int FreeD, UserTrack, AddTrack, + RecordTrackCOLMAP, + PlaybackTrackCOLMAP, + TrackTransform, Count }; void RegisterFreeDNode(nosNodeFunctions* functions); void RegisterController(nosNodeFunctions* functions); void RegisterAddTrack(nosNodeFunctions*); +void RegisterRecordTrackCOLMAP(nosNodeFunctions*); +void RegisterPlaybackTrackCOLMAP(nosNodeFunctions*); +void RegisterTrackTransform(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** outList) { @@ -40,7 +46,16 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou RegisterController(node); break; case TrackNode::AddTrack: - RegisterAddTrack(node); + RegisterAddTrack(node); + break; + case TrackNode::RecordTrackCOLMAP: + RegisterRecordTrackCOLMAP(node); + break; + case TrackNode::PlaybackTrackCOLMAP: + RegisterPlaybackTrackCOLMAP(node); + break; + case TrackNode::TrackTransform: + RegisterTrackTransform(node); break; } } diff --git a/Plugins/nosTrack/Source/TrackTransform.cpp b/Plugins/nosTrack/Source/TrackTransform.cpp new file mode 100644 index 00000000..0dabb1f1 --- /dev/null +++ b/Plugins/nosTrack/Source/TrackTransform.cpp @@ -0,0 +1,53 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include + +#include "CoordinateFrameConv.h" + +namespace nos::track +{ + +void RegisterTrackTransform(nosNodeFunctions* funcs) +{ + funcs->ClassName = NOS_NAME("TrackTransform"); + funcs->ExecuteNode = [](void*, nosNodeExecuteParams* params) { + auto pins = GetPinValues(params); + auto ids = GetPinIds(params); + + auto* inTrack = flatbuffers::GetMutableRoot(pins[NOS_NAME("In")]); + auto source = *static_cast(pins[NOS_NAME("Source")]); + auto target = *static_cast(pins[NOS_NAME("Target")]); + float worldScale = *static_cast(pins[NOS_NAME("WorldScale")]); + + nos::sys::track::TTrack out; + inTrack->UnPackTo(&out); + + const glm::dmat3 S_src = convention::BasisMatrix(source); + const glm::dmat3 S_tgt = convention::BasisMatrix(target); + const glm::dmat3 M = S_tgt * glm::inverse(S_src); + + // Location: basis change, then uniform world-scale. Other Track fields + // (rotation, fov, focus, sensor_size, lens_distortion, ...) are unaffected. + const auto& inLoc = *inTrack->location(); + glm::dvec3 loc(inLoc.x(), inLoc.y(), inLoc.z()); + glm::dvec3 outLoc = M * loc * static_cast(worldScale); + out.location.mutate_x(static_cast(outLoc.x)); + out.location.mutate_y(static_cast(outLoc.y)); + out.location.mutate_z(static_cast(outLoc.z)); + + // Rotation: build in source frame, conjugate by M, extract in target frame. + const auto& inRot = *inTrack->rotation(); + glm::dmat3 R_src = convention::EulerToMat(source, glm::dvec3(inRot.x(), inRot.y(), inRot.z())); + glm::dmat3 R_tgt = M * R_src * glm::transpose(M); + glm::dvec3 outRotDeg = convention::MatToEuler(target, R_tgt); + out.rotation.mutate_x(static_cast(outRotDeg.x)); + out.rotation.mutate_y(static_cast(outRotDeg.y)); + out.rotation.mutate_z(static_cast(outRotDeg.z)); + + return nosEngine.SetPinValue(ids[NOS_NAME("Out")], nos::Buffer::From(out)); + }; +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Track.noscfg b/Plugins/nosTrack/Track.noscfg index 861423c0..3810af05 100644 --- a/Plugins/nosTrack/Track.noscfg +++ b/Plugins/nosTrack/Track.noscfg @@ -2,21 +2,27 @@ "info": { "id": { "name": "nos.track", - "version": "1.10.0" + "version": "1.11.0" }, "display_name": "Track", "category": "Virtual Studio", "dependencies": [ { "name": "nos.sys.track", - "version": "1.0" + "version": "1.1" } ] }, "node_definitions": [ "Config/FreeD.nosdef", "Config/UserTrack.nosdef", - "Config/AddTrack.nosdef" + "Config/AddTrack.nosdef", + "Config/RecordTrackCOLMAP.nosdef", + "Config/PlaybackTrackCOLMAP.nosdef", + "Config/TrackTransform.nosdef" + ], + "custom_types": [ + "Config/PlaybackMode.fbs" ], "defaults": [ "Config/Defaults.json" diff --git a/Plugins/nosUtilities/Config/ChannelViewer.fbs b/Plugins/nosUtilities/Config/ChannelViewer.fbs index b5de7eb1..7042da21 100644 --- a/Plugins/nosUtilities/Config/ChannelViewer.fbs +++ b/Plugins/nosUtilities/Config/ChannelViewer.fbs @@ -9,9 +9,3 @@ enum ChannelViewerChannels : uint { Cb = 5, Cr = 6 } - -enum ChannelViewerFormats : uint { - Rec_601 = 0, - Rec_709 = 1, - Rec_2020 = 2 -} diff --git a/Plugins/nosUtilities/Config/ChannelViewer.nosdef b/Plugins/nosUtilities/Config/ChannelViewer.nosdef index 4a002d10..2722cba5 100644 --- a/Plugins/nosUtilities/Config/ChannelViewer.nosdef +++ b/Plugins/nosUtilities/Config/ChannelViewer.nosdef @@ -31,10 +31,10 @@ }, { "name": "Format", - "type_name": "nos.utilities.ChannelViewerFormats", + "type_name": "nos.mediaio.ColorSpace", "show_as": "PROPERTY", "can_show_as": "PROPERTY_ONLY", - "data": "Rec_709", + "data": "REC709", "description": "Sets the input texture color space,\nRequired for correct YCbCr conversion" }, { diff --git a/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef b/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef new file mode 100644 index 00000000..df1ef4c8 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef @@ -0,0 +1,56 @@ +{ + "nodes": [ + { + "class_name": "MultiBoundedQueue", + "menu_info": { + "category": "Utilities", + "display_name": "Multi Bounded Queue", + "name_aliases": [ "data structure", "algorithm", "circular", "multi", "fifo" ] + }, + "node": { + "class_name": "MultiBoundedQueue", + "display_name": "Multi Bounded Queue", + "contents_type": "Job", + "description": "Bounded FIFO queue with one or more independent input/output channel pairs sharing a single bound. Right-click the node to add a channel, right-click an Input_X or Output_X pin to remove its channel.", + "pins": [ + { + "name": "Thread", + "type_name": "nos.exe", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Size", + "type_name": "uint", + "data": 2, + "max": 120, + "min": 1, + "show_as": "PROPERTY", + "can_show_as": "PROPERTY_ONLY" + }, + { + "name": "Alignment", + "description": "Used for creating memory-aligned buffers in memory", + "type_name": "uint", + "def": 0, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Input_A", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_A", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Config/MultiLiveOut.nosdef b/Plugins/nosUtilities/Config/MultiLiveOut.nosdef new file mode 100644 index 00000000..36997973 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiLiveOut.nosdef @@ -0,0 +1,30 @@ +{ + "nodes": [ + { + "class_name": "MultiLiveOut", + "menu_info": { + "category": "Scheduling", + "display_name": "Multi Live Out" + }, + "node": { + "class_name": "MultiLiveOut", + "contents_type": "Job", + "pins": [ + { + "name": "Input_0", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_0", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef b/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef new file mode 100644 index 00000000..3fa2c4c3 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef @@ -0,0 +1,74 @@ +{ + "nodes": [ + { + "class_name": "MultiRingBuffer", + "menu_info": { + "category": "Utilities", + "display_name": "Multi Ring Buffer", + "name_aliases": [ "data structure", "algorithm", "circular", "multi" ] + }, + "node": { + "class_name": "MultiRingBuffer", + "display_name": "Multi Ring Buffer", + "contents_type": "Job", + "description": "Ring buffer with one or more independent input/output channel pairs sharing a single ring size. Right-click the node to add a channel, right-click an Input_X or Output_X pin to remove its channel.", + "pins": [ + { + "name": "Thread", + "type_name": "nos.exe", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Size", + "type_name": "uint", + "data": 2, + "max": 120, + "min": 1, + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Spare", + "type_name": "uint", + "data": 0, + "max": 119, + "min": 0, + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Alignment", + "description": "Used for creating memory-aligned buffers in memory", + "type_name": "uint", + "def": 0, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Input_A", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_A", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + }, + { + "name": "RepeatWhenFilling", + "display_name": "Repeat When Filling", + "type_name": "bool", + "description": "Serves the last value while the buffer is being filled instead of waiting & resets the ring on restart", + "def": true, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Source/ChannelViewer.cpp b/Plugins/nosUtilities/Source/ChannelViewer.cpp index a0d88f67..4fcc2e96 100644 --- a/Plugins/nosUtilities/Source/ChannelViewer.cpp +++ b/Plugins/nosUtilities/Source/ChannelViewer.cpp @@ -13,6 +13,41 @@ NOS_REGISTER_NAME_SPACED(Nos_Utilities_ChannelViewer, "nos.utilities.ChannelView namespace nos::utilities { +static nosResult MigrateNode(nosFbNodePtr nodePtr, nosBuffer* outBuffer) +{ + fb::TNode tNode; + nodePtr->UnPackTo(&tNode); + bool migrated = false; + for (auto& pin : tNode.pins) + { + if (!pin || pin->name != "Format") + continue; + bool legacyType = pin->type_name == "nos.utilities.ChannelViewerFormats" || + pin->type_name == "nos.fb.ChannelViewerFormats"; + const char* newValue = nullptr; + if (!pin->data.empty()) + { + std::string_view oldValue(reinterpret_cast(pin->data.data()), pin->data.size() - 1); + if (oldValue == "Rec_601") newValue = "REC601"; + else if (oldValue == "Rec_709") newValue = "REC709"; + else if (oldValue == "Rec_2020") newValue = "REC2020"; + } + if (!legacyType && !newValue) + continue; + pin->type_name = "nos.mediaio.ColorSpace"; + if (newValue) + { + std::string s = newValue; + pin->data = std::vector(s.c_str(), s.c_str() + s.size() + 1); + } + migrated = true; + } + if (!migrated) + return NOS_RESULT_SUCCESS; + *outBuffer = EngineBuffer::CopyFrom(tNode).Release(); + return NOS_RESULT_SUCCESS; +} + static nosResult ExecuteNode(void* ctx, nosNodeExecuteParams* pins) { auto values = GetPinValues(pins); @@ -25,7 +60,8 @@ static nosResult ExecuteNode(void* ctx, nosNodeExecuteParams* pins) glm::vec4 val{}; val[channel & 3] = 1; - constexpr glm::vec3 coeffs[3] = {{.299f, .587f, .114f}, {.2126f, .7152f, .0722f}, {.2627f, .678f, .0593f}}; + // Indexed by nos.mediaio.ColorSpace: REC709=0, REC601=1, REC2020=2 + constexpr glm::vec3 coeffs[3] = {{.2126f, .7152f, .0722f}, {.299f, .587f, .114f}, {.2627f, .678f, .0593f}}; glm::vec4 multipliers = glm::vec4(coeffs[format], channel > 3); std::vector bindings = { @@ -51,6 +87,7 @@ nosResult RegisterChannelViewer(nosNodeFunctions* out) { out->ClassName = NSN_Nos_Utilities_ChannelViewer; out->ExecuteNode = ExecuteNode; + out->MigrateNode = MigrateNode; fs::path root = nosEngine.Module->RootFolderPath; auto chViewerPath = (root / "Shaders" / "ChannelViewer.frag").generic_string(); diff --git a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp new file mode 100644 index 00000000..9f433574 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp @@ -0,0 +1,608 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#pragma once + +#include + +#include + +// External +#include +#include + +#include "MultiRing.h" +#include "Ring.h" +#include "nosUtil/Stopwatch.hpp" + +namespace nos::utilities +{ + +struct MultiBoundedQueueNodeContext : NodeContext +{ + static constexpr std::string_view CHANNEL_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + enum MenuCommandType : uint8_t + { + ADD_CHANNEL = 0, + REMOVE_CHANNEL = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t Letter; + MenuCommand(uint32_t cmd) + { + Type = static_cast(cmd & 0xFF); + Letter = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t letter) : Type(type), Letter(letter) {} + operator uint32_t() const { return (Letter << 8) | Type; } + }; + + struct Channel + { + char Letter; + nos::Name InputName; + nos::Name OutputName; + uuid InputId{}; + uuid OutputId{}; + nos::TypeInfo TypeInfo; + MultiRing::Channel* RingChannel = nullptr; + std::atomic_bool IsOutLive = false; + bool NeedsRecreation = false; + + Channel(char letter) + : Letter(letter), + InputName((std::string("Input_") + letter).c_str()), + OutputName((std::string("Output_") + letter).c_str()), + TypeInfo(NSN_Generic) + { + } + }; + + std::map> Channels; + std::unordered_map PinIdToLetter; + MultiRing Ring; + // Channels popped since the last SendScheduleRequest. One producer run + // pushes one slot per live channel, so we must only schedule again once + // every live channel has been popped — otherwise schedule requests pile + // up by a factor of N (channels) per consumer tick. + std::set PoppedSinceLastSchedule; + + std::optional RequestedRingSize = std::nullopt; + + std::string GetName() const { return "MultiBoundedQueue"; } + + static std::optional ParseLetter(std::string_view pinName) + { + auto pos = pinName.find_last_of('_'); + if (pos == std::string::npos || pos + 2 != pinName.size()) + return std::nullopt; + char c = pinName[pos + 1]; + if (c < 'A' || c > 'Z') + return std::nullopt; + return c; + } + + static bool IsInputPin(std::string_view pinName) { return pinName.starts_with("Input_"); } + static bool IsOutputPin(std::string_view pinName) { return pinName.starts_with("Output_"); } + + MultiBoundedQueueNodeContext(nosFbNodePtr node) : NodeContext(node) + { + std::vector pinsToUnorphan; + for (auto* pin : *node->pins()) + { + auto pinNameSv = pin->name()->string_view(); + if (!IsInputPin(pinNameSv) && !IsOutputPin(pinNameSv)) + continue; + auto letter = ParseLetter(pinNameSv); + if (!letter) + continue; + + auto& channel = Channels[*letter]; + if (!channel) + channel = std::make_unique(*letter); + + if (IsInputPin(pinNameSv)) + channel->InputId = uuid(*pin->id()); + else + { + channel->OutputId = uuid(*pin->id()); + channel->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + + nos::Name typeName(pin->type_name()->c_str()); + if (typeName != NSN_Generic && channel->TypeInfo->TypeName == NSN_Generic) + channel->TypeInfo = nos::TypeInfo(typeName); + + if (auto orphanState = pin->orphan_state()) + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(uuid(*pin->id())); + } + + for (auto& [_, ch] : Channels) + InitChannel(*ch); + + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); + + AddPinValueWatcher(NSN_Size, [this](nos::Buffer const& newSize, std::optional oldVal) { + uint32_t size = *newSize.As(); + if (oldVal && oldVal == newSize) + return; + RequestRingResize(size); + }); + AddPinValueWatcher(NSN_Alignment, [this](nos::Buffer const& newAlignment, std::optional oldVal) { + bool any = false; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) + { + nosEngine.SendPathRestart(ch->InputId); + ch->NeedsRecreation = true; + any = true; + } + } + if (any) + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } + }); + } + + ~MultiBoundedQueueNodeContext() override { Ring.Stop(); } + + void InitChannel(Channel& ch) + { + std::shared_ptr resource; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + resource = std::make_shared(); + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + resource = std::make_shared(); + else + resource = std::make_shared(); + + ch.RingChannel = &Ring.AddChannel(ch.Letter, std::move(resource), &ch); + } + + Channel* GetChannelByPinId(uuid const& id) + { + auto it = PinIdToLetter.find(id); + if (it == PinIdToLetter.end()) + return nullptr; + auto chIt = Channels.find(it->second); + return chIt != Channels.end() ? chIt->second.get() : nullptr; + } + + void RequestRingResize(uint32_t size) + { + if (size == 0) + { + nosEngine.LogW((GetName() + " size cannot be 0").c_str()); + return; + } + if (Ring.Size == size && (!RequestedRingSize.has_value() || *RequestedRingSize == size)) + return; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; + nosEngine.SendPathCommand(ch->InputId, ringSizeChange); + } + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + SendPathRestart(); + RequestedRingSize = size; + } + + void SendPathRestart() + { + for (auto& [_, ch] : Channels) + nosEngine.SendPathRestart(ch->InputId); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer value) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv)) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch || !ch->RingChannel) + return; + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) + { + nosEngine.SendPathRestart(ch->InputId); + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + ch->NeedsRecreation = true; + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinNameStr = nos::Name(params->InstigatorPinName).AsString(); + auto letter = ParseLetter(pinNameStr); + if (!letter) + return NOS_RESULT_FAILED; + auto chIt = Channels.find(*letter); + if (chIt == Channels.end()) + return NOS_RESULT_FAILED; + auto& ch = *chIt->second; + if (ch.TypeInfo->TypeName != NSN_Generic) + return NOS_RESULT_FAILED; + ch.TypeInfo = nos::TypeInfo(params->IncomingTypeName); + if (ch.RingChannel) + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + Ring.RemoveChannel(*letter); + ch.RingChannel = nullptr; + } + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pinInfo = params->Pins[i]; + if (pinInfo.Id == ch.InputId || pinInfo.Id == ch.OutputId) + pinInfo.OutResolvedTypeName = ch.TypeInfo->TypeName; + } + return NOS_RESULT_SUCCESS; + } + + void OnPinUpdated(const nosPinUpdate*) override + { + for (auto& [_, ch] : Channels) + if (!ch->RingChannel) + InitChannel(*ch); + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + auto it = PinIdToLetter.find(update->PinDeleted); + if (it == PinIdToLetter.end()) + return; + char letter = it->second; + PinIdToLetter.erase(it); + auto chIt = Channels.find(letter); + if (chIt == Channels.end()) + return; + auto& ch = *chIt->second; + bool inputAlive = PinIdToLetter.contains(ch.InputId); + bool outputAlive = PinIdToLetter.contains(ch.OutputId); + if (!inputAlive && !outputAlive) + { + if (ch.RingChannel) + { + Ring.RemoveChannel(letter); + ch.RingChannel = nullptr; + } + Channels.erase(chIt); + } + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto sv = pin->name()->string_view(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + auto& chPtr = Channels[*letter]; + if (!chPtr) + chPtr = std::make_unique(*letter); + if (IsInputPin(sv)) + chPtr->InputId = uuid(*pin->id()); + else + { + chPtr->OutputId = uuid(*pin->id()); + chPtr->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + if (!chPtr->RingChannel) + InitChannel(*chPtr); + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Channels.empty() || Ring.Exit) + return NOS_RESULT_FAILED; + + NodeExecuteParams pins(params); + uint32_t requestedSize = *pins.GetPinData(NSN_Size); + + struct Gathered + { + Channel* NodeCh; + MultiRing::Channel* RingCh; + void* Input; + }; + std::vector gathered; + gathered.reserve(Channels.size()); + std::vector wantedRings; + wantedRings.reserve(Channels.size()); + + uint32_t maxRequired = requestedSize; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel || ch->RingChannel->Resources.empty() || !ch->TypeInfo) + continue; + auto it = pins.find(ch->InputName); + if (it == pins.end()) + continue; + void* input = ch->RingChannel->ResInterface->GetPinInfo(it->second, false); + if (!input) + continue; + auto [required, _] = ch->RingChannel->ResInterface->GetRequiredRingSize(input, requestedSize); + if (required > maxRequired) + maxRequired = required; + gathered.push_back({ch.get(), ch->RingChannel, input}); + wantedRings.push_back(ch->RingChannel); + } + if (gathered.empty()) + { + SendScheduleRequest(0); + return NOS_RESULT_FAILED; + } + + if (Ring.Size != maxRequired) + { + RequestRingResize(maxRequired); + return NOS_RESULT_FAILED; + } + + std::vector slots; + if (!Ring.BeginPushSubset(100, wantedRings, slots)) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + for (size_t i = 0; i < gathered.size(); ++i) + { + auto& g = gathered[i]; + auto* slot = slots[i].second; + g.RingCh->ResInterface->Push(slot, g.Input, params, + NOS_NAME_STATIC("MultiBoundedQueue"), false); + if (!g.NodeCh->IsOutLive) + { + ChangePinLiveness(g.NodeCh->OutputName, true); + g.NodeCh->IsOutLive = true; + } + } + + Ring.EndPushAll(slots); + return NOS_RESULT_SUCCESS; + } + + nosResult CopyFrom(nosCopyInfo* cpy) override + { + auto* ch = GetChannelByPinId(cpy->ID); + if (!ch || !ch->RingChannel || Ring.Exit) + return NOS_RESULT_FAILED; + if (!ch->IsOutLive) + return NOS_RESULT_SUCCESS; + + ResourceInterface::ResourceBase* slot; + { + ScopedProfilerEvent _({.Name = "Wait For Filled Slot"}); + slot = Ring.BeginPop(*ch->RingChannel, 100); + } + if (!slot) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + // Propagate the slot resource's descriptor onto the output pin before + // Copy reads cpy->PinData as the destination — otherwise the GPU copy + // targets the stale (default-sized) output descriptor. + nos::Buffer outPinVal; + if (ch->RingChannel->ResInterface->BeginCopyFrom(slot, *cpy->PinData, outPinVal)) + nosEngine.SetPinValueByName(NodeId, ch->OutputName, outPinVal); + + ch->RingChannel->ResInterface->Copy(slot, cpy, NodeId); + + cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; + cpy->FrameNumber = slot->FrameNumber; + + Ring.EndPop(*ch->RingChannel, slot); + + PoppedSinceLastSchedule.insert(ch->Letter); + size_t liveCount = 0; + for (auto& [_, c] : Channels) + if (c->IsOutLive) + ++liveCount; + if (PoppedSinceLastSchedule.size() >= liveCount) + { + SendScheduleRequest(1); + PoppedSinceLastSchedule.clear(); + } + return NOS_RESULT_SUCCESS; + } + + void OnEndFrame(uuid const& pinId, nosEndFrameCause cause) override + { + if (cause != NOS_END_FRAME_FAILED) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch) + return; + if (pinId == ch->OutputId) + return; + if (!ch->IsOutLive) + return; + ChangePinLiveness(ch->OutputName, false); + ch->IsOutLive = false; + } + + void SendScheduleRequest(uint32_t count, bool reset = false) const + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = count, .Reset = reset}; + nosEngine.ScheduleNode(&schedule); + } + + void OnPathCommand(const nosPathCommand* command) override + { + switch (command->Event) + { + case NOS_RING_SIZE_CHANGE: + if (command->RingSize == 0) + return; + RequestedRingSize = command->RingSize; + nosEngine.SetPinValue(*GetPinId(NSN_Size), nos::Buffer::From(command->RingSize)); + break; + default: return; + } + } + + void OnPathStop() override + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } + + void OnPathStart() override + { + if (Channels.empty()) + return; + + PoppedSinceLastSchedule.clear(); + + Ring.ResetAll(false); + + if (RequestedRingSize) + { + Ring.ResizeAll(*RequestedRingSize); + for (auto& [_, ch] : Channels) + ch->NeedsRecreation = false; + RequestedRingSize = std::nullopt; + } + for (auto& [_, ch] : Channels) + { + if (ch->NeedsRecreation && ch->RingChannel) + { + Ring.RecreateChannelResources(*ch->RingChannel); + ch->NeedsRecreation = false; + } + } + + size_t totalSchedule = 0; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->Resources.empty()) + { + totalSchedule = std::max(totalSchedule, 1); + continue; + } + auto emptySlotCount = Ring.WritePoolSize(*ch->RingChannel); + totalSchedule = std::max(totalSchedule, emptySlotCount); + ch->RingChannel->ResInterface->OnPathStart(); + } + Ring.Start(); + if (totalSchedule > 0) + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = (uint32_t)totalSchedule}; + nosEngine.ScheduleNode(&schedule); + } + } + + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Channel", MenuCommand(ADD_CHANNEL, 0))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + if (Channels.size() <= 1) + return; + flatbuffers::FlatBufferBuilder fbb; + std::vector items = {nos::CreateContextMenuItemDirect( + fbb, "Remove Channel", MenuCommand(REMOVE_CHANNEL, static_cast(*letter)))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_CHANNEL: + { + char newLetter = 0; + for (char c : CHANNEL_LETTERS) + { + if (!Channels.contains(c)) + { + newLetter = c; + break; + } + } + if (newLetter == 0) + { + SetNodeStatusMessage("Maximum number of channels reached", fb::NodeStatusMessageType::WARNING); + return; + } + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = std::string("Input_") + newLetter; + inPin.type_name = "nos.Generic"; + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = std::string("Output_") + newLetter; + outPin.type_name = "nos.Generic"; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + outPin.live = true; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + case REMOVE_CHANNEL: + { + char letter = static_cast(command.Letter); + auto it = Channels.find(letter); + if (it == Channels.end()) + return; + auto& ch = *it->second; + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {ch.InputId, ch.OutputId}; + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + } + } +}; + +nosResult RegisterMultiBoundedQueue(nosNodeFunctions* functions) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiBoundedQueue"), MultiBoundedQueueNodeContext, functions) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/MultiLiveOut.cpp b/Plugins/nosUtilities/Source/MultiLiveOut.cpp new file mode 100644 index 00000000..c4d08d88 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiLiveOut.cpp @@ -0,0 +1,189 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include + +namespace nos::utilities +{ + +struct MultiLiveOutNode : NodeContext +{ + MultiLiveOutNode(nosFbNodePtr node) : NodeContext(node) + { + for (auto* pin : *node->pins()) + { + SetPinOrphanState(*pin->id(), nos::fb::PinOrphanStateType::ACTIVE); + auto index = GetPinIndex(pin->name()->string_view()); + if (!index) + { + nosEngine.LogE("Failed to parse index from pin name: %s", pin->name()->c_str()); + continue; + } + if (pin->show_as() == nosFbShowAs::OUTPUT_PIN) + IndexToPairs[*index].second = uuid(*pin->id()); + else + IndexToPairs[*index].first = uuid(*pin->id()); + } + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto index = GetPinIndex(pin->name()->string_view()); + if (!index) + return; + if (pin->show_as() == nosFbShowAs::OUTPUT_PIN) + IndexToPairs[*index].second = uuid(*pin->id()); + else + IndexToPairs[*index].first = uuid(*pin->id()); + } + else if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + for (auto it = IndexToPairs.begin(); it != IndexToPairs.end(); ++it) + { + if (it->second.first == update->PinDeleted || it->second.second == update->PinDeleted) + { + IndexToPairs.erase(it); + break; + } + } + } + } + + void OnMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector> items; + if (*request->item_id() == NodeId) + items.push_back(nos::CreateContextMenuItemDirect(fbb, "Add New Pair", 1)); + else + { + auto* pin = GetPin(*request->item_id()); + if (!pin) + return; + if (pin->Name == NOS_NAME("Input_0") || pin->Name == NOS_NAME("Output_0")) + return; + items.push_back(nos::CreateContextMenuItemDirect(fbb, "Remove Pair", 1)); + } + HandleEvent(CreateAppEvent( + fbb, CreateAppContextMenuUpdate( + fbb, request->item_id(), request->pos(), request->instigator(), fbb.CreateVector(items)))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + flatbuffers::FlatBufferBuilder fbb; + if (itemID == NodeId) + { + int index = 0; + for (; index < (int)IndexToPairs.size(); index++) + { + if (!IndexToPairs.contains(index)) + break; + } + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = "Output_" + std::to_string(index); + outPin.type_name = NOS_NAME("nos.Generic"); + outPin.live = true; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = "Input_" + std::to_string(index); + inPin.type_name = NOS_NAME("nos.Generic"); + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + IndexToPairs[index] = {uuid(inPin.id), uuid(outPin.id)}; + } + else + { + auto* pin = GetPin(itemID); + if (!pin) + return; + auto index = GetPinIndex(pin->Name.AsString()); + if (!index) + { + nosEngine.LogE("Failed to parse index from pin name: %s", pin->Name.AsCStr()); + return; + } + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {IndexToPairs[*index].first, IndexToPairs[*index].second}; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + IndexToPairs.erase(*index); + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinName = nos::Name(params->InstigatorPinName).AsString(); + auto index = GetPinIndex(pinName); + if (!index.has_value()) + { + strcpy(params->OutErrorMessage, "Failed to parse pin index from pin name."); + return NOS_RESULT_FAILED; + } + auto const& [firstId, secondId] = IndexToPairs[*index]; + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pin = params->Pins[i]; + if (pin.Id == firstId || pin.Id == secondId) + pin.OutResolvedTypeName = params->IncomingTypeName; + else + pin.OutResolvedTypeName = NOS_NAME("nos.Generic"); + } + return NOS_RESULT_SUCCESS; + } + + std::optional GetPinIndex(std::string_view pinName) const + { + auto indexPos = pinName.find_last_of('_'); + if (indexPos == std::string::npos) + return std::nullopt; + try + { + return std::stoi(std::string(pinName.substr(indexPos + 1))); + } + catch (...) + { + nosEngine.LogE("Failed to parse index from pin name: %s", std::string(pinName).c_str()); + return std::nullopt; + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + for (auto const& [_, idPair] : IndexToPairs) + { + for (size_t i = 0; i < params->PinCount; ++i) + { + auto& pin = params->Pins[i]; + if (pin.Id == idPair.first && pin.Data) + { + nosEngine.SetPinValue(idPair.second, *pin.Data); + break; + } + } + } + return NOS_RESULT_SUCCESS; + } + + std::unordered_map> IndexToPairs; +}; + +nosResult RegisterMultiLiveOut(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiLiveOut"), MultiLiveOutNode, fn) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/MultiRing.h b/Plugins/nosUtilities/Source/MultiRing.h new file mode 100644 index 00000000..c2d0ba14 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiRing.h @@ -0,0 +1,256 @@ +/* + * Copyright MediaZ Teknoloji A.S. All Rights Reserved. + */ + +#pragma once + +#include "Ring.h" + +namespace nos +{ + +// Ring that holds N independent channels under a single mutex / CV pair. +// Each channel still owns its own slot pools and Resources, but every push, +// pop, resize and reset goes through the shared synchronization, so an +// N-channel batch push is a single lock acquisition, not N. +struct MultiRing +{ + struct Channel + { + std::shared_ptr ResInterface; + std::vector> Resources; + std::deque WritePool; + std::deque ReadPool; + void* UserData = nullptr; + }; + + std::map> Channels; + std::mutex Mutex; + std::condition_variable WriteCV; + std::condition_variable ReadCV; + std::atomic_bool Exit = true; + uint32_t Size = 0; + + ~MultiRing() { Stop(); } + + void Stop() + { + { + std::unique_lock lock(Mutex); + Exit = true; + } + WriteCV.notify_all(); + ReadCV.notify_all(); + } + + void Start() + { + std::unique_lock lock(Mutex); + Exit = false; + } + + void AllocateChannelResourcesUnlocked(Channel& ch) + { + ch.WritePool.clear(); + ch.ReadPool.clear(); + ch.Resources.clear(); + for (uint32_t i = 0; i < Size; ++i) + { + auto res = ch.ResInterface->CreateResource(); + if (!res) + { + nosEngine.LogE("Failed to create resource for multi ring buffer."); + ch.Resources.clear(); + ch.WritePool.clear(); + ch.ReadPool.clear(); + Exit = true; + return; + } + ch.Resources.push_back(res); + ch.WritePool.push_back(res.get()); + } + } + + Channel& AddChannel(char key, std::shared_ptr resInterface, void* userData = nullptr) + { + std::unique_lock lock(Mutex); + auto& ch = Channels[key]; + if (!ch) + ch = std::make_unique(); + ch->ResInterface = std::move(resInterface); + ch->UserData = userData; + if (Size == 0) + Size = 1; + AllocateChannelResourcesUnlocked(*ch); + return *ch; + } + + void RemoveChannel(char key) + { + std::unique_lock lock(Mutex); + Channels.erase(key); + } + + void RecreateChannelResources(Channel& ch) + { + std::unique_lock lock(Mutex); + AllocateChannelResourcesUnlocked(ch); + } + + void ResizeAll(uint32_t newSize) + { + std::unique_lock lock(Mutex); + Size = newSize; + for (auto& [_, ch] : Channels) + AllocateChannelResourcesUnlocked(*ch); + } + + bool AreAllChannelsValid() + { + std::unique_lock lock(Mutex); + if (Channels.empty()) + return false; + for (auto& [_, ch] : Channels) + if (ch->Resources.empty()) + return false; + return true; + } + + // Move slots between pools for every channel. fill=false: read→write. + void ResetAll(bool fill) + { + std::unique_lock lock(Mutex); + for (auto& [_, ch] : Channels) + { + auto& from = fill ? ch->WritePool : ch->ReadPool; + auto& to = fill ? ch->ReadPool : ch->WritePool; + while (!from.empty()) + { + auto* slot = from.front(); + from.pop_front(); + ch->ResInterface->Reset(slot); + to.push_back(slot); + } + } + } + + // If this channel is full and its read pool is non-empty, hand one slot + // back to the write pool so the producer can start pushing again. + void MoveOneReadToWriteIfFull(Channel& ch) + { + std::unique_lock lock(Mutex); + if (ch.ReadPool.size() != ch.Resources.size() || ch.ReadPool.empty()) + return; + auto* slot = ch.ReadPool.front(); + ch.ReadPool.pop_front(); + ch.WritePool.push_back(slot); + } + + bool IsFull(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.size() == ch.Resources.size(); + } + + bool IsEmpty(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.empty(); + } + + size_t WritePoolSize(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.WritePool.size(); + } + + size_t ReadPoolSize(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.size(); + } + + using SlotPair = std::pair; + + // Atomically pop one slot from each requested channel's WritePool. + // Waits until every requested channel has at least one slot, or + // timeout/exit. The caller-supplied list typically excludes channels + // that don't have valid input data this frame. + bool BeginPushSubset(uint64_t timeoutMs, + std::vector const& wanted, + std::vector& outSlots) + { + std::unique_lock lock(Mutex); + auto pred = [&] { + if (Exit) + return true; + if (wanted.empty()) + return false; + for (auto* ch : wanted) + if (ch->WritePool.empty()) + return false; + return true; + }; + if (!WriteCV.wait_for(lock, std::chrono::milliseconds(timeoutMs), pred)) + return false; + if (Exit) + return false; + outSlots.clear(); + outSlots.reserve(wanted.size()); + for (auto* ch : wanted) + { + auto* slot = ch->WritePool.front(); + ch->WritePool.pop_front(); + outSlots.emplace_back(ch, slot); + } + return true; + } + + void EndPushAll(std::vector const& slots) + { + { + std::unique_lock lock(Mutex); + for (auto& [ch, slot] : slots) + ch->ReadPool.push_back(slot); + } + ReadCV.notify_all(); + } + + void CancelPushAll(std::vector const& slots) + { + { + std::unique_lock lock(Mutex); + for (auto& [ch, slot] : slots) + { + slot->FrameNumber = 0; + ch->WritePool.push_front(slot); + } + } + WriteCV.notify_all(); + } + + ResourceInterface::ResourceBase* BeginPop(Channel& ch, uint64_t timeoutMs) + { + std::unique_lock lock(Mutex); + if (!ReadCV.wait_for(lock, std::chrono::milliseconds(timeoutMs), + [&] { return !ch.ReadPool.empty() || Exit; })) + return nullptr; + if (Exit) + return nullptr; + auto* slot = ch.ReadPool.front(); + ch.ReadPool.pop_front(); + return slot; + } + + void EndPop(Channel& ch, ResourceInterface::ResourceBase* slot) + { + { + std::unique_lock lock(Mutex); + slot->FrameNumber = 0; + ch.WritePool.push_back(slot); + } + WriteCV.notify_all(); + } +}; + +} // namespace nos diff --git a/Plugins/nosUtilities/Source/MultiRingBuffer.cpp b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp new file mode 100644 index 00000000..2c446299 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp @@ -0,0 +1,731 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#pragma once + +#include + +#include + +// External +#include +#include + +#include "MultiRing.h" +#include "Ring.h" +#include "nosUtil/Stopwatch.hpp" + +namespace nos::utilities +{ + +struct MultiRingBufferNodeContext : NodeContext +{ + using RingMode = RingNodeBase::RingMode; + using OnRestartType = RingNodeBase::OnRestartType; + + static constexpr std::string_view CHANNEL_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + enum MenuCommandType : uint8_t + { + ADD_CHANNEL = 0, + REMOVE_CHANNEL = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t Letter; + MenuCommand(uint32_t cmd) + { + Type = static_cast(cmd & 0xFF); + Letter = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t letter) : Type(type), Letter(letter) {} + operator uint32_t() const { return (Letter << 8) | Type; } + }; + + struct Channel + { + char Letter; + nos::Name InputName; + nos::Name OutputName; + uuid InputId{}; + uuid OutputId{}; + nos::TypeInfo TypeInfo; + MultiRing::Channel* RingChannel = nullptr; + std::atomic_bool IsOutLive = false; + ResourceInterface::ResourceBase* LastPopped = nullptr; + bool NeedsRecreation = false; + std::size_t RemainingRepeatableCount = 0; + + Channel(char letter) + : Letter(letter), + InputName((std::string("Input_") + letter).c_str()), + OutputName((std::string("Output_") + letter).c_str()), + TypeInfo(NSN_Generic) + { + } + }; + + std::map> Channels; + std::unordered_map PinIdToLetter; + MultiRing Ring; + // Channels popped since the last SendScheduleRequest. One producer run + // pushes one slot per live channel, so we must only schedule again once + // every live channel has been popped — otherwise schedule requests pile + // up by a factor of N (channels) per consumer tick. + std::set PoppedSinceLastSchedule; + + OnRestartType OnRestart = OnRestartType::WAIT_UNTIL_FULL; + std::optional RequestedRingSize = std::nullopt; + std::atomic Mode = RingMode::CONSUME; + std::condition_variable ModeCV; + std::mutex ModeMutex; + std::atomic_bool RepeatWhenFilling = false; + + std::string GetName() const { return "MultiRingBuffer"; } + + static std::optional ParseLetter(std::string_view pinName) + { + auto pos = pinName.find_last_of('_'); + if (pos == std::string::npos || pos + 2 != pinName.size()) + return std::nullopt; + char c = pinName[pos + 1]; + if (c < 'A' || c > 'Z') + return std::nullopt; + return c; + } + + static bool IsInputPin(std::string_view pinName) { return pinName.starts_with("Input_"); } + static bool IsOutputPin(std::string_view pinName) { return pinName.starts_with("Output_"); } + + MultiRingBufferNodeContext(nosFbNodePtr node) : NodeContext(node) + { + std::vector pinsToUnorphan; + for (auto* pin : *node->pins()) + { + auto pinNameSv = pin->name()->string_view(); + if (!IsInputPin(pinNameSv) && !IsOutputPin(pinNameSv)) + continue; + auto letter = ParseLetter(pinNameSv); + if (!letter) + continue; + + auto& channel = Channels[*letter]; + if (!channel) + channel = std::make_unique(*letter); + + if (IsInputPin(pinNameSv)) + channel->InputId = uuid(*pin->id()); + else + { + channel->OutputId = uuid(*pin->id()); + channel->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + + nos::Name typeName(pin->type_name()->c_str()); + if (typeName != NSN_Generic && channel->TypeInfo->TypeName == NSN_Generic) + channel->TypeInfo = nos::TypeInfo(typeName); + + if (auto orphanState = pin->orphan_state()) + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(uuid(*pin->id())); + } + + for (auto& [_, ch] : Channels) + InitChannel(*ch); + + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); + + AddPinValueWatcher(NSN_Size, [this](nos::Buffer const& newSize, std::optional oldVal) { + uint32_t size = *newSize.As(); + if (oldVal && oldVal == newSize) + return; + RequestRingResize(size); + }); + AddPinValueWatcher(NSN_Alignment, [this](nos::Buffer const& newAlignment, std::optional oldVal) { + bool any = false; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) + { + nosEngine.SendPathRestart(ch->InputId); + ch->NeedsRecreation = true; + any = true; + } + } + if (any) + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } + }); + AddPinValueWatcher(NOS_NAME_STATIC("RepeatWhenFilling"), + [this](nos::Buffer const& newVal, std::optional oldVal) { + RepeatWhenFilling = *newVal.As(); + }); + } + + ~MultiRingBufferNodeContext() override + { + for (auto& [_, ch] : Channels) + NOS_SOFT_CHECK(ch->LastPopped == nullptr); + Ring.Stop(); + } + + void InitChannel(Channel& ch) + { + std::shared_ptr resource; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + resource = std::make_shared(); + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + resource = std::make_shared(); + else + resource = std::make_shared(); + + ch.RingChannel = &Ring.AddChannel(ch.Letter, std::move(resource), &ch); + } + + Channel* GetChannelByPinId(uuid const& id) + { + auto it = PinIdToLetter.find(id); + if (it == PinIdToLetter.end()) + return nullptr; + auto chIt = Channels.find(it->second); + return chIt != Channels.end() ? chIt->second.get() : nullptr; + } + + void SeedOutputPin(Channel& ch) + { + if (!ch.RingChannel || ch.RingChannel->Resources.empty()) + return; + auto* base = ch.RingChannel->Resources[0].get(); + if (!base) + return; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + { + if (auto* res = ResourceInterface::GetResource(base)) + nosEngine.SetPinValueByName(NodeId, ch.OutputName, res->VkRes.ToPinData()); + } + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + { + if (auto* res = ResourceInterface::GetResource(base)) + { + sys::vulkan::TTexture texDef = vkss::ConvertTextureInfo(res->VkRes); + texDef.unscaled = true; + nosEngine.SetPinValueByName(NodeId, ch.OutputName, nos::Buffer::From(texDef)); + } + } + } + + void RequestRingResize(uint32_t size) + { + if (size == 0) + { + nosEngine.LogW((GetName() + " size cannot be 0").c_str()); + return; + } + if (Ring.Size == size && (!RequestedRingSize.has_value() || *RequestedRingSize == size)) + return; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; + nosEngine.SendPathCommand(ch->InputId, ringSizeChange); + } + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + SendPathRestart(); + RequestedRingSize = size; + } + + void SendPathRestart() + { + for (auto& [_, ch] : Channels) + nosEngine.SendPathRestart(ch->InputId); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer value) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv)) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch || !ch->RingChannel) + return; + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) + { + nosEngine.SendPathRestart(ch->InputId); + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + ch->NeedsRecreation = true; + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinNameStr = nos::Name(params->InstigatorPinName).AsString(); + auto letter = ParseLetter(pinNameStr); + if (!letter) + return NOS_RESULT_FAILED; + auto chIt = Channels.find(*letter); + if (chIt == Channels.end()) + return NOS_RESULT_FAILED; + auto& ch = *chIt->second; + if (ch.TypeInfo->TypeName != NSN_Generic) + return NOS_RESULT_FAILED; + ch.TypeInfo = nos::TypeInfo(params->IncomingTypeName); + // Drop the Generic-fallback ring channel so OnPinUpdated re-inits with the resolved type. + if (ch.RingChannel) + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + Ring.RemoveChannel(*letter); + ch.RingChannel = nullptr; + } + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pinInfo = params->Pins[i]; + if (pinInfo.Id == ch.InputId || pinInfo.Id == ch.OutputId) + pinInfo.OutResolvedTypeName = ch.TypeInfo->TypeName; + } + return NOS_RESULT_SUCCESS; + } + + void OnPinUpdated(const nosPinUpdate*) override + { + for (auto& [_, ch] : Channels) + if (!ch->RingChannel) + InitChannel(*ch); + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + auto it = PinIdToLetter.find(update->PinDeleted); + if (it == PinIdToLetter.end()) + return; + char letter = it->second; + PinIdToLetter.erase(it); + auto chIt = Channels.find(letter); + if (chIt == Channels.end()) + return; + auto& ch = *chIt->second; + bool inputAlive = PinIdToLetter.contains(ch.InputId); + bool outputAlive = PinIdToLetter.contains(ch.OutputId); + if (!inputAlive && !outputAlive) + { + if (ch.RingChannel) + { + Ring.RemoveChannel(letter); + ch.RingChannel = nullptr; + } + Channels.erase(chIt); + } + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto sv = pin->name()->string_view(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + auto& chPtr = Channels[*letter]; + if (!chPtr) + chPtr = std::make_unique(*letter); + if (IsInputPin(sv)) + chPtr->InputId = uuid(*pin->id()); + else + { + chPtr->OutputId = uuid(*pin->id()); + chPtr->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + if (!chPtr->RingChannel) + InitChannel(*chPtr); + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Channels.empty() || Ring.Exit) + return NOS_RESULT_FAILED; + + NodeExecuteParams pins(params); + uint32_t requestedSize = *pins.GetPinData(NSN_Size); + + struct Gathered + { + Channel* NodeCh; + MultiRing::Channel* RingCh; + void* Input; + }; + std::vector gathered; + gathered.reserve(Channels.size()); + std::vector wantedRings; + wantedRings.reserve(Channels.size()); + + uint32_t maxRequired = requestedSize; + std::string adjustMessage; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel || ch->RingChannel->Resources.empty() || !ch->TypeInfo) + continue; + auto it = pins.find(ch->InputName); + if (it == pins.end()) + continue; + void* input = ch->RingChannel->ResInterface->GetPinInfo(it->second, true); + if (!input) + continue; + auto [required, message] = ch->RingChannel->ResInterface->GetRequiredRingSize(input, requestedSize); + if (required > maxRequired) + { + maxRequired = required; + adjustMessage = message; + } + gathered.push_back({ch.get(), ch->RingChannel, input}); + wantedRings.push_back(ch->RingChannel); + } + if (gathered.empty()) + { + SendScheduleRequest(0); + return NOS_RESULT_FAILED; + } + + bool effectiveSizeAdjusted = maxRequired != requestedSize; + ClearNodeStatusMessages(); + if (effectiveSizeAdjusted) + SetNodeStatusMessage(adjustMessage, fb::NodeStatusMessageType::WARNING); + + if (Ring.Size != maxRequired) + { + RequestRingResize(maxRequired); + if (effectiveSizeAdjusted) + nosEngine.LogW("%s", adjustMessage.c_str()); + return NOS_RESULT_FAILED; + } + + std::vector slots; + if (!Ring.BeginPushSubset(100, wantedRings, slots)) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + // Push outside the lock — Vulkan command recording can be slow. + for (size_t i = 0; i < gathered.size(); ++i) + { + auto& g = gathered[i]; + auto* slot = slots[i].second; + g.RingCh->ResInterface->Push(slot, g.Input, params, + NOS_NAME_STATIC("MultiRingBuffer"), true); + if (!g.NodeCh->IsOutLive) + { + ChangePinLiveness(g.NodeCh->OutputName, true); + g.NodeCh->IsOutLive = true; + } + } + + Ring.EndPushAll(slots); + + if (Mode == RingMode::FILL) + { + bool isFillComplete = true; + for (auto* rc : wantedRings) + if (Ring.WritePoolSize(*rc) != 0) + { + isFillComplete = false; + break; + } + if (isFillComplete) + { + Mode = RingMode::CONSUME; + ModeCV.notify_all(); + } + } + + return NOS_RESULT_SUCCESS; + } + + nosResult CopyFrom(nosCopyInfo* cpy) override + { + auto* ch = GetChannelByPinId(cpy->ID); + if (!ch || !ch->RingChannel || Ring.Exit) + return NOS_RESULT_FAILED; + if (!ch->IsOutLive) + return NOS_RESULT_SUCCESS; + + // EndPop the previous frame's slot before popping a new one. We can't + // rely on OnEndFrame: the engine only fires it on the path's primary + // source pin, so live secondary outputs (e.g. a second channel feeding + // the same consumer) never receive it. By the time the consumer asks + // for the next frame on this pin, it's done with the previous one. + if (ch->LastPopped) + { + Ring.EndPop(*ch->RingChannel, ch->LastPopped); + ch->LastPopped = nullptr; + } + + if (OnRestart == OnRestartType::WAIT_UNTIL_FULL && RepeatWhenFilling) + { + if (ch->RemainingRepeatableCount > 0) + { + ch->RingChannel->ResInterface->OnRepeatPinValue(cpy); + ch->RemainingRepeatableCount--; + return NOS_RESULT_SUCCESS; + } + } + else if (Mode == RingMode::FILL) + { + std::unique_lock lock(ModeMutex); + if (!ModeCV.wait_for(lock, std::chrono::milliseconds(100), + [this] { return Mode != RingMode::FILL; })) + return NOS_RESULT_PENDING; + } + + ResourceInterface::ResourceBase* slot; + { + ScopedProfilerEvent _({.Name = "Wait For Filled Slot"}); + slot = Ring.BeginPop(*ch->RingChannel, 100); + } + if (!slot) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + nos::Buffer outPinVal; + bool changePinValue = ch->RingChannel->ResInterface->BeginCopyFrom(slot, *cpy->PinData, outPinVal); + if (changePinValue) + nosEngine.SetPinValueByName(NodeId, ch->OutputName, outPinVal); + + ch->RingChannel->ResInterface->WaitForDownloadToEnd(slot, "MultiRingBuffer", NodeName.AsString(), cpy); + + cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; + cpy->FrameNumber = slot->FrameNumber; + + ch->LastPopped = slot; + + PoppedSinceLastSchedule.insert(ch->Letter); + size_t liveCount = 0; + for (auto& [_, c] : Channels) + if (c->IsOutLive) + ++liveCount; + if (PoppedSinceLastSchedule.size() >= liveCount) + { + SendScheduleRequest(1); + PoppedSinceLastSchedule.clear(); + } + return NOS_RESULT_SUCCESS; + } + + void OnEndFrame(uuid const& pinId, nosEndFrameCause cause) override + { + auto* ch = GetChannelByPinId(pinId); + if (!ch) + return; + + if (cause == NOS_END_FRAME_FAILED) + { + if (pinId == ch->OutputId) + return; + if (!ch->IsOutLive) + return; + ChangePinLiveness(ch->OutputName, false); + ch->IsOutLive = false; + } + // EndPop happens at the start of the next CopyFrom for this channel + // rather than here, because OnEndFrame is unreliable for secondary + // live outputs. + } + + void SendScheduleRequest(uint32_t count, bool reset = false) const + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = count, .Reset = reset}; + nosEngine.ScheduleNode(&schedule); + } + + void OnPathCommand(const nosPathCommand* command) override + { + switch (command->Event) + { + case NOS_RING_SIZE_CHANGE: + if (command->RingSize == 0) + return; + RequestedRingSize = command->RingSize; + nosEngine.SetPinValue(*GetPinId(NSN_Size), nos::Buffer::From(command->RingSize)); + break; + default: return; + } + } + + void OnPathStop() override + { + if (OnRestart == OnRestartType::WAIT_UNTIL_FULL) + Mode = RingMode::FILL; + for (auto& [_, ch] : Channels) + { + if (ch->LastPopped && ch->RingChannel) + { + Ring.EndPop(*ch->RingChannel, ch->LastPopped); + ch->LastPopped = nullptr; + } + } + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } + + void OnPathStart() override + { + if (Channels.empty()) + return; + + PoppedSinceLastSchedule.clear(); + + if (OnRestart == OnRestartType::RESET || RepeatWhenFilling) + Ring.ResetAll(false); + else + { + for (auto& [_, ch] : Channels) + if (ch->RingChannel) + Ring.MoveOneReadToWriteIfFull(*ch->RingChannel); + } + + if (RequestedRingSize) + { + Ring.ResizeAll(*RequestedRingSize); + for (auto& [_, ch] : Channels) + ch->NeedsRecreation = false; + RequestedRingSize = std::nullopt; + } + for (auto& [_, ch] : Channels) + { + if (ch->NeedsRecreation && ch->RingChannel) + { + Ring.RecreateChannelResources(*ch->RingChannel); + ch->NeedsRecreation = false; + } + } + + size_t totalSchedule = 0; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->Resources.empty()) + { + totalSchedule = std::max(totalSchedule, 1); + continue; + } + auto emptySlotCount = Ring.WritePoolSize(*ch->RingChannel); + if (RepeatWhenFilling) + ch->RemainingRepeatableCount = std::max(emptySlotCount, (size_t)1) - 1; + totalSchedule = std::max(totalSchedule, emptySlotCount); + ch->RingChannel->ResInterface->OnPathStart(); + SeedOutputPin(*ch); + } + Ring.Start(); + if (totalSchedule > 0) + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = (uint32_t)totalSchedule}; + nosEngine.ScheduleNode(&schedule); + } + } + + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Channel", MenuCommand(ADD_CHANNEL, 0))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + if (Channels.size() <= 1) + return; + flatbuffers::FlatBufferBuilder fbb; + std::vector items = {nos::CreateContextMenuItemDirect( + fbb, "Remove Channel", MenuCommand(REMOVE_CHANNEL, static_cast(*letter)))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_CHANNEL: + { + char newLetter = 0; + for (char c : CHANNEL_LETTERS) + { + if (!Channels.contains(c)) + { + newLetter = c; + break; + } + } + if (newLetter == 0) + { + SetNodeStatusMessage("Maximum number of channels reached", fb::NodeStatusMessageType::WARNING); + return; + } + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = std::string("Input_") + newLetter; + inPin.type_name = "nos.Generic"; + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = std::string("Output_") + newLetter; + outPin.type_name = "nos.Generic"; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + outPin.live = true; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + case REMOVE_CHANNEL: + { + char letter = static_cast(command.Letter); + auto it = Channels.find(letter); + if (it == Channels.end()) + return; + auto& ch = *it->second; + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {ch.InputId, ch.OutputId}; + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + } + } +}; + +nosResult RegisterMultiRingBuffer(nosNodeFunctions* functions) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiRingBuffer"), MultiRingBufferNodeContext, functions) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/Sink.cpp b/Plugins/nosUtilities/Source/Sink.cpp index 043982f3..28294d7e 100644 --- a/Plugins/nosUtilities/Source/Sink.cpp +++ b/Plugins/nosUtilities/Source/Sink.cpp @@ -5,6 +5,7 @@ // stl #include +#include #include #include "Sink_generated.h" @@ -18,6 +19,33 @@ constexpr uint64_t VULKAN_TIMEOUT_BEFORE_LEAK = struct SinkNode : NodeContext { + enum MenuCommandType : uint8_t + { + ADD_INPUT = 0, + REMOVE_INPUT = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t InputIndex; + MenuCommand(uint32_t cmd) { + Type = static_cast(cmd & 0xFF); + InputIndex = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t inputIndex) : Type(type), InputIndex(inputIndex) {} + operator uint32_t() const { return (InputIndex << 8) | Type; } + }; + + static const std::unordered_set& StaticPinNames() + { + static const std::unordered_set names = { + "InExe", "Sink Input", "Sink FPS", "HasGPUWork", "GPUFrameBuffering", + "AcceptsRepeat", "SinkMode", "LatencyBudget" + }; + return names; + } + std::mutex Mutex; std::atomic ShouldStop = false; std::atomic Fps = 1000.0f / 60.0f; @@ -31,9 +59,27 @@ struct SinkNode : NodeContext std::optional> GPUFrameSyncEvents = std::nullopt; size_t GPUFrameBuffering = 1; uint64_t CurrentGPUEventIndex = 0; + std::vector DynamicInputs; SinkNode(nosFbNodePtr inNode) : NodeContext(inNode) { + std::list pinsToUnorphan; + for (auto i = 0; i < inNode->pins()->size(); i++) + { + auto pin = inNode->pins()->Get(i); + if (pin->show_as() != fb::ShowAs::INPUT_PIN) + continue; + if (StaticPinNames().contains(pin->name()->string_view())) + continue; + DynamicInputs.push_back(*pin->id()); + if (auto orphanState = pin->orphan_state()) + { + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(*pin->id()); + } + } + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); AddPinValueWatcher(NOS_NAME("HasGPUWork"), [this](nosBuffer const& newVal, std::optional oldValue) { bool hasGpuWork = *static_cast(newVal.Data); @@ -255,6 +301,93 @@ struct SinkNode : NodeContext } } + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + uint32_t cmd = MenuCommand(ADD_INPUT, 0); + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Sink", cmd, nullptr) + }; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + if (StaticPinNames().contains(pinName.AsString())) + return; + auto pinId = GetPinId(pinName); + if (!pinId) + return; + auto it = std::find(DynamicInputs.begin(), DynamicInputs.end(), *pinId); + if (it == DynamicInputs.end()) + return; + auto index = std::distance(DynamicInputs.begin(), it); + uint32_t cmd = MenuCommand(REMOVE_INPUT, static_cast(index)); + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Remove Input", cmd, nullptr) + }; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_INPUT: + { + std::string pinName; + for (size_t i = 2;; i++) + { + auto candidate = "Sink Input " + std::to_string(i); + if (!GetPinId(nos::Name(candidate))) + { + pinName = std::move(candidate); + break; + } + } + flatbuffers::FlatBufferBuilder fbb; + uuid pinId = nosEngine.GenerateID(); + std::vector pins = { + fb::CreatePinDirect(fbb, &pinId, pinName.c_str(), "nos.Generic", + fb::ShowAs::INPUT_PIN, fb::CanShowAs::INPUT_PIN_ONLY) + }; + HandleEvent(CreateAppEvent(fbb, CreatePartialNodeUpdateDirect(fbb, &NodeId, ClearFlags::NONE, 0, &pins))); + break; + } + case REMOVE_INPUT: + { + if (command.InputIndex >= DynamicInputs.size()) + return; + auto pinId = DynamicInputs[command.InputIndex]; + flatbuffers::FlatBufferBuilder fbb; + std::vector pinsToRemove = { *&pinId }; + HandleEvent(CreateAppEvent(fbb, CreatePartialNodeUpdateDirect(fbb, &NodeId, ClearFlags::NONE, &pinsToRemove))); + break; + } + } + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + std::erase_if(DynamicInputs, [&](auto id) { return id == update->PinDeleted; }); + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + if (pin->show_as() != fb::ShowAs::INPUT_PIN) + return; + if (StaticPinNames().contains(pin->name()->string_view())) + return; + DynamicInputs.push_back(*pin->id()); + } + } + void GetScheduleInfo(nosScheduleInfo* info) override { info->Type = NOS_SCHEDULE_TYPE_ON_DEMAND; diff --git a/Plugins/nosUtilities/Source/UtilitiesMain.cpp b/Plugins/nosUtilities/Source/UtilitiesMain.cpp index c3d3e24a..826732ca 100644 --- a/Plugins/nosUtilities/Source/UtilitiesMain.cpp +++ b/Plugins/nosUtilities/Source/UtilitiesMain.cpp @@ -41,7 +41,9 @@ enum Utilities : int PropagateExecution, UploadBufferProvider, BoundedQueue, + MultiBoundedQueue, RingBuffer, + MultiRingBuffer, Host, DeinterlacedBoundedTextureQueue, DeinterlacedBufferRing, @@ -57,6 +59,7 @@ enum Utilities : int GridOutputLayout, LoadCubeLUT, RepeatingJunction, + MultiLiveOut, Count }; @@ -75,7 +78,9 @@ nosResult RegisterSink(nosNodeFunctions*); nosResult RegisterPropagateExecution(nosNodeFunctions*); nosResult RegisterUploadBufferProvider(nosNodeFunctions*); nosResult RegisterBoundedQueue(nosNodeFunctions*); +nosResult RegisterMultiBoundedQueue(nosNodeFunctions*); nosResult RegisterRingBuffer(nosNodeFunctions*); +nosResult RegisterMultiRingBuffer(nosNodeFunctions*); nosResult RegisterHost(nosNodeFunctions*); nosResult RegisterPin2Json(nosNodeFunctions*); nosResult RegisterJson2Pin(nosNodeFunctions*); @@ -93,6 +98,7 @@ nosResult RegisterFreeOutputLayout(nosNodeFunctions*); nosResult RegisterGridOutputLayout(nosNodeFunctions*); nosResult RegisterLoadCubeLUT(nosNodeFunctions*); nosResult RegisterRepeatingJunction(nosNodeFunctions*); +nosResult RegisterMultiLiveOut(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** outList) { @@ -129,7 +135,9 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou GEN_CASE_NODE(PropagateExecution) GEN_CASE_NODE(UploadBufferProvider) GEN_CASE_NODE(BoundedQueue) + GEN_CASE_NODE(MultiBoundedQueue) GEN_CASE_NODE(RingBuffer) + GEN_CASE_NODE(MultiRingBuffer) GEN_CASE_NODE(Host) GEN_CASE_NODE(DeinterlacedBoundedTextureQueue) GEN_CASE_NODE(DeinterlacedBufferRing) @@ -145,6 +153,7 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou GEN_CASE_NODE(GridOutputLayout) GEN_CASE_NODE(LoadCubeLUT) GEN_CASE_NODE(RepeatingJunction) + GEN_CASE_NODE(MultiLiveOut) } } return NOS_RESULT_SUCCESS; @@ -163,7 +172,7 @@ NOSAPI_ATTR nosResult NOSAPI_CALL nosExportPlugin(nosPluginFunctions* out) } // clang-format off outRenamedFrom[0] = NOS_NAME("nos.fb.ChannelViewerChannels"); outRenamedTo[0] = NOS_NAME("nos.utilities.ChannelViewerChannels"); - outRenamedFrom[1] = NOS_NAME("nos.fb.ChannelViewerFormats"); outRenamedTo[1] = NOS_NAME("nos.utilities.ChannelViewerFormats"); + outRenamedFrom[1] = NOS_NAME("nos.fb.ChannelViewerFormats"); outRenamedTo[1] = NOS_NAME("nos.mediaio.ColorSpace"); outRenamedFrom[2] = NOS_NAME("nos.fb.GradientKind"); outRenamedTo[2] = NOS_NAME("nos.utilities.GradientKind"); outRenamedFrom[3] = NOS_NAME("nos.fb.BlendMode"); outRenamedTo[3] = NOS_NAME("nos.utilities.BlendMode"); outRenamedFrom[4] = NOS_NAME("nos.fb.ResizeMethod"); outRenamedTo[4] = NOS_NAME("nos.utilities.ResizeMethod"); diff --git a/Plugins/nosUtilities/Utilities.noscfg b/Plugins/nosUtilities/Utilities.noscfg index 79883ae9..b83efd55 100644 --- a/Plugins/nosUtilities/Utilities.noscfg +++ b/Plugins/nosUtilities/Utilities.noscfg @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.utilities", - "version": "3.14.8" + "version": "3.15.0" }, "description": "Various utility nodes.", "display_name": "Utilities", @@ -43,7 +43,9 @@ "Config/UploadBufferProvider.nosdef", "Config/TimedFunctionSignaller.nosdef", "Config/RingBuffer.nosdef", + "Config/MultiRingBuffer.nosdef", "Config/BoundedQueue.nosdef", + "Config/MultiBoundedQueue.nosdef", "Config/Host.nosdef", "Config/AutoResize.nosdef", "Config/ExecDepend.nosdef", @@ -63,7 +65,8 @@ "Config/CalculateDispatchSize.nosdef", "Config/YADIF.nosdef", "Config/YADIFWithAutoDispatchSize.nosdef", - "Config/RepeatingJunction.nosdef" + "Config/RepeatingJunction.nosdef", + "Config/MultiLiveOut.nosdef" ], "custom_types": [ "Config/Merge.fbs", diff --git a/Subsystems/nosTrackSubsystem/Config/Track.fbs b/Subsystems/nosTrackSubsystem/Config/Track.fbs index e1dcce23..f36f2818 100644 --- a/Subsystems/nosTrackSubsystem/Config/Track.fbs +++ b/Subsystems/nosTrackSubsystem/Config/Track.fbs @@ -42,3 +42,18 @@ enum RotationSystem : uint { RPT = 4, PRT = 5, } + +// World coordinate frame convention used by a Track endpoint. Encodes axis +// assignments to world-semantic directions (forward, right, up), the implied +// handedness, and the Euler convention for the Track.rotation field. +enum CoordinateFrame : ubyte { + // Left-handed, Z-up. +X forward, +Y right, +Z up. + // Rotation: rot.x = roll (X), rot.y = pitch (Y), rot.z = yaw (Z), + // intrinsic ZYX => R = Rz(yaw) * Ry(pitch) * Rx(roll). + LH_ZUp_FwdX_RightY = 0, + + // Right-handed, Y-up. +X right, +Y up, -Z forward. + // Rotation: rot.x = pitch (X), rot.y = yaw (Y), rot.z = roll (Z), + // intrinsic YXZ => R = Ry(yaw) * Rx(pitch) * Rz(roll). + RH_YUp_FwdNegZ_RightX = 1, +} diff --git a/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys b/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys index d2f6b9cb..625fd3fe 100644 --- a/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys +++ b/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.sys.track", - "version": "1.0.0" + "version": "1.1.0" }, "display_name": "Track Subsystem", "dependencies": [