From ea0eaf8d48cbe0baac2340671d0bc497c746ec15 Mon Sep 17 00:00:00 2001 From: Johanness Date: Mon, 25 May 2026 22:21:24 +0300 Subject: [PATCH] initial commit --- .gitignore | 1 + .gitmodules | 3 + DMF_STUDIO_PLAN.md | 409 +++++++++++++++++++++++++++++++++++++++++++++ mxl | 1 + 4 files changed, 414 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 DMF_STUDIO_PLAN.md create mode 160000 mxl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..95b4aa7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +ref_arch.pdf diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0162c50 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "mxl"] + path = mxl + url = https://github.com/dmf-mxl/mxl.git diff --git a/DMF_STUDIO_PLAN.md b/DMF_STUDIO_PLAN.md new file mode 100644 index 0000000..1605ad7 --- /dev/null +++ b/DMF_STUDIO_PLAN.md @@ -0,0 +1,409 @@ +# DMF Studio — Implementation Plan + +## Product Definition + +**DMF Studio** is a node-based visual production platform for the Dynamic Media Facility architecture. It provides a canvas where users create, connect, and control containerized Media Functions that exchange live video via the MXL shared-memory layer. + +## Technology Stack + +| Component | Choice | +|---|---| +| Language | C++20 | +| Build | CMake 3.24+, vcpkg | +| MXL SDK | Git submodule (github.com/dmf-mxl/mxl) | +| WebSocket | libwebsockets | +| JSON | nlohmann/json | +| Logging | spdlog | +| GPU compute | Vulkan (headless render + compute shaders) | +| SDI I/O | Blackmagic DeckLink SDK | +| Color conversion | libyuv | +| JPEG encoding | libjpeg-turbo | +| Testing | Catch2 | +| Frontend (phase 4) | Vue 3 + Vue Flow + Vite + Pinia | + +## Architecture Principles + +1. **Separate processes per node** — crash isolation, true DMF model +2. **MXL is the ONLY data path between nodes** — zero-copy shared memory +3. **Engine owns the graph, nodes own the media** — clean separation +4. **Engine generates flow IDs, nodes create MXL readers/writers** — centralized routing +5. **WebSocket control per node** — NMOS IS-12 aligned, bidirectional +6. **All nodes use `libdmf-node` skeleton** — consistent lifecycle +7. **Node binary = 1 C++ class + 1 main.cpp** — minimal per-node code +8. **Vulkan for GPU, CPU fallback available** — GPU-accelerated but not GPU-required +9. **Video only for MVP** — audio deferred +10. **Backend first, frontend last** — validate with curl + minimal HTML before building UI + +## Project Structure + +``` +dmf-studio/ +├── CMakeLists.txt +├── vcpkg.json +├── vcpkg-configuration.json +│ +├── extern/ +│ └── mxl/ # git submodule +│ +├── libs/ +│ ├── dmf-node/ # Node skeleton library (static) +│ │ ├── CMakeLists.txt +│ │ ├── include/dmf-node/ +│ │ │ ├── node.hpp # Abstract Node interface +│ │ │ ├── port.hpp # PortDef, PortDirection, MediaType +│ │ │ ├── node_runner.hpp # CLI, MXL init, control WS, main loop +│ │ │ ├── control_server.hpp # WS server per-node +│ │ │ └── types.hpp # NodeId, PortId, FlowId aliases +│ │ └── src/ +│ │ ├── node_runner.cpp +│ │ └── control_server.cpp +│ │ +│ └── dmf-engine/ # Engine library (static) +│ ├── CMakeLists.txt +│ ├── include/dmf-engine/ +│ │ ├── graph.hpp # Graph model: nodes, edges +│ │ ├── flow_manager.hpp # Flow UUID gen, NMOS flow defs +│ │ ├── process_manager.hpp # Spawn/monitor/kill node processes +│ │ ├── api_server.hpp # REST + WS to browser +│ │ └── types.hpp # Graph-level types +│ └── src/ +│ ├── graph.cpp +│ ├── flow_manager.cpp +│ ├── process_manager.cpp +│ └── api_server.cpp +│ +├── nodes/ +│ ├── passthrough/ +│ │ ├── CMakeLists.txt +│ │ └── src/ +│ │ ├── main.cpp +│ │ ├── passthrough_node.hpp +│ │ └── passthrough_node.cpp +│ │ +│ ├── test-source/ +│ │ ├── CMakeLists.txt +│ │ └── src/ +│ │ ├── main.cpp +│ │ ├── test_source_node.hpp +│ │ ├── test_source_node.cpp +│ │ ├── vulkan_context.hpp +│ │ ├── vulkan_context.cpp +│ │ ├── vulkan_renderer.hpp +│ │ ├── vulkan_renderer.cpp +│ │ └── shaders/ +│ │ ├── pattern.vert +│ │ ├── pattern_bars.frag +│ │ ├── pattern_ramp.frag +│ │ ├── pattern_solid.frag +│ │ └── rgb_to_v210.comp +│ │ +│ ├── decklink-in/ +│ │ ├── CMakeLists.txt +│ │ └── src/ +│ │ ├── main.cpp +│ │ ├── decklink_in_node.hpp +│ │ └── decklink_in_node.cpp +│ │ +│ ├── decklink-out/ +│ │ ├── CMakeLists.txt +│ │ └── src/ +│ │ ├── main.cpp +│ │ ├── decklink_out_node.hpp +│ │ └── decklink_out_node.cpp +│ │ +│ ├── preview/ +│ │ ├── CMakeLists.txt +│ │ └── src/ +│ │ ├── main.cpp +│ │ ├── preview_node.hpp +│ │ └── preview_node.cpp +│ │ +│ └── mixer/ +│ ├── CMakeLists.txt +│ └── src/ +│ ├── main.cpp +│ ├── mixer_node.hpp +│ └── mixer_node.cpp +│ +├── engine/ +│ ├── CMakeLists.txt +│ └── src/ +│ └── main.cpp +│ +├── web/ # Phase 4 — Vue 3 frontend +│ ├── package.json +│ ├── vite.config.ts +│ ├── tsconfig.json +│ └── src/ +│ ├── App.vue +│ ├── main.ts +│ ├── api/ +│ │ ├── rest.ts +│ │ └── websocket.ts +│ ├── stores/ +│ │ ├── graph.ts +│ │ └── connection.ts +│ ├── canvas/ +│ │ ├── StudioCanvas.vue +│ │ ├── nodes/ +│ │ │ ├── SourceNode.vue +│ │ │ ├── ProcessorNode.vue +│ │ │ ├── OutputNode.vue +│ │ │ └── NodeThumbnail.vue +│ │ └── edges/ +│ │ └── FlowEdge.vue +│ ├── panels/ +│ │ ├── NodeProperties.vue +│ │ ├── Toolbar.vue +│ │ └── StatusBar.vue +│ └── templates/ +│ └── templateLoader.ts +│ +├── templates/ +│ ├── loopback.json +│ ├── sdi_monitor.json +│ ├── sdi_pass.json +│ └── simple_switcher.json +│ +├── docker/ +│ ├── Dockerfile.engine +│ ├── Dockerfile.node.test-source +│ ├── Dockerfile.node.decklink-in +│ ├── Dockerfile.node.decklink-out +│ ├── Dockerfile.node.preview +│ ├── Dockerfile.node.passthrough +│ └── docker-compose.yaml +│ +└── tests/ + ├── CMakeLists.txt + ├── test_graph.cpp + ├── test_flow_manager.cpp + └── test_control_protocol.cpp +``` + +## Node Interface + +```cpp +class Node { +public: + virtual ~Node() = default; + virtual std::string type() const = 0; + virtual std::vector input_ports() const = 0; + virtual std::vector output_ports() const = 0; + virtual void configure(const nlohmann::json& params) = 0; + virtual void on_add_writer(const std::string& port_id, mxlFlowWriter writer) = 0; + virtual void on_add_reader(const std::string& port_id, mxlFlowReader reader) = 0; + virtual void on_remove_writer(const std::string& port_id) = 0; + virtual void on_remove_reader(const std::string& port_id) = 0; + virtual void process() = 0; + virtual nlohmann::json status() const = 0; +}; +``` + +## Control Protocol (Engine ↔ Node via WebSocket) + +Engine → Node: + +```json +{ "cmd": "add_writer", "flow_id": "uuid", "flow_def": { ... } } +{ "cmd": "add_reader", "flow_id": "uuid" } +{ "cmd": "remove_writer", "port_id": "out_0" } +{ "cmd": "remove_reader", "port_id": "in_0" } +{ "cmd": "configure", "params": { ... } } +{ "cmd": "shutdown" } +``` + +Node → Engine: + +```json +{ "event": "ready", "type": "test_source", "ports": { ... } } +{ "event": "flow_status", "port_id": "out_0", "writing": true, "fps": 50 } +{ "event": "error", "message": "..." } +{ "event": "grain_stats", "port_id": "out_0", "grains_written": 1234, "latency_us": 120 } +``` + +## Flow Connection Sequence + +``` +User draws edge in UI (or REST API call): + 1. Engine generates flow UUID + NMOS flow definition + 2. Engine → Node A: { "cmd": "add_writer", "flow_id": ..., "flow_def": ... } + 3. Node A creates mxlFlowWriter, starts writing grains + 4. Node A → Engine: { "event": "flow_status", "writing": true } + 5. Engine → Node B: { "cmd": "add_reader", "flow_id": ... } + 6. Node B creates mxlFlowReader, starts reading grains + 7. Node B → Engine: { "event": "flow_status", "reading": true } + +User removes edge: + 1. Engine → Node B: { "cmd": "remove_reader", "port_id": ... } + 2. Node B destroys mxlFlowReader + 3. Engine → Node A: { "cmd": "remove_writer", "port_id": ... } + 4. Node A destroys mxlFlowWriter +``` + +## Engine REST API + +``` +GET /api/graph # Current node graph +POST /api/graph/nodes # Add node { type, id? } +DELETE /api/graph/nodes/:id # Remove node +POST /api/graph/edges # Connect { from_node:from_port, to_node:to_port } +DELETE /api/graph/edges/:id # Disconnect +PUT /api/graph/nodes/:id # Configure node +POST /api/graph/start # Start all node processes + create flows +POST /api/graph/stop # Stop all +GET /api/templates # List templates +POST /api/templates/:name/load # Load template as graph +GET /api/nodes/types # List available node types +``` + +## Node Designs + +### Test Source (Vulkan) + +- Headless Vulkan: render 2 fullscreen triangles with pattern fragment shader → RGBA8 image +- Compute shader `rgb_to_v210.comp`: RGBA8 → BT.709 YCbCr → 4:2:2 downsample → V210 packing +- vkMapMemory → memcpy to MXL grain → commit +- Patterns: SMPTE bars, luma ramp, solid color +- Frame pacing to target 50fps +- 1 output port: `video_out` (V210) + +### Passthrough + +- Read 1 MXL grain → memcpy → write 1 MXL grain +- 1 input: `video_in`, 1 output: `video_out` +- Hello-world node for validating the pipeline + +### Preview + +- Read V210 MXL grain → libyuv convert to RGBA → downsample to 480×270 → libjpeg-turbo → JPEG +- Push JPEG frames to subscribed WS clients (throttled to ~15fps) +- 1 input: `video_in` + +### DeckLink In + +- IDeckLinkInputCallback::VideoInputFrameArrived → GetBytes (V210) → memcpy to MXL grain → commit +- V210 native match = straight copy, no conversion +- 1 output: `video_out` +- Config: device index, connection type + +### DeckLink Out + +- IDeckLinkOutputCallback::ScheduledFrameCompleted → read MXL grain → memcpy to DeckLink buffer → schedule frame +- 1 input: `video_in` +- Config: device index, connection type + +### Mixer + +- 2 inputs: `video_a`, `video_b` (V210) → 1 output: `program` (V210) +- Cut: select active input +- Dissolve: blend A×(1-t) + B×t over N frames +- V210 blending: unpack 10-bit → linear blend → repack +- Config: active_input, transition_type, transition_duration_frames + +## Implementation Phases + +### Phase 1: Framework + Passthrough (Week 1-2) + +**Goal: Two processes exchanging video via MXL, controlled by engine** + +| Step | Task | Test | +|---|---|---| +| 1.1 | Repo init, CMake top-level, vcpkg, MXL submodule | Build succeeds | +| 1.2 | Verify MXL SDK: `mxl-gst-testsrc` + `mxl-gst-sink` work | Video displays | +| 1.3 | Build `libdmf-node`: node.hpp, port.hpp, control_server, node_runner | Unit tests pass | +| 1.4 | Build `passthrough` node binary | Starts, connects to MXL | +| 1.5 | Build `libdmf-engine`: graph, flow_manager, process_manager, api_server | Unit tests pass | +| 1.6 | Build engine binary with minimal REST | `curl` works | +| 1.7 | Integration: engine starts 2 passthrough processes, creates flow between them | Frames flow A→B | + +**Testing method**: curl + mxl-gst-testsrc/sink as external I/O + +### Phase 2: Test Source + Preview (Week 3-4) + +**Goal: GPU-generated color bars visible in browser** + +| Step | Task | Test | +|---|---|---| +| 2.1 | Vulkan headless context (VkDevice, queues, command pool) | Device created | +| 2.2 | Pattern fragment shaders (bars, ramp, solid) + fullscreen vert | Render to RGBA8 | +| 2.3 | `rgb_to_v210.comp` compute shader | V210 output verified | +| 2.4 | `test-source` node: render → compute → MXL write | MXL flow active | +| 2.5 | `preview` node: MXL read → libyuv → JPEG → WS | Browser sees frames | +| 2.6 | Engine integration: test-source → preview | curl starts graph, browser shows bars | + +**Testing method**: curl to start graph, open browser to preview WS endpoint, see live color bars + +### Phase 3: DeckLink I/O (Week 5-6) + +**Goal: Real SDI signal flowing through DMF** + +| Step | Task | Test | +|---|---|---| +| 3.1 | `decklink-in` node: DeckLink callback → MXL writer | SDI camera → MXL flow | +| 3.2 | `decklink-out` node: MXL reader → DeckLink playback | MXL flow → SDI monitor | +| 3.3 | Templates: sdi_monitor, sdi_loopback, test_output | Load and run via curl | +| 3.4 | Full pipeline: SDI in → preview → browser | Camera visible in browser | + +**Testing method**: curl to start graph, real SDI camera + monitor + +### Phase 4: Frontend (Week 7-8) + +**Goal: Visual graph editor on DMF TV** + +| Step | Task | Test | +|---|---|---| +| 4.1 | Vue 3 + Vite project setup, Vue Flow integration | Dev server runs | +| 4.2 | Pinia stores: graph model, WS connection | State management works | +| 4.3 | StudioCanvas: drag-add nodes, draw edges, delete | Graph editing works | +| 4.4 | Custom node components: type icon, name, status, live thumbnail | Nodes render | +| 4.5 | FlowEdge: color-coded by flow status | Edges render | +| 4.6 | NodeProperties panel: dynamic config from node schema | Config changes work | +| 4.7 | Toolbar: New/Load/Save/Run/Stop + template menu | Full workflow works | +| 4.8 | StatusBar: node count, flow count, running state | Status updates live | +| 4.9 | Test on DMF TV browser | Everything renders and works | + +**Testing method**: Full end-to-end — build graph in UI, run, see live video + +### Phase 5: Mixer Node (Week 9-10) + +**Goal: DMF Studio is a production tool** + +| Step | Task | Test | +|---|---|---| +| 5.1 | `mixer` node: 2-in, 1-out, cut + dissolve | Cut/dissolve work via curl | +| 5.2 | Mixer control panel in Vue: PGM/PVW buses, CUT, AUTO TRANS, T-bar | UI controls work | +| 5.3 | Template: Simple Switcher (2 sources → mixer → preview + SDI out) | Full production workflow | +| 5.4 | Polish, bug fixes, edge cases | Stable demo | + +## vcpkg.json Dependencies + +```json +{ + "name": "dmf-studio", + "version": "0.1.0", + "dependencies": [ + "libwebsockets", + "nlohmann-json", + "spdlog", + "fmt", + "libyuv", + "libjpeg-turbo", + "catch2" + ] +} +``` + +Vulkan: `find_package(Vulkan)` — system install, not vcpkg. +DeckLink SDK: vendor headers linked manually, not in vcpkg. + +## Key Risks and Mitigations + +| Risk | Mitigation | +|---|---| +| MXL SDK API changes | Pin submodule to v1.0.1 tag, test before upgrading | +| V210 compute shader correctness | Validate against known reference frames, compare with libyuv output | +| DeckLink SDK availability on dev machine | Develop without hardware using test-source; DeckLink nodes only need testing on machines with cards | +| Single-process engine becomes bottleneck | Engine never touches media data; it only manages metadata and process lifecycle | +| WebSocket control latency | All control is localhost; sub-ms latency. Remote control is future work | +| Vulkan driver compatibility | Start with standard Vulkan 1.2; no extensions required for headless compute | diff --git a/mxl b/mxl new file mode 160000 index 0000000..580abf7 --- /dev/null +++ b/mxl @@ -0,0 +1 @@ +Subproject commit 580abf71f4a35c2cbbcfe532e4f8e3f0803af45a