itten/dmf-studio-rnd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: NDI in node receives and writes audio flow

Browse Source 
- FlowDef.hpp: add make_audio_flow_def (NMOS IS-04 audio/Lnn flow)
- NDIReceiver: replace capture_v210 with capture() — single recv call
  dispatches to FrameKind::{Video,Audio,None}; AudioInfo struct carries
  sample_rate/channels/samples/channel_stride
- ndiin: optional audio writer when graph wires audio_flow_id; single
  loop with 5ms NDI poll; audio grains written immediately on arrival;
  video grains written when mxlGetCurrentIndex reaches video_index
- build_graph: rename ndiin video port flow_id → video_flow_id; add
  audio_flow_id edge (no sink yet — wired to ndiin only)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
JohannesItten
2026-07-01 12:39:35 +03:00
parent ee47d85cbd
commit cc1011cced
4 changed files with 196 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodes/ndiin/main.cpp
+97 -44
View File
@@ -1,3 +1,4 @@
#include <algorithm>
#include <cstring> #include <cstring>
#include <string> #include <string>
#include <vector> #include <vector>
@@ -25,77 +26,129 @@ class NDIInNode : public dmf::NodeBase {
return; return;
} }
const auto flow_info = config().at("flow_id"); // --- video flow ---
const auto flow_id = flow_info.at("id").get<std::string>(); const auto video_flow_info = config().at("video_flow_id");
const int width = flow_info.value("width", src.width); const auto video_flow_id = video_flow_info.at("id").get<std::string>();
const int height = flow_info.value("height", src.height); const int width = video_flow_info.value("width", src.width);
const int fps_num = flow_info.value("fps_num", src.fps_num); const int height = video_flow_info.value("height", src.height);
const int fps_den = flow_info.value("fps_den", src.fps_den); const int fps_num = video_flow_info.value("fps_num", src.fps_num);
const int fps_den = video_flow_info.value("fps_den", src.fps_den);
log("flow=%s %dx%d @ %d/%d fps", flow_id.c_str(), width, height, fps_num, fps_den); log("video flow=%s %dx%d @ %d/%d fps", video_flow_id.c_str(), width, height, fps_num, fps_den);
const std::string flow_def = mxlFlowWriter video_writer{};
dmf::make_video_flow_def(flow_id, node_id(), width, height, fps_num, fps_den); mxlFlowConfigInfo video_cfg{};
mxlFlowWriter writer{};
mxlFlowConfigInfo cfg_info{};
bool created = false; bool created = false;
mxlStatus st = mxlCreateFlowWriter( mxlStatus st = mxlCreateFlowWriter(
instance(), flow_def.c_str(), nullptr, &writer, &cfg_info, &created); instance(),
if (st != MXL_STATUS_OK) { dmf::make_video_flow_def(video_flow_id, node_id(), width, height, fps_num, fps_den).c_str(),
log("mxlCreateFlowWriter failed (status=%d)", st); nullptr, &video_writer, &video_cfg, &created);
return; if (st != MXL_STATUS_OK) { log("video mxlCreateFlowWriter failed (status=%d)", st); return; }
const uint32_t video_stride = video_cfg.discrete.sliceSizes[0];
log("video stride=%u B/line grain=%u B ring=%u grains",
video_stride, video_stride * static_cast<uint32_t>(height), video_cfg.discrete.grainCount);
// --- audio flow (optional — only created when graph wires audio_flow_id) ---
mxlFlowWriter audio_writer{};
mxlFlowConfigInfo audio_cfg{};
int sample_rate = 0;
int channels = 0;
int bit_depth = 32;
bool has_audio = config().contains("audio_flow_id");
if (has_audio) {
const auto audio_flow_info = config().at("audio_flow_id");
const auto audio_flow_id = audio_flow_info.at("id").get<std::string>();
sample_rate = audio_flow_info.value("sample_rate", 48000);
channels = audio_flow_info.value("channels", 2);
bit_depth = audio_flow_info.value("bit_depth", 32);
log("audio flow=%s %d Hz %dch %d-bit", audio_flow_id.c_str(), sample_rate, channels, bit_depth);
mxlStatus ast = mxlCreateFlowWriter(
instance(),
dmf::make_audio_flow_def(audio_flow_id, node_id(), sample_rate, channels, bit_depth,
fps_num, fps_den).c_str(),
nullptr, &audio_writer, &audio_cfg, &created);
if (ast != MXL_STATUS_OK) {
log("audio mxlCreateFlowWriter failed (status=%d) — continuing without audio", ast);
has_audio = false;
} else {
log("audio grain=%u B", audio_cfg.discrete.sliceSizes[0]);
}
} }
const uint32_t stride = cfg_info.discrete.sliceSizes[0]; // --- main loop ---
log("stride=%u B/line grain=%u B ring=%u grains", const mxlRational video_rate = {fps_num, fps_den};
stride, stride * static_cast<uint32_t>(height), cfg_info.discrete.grainCount); const mxlRational audio_rate = {sample_rate, 1};
const mxlRational rate = {fps_num, fps_den}; uint64_t video_index = mxlGetCurrentIndex(&video_rate);
uint64_t index = mxlGetCurrentIndex(&rate); uint64_t audio_index = has_audio ? mxlGetCurrentIndex(&audio_rate) : 0;
log("start index=%llu", index); log("start video_index=%llu", video_index);
// NDI can drop to 1fps for static content — hold last valid frame std::vector<uint8_t> latest_video(video_stride * height);
std::vector<uint8_t> latest_frame(stride * height); bool have_video = false;
bool have_frame = false; std::vector<float> audio_buf;
dmf::NDIReceiver::AudioInfo audio_info;
while (dmf::g_running.load(std::memory_order_relaxed)) { while (dmf::g_running.load(std::memory_order_relaxed)) {
dmf::NDIReceiver::FrameKind kind;
try { try {
if (ndi.capture_v210(latest_frame.data(), stride)) kind = ndi.capture(latest_video.data(), video_stride, audio_buf, audio_info);
have_frame = true;
} catch (const std::runtime_error& e) { } catch (const std::runtime_error& e) {
log("NDI error: %s — stopping", e.what()); log("NDI error: %s — stopping", e.what());
break; break;
} }
if (kind == dmf::NDIReceiver::FrameKind::Video) {
have_video = true;
} else if (kind == dmf::NDIReceiver::FrameKind::Audio && has_audio) {
mxlGrainInfo grain{}; mxlGrainInfo grain{};
uint8_t* buf = nullptr; uint8_t* buf = nullptr;
if (mxlFlowWriterOpenGrain(audio_writer, audio_index, &grain, &buf) == MXL_STATUS_OK) {
st = mxlFlowWriterOpenGrain(writer, index, &grain, &buf); // Convert float32 planar → int32 interleaved PCM
if (st != MXL_STATUS_OK) { const int audio_grain_size = audio_cfg.discrete.sliceSizes[0];
log("OpenGrain failed (status=%d), skipping index=%llu", st, index); const int capacity = audio_grain_size / (channels * static_cast<int>(sizeof(int32_t)));
index++; const int to_write = std::min(audio_info.samples, capacity);
continue; auto* out = reinterpret_cast<int32_t*>(buf);
for (int s = 0; s < to_write; ++s)
for (int c = 0; c < audio_info.channels; ++c) {
float f = audio_buf[c * audio_info.channel_stride + s];
f = std::max(-1.0f, std::min(1.0f, f));
*out++ = static_cast<int32_t>(f * 2147483647.0f);
}
grain.validSlices = grain.totalSlices;
mxlFlowWriterCommitGrain(audio_writer, &grain);
}
audio_index += audio_info.samples;
} }
if (have_frame) { // Write video grain whenever the MXL clock has reached video_index
std::memcpy(buf, latest_frame.data(), latest_frame.size()); const uint64_t current = mxlGetCurrentIndex(&video_rate);
if (current >= video_index) {
mxlGrainInfo grain{};
uint8_t* buf = nullptr;
st = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &buf);
if (st == MXL_STATUS_OK) {
if (have_video) {
std::memcpy(buf, latest_video.data(), latest_video.size());
grain.flags = 0; grain.flags = 0;
} else { } else {
grain.flags = MXL_GRAIN_FLAG_INVALID; grain.flags = MXL_GRAIN_FLAG_INVALID;
} }
grain.validSlices = grain.totalSlices; grain.validSlices = grain.totalSlices;
mxlFlowWriterCommitGrain(writer, &grain); mxlFlowWriterCommitGrain(video_writer, &grain);
} else {
const uint64_t ns = mxlGetNsUntilIndex(index + 1, &rate); log("video OpenGrain failed (status=%d) at index=%llu", st, video_index);
if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns); }
index++; video_index = current + 1;
}
} }
log("stopped at index=%llu", index); log("stopped at video_index=%llu", video_index);
mxlReleaseFlowWriter(instance(), writer); mxlReleaseFlowWriter(instance(), video_writer);
if (has_audio) mxlReleaseFlowWriter(instance(), audio_writer);
} }
}; };
shared/FlowDef.hpp
+32
View File
@@ -36,4 +36,36 @@ inline std::string make_video_flow_def(
}.dump(); }.dump();
} }
// Generates a minimal but valid NMOS IS-04 flow definition JSON string
// for a raw PCM audio flow. grain_rate_num/den sets the grain delivery rate
// (default 25/1 = one grain per video frame). MXL computes grain size as
// sample_rate / grain_rate samples per grain.
inline std::string make_audio_flow_def(
const std::string& flow_id,
const std::string& label,
int sample_rate,
int channels,
int bit_depth = 32,
int grain_rate_num = 25,
int grain_rate_den = 1)
{
using json = nlohmann::json;
static const char* ch_labels[] = {"L","R","C","LFE","Ls","Rs","Lss","Rss"};
json ch_arr = json::array();
for (int i = 0; i < channels; ++i)
ch_arr.push_back({{"label", i < 8 ? ch_labels[i] : ("Ch" + std::to_string(i + 1))}});
return json{
{"id", flow_id},
{"format", "urn:x-nmos:format:audio"},
{"label", label},
{"description", label + " MXL Audio Flow"},
{"media_type", "audio/L" + std::to_string(bit_depth)},
{"parents", json::array()},
{"grain_rate", {{"numerator", grain_rate_num}, {"denominator", grain_rate_den}}},
{"sample_rate", {{"numerator", sample_rate}, {"denominator", 1}}},
{"channels", ch_arr},
{"bit_depth", bit_depth},
}.dump();
}
} // namespace dmf } // namespace dmf
shared/NDIReceiver.hpp
+39 -15
View File
@@ -112,12 +112,24 @@ public:
throw std::runtime_error("Interrupted during probe"); throw std::runtime_error("Interrupted during probe");
} }
// Captures one video frame and converts it to V210 in frame_buffer. enum class FrameKind { None, Video, Audio };
// Returns false if no frame was available this tick (caller should repeat last frame).
// Throws on source lost or unsupported format. struct AudioInfo {
bool capture_v210(uint8_t* frame_buffer, uint32_t frame_stride) { int sample_rate = 0;
NDIlib_video_frame_v2_t frame; int channels = 0;
auto type = NDIlib_recv_capture_v3(recv_, &frame, nullptr, nullptr, 5); int samples = 0;
int channel_stride = 0; // floats between channel planes (NDI planar layout)
};
// Receives one NDI frame. On video: converts to V210 in frame_buffer/frame_stride.
// On audio: copies float32 planar samples into audio_out and fills audio_info.
// Returns FrameKind::None on timeout or non-A/V frames.
// Throws on source lost or video format change.
FrameKind capture(uint8_t* frame_buffer, uint32_t frame_stride,
std::vector<float>& audio_out, AudioInfo& audio_info) {
NDIlib_video_frame_v2_t video_frame{};
NDIlib_audio_frame_v3_t audio_frame{};
auto type = NDIlib_recv_capture_v3(recv_, &video_frame, &audio_frame, nullptr, 5);
if (type == NDIlib_frame_type_error) if (type == NDIlib_frame_type_error)
throw std::runtime_error("NDI source lost"); throw std::runtime_error("NDI source lost");
@@ -135,30 +147,42 @@ public:
std::to_string(info_.width) + "x" + std::to_string(info_.height) + std::to_string(info_.width) + "x" + std::to_string(info_.height) +
" @" + std::to_string(info_.fps_num) + "/" + std::to_string(info_.fps_den)); " @" + std::to_string(info_.fps_num) + "/" + std::to_string(info_.fps_den));
} }
return false; return FrameKind::None;
} }
if (type != NDIlib_frame_type_video) if (type == NDIlib_frame_type_video) {
return false;
switch (info_.fourcc) { switch (info_.fourcc) {
case NDIlib_FourCC_type_UYVY: case NDIlib_FourCC_type_UYVY:
v210::UYVYtoV210(frame.p_data, frame_buffer, v210::UYVYtoV210(video_frame.p_data, frame_buffer,
info_.width, info_.height, info_.stride, frame_stride); info_.width, info_.height, info_.stride, frame_stride);
break; break;
case NDIlib_FourCC_type_P216: { case NDIlib_FourCC_type_P216: {
NDIlib_video_frame_v2_t dst{}; NDIlib_video_frame_v2_t dst{};
dst.p_data = frame_buffer; dst.p_data = frame_buffer;
dst.line_stride_in_bytes = frame_stride; dst.line_stride_in_bytes = frame_stride;
NDIlib_util_P216_to_V210(&frame, &dst); NDIlib_util_P216_to_V210(&video_frame, &dst);
break; break;
} }
default: default:
NDIlib_recv_free_video_v2(recv_, &frame); NDIlib_recv_free_video_v2(recv_, &video_frame);
throw std::runtime_error("Unsupported NDI color format: " + fourcc_str(info_.fourcc)); throw std::runtime_error("Unsupported NDI color format: " + fourcc_str(info_.fourcc));
} }
NDIlib_recv_free_video_v2(recv_, &frame); NDIlib_recv_free_video_v2(recv_, &video_frame);
return true; return FrameKind::Video;
}
if (type == NDIlib_frame_type_audio) {
audio_info.sample_rate = audio_frame.sample_rate;
audio_info.channels = audio_frame.no_channels;
audio_info.samples = audio_frame.no_samples;
audio_info.channel_stride = audio_frame.channel_stride_in_bytes / sizeof(float);
const int total = audio_frame.no_channels * audio_info.channel_stride;
audio_out.assign(audio_frame.p_data, audio_frame.p_data + total);
NDIlib_recv_free_audio_v3(recv_, &audio_frame);
return FrameKind::Audio;
}
return FrameKind::None;
} }
private: private:
studio-manager/main.cpp
+11 -4
View File
@@ -39,16 +39,23 @@ static std::string gen_uuid() {
static dmf::FlowGraph build_graph() { static dmf::FlowGraph build_graph() {
dmf::FlowGraph g; dmf::FlowGraph g;
const std::string video_flow = gen_uuid(); const std::string video_flow = gen_uuid();
const std::string audio_flow = gen_uuid();
g.nodes = { g.nodes = {
{ "ndiin", "ndiin", {} }, { "ndiin", "ndiin", {} },
{ "fakesink", "fakesink", {} }, { "fakesink", "fakesink", {} },
{ "ndiout", "ndiout", {} }, { "ndiout", "ndiout", {} },
}; };
const nlohmann::json video_fmt = {
{"kind","video"}, {"width",1920}, {"height",1080}, {"fps_num",25}, {"fps_den",1}
};
const nlohmann::json audio_fmt = {
{"kind","audio"}, {"sample_rate",48000}, {"channels",2}, {"bit_depth",32}
};
g.edges = { g.edges = {
{ video_flow, "ndiin", "flow_id", "fakesink", "flow_id", { video_flow, "ndiin", "video_flow_id", "fakesink", "flow_id", video_fmt },
{ {"kind","video"}, {"width",1920}, {"height",1080}, {"fps_num",25}, {"fps_den",1} } }, { video_flow, "ndiin", "video_flow_id", "ndiout", "flow_id", video_fmt },
{ video_flow, "ndiin", "flow_id", "ndiout", "flow_id", // audio_flow_id wired to ndiin only — no sink node yet, readers added later
{ {"kind","video"}, {"width",1920}, {"height",1080}, {"fps_num",25}, {"fps_den",1} } }, { audio_flow, "ndiin", "audio_flow_id", "", "", audio_fmt },
}; };
return g; return g;
} }