itten/dmf-studio-rnd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: NDI in node receives and writes audio flow

Browse Source 
- FlowDef.hpp: add make_audio_flow_def (NMOS IS-04 audio/Lnn flow)
- NDIReceiver: replace capture_v210 with capture() — single recv call
  dispatches to FrameKind::{Video,Audio,None}; AudioInfo struct carries
  sample_rate/channels/samples/channel_stride
- ndiin: optional audio writer when graph wires audio_flow_id; single
  loop with 5ms NDI poll; audio grains written immediately on arrival;
  video grains written when mxlGetCurrentIndex reaches video_index
- build_graph: rename ndiin video port flow_id → video_flow_id; add
  audio_flow_id edge (no sink yet — wired to ndiin only)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
JohannesItten
2026-07-01 12:39:35 +03:00
parent ee47d85cbd
commit cc1011cced
4 changed files with 196 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodes/ndiin/main.cpp
+103 -50
View File
@@ -1,3 +1,4 @@
#include <algorithm>
#include <cstring>
#include <string>
#include <vector>
@@ -25,77 +26,129 @@ class NDIInNode : public dmf::NodeBase {
return;
}
const auto flow_info = config().at("flow_id");
const auto flow_id = flow_info.at("id").get<std::string>();
const int width = flow_info.value("width", src.width);
const int height = flow_info.value("height", src.height);
const int fps_num = flow_info.value("fps_num", src.fps_num);
const int fps_den = flow_info.value("fps_den", src.fps_den);
// --- video flow ---
const auto video_flow_info = config().at("video_flow_id");
const auto video_flow_id = video_flow_info.at("id").get<std::string>();
const int width = video_flow_info.value("width", src.width);
const int height = video_flow_info.value("height", src.height);
const int fps_num = video_flow_info.value("fps_num", src.fps_num);
const int fps_den = video_flow_info.value("fps_den", src.fps_den);
log("flow=%s %dx%d @ %d/%d fps", flow_id.c_str(), width, height, fps_num, fps_den);
log("video flow=%s %dx%d @ %d/%d fps", video_flow_id.c_str(), width, height, fps_num, fps_den);
const std::string flow_def =
dmf::make_video_flow_def(flow_id, node_id(), width, height, fps_num, fps_den);
mxlFlowWriter writer{};
mxlFlowConfigInfo cfg_info{};
mxlFlowWriter video_writer{};
mxlFlowConfigInfo video_cfg{};
bool created = false;
mxlStatus st = mxlCreateFlowWriter(
instance(), flow_def.c_str(), nullptr, &writer, &cfg_info, &created);
if (st != MXL_STATUS_OK) {
log("mxlCreateFlowWriter failed (status=%d)", st);
return;
instance(),
dmf::make_video_flow_def(video_flow_id, node_id(), width, height, fps_num, fps_den).c_str(),
nullptr, &video_writer, &video_cfg, &created);
if (st != MXL_STATUS_OK) { log("video mxlCreateFlowWriter failed (status=%d)", st); return; }
const uint32_t video_stride = video_cfg.discrete.sliceSizes[0];
log("video stride=%u B/line grain=%u B ring=%u grains",
video_stride, video_stride * static_cast<uint32_t>(height), video_cfg.discrete.grainCount);
// --- audio flow (optional — only created when graph wires audio_flow_id) ---
mxlFlowWriter audio_writer{};
mxlFlowConfigInfo audio_cfg{};
int sample_rate = 0;
int channels = 0;
int bit_depth = 32;
bool has_audio = config().contains("audio_flow_id");
if (has_audio) {
const auto audio_flow_info = config().at("audio_flow_id");
const auto audio_flow_id = audio_flow_info.at("id").get<std::string>();
sample_rate = audio_flow_info.value("sample_rate", 48000);
channels = audio_flow_info.value("channels", 2);
bit_depth = audio_flow_info.value("bit_depth", 32);
log("audio flow=%s %d Hz %dch %d-bit", audio_flow_id.c_str(), sample_rate, channels, bit_depth);
mxlStatus ast = mxlCreateFlowWriter(
instance(),
dmf::make_audio_flow_def(audio_flow_id, node_id(), sample_rate, channels, bit_depth,
fps_num, fps_den).c_str(),
nullptr, &audio_writer, &audio_cfg, &created);
if (ast != MXL_STATUS_OK) {
log("audio mxlCreateFlowWriter failed (status=%d) — continuing without audio", ast);
has_audio = false;
} else {
log("audio grain=%u B", audio_cfg.discrete.sliceSizes[0]);
}
}
const uint32_t stride = cfg_info.discrete.sliceSizes[0];
log("stride=%u B/line grain=%u B ring=%u grains",
stride, stride * static_cast<uint32_t>(height), cfg_info.discrete.grainCount);
// --- main loop ---
const mxlRational video_rate = {fps_num, fps_den};
const mxlRational audio_rate = {sample_rate, 1};
const mxlRational rate = {fps_num, fps_den};
uint64_t index = mxlGetCurrentIndex(&rate);
log("start index=%llu", index);
uint64_t video_index = mxlGetCurrentIndex(&video_rate);
uint64_t audio_index = has_audio ? mxlGetCurrentIndex(&audio_rate) : 0;
log("start video_index=%llu", video_index);
// NDI can drop to 1fps for static content — hold last valid frame
std::vector<uint8_t> latest_frame(stride * height);
bool have_frame = false;
std::vector<uint8_t> latest_video(video_stride * height);
bool have_video = false;
std::vector<float> audio_buf;
dmf::NDIReceiver::AudioInfo audio_info;
while (dmf::g_running.load(std::memory_order_relaxed)) {
dmf::NDIReceiver::FrameKind kind;
try {
if (ndi.capture_v210(latest_frame.data(), stride))
have_frame = true;
kind = ndi.capture(latest_video.data(), video_stride, audio_buf, audio_info);
} catch (const std::runtime_error& e) {
log("NDI error: %s — stopping", e.what());
break;
}
mxlGrainInfo grain{};
uint8_t* buf = nullptr;
st = mxlFlowWriterOpenGrain(writer, index, &grain, &buf);
if (st != MXL_STATUS_OK) {
log("OpenGrain failed (status=%d), skipping index=%llu", st, index);
index++;
continue;
if (kind == dmf::NDIReceiver::FrameKind::Video) {
have_video = true;
} else if (kind == dmf::NDIReceiver::FrameKind::Audio && has_audio) {
mxlGrainInfo grain{};
uint8_t* buf = nullptr;
if (mxlFlowWriterOpenGrain(audio_writer, audio_index, &grain, &buf) == MXL_STATUS_OK) {
// Convert float32 planar → int32 interleaved PCM
const int audio_grain_size = audio_cfg.discrete.sliceSizes[0];
const int capacity = audio_grain_size / (channels * static_cast<int>(sizeof(int32_t)));
const int to_write = std::min(audio_info.samples, capacity);
auto* out = reinterpret_cast<int32_t*>(buf);
for (int s = 0; s < to_write; ++s)
for (int c = 0; c < audio_info.channels; ++c) {
float f = audio_buf[c * audio_info.channel_stride + s];
f = std::max(-1.0f, std::min(1.0f, f));
*out++ = static_cast<int32_t>(f * 2147483647.0f);
}
grain.validSlices = grain.totalSlices;
mxlFlowWriterCommitGrain(audio_writer, &grain);
}
audio_index += audio_info.samples;
}
if (have_frame) {
std::memcpy(buf, latest_frame.data(), latest_frame.size());
grain.flags = 0;
} else {
grain.flags = MXL_GRAIN_FLAG_INVALID;
// Write video grain whenever the MXL clock has reached video_index
const uint64_t current = mxlGetCurrentIndex(&video_rate);
if (current >= video_index) {
mxlGrainInfo grain{};
uint8_t* buf = nullptr;
st = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &buf);
if (st == MXL_STATUS_OK) {
if (have_video) {
std::memcpy(buf, latest_video.data(), latest_video.size());
grain.flags = 0;
} else {
grain.flags = MXL_GRAIN_FLAG_INVALID;
}
grain.validSlices = grain.totalSlices;
mxlFlowWriterCommitGrain(video_writer, &grain);
} else {
log("video OpenGrain failed (status=%d) at index=%llu", st, video_index);
}
video_index = current + 1;
}
grain.validSlices = grain.totalSlices;
mxlFlowWriterCommitGrain(writer, &grain);
const uint64_t ns = mxlGetNsUntilIndex(index + 1, &rate);
if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns);
index++;
}
log("stopped at index=%llu", index);
mxlReleaseFlowWriter(instance(), writer);
log("stopped at video_index=%llu", video_index);
mxlReleaseFlowWriter(instance(), video_writer);
if (has_audio) mxlReleaseFlowWriter(instance(), audio_writer);
}
};