itten/dmf-studio-rnd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

audio works, holy shit

Browse Source
This commit is contained in:
itten
2026-07-04 03:24:20 +03:00
parent 96475acf62
commit b2534efb0f
4 changed files with 258 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+1
View File
@@ -82,6 +82,7 @@ pkg_check_modules(FFMPEG REQUIRED
libavcodec
libswscale
libavutil
libswresample
)
# Create an interface library for FFmpeg
nodes/videoin/main.cpp
+95 -14
View File
@@ -25,10 +25,10 @@ class VideoInNode : public dmf::NodeBase {
const auto video_flow_info = config().at("video_flow_id");
const auto video_flow_id = video_flow_info.at("id").get<std::string>();
const int width = video_flow_info.value("width", video_reader.source_info.width);
const int height = video_flow_info.value("height", video_reader.source_info.height);
const int fps_num = video_flow_info.value("fps_num", video_reader.source_info.fps_num);
const int fps_den = video_flow_info.value("fps_den", video_reader.source_info.fps_den);
const int width = video_flow_info.value("width", video_reader.video_info.width);
const int height = video_flow_info.value("height", video_reader.video_info.height);
const int fps_num = video_flow_info.value("fps_num", video_reader.video_info.fps_num);
const int fps_den = video_flow_info.value("fps_den", video_reader.video_info.fps_den);
log("video flow=%s %dx%d @ %d/%d fps", video_flow_id.c_str(), width, height, fps_num, fps_den);
@@ -47,26 +47,107 @@ class VideoInNode : public dmf::NodeBase {
log("video stride=%u B/line grain=%u B ring=%u grains",
video_stride, video_stride * static_cast<uint32_t>(height), video_cfg.discrete.grainCount);
mxlFlowWriter audio_writer{};
mxlFlowConfigInfo audio_cfg{};
int sample_rate = video_reader.audio_info.sample_rate;
int channels = video_reader.audio_info.channels;
int bit_depth = 32;
bool has_audio = config().contains("audio_flow_id") && video_reader.has_audio;
int max_audio_samples = 0;
if (has_audio) {
const auto audio_flow_info = config().at("audio_flow_id");
const auto audio_flow_id = audio_flow_info.at("id").get<std::string>();
log("audio flow=%s %d Hz %dch %d-bit", audio_flow_id.c_str(), sample_rate, channels, bit_depth);
mxlStatus ast = mxlCreateFlowWriter(
instance(),
dmf::make_audio_flow_def(audio_flow_id, node_id(), sample_rate, channels, bit_depth,
fps_num, fps_den).c_str(),
"", &audio_writer, &audio_cfg, &created);
if (ast != MXL_STATUS_OK) {
log("audio mxlCreateFlowWriter failed (%s) — continuing without audio", dmf::mxl_status_str(ast));
has_audio = false;
} else {
log("audio channels=%u buffer=%u samples",
audio_cfg.continuous.channelCount, audio_cfg.continuous.bufferLength);
}
size_t max_write = 0;
mxlFlowWriterGetMaxWriteLengthSamples(audio_writer, &max_write);
max_audio_samples = static_cast<int>(max_write);
}
std::vector<uint8_t> audio_temp(max_audio_samples * channels * sizeof(float));
const mxlRational video_rate = {fps_num, fps_den};
uint64_t audio_index = 0;
if (has_audio) {
const mxlRational audio_rate = {sample_rate, 1};
audio_index = mxlGetCurrentIndex(&audio_rate);
}
uint64_t video_index = mxlGetCurrentIndex(&video_rate);
while (dmf::g_running.load(std::memory_order_relaxed)) {
uint8_t* buf = nullptr;
uint8_t* video_buf = nullptr;
mxlGrainInfo grain{};
vst = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &buf);
if (vst == MXL_STATUS_OK) {
if (!video_reader.get_next_frame(buf, video_stride)) break;
grain.flags = 0;
grain.validSlices = grain.totalSlices;
mxlFlowWriterCommitGrain(video_writer, &grain);
vst = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &video_buf);
int out_samples_written = 0;
dmf::VideoReader::FrameKind frame_kind = video_reader.get_next_frame(
video_buf,
video_stride,
has_audio ? audio_temp.data() : nullptr,
max_audio_samples,
out_samples_written
);
if (frame_kind == dmf::VideoReader::FrameKind::None) break;
if (frame_kind == dmf::VideoReader::FrameKind::Video) {
if (vst == MXL_STATUS_OK) {
grain.flags = 0;
grain.validSlices = grain.totalSlices;
mxlFlowWriterCommitGrain(video_writer, &grain);
}
const uint64_t ns = mxlGetNsUntilIndex(video_index + 1, &video_rate);
if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns);
video_index++;
} else if (frame_kind == dmf::VideoReader::FrameKind::Audio) {
mxlFlowWriterCancelGrain(video_writer);
mxlMutableWrappedMultiBufferSlice slice{};
mxlStatus ast = mxlFlowWriterOpenSamples(audio_writer, audio_index, out_samples_written, &slice);
if (ast != MXL_STATUS_OK) {
log("audio OpenSamples failed (%s) index=%llu", dmf::mxl_status_str(ast), audio_index);
continue;
}
// copy per channel from audio_temp into slice
for (int ch = 0; ch < channels; ch++) {
uint8_t* dst0 = static_cast<uint8_t*>(slice.base.fragments[0].pointer)
+ ch * slice.stride;
uint8_t* src = audio_temp.data() + ch * max_audio_samples * sizeof(float);
size_t frag0_bytes = slice.base.fragments[0].size;
size_t total_bytes = out_samples_written * sizeof(float);
if (total_bytes <= frag0_bytes) {
std::memcpy(dst0, src, total_bytes);
} else {
// Wrapped — copy first fragment, then second
std::memcpy(dst0, src, frag0_bytes);
uint8_t* dst1 = static_cast<uint8_t*>(slice.base.fragments[1].pointer)
+ ch * slice.stride;
std::memcpy(dst1, src + frag0_bytes, total_bytes - frag0_bytes);
}
}
mxlFlowWriterCommitSamples(audio_writer);
audio_index += out_samples_written;
}
const uint64_t ns = mxlGetNsUntilIndex(video_index + 1, &video_rate);
if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns);
video_index++;
}
log("stopped at video_index=%llu", video_index);
mxlReleaseFlowWriter(instance(), video_writer);
mxlReleaseFlowWriter(instance(), audio_writer);
}
};
shared/VideoReader.hpp
+155 -47
View File
@@ -6,18 +6,23 @@ extern "C" {
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
#include <libswresample/swresample.h>
#include <libavutil/opt.h>
}
#include <stdexcept>
#include <string>
#include <vector>
#include "Signal.hpp"
#include "V210.hpp"
#include <iostream>
namespace dmf {
class VideoReader {
public:
struct SourceInfo {
struct VideoInfo {
int width = 0;
int height = 0;
int fps_num = 0;
@@ -25,7 +30,17 @@ public:
AVPixelFormat pix_fmt{};
};
SourceInfo source_info{};
struct AudioInfo {
int sample_rate = 0;
int channels = 0;
int samples = 0;
int channel_stride = 0; // floats between channel planes (NDI planar layout)
};
enum class FrameKind { None, Video, Audio };
VideoInfo video_info{};
AudioInfo audio_info{};
bool has_audio = false;
bool have_video = false;
@@ -34,31 +49,42 @@ public:
return;
if (have_video) {
get_source_info();
allocate_conversion_buffers();
if (have_video) {
allocate_video_conversion_buffers();
}
if (has_audio) {
allocate_audio_conversion_buffers();
}
}
}
~VideoReader() {
avcodec_free_context(&codec_context);
avcodec_free_context(&video_codec_context);
avformat_close_input(&format_context);
sws_freeContext(sws_ctx);
sws_freeContext(sws_video_ctx);
swr_free(&swr_audio_ctx);
av_freep(&p10_data[0]);
av_frame_free(&frame);
av_frame_free(&video_frame);
av_frame_free(&audio_frame);
av_packet_free(&packet);
}
// Returns true when a frame was decoded and written into video_buf.
// Returns false when g_running goes false.
bool get_next_frame(uint8_t* video_buf, uint32_t mxl_stride) {
FrameKind get_next_frame(uint8_t* video_buf, uint32_t mxl_stride, uint8_t* audio_buf, int max_audio_samples, int& out_samples_written) {
while (dmf::g_running.load(std::memory_order_relaxed)) {
// Drain any frames buffered in the decoder first
if (avcodec_receive_frame(codec_context, frame) == 0) {
if (avcodec_receive_frame(video_codec_context, video_frame) == 0) {
if (!video_buf) {
av_frame_unref(video_frame);
continue; // nowhere to write — discard frame
}
sws_scale(
sws_ctx,
frame->data,
frame->linesize,
sws_video_ctx,
video_frame->data,
video_frame->linesize,
0,
source_info.height,
video_info.height,
p10_data,
p10_linesizes
);
@@ -67,15 +93,48 @@ public:
reinterpret_cast<uint16_t*>(p10_data[1]),
reinterpret_cast<uint16_t*>(p10_data[2]),
video_buf,
source_info.width,
source_info.height,
video_info.width,
video_info.height,
p10_linesizes[0],
p10_linesizes[1],
p10_linesizes[2],
mxl_stride
);
av_frame_unref(frame);
return true;
av_frame_unref(video_frame);
return FrameKind::Video;
}
if (avcodec_receive_frame(audio_codec_context, audio_frame) == 0) {
if (!audio_buf) {
av_frame_unref(audio_frame);
continue; // nowhere to write — discard frame
}
int dst_nb_samples = av_rescale_rnd(
swr_get_delay(swr_audio_ctx, audio_codec_context->sample_rate) + audio_frame->nb_samples,
audio_codec_context->sample_rate, audio_codec_context->sample_rate, AV_ROUND_UP
);
// Guard against buffer overflows
if (dst_nb_samples > max_audio_samples) {
dst_nb_samples = max_audio_samples;
}
std::vector<uint8_t*> dst(audio_info.channels);
for (int ch = 0; ch < audio_info.channels; ch++) {
dst[ch] = audio_buf + ch * max_audio_samples * sizeof(float);
}
// Convert/Resample the audio layout and sample format
int converted_samples = swr_convert(
swr_audio_ctx,
dst.data(), dst_nb_samples,
(const uint8_t**)audio_frame->data, audio_frame->nb_samples
);
out_samples_written = converted_samples;
av_frame_unref(audio_frame);
return FrameKind::Audio;
}
// No buffered frame — read next packet
@@ -83,27 +142,38 @@ public:
if (av_read_frame(format_context, packet) < 0) {
// EOF — loop back to start
avformat_seek_file(format_context, -1, 0, 0, 0, AVSEEK_FLAG_BACKWARD);
avcodec_flush_buffers(codec_context);
avcodec_flush_buffers(video_codec_context);
swr_close(swr_audio_ctx);
swr_init(swr_audio_ctx);
continue;
}
if (packet->stream_index != video_stream_index) continue;
avcodec_send_packet(codec_context, packet);
if (packet->stream_index == video_stream_index) {
avcodec_send_packet(video_codec_context, packet);
} else if (packet->stream_index == audio_stream_index) {
avcodec_send_packet(audio_codec_context, packet);
}
}
return false;
return FrameKind::None;
}
private:
AVFormatContext* format_context = nullptr;
AVCodecContext* codec_context = nullptr;
AVPacket* packet = av_packet_alloc();
AVFrame* frame = av_frame_alloc();
int video_stream_index = -1;
int audio_stream_index = -1;
SwsContext* sws_ctx = nullptr;
AVFormatContext* format_context = nullptr;
AVPacket* packet = av_packet_alloc();
AVCodecContext* video_codec_context = nullptr;
AVFrame* video_frame = av_frame_alloc();
SwsContext* sws_video_ctx = nullptr;
int p10_linesizes[4] = {0, 0, 0, 0};
uint8_t* p10_data[4] = {nullptr, nullptr, nullptr, nullptr};
AVCodecContext* audio_codec_context = nullptr;
AVFrame* audio_frame = av_frame_alloc();
SwrContext* swr_audio_ctx = nullptr;
bool open_file(const std::string& filename) {
if (avformat_open_input(&format_context, filename.c_str(), nullptr, nullptr) != 0)
throw std::runtime_error("Could not open file: " + filename);
@@ -133,45 +203,83 @@ private:
}
void get_source_info() {
AVCodecParameters* codec_params = format_context->streams[video_stream_index]->codecpar;
const AVCodec* codec = avcodec_find_decoder(codec_params->codec_id);
if (!codec)
throw std::runtime_error("Unsupported codec");
AVCodecParameters* video_codec_params = format_context->streams[video_stream_index]->codecpar;
const AVCodec* video_codec = avcodec_find_decoder(video_codec_params->codec_id);
if (!video_codec)
throw std::runtime_error("Unsupported video codec");
codec_context = avcodec_alloc_context3(codec);
if (avcodec_parameters_to_context(codec_context, codec_params) < 0)
throw std::runtime_error("Could not copy codec parameters");
video_codec_context = avcodec_alloc_context3(video_codec);
if (avcodec_parameters_to_context(video_codec_context, video_codec_params) < 0)
throw std::runtime_error("Could not copy video codec parameters");
if (avcodec_open2(codec_context, codec, nullptr) < 0) {
avcodec_free_context(&codec_context);
throw std::runtime_error("Could not open codec");
if (avcodec_open2(video_codec_context, video_codec, nullptr) < 0) {
avcodec_free_context(&video_codec_context);
throw std::runtime_error("Could not open video codec");
}
AVRational fps = codec_context->framerate;
AVRational fps = video_codec_context->framerate;
if (fps.num == 0 || fps.den == 0)
fps = format_context->streams[video_stream_index]->avg_frame_rate;
source_info.width = codec_context->width;
source_info.height = codec_context->height;
source_info.fps_num = fps.num;
source_info.fps_den = fps.den;
source_info.pix_fmt = codec_context->pix_fmt;
video_info.width = video_codec_context->width;
video_info.height = video_codec_context->height;
video_info.fps_num = fps.num;
video_info.fps_den = fps.den;
video_info.pix_fmt = video_codec_context->pix_fmt;
// audio part
if (audio_stream_index == -1) return;
AVCodecParameters* audio_codec_params = format_context->streams[audio_stream_index]->codecpar;
const AVCodec* audio_codec = avcodec_find_decoder(audio_codec_params->codec_id);
if (!audio_codec)
throw std::runtime_error("Unsupported audio codec");
audio_codec_context = avcodec_alloc_context3(audio_codec);
if (avcodec_parameters_to_context(audio_codec_context, audio_codec_params) < 0)
throw std::runtime_error("Could not copy audio codec parameters");
if (avcodec_open2(audio_codec_context, audio_codec, nullptr) < 0) {
avcodec_free_context(&audio_codec_context);
throw std::runtime_error("Could not open audio codec");
}
audio_info.sample_rate = audio_codec_context->sample_rate;
audio_info.channels = audio_codec_context->ch_layout.nb_channels;
}
void allocate_conversion_buffers() {
sws_ctx = sws_getContext(
source_info.width, source_info.height, source_info.pix_fmt,
source_info.width, source_info.height, AV_PIX_FMT_YUV422P10LE,
void allocate_video_conversion_buffers() {
sws_video_ctx = sws_getContext(
video_info.width, video_info.height, video_info.pix_fmt,
video_info.width, video_info.height, AV_PIX_FMT_YUV422P10LE,
SWS_BILINEAR, nullptr, nullptr, nullptr
);
if (!sws_ctx)
if (!sws_video_ctx)
throw std::runtime_error("Failed to create SwsContext");
if (av_image_alloc(p10_data, p10_linesizes,
source_info.width, source_info.height,
video_info.width, video_info.height,
AV_PIX_FMT_YUV422P10LE, 64) < 0)
throw std::runtime_error("Failed to allocate YUV422P10 buffer");
}
void allocate_audio_conversion_buffers() {
swr_audio_ctx = swr_alloc();
// Set input options
av_opt_set_chlayout(swr_audio_ctx, "in_chlayout", &audio_codec_context->ch_layout, 0);
av_opt_set_int(swr_audio_ctx, "in_sample_rate", audio_info.sample_rate, 0);
av_opt_set_sample_fmt(swr_audio_ctx, "in_sample_fmt", audio_codec_context->sample_fmt, 0);
// Set output options
av_opt_set_chlayout(swr_audio_ctx, "out_chlayout", &audio_codec_context->ch_layout, 0);
av_opt_set_int(swr_audio_ctx, "out_sample_rate", audio_info.sample_rate, 0);
av_opt_set_sample_fmt(swr_audio_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
// Initialize the context
if (swr_init(swr_audio_ctx) < 0) {
throw std::runtime_error("Failed to create SwrContext");
}
}
};
} // namespace dmf
video-ndi.json
+7 -2
View File
@@ -1,13 +1,18 @@
{
"nodes": [
{ "id": "videoin", "type": "videoin", "params": {} },
{ "id": "videoin", "type": "videoin", "params": {"file": "/home/itten/test-vid/2.ts"} },
{ "id": "ndiout", "type": "ndiout", "params": {} }
],
"edges": [
{
"from": "videoin", "from_port": "video_flow_id",
"to": "ndiout", "to_port": "video_flow_id",
"format": { "kind": "video", "width": 1920, "height": 1080, "fps_num": 25, "fps_den": 1 }
"format": { "kind": "video", "width": 1920, "height": 1080, "fps_num": 60, "fps_den": 1 }
},
{
"from": "videoin", "from_port": "audio_flow_id",
"to": "ndiout", "to_port": "audio_flow_id",
"format": { "kind": "audio", "sample_rate": 44100, "channels": 2, "bit_depth": 32 }
}
]
}