From b2534efb0f3f107f20df6e2f3a350689aaebb6bd Mon Sep 17 00:00:00 2001 From: itten Date: Sat, 4 Jul 2026 03:24:20 +0300 Subject: [PATCH] audio works, holy shit --- CMakeLists.txt | 1 + nodes/videoin/main.cpp | 109 +++++++++++++++++++--- shared/VideoReader.hpp | 202 +++++++++++++++++++++++++++++++---------- video-ndi.json | 9 +- 4 files changed, 258 insertions(+), 63 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a30194..967131f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,6 +82,7 @@ pkg_check_modules(FFMPEG REQUIRED libavcodec libswscale libavutil + libswresample ) # Create an interface library for FFmpeg diff --git a/nodes/videoin/main.cpp b/nodes/videoin/main.cpp index d39893c..5549bbf 100644 --- a/nodes/videoin/main.cpp +++ b/nodes/videoin/main.cpp @@ -25,10 +25,10 @@ class VideoInNode : public dmf::NodeBase { const auto video_flow_info = config().at("video_flow_id"); const auto video_flow_id = video_flow_info.at("id").get(); - const int width = video_flow_info.value("width", video_reader.source_info.width); - const int height = video_flow_info.value("height", video_reader.source_info.height); - const int fps_num = video_flow_info.value("fps_num", video_reader.source_info.fps_num); - const int fps_den = video_flow_info.value("fps_den", video_reader.source_info.fps_den); + const int width = video_flow_info.value("width", video_reader.video_info.width); + const int height = video_flow_info.value("height", video_reader.video_info.height); + const int fps_num = video_flow_info.value("fps_num", video_reader.video_info.fps_num); + const int fps_den = video_flow_info.value("fps_den", video_reader.video_info.fps_den); log("video flow=%s %dx%d @ %d/%d fps", video_flow_id.c_str(), width, height, fps_num, fps_den); @@ -47,26 +47,107 @@ class VideoInNode : public dmf::NodeBase { log("video stride=%u B/line grain=%u B ring=%u grains", video_stride, video_stride * static_cast(height), video_cfg.discrete.grainCount); + mxlFlowWriter audio_writer{}; + mxlFlowConfigInfo audio_cfg{}; + int sample_rate = video_reader.audio_info.sample_rate; + int channels = video_reader.audio_info.channels; + int bit_depth = 32; + bool has_audio = config().contains("audio_flow_id") && video_reader.has_audio; + + int max_audio_samples = 0; + if (has_audio) { + const auto audio_flow_info = config().at("audio_flow_id"); + const auto audio_flow_id = audio_flow_info.at("id").get(); + + log("audio flow=%s %d Hz %dch %d-bit", audio_flow_id.c_str(), sample_rate, channels, bit_depth); + + mxlStatus ast = mxlCreateFlowWriter( + instance(), + dmf::make_audio_flow_def(audio_flow_id, node_id(), sample_rate, channels, bit_depth, + fps_num, fps_den).c_str(), + "", &audio_writer, &audio_cfg, &created); + if (ast != MXL_STATUS_OK) { + log("audio mxlCreateFlowWriter failed (%s) — continuing without audio", dmf::mxl_status_str(ast)); + has_audio = false; + } else { + log("audio channels=%u buffer=%u samples", + audio_cfg.continuous.channelCount, audio_cfg.continuous.bufferLength); + } + + size_t max_write = 0; + mxlFlowWriterGetMaxWriteLengthSamples(audio_writer, &max_write); + max_audio_samples = static_cast(max_write); + } + std::vector audio_temp(max_audio_samples * channels * sizeof(float)); + const mxlRational video_rate = {fps_num, fps_den}; + uint64_t audio_index = 0; + if (has_audio) { + const mxlRational audio_rate = {sample_rate, 1}; + audio_index = mxlGetCurrentIndex(&audio_rate); + } uint64_t video_index = mxlGetCurrentIndex(&video_rate); while (dmf::g_running.load(std::memory_order_relaxed)) { - uint8_t* buf = nullptr; + uint8_t* video_buf = nullptr; mxlGrainInfo grain{}; - vst = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &buf); - if (vst == MXL_STATUS_OK) { - if (!video_reader.get_next_frame(buf, video_stride)) break; - grain.flags = 0; - grain.validSlices = grain.totalSlices; - mxlFlowWriterCommitGrain(video_writer, &grain); + vst = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &video_buf); + + int out_samples_written = 0; + dmf::VideoReader::FrameKind frame_kind = video_reader.get_next_frame( + video_buf, + video_stride, + has_audio ? audio_temp.data() : nullptr, + max_audio_samples, + out_samples_written + ); + + if (frame_kind == dmf::VideoReader::FrameKind::None) break; + + if (frame_kind == dmf::VideoReader::FrameKind::Video) { + if (vst == MXL_STATUS_OK) { + grain.flags = 0; + grain.validSlices = grain.totalSlices; + mxlFlowWriterCommitGrain(video_writer, &grain); + } + const uint64_t ns = mxlGetNsUntilIndex(video_index + 1, &video_rate); + if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns); + video_index++; + } else if (frame_kind == dmf::VideoReader::FrameKind::Audio) { + mxlFlowWriterCancelGrain(video_writer); + mxlMutableWrappedMultiBufferSlice slice{}; + mxlStatus ast = mxlFlowWriterOpenSamples(audio_writer, audio_index, out_samples_written, &slice); + if (ast != MXL_STATUS_OK) { + log("audio OpenSamples failed (%s) index=%llu", dmf::mxl_status_str(ast), audio_index); + continue; + } + // copy per channel from audio_temp into slice + for (int ch = 0; ch < channels; ch++) { + uint8_t* dst0 = static_cast(slice.base.fragments[0].pointer) + + ch * slice.stride; + uint8_t* src = audio_temp.data() + ch * max_audio_samples * sizeof(float); + + size_t frag0_bytes = slice.base.fragments[0].size; + size_t total_bytes = out_samples_written * sizeof(float); + + if (total_bytes <= frag0_bytes) { + std::memcpy(dst0, src, total_bytes); + } else { + // Wrapped — copy first fragment, then second + std::memcpy(dst0, src, frag0_bytes); + uint8_t* dst1 = static_cast(slice.base.fragments[1].pointer) + + ch * slice.stride; + std::memcpy(dst1, src + frag0_bytes, total_bytes - frag0_bytes); + } + } + mxlFlowWriterCommitSamples(audio_writer); + audio_index += out_samples_written; } - const uint64_t ns = mxlGetNsUntilIndex(video_index + 1, &video_rate); - if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns); - video_index++; } log("stopped at video_index=%llu", video_index); mxlReleaseFlowWriter(instance(), video_writer); + mxlReleaseFlowWriter(instance(), audio_writer); } }; diff --git a/shared/VideoReader.hpp b/shared/VideoReader.hpp index 65f7925..1862bea 100644 --- a/shared/VideoReader.hpp +++ b/shared/VideoReader.hpp @@ -6,18 +6,23 @@ extern "C" { #include #include #include + #include + #include } #include #include +#include #include "Signal.hpp" #include "V210.hpp" +#include + namespace dmf { class VideoReader { public: - struct SourceInfo { + struct VideoInfo { int width = 0; int height = 0; int fps_num = 0; @@ -25,7 +30,17 @@ public: AVPixelFormat pix_fmt{}; }; - SourceInfo source_info{}; + struct AudioInfo { + int sample_rate = 0; + int channels = 0; + int samples = 0; + int channel_stride = 0; // floats between channel planes (NDI planar layout) + }; + + enum class FrameKind { None, Video, Audio }; + + VideoInfo video_info{}; + AudioInfo audio_info{}; bool has_audio = false; bool have_video = false; @@ -34,31 +49,42 @@ public: return; if (have_video) { get_source_info(); - allocate_conversion_buffers(); + if (have_video) { + allocate_video_conversion_buffers(); + } + if (has_audio) { + allocate_audio_conversion_buffers(); + } } } ~VideoReader() { - avcodec_free_context(&codec_context); + avcodec_free_context(&video_codec_context); avformat_close_input(&format_context); - sws_freeContext(sws_ctx); + sws_freeContext(sws_video_ctx); + swr_free(&swr_audio_ctx); av_freep(&p10_data[0]); - av_frame_free(&frame); + av_frame_free(&video_frame); + av_frame_free(&audio_frame); av_packet_free(&packet); } // Returns true when a frame was decoded and written into video_buf. // Returns false when g_running goes false. - bool get_next_frame(uint8_t* video_buf, uint32_t mxl_stride) { + FrameKind get_next_frame(uint8_t* video_buf, uint32_t mxl_stride, uint8_t* audio_buf, int max_audio_samples, int& out_samples_written) { while (dmf::g_running.load(std::memory_order_relaxed)) { // Drain any frames buffered in the decoder first - if (avcodec_receive_frame(codec_context, frame) == 0) { + if (avcodec_receive_frame(video_codec_context, video_frame) == 0) { + if (!video_buf) { + av_frame_unref(video_frame); + continue; // nowhere to write — discard frame + } sws_scale( - sws_ctx, - frame->data, - frame->linesize, + sws_video_ctx, + video_frame->data, + video_frame->linesize, 0, - source_info.height, + video_info.height, p10_data, p10_linesizes ); @@ -67,15 +93,48 @@ public: reinterpret_cast(p10_data[1]), reinterpret_cast(p10_data[2]), video_buf, - source_info.width, - source_info.height, + video_info.width, + video_info.height, p10_linesizes[0], p10_linesizes[1], p10_linesizes[2], mxl_stride ); - av_frame_unref(frame); - return true; + av_frame_unref(video_frame); + return FrameKind::Video; + } + + if (avcodec_receive_frame(audio_codec_context, audio_frame) == 0) { + if (!audio_buf) { + av_frame_unref(audio_frame); + continue; // nowhere to write — discard frame + } + + int dst_nb_samples = av_rescale_rnd( + swr_get_delay(swr_audio_ctx, audio_codec_context->sample_rate) + audio_frame->nb_samples, + audio_codec_context->sample_rate, audio_codec_context->sample_rate, AV_ROUND_UP + ); + + // Guard against buffer overflows + if (dst_nb_samples > max_audio_samples) { + dst_nb_samples = max_audio_samples; + } + + std::vector dst(audio_info.channels); + for (int ch = 0; ch < audio_info.channels; ch++) { + dst[ch] = audio_buf + ch * max_audio_samples * sizeof(float); + } + + // Convert/Resample the audio layout and sample format + int converted_samples = swr_convert( + swr_audio_ctx, + dst.data(), dst_nb_samples, + (const uint8_t**)audio_frame->data, audio_frame->nb_samples + ); + out_samples_written = converted_samples; + + av_frame_unref(audio_frame); + return FrameKind::Audio; } // No buffered frame — read next packet @@ -83,27 +142,38 @@ public: if (av_read_frame(format_context, packet) < 0) { // EOF — loop back to start avformat_seek_file(format_context, -1, 0, 0, 0, AVSEEK_FLAG_BACKWARD); - avcodec_flush_buffers(codec_context); + avcodec_flush_buffers(video_codec_context); + swr_close(swr_audio_ctx); + swr_init(swr_audio_ctx); continue; } - if (packet->stream_index != video_stream_index) continue; - avcodec_send_packet(codec_context, packet); + + if (packet->stream_index == video_stream_index) { + avcodec_send_packet(video_codec_context, packet); + } else if (packet->stream_index == audio_stream_index) { + avcodec_send_packet(audio_codec_context, packet); + } } - return false; + return FrameKind::None; } private: - AVFormatContext* format_context = nullptr; - AVCodecContext* codec_context = nullptr; - AVPacket* packet = av_packet_alloc(); - AVFrame* frame = av_frame_alloc(); int video_stream_index = -1; int audio_stream_index = -1; - SwsContext* sws_ctx = nullptr; + AVFormatContext* format_context = nullptr; + AVPacket* packet = av_packet_alloc(); + + AVCodecContext* video_codec_context = nullptr; + AVFrame* video_frame = av_frame_alloc(); + SwsContext* sws_video_ctx = nullptr; int p10_linesizes[4] = {0, 0, 0, 0}; uint8_t* p10_data[4] = {nullptr, nullptr, nullptr, nullptr}; + AVCodecContext* audio_codec_context = nullptr; + AVFrame* audio_frame = av_frame_alloc(); + SwrContext* swr_audio_ctx = nullptr; + bool open_file(const std::string& filename) { if (avformat_open_input(&format_context, filename.c_str(), nullptr, nullptr) != 0) throw std::runtime_error("Could not open file: " + filename); @@ -133,45 +203,83 @@ private: } void get_source_info() { - AVCodecParameters* codec_params = format_context->streams[video_stream_index]->codecpar; - const AVCodec* codec = avcodec_find_decoder(codec_params->codec_id); - if (!codec) - throw std::runtime_error("Unsupported codec"); + AVCodecParameters* video_codec_params = format_context->streams[video_stream_index]->codecpar; + const AVCodec* video_codec = avcodec_find_decoder(video_codec_params->codec_id); + if (!video_codec) + throw std::runtime_error("Unsupported video codec"); - codec_context = avcodec_alloc_context3(codec); - if (avcodec_parameters_to_context(codec_context, codec_params) < 0) - throw std::runtime_error("Could not copy codec parameters"); + video_codec_context = avcodec_alloc_context3(video_codec); + if (avcodec_parameters_to_context(video_codec_context, video_codec_params) < 0) + throw std::runtime_error("Could not copy video codec parameters"); - if (avcodec_open2(codec_context, codec, nullptr) < 0) { - avcodec_free_context(&codec_context); - throw std::runtime_error("Could not open codec"); + if (avcodec_open2(video_codec_context, video_codec, nullptr) < 0) { + avcodec_free_context(&video_codec_context); + throw std::runtime_error("Could not open video codec"); } - AVRational fps = codec_context->framerate; + AVRational fps = video_codec_context->framerate; if (fps.num == 0 || fps.den == 0) fps = format_context->streams[video_stream_index]->avg_frame_rate; - source_info.width = codec_context->width; - source_info.height = codec_context->height; - source_info.fps_num = fps.num; - source_info.fps_den = fps.den; - source_info.pix_fmt = codec_context->pix_fmt; + video_info.width = video_codec_context->width; + video_info.height = video_codec_context->height; + video_info.fps_num = fps.num; + video_info.fps_den = fps.den; + video_info.pix_fmt = video_codec_context->pix_fmt; + + // audio part + if (audio_stream_index == -1) return; + AVCodecParameters* audio_codec_params = format_context->streams[audio_stream_index]->codecpar; + const AVCodec* audio_codec = avcodec_find_decoder(audio_codec_params->codec_id); + if (!audio_codec) + throw std::runtime_error("Unsupported audio codec"); + + audio_codec_context = avcodec_alloc_context3(audio_codec); + if (avcodec_parameters_to_context(audio_codec_context, audio_codec_params) < 0) + throw std::runtime_error("Could not copy audio codec parameters"); + + if (avcodec_open2(audio_codec_context, audio_codec, nullptr) < 0) { + avcodec_free_context(&audio_codec_context); + throw std::runtime_error("Could not open audio codec"); + } + audio_info.sample_rate = audio_codec_context->sample_rate; + audio_info.channels = audio_codec_context->ch_layout.nb_channels; } - void allocate_conversion_buffers() { - sws_ctx = sws_getContext( - source_info.width, source_info.height, source_info.pix_fmt, - source_info.width, source_info.height, AV_PIX_FMT_YUV422P10LE, + void allocate_video_conversion_buffers() { + sws_video_ctx = sws_getContext( + video_info.width, video_info.height, video_info.pix_fmt, + video_info.width, video_info.height, AV_PIX_FMT_YUV422P10LE, SWS_BILINEAR, nullptr, nullptr, nullptr ); - if (!sws_ctx) + if (!sws_video_ctx) throw std::runtime_error("Failed to create SwsContext"); if (av_image_alloc(p10_data, p10_linesizes, - source_info.width, source_info.height, + video_info.width, video_info.height, AV_PIX_FMT_YUV422P10LE, 64) < 0) throw std::runtime_error("Failed to allocate YUV422P10 buffer"); } + + void allocate_audio_conversion_buffers() { + swr_audio_ctx = swr_alloc(); + + // Set input options + av_opt_set_chlayout(swr_audio_ctx, "in_chlayout", &audio_codec_context->ch_layout, 0); + av_opt_set_int(swr_audio_ctx, "in_sample_rate", audio_info.sample_rate, 0); + av_opt_set_sample_fmt(swr_audio_ctx, "in_sample_fmt", audio_codec_context->sample_fmt, 0); + + // Set output options + av_opt_set_chlayout(swr_audio_ctx, "out_chlayout", &audio_codec_context->ch_layout, 0); + av_opt_set_int(swr_audio_ctx, "out_sample_rate", audio_info.sample_rate, 0); + av_opt_set_sample_fmt(swr_audio_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0); + + // Initialize the context + if (swr_init(swr_audio_ctx) < 0) { + throw std::runtime_error("Failed to create SwrContext"); + } + + } }; } // namespace dmf diff --git a/video-ndi.json b/video-ndi.json index c908c44..1e354a5 100644 --- a/video-ndi.json +++ b/video-ndi.json @@ -1,13 +1,18 @@ { "nodes": [ - { "id": "videoin", "type": "videoin", "params": {} }, + { "id": "videoin", "type": "videoin", "params": {"file": "/home/itten/test-vid/2.ts"} }, { "id": "ndiout", "type": "ndiout", "params": {} } ], "edges": [ { "from": "videoin", "from_port": "video_flow_id", "to": "ndiout", "to_port": "video_flow_id", - "format": { "kind": "video", "width": 1920, "height": 1080, "fps_num": 25, "fps_den": 1 } + "format": { "kind": "video", "width": 1920, "height": 1080, "fps_num": 60, "fps_den": 1 } + }, + { + "from": "videoin", "from_port": "audio_flow_id", + "to": "ndiout", "to_port": "audio_flow_id", + "format": { "kind": "audio", "sample_rate": 44100, "channels": 2, "bit_depth": 32 } } ] }