From 166efc3c595d092720be168dea4538eb6b79427b Mon Sep 17 00:00:00 2001 From: itten Date: Sun, 5 Jul 2026 12:39:06 +0300 Subject: [PATCH] Some fixes. For a now we fully support only .ts, due to huge problem with h264/hevc audio bursts --- nodes/videoin/main.cpp | 114 +++++++++++++++++++++++++---------------- shared/VideoReader.hpp | 65 +++++++++++------------ video-ndi.json | 6 +-- 3 files changed, 106 insertions(+), 79 deletions(-) diff --git a/nodes/videoin/main.cpp b/nodes/videoin/main.cpp index 5549bbf..315b485 100644 --- a/nodes/videoin/main.cpp +++ b/nodes/videoin/main.cpp @@ -21,31 +21,40 @@ class VideoInNode : public dmf::NodeBase { log("file: %s", filename.c_str()); dmf::VideoReader video_reader(filename); - if (!video_reader.have_video) { log("no video stream found"); return; } - - const auto video_flow_info = config().at("video_flow_id"); - const auto video_flow_id = video_flow_info.at("id").get(); - const int width = video_flow_info.value("width", video_reader.video_info.width); - const int height = video_flow_info.value("height", video_reader.video_info.height); - const int fps_num = video_flow_info.value("fps_num", video_reader.video_info.fps_num); - const int fps_den = video_flow_info.value("fps_den", video_reader.video_info.fps_den); - - log("video flow=%s %dx%d @ %d/%d fps", video_flow_id.c_str(), width, height, fps_num, fps_den); - - mxlFlowWriter video_writer = nullptr; - mxlFlowConfigInfo video_cfg = {}; - bool created = false; - mxlStatus vst = mxlCreateFlowWriter( - instance(), - dmf::make_video_flow_def(video_flow_id, node_id(), width, height, fps_num, fps_den).c_str(), - "", &video_writer, &video_cfg, &created); - if (vst != MXL_STATUS_OK) { - log("mxlCreateFlowWriter failed (%s)", dmf::mxl_status_str(vst)); - return; + if (!video_reader.have_video && !video_reader.has_audio) { + log("no video or audio stream found"); return; + } + + const bool has_video = config().contains("video_flow_id") && video_reader.have_video; + mxlFlowWriter video_writer = nullptr; + mxlFlowConfigInfo video_cfg{}; + uint32_t video_stride = 0; + int width = 0, height = 0, fps_num = 25, fps_den = 1; + std::string video_flow_id; + + if (has_video) { + const auto video_flow_info = config().at("video_flow_id"); + video_flow_id = video_flow_info.at("id").get(); + width = video_flow_info.value("width", video_reader.video_info.width); + height = video_flow_info.value("height", video_reader.video_info.height); + fps_num = video_flow_info.value("fps_num", video_reader.video_info.fps_num); + fps_den = video_flow_info.value("fps_den", video_reader.video_info.fps_den); + + log("video flow=%s %dx%d @ %d/%d fps", video_flow_id.c_str(), width, height, fps_num, fps_den); + + bool created = false; + mxlStatus vst = mxlCreateFlowWriter( + instance(), + dmf::make_video_flow_def(video_flow_id, node_id(), width, height, fps_num, fps_den).c_str(), + "", &video_writer, &video_cfg, &created); + if (vst != MXL_STATUS_OK) { + log("mxlCreateFlowWriter failed (%s)", dmf::mxl_status_str(vst)); + return; + } + video_stride = video_cfg.discrete.sliceSizes[0]; + log("video stride=%u B/line grain=%u B ring=%u grains", + video_stride, video_stride * static_cast(height), video_cfg.discrete.grainCount); } - const uint32_t video_stride = video_cfg.discrete.sliceSizes[0]; - log("video stride=%u B/line grain=%u B ring=%u grains", - video_stride, video_stride * static_cast(height), video_cfg.discrete.grainCount); mxlFlowWriter audio_writer{}; mxlFlowConfigInfo audio_cfg{}; @@ -53,7 +62,7 @@ class VideoInNode : public dmf::NodeBase { int channels = video_reader.audio_info.channels; int bit_depth = 32; bool has_audio = config().contains("audio_flow_id") && video_reader.has_audio; - + int max_audio_samples = 0; if (has_audio) { const auto audio_flow_info = config().at("audio_flow_id"); @@ -61,6 +70,7 @@ class VideoInNode : public dmf::NodeBase { log("audio flow=%s %d Hz %dch %d-bit", audio_flow_id.c_str(), sample_rate, channels, bit_depth); + bool created = false; mxlStatus ast = mxlCreateFlowWriter( instance(), dmf::make_audio_flow_def(audio_flow_id, node_id(), sample_rate, channels, bit_depth, @@ -81,55 +91,71 @@ class VideoInNode : public dmf::NodeBase { std::vector audio_temp(max_audio_samples * channels * sizeof(float)); const mxlRational video_rate = {fps_num, fps_den}; + const mxlRational audio_rate = {sample_rate, 1}; uint64_t audio_index = 0; if (has_audio) { - const mxlRational audio_rate = {sample_rate, 1}; audio_index = mxlGetCurrentIndex(&audio_rate); } - uint64_t video_index = mxlGetCurrentIndex(&video_rate); + uint64_t video_index = has_video ? mxlGetCurrentIndex(&video_rate) : 0; while (dmf::g_running.load(std::memory_order_relaxed)) { - uint8_t* video_buf = nullptr; - mxlGrainInfo grain{}; - vst = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &video_buf); + uint8_t* video_buf = nullptr; + mxlGrainInfo grain{}; + mxlStatus vst = MXL_ERR_UNSUPPORTED_OPERATION; + if (has_video) { + vst = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &video_buf); + } int out_samples_written = 0; dmf::VideoReader::FrameKind frame_kind = video_reader.get_next_frame( - video_buf, + has_video ? video_buf : nullptr, video_stride, has_audio ? audio_temp.data() : nullptr, max_audio_samples, out_samples_written ); - + if (frame_kind == dmf::VideoReader::FrameKind::None) break; if (frame_kind == dmf::VideoReader::FrameKind::Video) { - if (vst == MXL_STATUS_OK) { + if (has_video && vst == MXL_STATUS_OK) { grain.flags = 0; grain.validSlices = grain.totalSlices; mxlFlowWriterCommitGrain(video_writer, &grain); } - const uint64_t ns = mxlGetNsUntilIndex(video_index + 1, &video_rate); - if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns); - video_index++; + if (has_video) { + const uint64_t ns = mxlGetNsUntilIndex(video_index + 1, &video_rate); + if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns); + video_index++; + } } else if (frame_kind == dmf::VideoReader::FrameKind::Audio) { - mxlFlowWriterCancelGrain(video_writer); + if (has_video && vst == MXL_STATUS_OK) mxlFlowWriterCancelGrain(video_writer); + // Wait for audio clock to catch up — prevents TOO_EARLY and sample loss + while (dmf::g_running.load(std::memory_order_relaxed)) { + const uint64_t audio_now = mxlGetCurrentIndex(&audio_rate); + if (audio_index + static_cast(out_samples_written) <= audio_now) break; + const uint64_t ns = mxlGetNsUntilIndex(audio_index + out_samples_written, &audio_rate); + if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns); + else break; + } + if (!dmf::g_running.load(std::memory_order_relaxed)) break; + mxlMutableWrappedMultiBufferSlice slice{}; mxlStatus ast = mxlFlowWriterOpenSamples(audio_writer, audio_index, out_samples_written, &slice); if (ast != MXL_STATUS_OK) { - log("audio OpenSamples failed (%s) index=%llu", dmf::mxl_status_str(ast), audio_index); + log("audio OpenSamples failed (%s) index=%llu — skipping", dmf::mxl_status_str(ast), audio_index); + audio_index += out_samples_written; // advance even on failure — keeps alignment continue; } // copy per channel from audio_temp into slice for (int ch = 0; ch < channels; ch++) { - uint8_t* dst0 = static_cast(slice.base.fragments[0].pointer) + uint8_t* dst0 = static_cast(slice.base.fragments[0].pointer) + ch * slice.stride; uint8_t* src = audio_temp.data() + ch * max_audio_samples * sizeof(float); - + size_t frag0_bytes = slice.base.fragments[0].size; size_t total_bytes = out_samples_written * sizeof(float); - + if (total_bytes <= frag0_bytes) { std::memcpy(dst0, src, total_bytes); } else { @@ -145,9 +171,9 @@ class VideoInNode : public dmf::NodeBase { } } - log("stopped at video_index=%llu", video_index); - mxlReleaseFlowWriter(instance(), video_writer); - mxlReleaseFlowWriter(instance(), audio_writer); + log("stopped at video_index=%llu", static_cast(video_index)); + if (has_video) mxlReleaseFlowWriter(instance(), video_writer); + if (has_audio) mxlReleaseFlowWriter(instance(), audio_writer); } }; diff --git a/shared/VideoReader.hpp b/shared/VideoReader.hpp index 1862bea..3c620fc 100644 --- a/shared/VideoReader.hpp +++ b/shared/VideoReader.hpp @@ -47,19 +47,15 @@ public: explicit VideoReader(const std::string& filename) { if (!open_file(filename)) return; - if (have_video) { - get_source_info(); - if (have_video) { - allocate_video_conversion_buffers(); - } - if (has_audio) { - allocate_audio_conversion_buffers(); - } - } + if (!have_video && !has_audio) return; + get_source_info(); + if (have_video) allocate_video_conversion_buffers(); + if (has_audio) allocate_audio_conversion_buffers(); } ~VideoReader() { avcodec_free_context(&video_codec_context); + avcodec_free_context(&audio_codec_context); avformat_close_input(&format_context); sws_freeContext(sws_video_ctx); swr_free(&swr_audio_ctx); @@ -74,7 +70,7 @@ public: FrameKind get_next_frame(uint8_t* video_buf, uint32_t mxl_stride, uint8_t* audio_buf, int max_audio_samples, int& out_samples_written) { while (dmf::g_running.load(std::memory_order_relaxed)) { // Drain any frames buffered in the decoder first - if (avcodec_receive_frame(video_codec_context, video_frame) == 0) { + if (have_video && avcodec_receive_frame(video_codec_context, video_frame) == 0) { if (!video_buf) { av_frame_unref(video_frame); continue; // nowhere to write — discard frame @@ -104,7 +100,7 @@ public: return FrameKind::Video; } - if (avcodec_receive_frame(audio_codec_context, audio_frame) == 0) { + if (has_audio && avcodec_receive_frame(audio_codec_context, audio_frame) == 0) { if (!audio_buf) { av_frame_unref(audio_frame); continue; // nowhere to write — discard frame @@ -142,7 +138,8 @@ public: if (av_read_frame(format_context, packet) < 0) { // EOF — loop back to start avformat_seek_file(format_context, -1, 0, 0, 0, AVSEEK_FLAG_BACKWARD); - avcodec_flush_buffers(video_codec_context); + if (have_video) avcodec_flush_buffers(video_codec_context); + if (has_audio) avcodec_flush_buffers(audio_codec_context); swr_close(swr_audio_ctx); swr_init(swr_audio_ctx); continue; @@ -203,30 +200,34 @@ private: } void get_source_info() { - AVCodecParameters* video_codec_params = format_context->streams[video_stream_index]->codecpar; - const AVCodec* video_codec = avcodec_find_decoder(video_codec_params->codec_id); - if (!video_codec) - throw std::runtime_error("Unsupported video codec"); + if (video_stream_index != -1) { + AVCodecParameters* video_codec_params = format_context->streams[video_stream_index]->codecpar; + const AVCodec* video_codec = avcodec_find_decoder(video_codec_params->codec_id); + if (!video_codec) + throw std::runtime_error("Unsupported video codec"); - video_codec_context = avcodec_alloc_context3(video_codec); - if (avcodec_parameters_to_context(video_codec_context, video_codec_params) < 0) - throw std::runtime_error("Could not copy video codec parameters"); + video_codec_context = avcodec_alloc_context3(video_codec); + if (avcodec_parameters_to_context(video_codec_context, video_codec_params) < 0) + throw std::runtime_error("Could not copy video codec parameters"); - if (avcodec_open2(video_codec_context, video_codec, nullptr) < 0) { - avcodec_free_context(&video_codec_context); - throw std::runtime_error("Could not open video codec"); + if (avcodec_open2(video_codec_context, video_codec, nullptr) < 0) { + avcodec_free_context(&video_codec_context); + throw std::runtime_error("Could not open video codec"); + } + + AVRational fps = video_codec_context->framerate; + if (fps.num == 0 || fps.den == 0) + fps = format_context->streams[video_stream_index]->avg_frame_rate; + + video_info.width = video_codec_context->width; + video_info.height = video_codec_context->height; + video_info.fps_num = fps.num; + video_info.fps_den = fps.den; + video_info.pix_fmt = video_codec_context->pix_fmt; + } else { + have_video = false; } - AVRational fps = video_codec_context->framerate; - if (fps.num == 0 || fps.den == 0) - fps = format_context->streams[video_stream_index]->avg_frame_rate; - - video_info.width = video_codec_context->width; - video_info.height = video_codec_context->height; - video_info.fps_num = fps.num; - video_info.fps_den = fps.den; - video_info.pix_fmt = video_codec_context->pix_fmt; - // audio part if (audio_stream_index == -1) return; AVCodecParameters* audio_codec_params = format_context->streams[audio_stream_index]->codecpar; diff --git a/video-ndi.json b/video-ndi.json index 1e354a5..c2b8d0b 100644 --- a/video-ndi.json +++ b/video-ndi.json @@ -1,18 +1,18 @@ { "nodes": [ - { "id": "videoin", "type": "videoin", "params": {"file": "/home/itten/test-vid/2.ts"} }, + { "id": "videoin", "type": "videoin", "params": {"file": "/home/itten/test-vid/1.ts"} }, { "id": "ndiout", "type": "ndiout", "params": {} } ], "edges": [ { "from": "videoin", "from_port": "video_flow_id", "to": "ndiout", "to_port": "video_flow_id", - "format": { "kind": "video", "width": 1920, "height": 1080, "fps_num": 60, "fps_den": 1 } + "format": { "kind": "video", "width": 1920, "height": 1080, "fps_num": 24, "fps_den": 1 } }, { "from": "videoin", "from_port": "audio_flow_id", "to": "ndiout", "to_port": "audio_flow_id", - "format": { "kind": "audio", "sample_rate": 44100, "channels": 2, "bit_depth": 32 } + "format": { "kind": "audio", "sample_rate": 48000, "channels": 6, "bit_depth": 32 } } ] }