audio works, holy shit

CMakeLists.txt

@@ -82,6 +82,7 @@ pkg_check_modules(FFMPEG REQUIRED
     libavcodec
     libswscale
     libavutil
+    libswresample
 )
 
 # Create an interface library for FFmpeg

nodes/videoin/main.cpp

@@ -25,10 +25,10 @@ class VideoInNode : public dmf::NodeBase {
 
         const auto video_flow_info = config().at("video_flow_id");
         const auto video_flow_id   = video_flow_info.at("id").get<std::string>();
-        const int  width    = video_flow_info.value("width",   video_reader.source_info.width);
+        const int  width    = video_flow_info.value("width",   video_reader.video_info.width);
-        const int  height   = video_flow_info.value("height",  video_reader.source_info.height);
+        const int  height   = video_flow_info.value("height",  video_reader.video_info.height);
-        const int  fps_num  = video_flow_info.value("fps_num", video_reader.source_info.fps_num);
+        const int  fps_num  = video_flow_info.value("fps_num", video_reader.video_info.fps_num);
-        const int  fps_den  = video_flow_info.value("fps_den", video_reader.source_info.fps_den);
+        const int  fps_den  = video_flow_info.value("fps_den", video_reader.video_info.fps_den);
 
         log("video flow=%s  %dx%d @ %d/%d fps", video_flow_id.c_str(), width, height, fps_num, fps_den);
 
@@ -47,15 +47,65 @@ class VideoInNode : public dmf::NodeBase {
         log("video stride=%u B/line  grain=%u B  ring=%u grains",
             video_stride, video_stride * static_cast<uint32_t>(height), video_cfg.discrete.grainCount);
 
+        mxlFlowWriter audio_writer{};
+        mxlFlowConfigInfo audio_cfg{};
+        int  sample_rate  = video_reader.audio_info.sample_rate;
+        int  channels     = video_reader.audio_info.channels;
+        int  bit_depth    = 32;
+        bool has_audio    = config().contains("audio_flow_id") && video_reader.has_audio;
+        
+        int max_audio_samples = 0;
+        if (has_audio) {
+            const auto audio_flow_info = config().at("audio_flow_id");
+            const auto audio_flow_id   = audio_flow_info.at("id").get<std::string>();
+
+            log("audio flow=%s  %d Hz  %dch  %d-bit", audio_flow_id.c_str(), sample_rate, channels, bit_depth);
+
+            mxlStatus ast = mxlCreateFlowWriter(
+                instance(),
+                dmf::make_audio_flow_def(audio_flow_id, node_id(), sample_rate, channels, bit_depth,
+                                         fps_num, fps_den).c_str(),
+                "", &audio_writer, &audio_cfg, &created);
+            if (ast != MXL_STATUS_OK) {
+                log("audio mxlCreateFlowWriter failed (%s) — continuing without audio", dmf::mxl_status_str(ast));
+                has_audio = false;
+            } else {
+                log("audio channels=%u  buffer=%u samples",
+                    audio_cfg.continuous.channelCount, audio_cfg.continuous.bufferLength);
+            }
+
+            size_t max_write = 0;
+            mxlFlowWriterGetMaxWriteLengthSamples(audio_writer, &max_write);
+            max_audio_samples = static_cast<int>(max_write);
+        }
+        std::vector<uint8_t> audio_temp(max_audio_samples * channels * sizeof(float));
+
         const mxlRational video_rate = {fps_num, fps_den};
+        uint64_t audio_index = 0;
+        if (has_audio) {
+            const mxlRational audio_rate = {sample_rate, 1};
+            audio_index = mxlGetCurrentIndex(&audio_rate);
+        }
         uint64_t video_index = mxlGetCurrentIndex(&video_rate);
 
         while (dmf::g_running.load(std::memory_order_relaxed)) {
-            uint8_t*     buf = nullptr;
+            uint8_t* video_buf = nullptr;
             mxlGrainInfo grain{};
-            vst = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &buf);
+            vst = mxlFlowWriterOpenGrain(video_writer, video_index, &grain, &video_buf);
+
+            int out_samples_written = 0;
+            dmf::VideoReader::FrameKind frame_kind = video_reader.get_next_frame(
+                video_buf,
+                video_stride,
+                has_audio ? audio_temp.data() : nullptr,
+                max_audio_samples,
+                out_samples_written
+            );
+            
+            if (frame_kind == dmf::VideoReader::FrameKind::None) break;
+
+            if (frame_kind == dmf::VideoReader::FrameKind::Video) {
                 if (vst == MXL_STATUS_OK) {
-                if (!video_reader.get_next_frame(buf, video_stride)) break;
                     grain.flags       = 0;
                     grain.validSlices = grain.totalSlices;
                     mxlFlowWriterCommitGrain(video_writer, &grain);
@@ -63,10 +113,41 @@ class VideoInNode : public dmf::NodeBase {
                 const uint64_t ns = mxlGetNsUntilIndex(video_index + 1, &video_rate);
                 if (ns > 0 && ns < 2'000'000'000ULL) mxlSleepForNs(ns);
                 video_index++;
+            } else if (frame_kind == dmf::VideoReader::FrameKind::Audio) {
+                mxlFlowWriterCancelGrain(video_writer);
+                mxlMutableWrappedMultiBufferSlice slice{};
+                mxlStatus ast = mxlFlowWriterOpenSamples(audio_writer, audio_index, out_samples_written, &slice);
+                if (ast != MXL_STATUS_OK) {
+                    log("audio OpenSamples failed (%s) index=%llu", dmf::mxl_status_str(ast), audio_index);
+                    continue;
+                }
+                // copy per channel from audio_temp into slice
+                for (int ch = 0; ch < channels; ch++) {
+                    uint8_t* dst0 = static_cast<uint8_t*>(slice.base.fragments[0].pointer) 
+                        + ch * slice.stride;
+                    uint8_t* src = audio_temp.data() + ch * max_audio_samples * sizeof(float);
+                    
+                    size_t frag0_bytes = slice.base.fragments[0].size;
+                    size_t total_bytes = out_samples_written * sizeof(float);
+                    
+                    if (total_bytes <= frag0_bytes) {
+                        std::memcpy(dst0, src, total_bytes);
+                    } else {
+                        // Wrapped — copy first fragment, then second
+                        std::memcpy(dst0, src, frag0_bytes);
+                        uint8_t* dst1 = static_cast<uint8_t*>(slice.base.fragments[1].pointer)
+                                    + ch * slice.stride;
+                        std::memcpy(dst1, src + frag0_bytes, total_bytes - frag0_bytes);
+                    }
+                }
+                mxlFlowWriterCommitSamples(audio_writer);
+                audio_index += out_samples_written;
+            }
         }
 
         log("stopped at video_index=%llu", video_index);
         mxlReleaseFlowWriter(instance(), video_writer);
+        mxlReleaseFlowWriter(instance(), audio_writer);
     }
 };
 

shared/VideoReader.hpp

@@ -6,18 +6,23 @@ extern "C" {
     #include <libavutil/avutil.h>
     #include <libavutil/imgutils.h>
     #include <libswscale/swscale.h>
+    #include <libswresample/swresample.h>
+    #include <libavutil/opt.h>
 }
 
 #include <stdexcept>
 #include <string>
+#include <vector>
 #include "Signal.hpp"
 #include "V210.hpp"
 
+#include <iostream>
+
 namespace dmf {
 
 class VideoReader {
 public:
-    struct SourceInfo {
+    struct VideoInfo {
         int width   = 0;
         int height  = 0;
         int fps_num = 0;
@@ -25,7 +30,17 @@ public:
         AVPixelFormat pix_fmt{};
     };
 
-    SourceInfo source_info{};
+    struct AudioInfo {
+        int sample_rate    = 0;
+        int channels       = 0;
+        int samples        = 0;
+        int channel_stride = 0;  // floats between channel planes (NDI planar layout)
+    };
+
+    enum class FrameKind { None, Video, Audio };
+
+    VideoInfo video_info{};
+    AudioInfo audio_info{};
     bool has_audio  = false;
     bool have_video = false;
 
@@ -34,31 +49,42 @@ public:
             return;
         if (have_video) {
             get_source_info();
-            allocate_conversion_buffers();
+            if (have_video) {
+                allocate_video_conversion_buffers();
+            }
+            if (has_audio) {
+                allocate_audio_conversion_buffers();
+            }
         }
     }
 
     ~VideoReader() {
-        avcodec_free_context(&codec_context);
+        avcodec_free_context(&video_codec_context);
         avformat_close_input(&format_context);
-        sws_freeContext(sws_ctx);
+        sws_freeContext(sws_video_ctx);
+        swr_free(&swr_audio_ctx);
         av_freep(&p10_data[0]);
-        av_frame_free(&frame);
+        av_frame_free(&video_frame);
+        av_frame_free(&audio_frame);
         av_packet_free(&packet);
     }
 
     // Returns true when a frame was decoded and written into video_buf.
     // Returns false when g_running goes false.
-    bool get_next_frame(uint8_t* video_buf, uint32_t mxl_stride) {
+    FrameKind get_next_frame(uint8_t* video_buf, uint32_t mxl_stride, uint8_t* audio_buf, int max_audio_samples, int& out_samples_written) {
         while (dmf::g_running.load(std::memory_order_relaxed)) {
             // Drain any frames buffered in the decoder first
-            if (avcodec_receive_frame(codec_context, frame) == 0) {
+            if (avcodec_receive_frame(video_codec_context, video_frame) == 0) {
+                if (!video_buf) {
+                    av_frame_unref(video_frame);
+                    continue;  // nowhere to write — discard frame
+                }
                 sws_scale(
-                    sws_ctx,
+                    sws_video_ctx,
-                    frame->data,
+                    video_frame->data,
-                    frame->linesize,
+                    video_frame->linesize,
                     0,
-                    source_info.height,
+                    video_info.height,
                     p10_data,
                     p10_linesizes
                 );
@@ -67,15 +93,48 @@ public:
                     reinterpret_cast<uint16_t*>(p10_data[1]),
                     reinterpret_cast<uint16_t*>(p10_data[2]),
                     video_buf,
-                    source_info.width,
+                    video_info.width,
-                    source_info.height,
+                    video_info.height,
                     p10_linesizes[0],
                     p10_linesizes[1],
                     p10_linesizes[2],
                     mxl_stride
                 );
-                av_frame_unref(frame);
+                av_frame_unref(video_frame);
-                return true;
+                return FrameKind::Video;
+            }
+
+            if (avcodec_receive_frame(audio_codec_context, audio_frame) == 0) {
+                if (!audio_buf) {
+                    av_frame_unref(audio_frame);
+                    continue;  // nowhere to write — discard frame
+                }
+
+                int dst_nb_samples = av_rescale_rnd(
+                    swr_get_delay(swr_audio_ctx, audio_codec_context->sample_rate) + audio_frame->nb_samples,
+                    audio_codec_context->sample_rate, audio_codec_context->sample_rate, AV_ROUND_UP
+                );
+
+                // Guard against buffer overflows
+                if (dst_nb_samples > max_audio_samples) {
+                    dst_nb_samples = max_audio_samples;
+                }
+
+                std::vector<uint8_t*> dst(audio_info.channels);
+                for (int ch = 0; ch < audio_info.channels; ch++) {
+                    dst[ch] = audio_buf + ch * max_audio_samples * sizeof(float);
+                }
+
+                // Convert/Resample the audio layout and sample format
+                int converted_samples = swr_convert(
+                    swr_audio_ctx,
+                    dst.data(), dst_nb_samples,
+                    (const uint8_t**)audio_frame->data, audio_frame->nb_samples
+                );
+                out_samples_written = converted_samples;
+
+                av_frame_unref(audio_frame);
+                return FrameKind::Audio;
             }
 
             // No buffered frame — read next packet
@@ -83,27 +142,38 @@ public:
             if (av_read_frame(format_context, packet) < 0) {
                 // EOF — loop back to start
                 avformat_seek_file(format_context, -1, 0, 0, 0, AVSEEK_FLAG_BACKWARD);
-                avcodec_flush_buffers(codec_context);
+                avcodec_flush_buffers(video_codec_context);
+                swr_close(swr_audio_ctx);
+                swr_init(swr_audio_ctx);
                 continue;
             }
-            if (packet->stream_index != video_stream_index) continue;
+
-            avcodec_send_packet(codec_context, packet);
+            if (packet->stream_index == video_stream_index) {
+                avcodec_send_packet(video_codec_context, packet);
+            } else if (packet->stream_index == audio_stream_index) {
+                avcodec_send_packet(audio_codec_context, packet);
             }
-        return false;
+        }
+        return FrameKind::None;
     }
 
 private:
-    AVFormatContext* format_context  = nullptr;
-    AVCodecContext*  codec_context   = nullptr;
-    AVPacket*        packet          = av_packet_alloc();
-    AVFrame*         frame           = av_frame_alloc();
     int video_stream_index = -1;
     int audio_stream_index = -1;
 
-    SwsContext* sws_ctx         = nullptr;
+    AVFormatContext* format_context        = nullptr;
+    AVPacket*        packet                = av_packet_alloc();
+
+    AVCodecContext*  video_codec_context   = nullptr;
+    AVFrame*         video_frame           = av_frame_alloc();
+    SwsContext* sws_video_ctx         = nullptr;
     int         p10_linesizes[4] = {0, 0, 0, 0};
     uint8_t*    p10_data[4]      = {nullptr, nullptr, nullptr, nullptr};
 
+    AVCodecContext* audio_codec_context = nullptr;
+    AVFrame*        audio_frame        = av_frame_alloc();
+    SwrContext* swr_audio_ctx = nullptr;
+
     bool open_file(const std::string& filename) {
         if (avformat_open_input(&format_context, filename.c_str(), nullptr, nullptr) != 0)
             throw std::runtime_error("Could not open file: " + filename);
@@ -133,45 +203,83 @@ private:
     }
 
     void get_source_info() {
-        AVCodecParameters* codec_params = format_context->streams[video_stream_index]->codecpar;
+        AVCodecParameters* video_codec_params = format_context->streams[video_stream_index]->codecpar;
-        const AVCodec* codec = avcodec_find_decoder(codec_params->codec_id);
+        const AVCodec* video_codec = avcodec_find_decoder(video_codec_params->codec_id);
-        if (!codec)
+        if (!video_codec)
-            throw std::runtime_error("Unsupported codec");
+            throw std::runtime_error("Unsupported video codec");
 
-        codec_context = avcodec_alloc_context3(codec);
+        video_codec_context = avcodec_alloc_context3(video_codec);
-        if (avcodec_parameters_to_context(codec_context, codec_params) < 0)
+        if (avcodec_parameters_to_context(video_codec_context, video_codec_params) < 0)
-            throw std::runtime_error("Could not copy codec parameters");
+            throw std::runtime_error("Could not copy video codec parameters");
 
-        if (avcodec_open2(codec_context, codec, nullptr) < 0) {
+        if (avcodec_open2(video_codec_context, video_codec, nullptr) < 0) {
-            avcodec_free_context(&codec_context);
+            avcodec_free_context(&video_codec_context);
-            throw std::runtime_error("Could not open codec");
+            throw std::runtime_error("Could not open video codec");
         }
 
-        AVRational fps = codec_context->framerate;
+        AVRational fps = video_codec_context->framerate;
         if (fps.num == 0 || fps.den == 0)
             fps = format_context->streams[video_stream_index]->avg_frame_rate;
 
-        source_info.width   = codec_context->width;
+        video_info.width   = video_codec_context->width;
-        source_info.height  = codec_context->height;
+        video_info.height  = video_codec_context->height;
-        source_info.fps_num = fps.num;
+        video_info.fps_num = fps.num;
-        source_info.fps_den = fps.den;
+        video_info.fps_den = fps.den;
-        source_info.pix_fmt = codec_context->pix_fmt;
+        video_info.pix_fmt = video_codec_context->pix_fmt;
+
+        // audio part
+        if (audio_stream_index == -1) return;
+        AVCodecParameters* audio_codec_params = format_context->streams[audio_stream_index]->codecpar;
+        const AVCodec* audio_codec = avcodec_find_decoder(audio_codec_params->codec_id);
+        if (!audio_codec)
+            throw std::runtime_error("Unsupported audio codec");
+
+        audio_codec_context = avcodec_alloc_context3(audio_codec);
+        if (avcodec_parameters_to_context(audio_codec_context, audio_codec_params) < 0)
+            throw std::runtime_error("Could not copy audio codec parameters");
+
+        if (avcodec_open2(audio_codec_context, audio_codec, nullptr) < 0) {
+            avcodec_free_context(&audio_codec_context);
+            throw std::runtime_error("Could not open audio codec");
+        }
+        audio_info.sample_rate = audio_codec_context->sample_rate;
+        audio_info.channels = audio_codec_context->ch_layout.nb_channels;
     }
 
-    void allocate_conversion_buffers() {
+    void allocate_video_conversion_buffers() {
-        sws_ctx = sws_getContext(
+        sws_video_ctx = sws_getContext(
-            source_info.width, source_info.height, source_info.pix_fmt,
+            video_info.width, video_info.height, video_info.pix_fmt,
-            source_info.width, source_info.height, AV_PIX_FMT_YUV422P10LE,
+            video_info.width, video_info.height, AV_PIX_FMT_YUV422P10LE,
             SWS_BILINEAR, nullptr, nullptr, nullptr
         );
-        if (!sws_ctx)
+        if (!sws_video_ctx)
             throw std::runtime_error("Failed to create SwsContext");
 
         if (av_image_alloc(p10_data, p10_linesizes,
-                source_info.width, source_info.height,
+                video_info.width, video_info.height,
                 AV_PIX_FMT_YUV422P10LE, 64) < 0)
             throw std::runtime_error("Failed to allocate YUV422P10 buffer");
     }
+
+    void allocate_audio_conversion_buffers() {
+        swr_audio_ctx = swr_alloc();
+
+        // Set input options
+        av_opt_set_chlayout(swr_audio_ctx, "in_chlayout",    &audio_codec_context->ch_layout, 0);
+        av_opt_set_int(swr_audio_ctx,      "in_sample_rate", audio_info.sample_rate, 0);
+        av_opt_set_sample_fmt(swr_audio_ctx, "in_sample_fmt", audio_codec_context->sample_fmt, 0);
+
+        // Set output options
+        av_opt_set_chlayout(swr_audio_ctx, "out_chlayout",   &audio_codec_context->ch_layout, 0);
+        av_opt_set_int(swr_audio_ctx,      "out_sample_rate", audio_info.sample_rate, 0);
+        av_opt_set_sample_fmt(swr_audio_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
+
+        // Initialize the context
+        if (swr_init(swr_audio_ctx) < 0) {
+            throw std::runtime_error("Failed to create SwrContext");
+        }
+
+    }
 };
 
 } // namespace dmf

video-ndi.json

@@ -1,13 +1,18 @@
 {
   "nodes": [
-    { "id": "videoin", "type": "videoin", "params": {} },
+    { "id": "videoin", "type": "videoin", "params": {"file": "/home/itten/test-vid/2.ts"} },
     { "id": "ndiout",  "type": "ndiout",  "params": {} }
   ],
   "edges": [
     {
       "from": "videoin", "from_port": "video_flow_id",
       "to":   "ndiout",  "to_port":   "video_flow_id",
-      "format": { "kind": "video", "width": 1920, "height": 1080, "fps_num": 25, "fps_den": 1 }
+      "format": { "kind": "video", "width": 1920, "height": 1080, "fps_num": 60, "fps_den": 1 }
+    },
+    {
+      "from": "videoin", "from_port": "audio_flow_id",
+      "to":   "ndiout",  "to_port":   "audio_flow_id",
+      "format": { "kind": "audio", "sample_rate": 44100, "channels": 2, "bit_depth": 32 }
     }
   ]
 }