audio works, holy shit

2026-07-04 03:24:20 +03:00
parent 96475acf62
commit b2534efb0f
4 changed files with 258 additions and 63 deletions
@@ -6,18 +6,23 @@ extern "C" {
    #include <libavutil/avutil.h>
    #include <libavutil/imgutils.h>
    #include <libswscale/swscale.h>
+    #include <libswresample/swresample.h>
+    #include <libavutil/opt.h>
 }

 #include <stdexcept>
 #include <string>
+#include <vector>
 #include "Signal.hpp"
 #include "V210.hpp"

+#include <iostream>
+
 namespace dmf {

 class VideoReader {
 public:
-    struct SourceInfo {
+    struct VideoInfo {
        int width   = 0;
        int height  = 0;
        int fps_num = 0;
@@ -25,7 +30,17 @@ public:
        AVPixelFormat pix_fmt{};
    };

-    SourceInfo source_info{};
+    struct AudioInfo {
+        int sample_rate    = 0;
+        int channels       = 0;
+        int samples        = 0;
+        int channel_stride = 0;  // floats between channel planes (NDI planar layout)
+    };
+
+    enum class FrameKind { None, Video, Audio };
+
+    VideoInfo video_info{};
+    AudioInfo audio_info{};
    bool has_audio  = false;
    bool have_video = false;

@@ -34,31 +49,42 @@ public:
            return;
        if (have_video) {
            get_source_info();
-            allocate_conversion_buffers();
+            if (have_video) {
+                allocate_video_conversion_buffers();
+            }
+            if (has_audio) {
+                allocate_audio_conversion_buffers();
+            }
        }
    }

    ~VideoReader() {
-        avcodec_free_context(&codec_context);
+        avcodec_free_context(&video_codec_context);
        avformat_close_input(&format_context);
-        sws_freeContext(sws_ctx);
+        sws_freeContext(sws_video_ctx);
+        swr_free(&swr_audio_ctx);
        av_freep(&p10_data[0]);
-        av_frame_free(&frame);
+        av_frame_free(&video_frame);
+        av_frame_free(&audio_frame);
        av_packet_free(&packet);
    }

    // Returns true when a frame was decoded and written into video_buf.
    // Returns false when g_running goes false.
-    bool get_next_frame(uint8_t* video_buf, uint32_t mxl_stride) {
+    FrameKind get_next_frame(uint8_t* video_buf, uint32_t mxl_stride, uint8_t* audio_buf, int max_audio_samples, int& out_samples_written) {
        while (dmf::g_running.load(std::memory_order_relaxed)) {
            // Drain any frames buffered in the decoder first
-            if (avcodec_receive_frame(codec_context, frame) == 0) {
+            if (avcodec_receive_frame(video_codec_context, video_frame) == 0) {
+                if (!video_buf) {
+                    av_frame_unref(video_frame);
+                    continue;  // nowhere to write — discard frame
+                }
                sws_scale(
-                    sws_ctx,
-                    frame->data,
-                    frame->linesize,
+                    sws_video_ctx,
+                    video_frame->data,
+                    video_frame->linesize,
                    0,
-                    source_info.height,
+                    video_info.height,
                    p10_data,
                    p10_linesizes
                );
@@ -67,15 +93,48 @@ public:
                    reinterpret_cast<uint16_t*>(p10_data[1]),
                    reinterpret_cast<uint16_t*>(p10_data[2]),
                    video_buf,
-                    source_info.width,
-                    source_info.height,
+                    video_info.width,
+                    video_info.height,
                    p10_linesizes[0],
                    p10_linesizes[1],
                    p10_linesizes[2],
                    mxl_stride
                );
-                av_frame_unref(frame);
-                return true;
+                av_frame_unref(video_frame);
+                return FrameKind::Video;
+            }
+
+            if (avcodec_receive_frame(audio_codec_context, audio_frame) == 0) {
+                if (!audio_buf) {
+                    av_frame_unref(audio_frame);
+                    continue;  // nowhere to write — discard frame
+                }
+
+                int dst_nb_samples = av_rescale_rnd(
+                    swr_get_delay(swr_audio_ctx, audio_codec_context->sample_rate) + audio_frame->nb_samples,
+                    audio_codec_context->sample_rate, audio_codec_context->sample_rate, AV_ROUND_UP
+                );
+
+                // Guard against buffer overflows
+                if (dst_nb_samples > max_audio_samples) {
+                    dst_nb_samples = max_audio_samples;
+                }
+
+                std::vector<uint8_t*> dst(audio_info.channels);
+                for (int ch = 0; ch < audio_info.channels; ch++) {
+                    dst[ch] = audio_buf + ch * max_audio_samples * sizeof(float);
+                }
+
+                // Convert/Resample the audio layout and sample format
+                int converted_samples = swr_convert(
+                    swr_audio_ctx,
+                    dst.data(), dst_nb_samples,
+                    (const uint8_t**)audio_frame->data, audio_frame->nb_samples
+                );
+                out_samples_written = converted_samples;
+
+                av_frame_unref(audio_frame);
+                return FrameKind::Audio;
            }

            // No buffered frame — read next packet
@@ -83,27 +142,38 @@ public:
            if (av_read_frame(format_context, packet) < 0) {
                // EOF — loop back to start
                avformat_seek_file(format_context, -1, 0, 0, 0, AVSEEK_FLAG_BACKWARD);
-                avcodec_flush_buffers(codec_context);
+                avcodec_flush_buffers(video_codec_context);
+                swr_close(swr_audio_ctx);
+                swr_init(swr_audio_ctx);
                continue;
            }
-            if (packet->stream_index != video_stream_index) continue;
-            avcodec_send_packet(codec_context, packet);
+
+            if (packet->stream_index == video_stream_index) {
+                avcodec_send_packet(video_codec_context, packet);
+            } else if (packet->stream_index == audio_stream_index) {
+                avcodec_send_packet(audio_codec_context, packet);
+            }
        }
-        return false;
+        return FrameKind::None;
    }

 private:
-    AVFormatContext* format_context  = nullptr;
-    AVCodecContext*  codec_context   = nullptr;
-    AVPacket*        packet          = av_packet_alloc();
-    AVFrame*         frame           = av_frame_alloc();
    int video_stream_index = -1;
    int audio_stream_index = -1;

-    SwsContext* sws_ctx         = nullptr;
+    AVFormatContext* format_context        = nullptr;
+    AVPacket*        packet                = av_packet_alloc();
+
+    AVCodecContext*  video_codec_context   = nullptr;
+    AVFrame*         video_frame           = av_frame_alloc();
+    SwsContext* sws_video_ctx         = nullptr;
    int         p10_linesizes[4] = {0, 0, 0, 0};
    uint8_t*    p10_data[4]      = {nullptr, nullptr, nullptr, nullptr};

+    AVCodecContext* audio_codec_context = nullptr;
+    AVFrame*        audio_frame        = av_frame_alloc();
+    SwrContext* swr_audio_ctx = nullptr;
+
    bool open_file(const std::string& filename) {
        if (avformat_open_input(&format_context, filename.c_str(), nullptr, nullptr) != 0)
            throw std::runtime_error("Could not open file: " + filename);
@@ -133,45 +203,83 @@ private:
    }

    void get_source_info() {
-        AVCodecParameters* codec_params = format_context->streams[video_stream_index]->codecpar;
-        const AVCodec* codec = avcodec_find_decoder(codec_params->codec_id);
-        if (!codec)
-            throw std::runtime_error("Unsupported codec");
+        AVCodecParameters* video_codec_params = format_context->streams[video_stream_index]->codecpar;
+        const AVCodec* video_codec = avcodec_find_decoder(video_codec_params->codec_id);
+        if (!video_codec)
+            throw std::runtime_error("Unsupported video codec");

-        codec_context = avcodec_alloc_context3(codec);
-        if (avcodec_parameters_to_context(codec_context, codec_params) < 0)
-            throw std::runtime_error("Could not copy codec parameters");
+        video_codec_context = avcodec_alloc_context3(video_codec);
+        if (avcodec_parameters_to_context(video_codec_context, video_codec_params) < 0)
+            throw std::runtime_error("Could not copy video codec parameters");

-        if (avcodec_open2(codec_context, codec, nullptr) < 0) {
-            avcodec_free_context(&codec_context);
-            throw std::runtime_error("Could not open codec");
+        if (avcodec_open2(video_codec_context, video_codec, nullptr) < 0) {
+            avcodec_free_context(&video_codec_context);
+            throw std::runtime_error("Could not open video codec");
        }

-        AVRational fps = codec_context->framerate;
+        AVRational fps = video_codec_context->framerate;
        if (fps.num == 0 || fps.den == 0)
            fps = format_context->streams[video_stream_index]->avg_frame_rate;

-        source_info.width   = codec_context->width;
-        source_info.height  = codec_context->height;
-        source_info.fps_num = fps.num;
-        source_info.fps_den = fps.den;
-        source_info.pix_fmt = codec_context->pix_fmt;
+        video_info.width   = video_codec_context->width;
+        video_info.height  = video_codec_context->height;
+        video_info.fps_num = fps.num;
+        video_info.fps_den = fps.den;
+        video_info.pix_fmt = video_codec_context->pix_fmt;
+
+        // audio part
+        if (audio_stream_index == -1) return;
+        AVCodecParameters* audio_codec_params = format_context->streams[audio_stream_index]->codecpar;
+        const AVCodec* audio_codec = avcodec_find_decoder(audio_codec_params->codec_id);
+        if (!audio_codec)
+            throw std::runtime_error("Unsupported audio codec");
+
+        audio_codec_context = avcodec_alloc_context3(audio_codec);
+        if (avcodec_parameters_to_context(audio_codec_context, audio_codec_params) < 0)
+            throw std::runtime_error("Could not copy audio codec parameters");
+
+        if (avcodec_open2(audio_codec_context, audio_codec, nullptr) < 0) {
+            avcodec_free_context(&audio_codec_context);
+            throw std::runtime_error("Could not open audio codec");
+        }
+        audio_info.sample_rate = audio_codec_context->sample_rate;
+        audio_info.channels = audio_codec_context->ch_layout.nb_channels;
    }

-    void allocate_conversion_buffers() {
-        sws_ctx = sws_getContext(
-            source_info.width, source_info.height, source_info.pix_fmt,
-            source_info.width, source_info.height, AV_PIX_FMT_YUV422P10LE,
+    void allocate_video_conversion_buffers() {
+        sws_video_ctx = sws_getContext(
+            video_info.width, video_info.height, video_info.pix_fmt,
+            video_info.width, video_info.height, AV_PIX_FMT_YUV422P10LE,
            SWS_BILINEAR, nullptr, nullptr, nullptr
        );
-        if (!sws_ctx)
+        if (!sws_video_ctx)
            throw std::runtime_error("Failed to create SwsContext");

        if (av_image_alloc(p10_data, p10_linesizes,
-                source_info.width, source_info.height,
+                video_info.width, video_info.height,
                AV_PIX_FMT_YUV422P10LE, 64) < 0)
            throw std::runtime_error("Failed to allocate YUV422P10 buffer");
    }
+
+    void allocate_audio_conversion_buffers() {
+        swr_audio_ctx = swr_alloc();
+
+        // Set input options
+        av_opt_set_chlayout(swr_audio_ctx, "in_chlayout",    &audio_codec_context->ch_layout, 0);
+        av_opt_set_int(swr_audio_ctx,      "in_sample_rate", audio_info.sample_rate, 0);
+        av_opt_set_sample_fmt(swr_audio_ctx, "in_sample_fmt", audio_codec_context->sample_fmt, 0);
+
+        // Set output options
+        av_opt_set_chlayout(swr_audio_ctx, "out_chlayout",   &audio_codec_context->ch_layout, 0);
+        av_opt_set_int(swr_audio_ctx,      "out_sample_rate", audio_info.sample_rate, 0);
+        av_opt_set_sample_fmt(swr_audio_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
+
+        // Initialize the context
+        if (swr_init(swr_audio_ctx) < 0) {
+            throw std::runtime_error("Failed to create SwrContext");
+        }
+
+    }
 };

 } // namespace dmf