itten/dmf-studio-rnd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

audio works, holy shit

Browse Source
This commit is contained in:
itten
2026-07-04 03:24:20 +03:00
parent 96475acf62
commit b2534efb0f
4 changed files with 258 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
shared/VideoReader.hpp
+155 -47
View File
@@ -6,18 +6,23 @@ extern "C" {
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
#include <libswresample/swresample.h>
#include <libavutil/opt.h>
}
#include <stdexcept>
#include <string>
#include <vector>
#include "Signal.hpp"
#include "V210.hpp"
#include <iostream>
namespace dmf {
class VideoReader {
public:
struct SourceInfo {
struct VideoInfo {
int width = 0;
int height = 0;
int fps_num = 0;
@@ -25,7 +30,17 @@ public:
AVPixelFormat pix_fmt{};
};
SourceInfo source_info{};
struct AudioInfo {
int sample_rate = 0;
int channels = 0;
int samples = 0;
int channel_stride = 0; // floats between channel planes (NDI planar layout)
};
enum class FrameKind { None, Video, Audio };
VideoInfo video_info{};
AudioInfo audio_info{};
bool has_audio = false;
bool have_video = false;
@@ -34,31 +49,42 @@ public:
return;
if (have_video) {
get_source_info();
allocate_conversion_buffers();
if (have_video) {
allocate_video_conversion_buffers();
}
if (has_audio) {
allocate_audio_conversion_buffers();
}
}
}
~VideoReader() {
avcodec_free_context(&codec_context);
avcodec_free_context(&video_codec_context);
avformat_close_input(&format_context);
sws_freeContext(sws_ctx);
sws_freeContext(sws_video_ctx);
swr_free(&swr_audio_ctx);
av_freep(&p10_data[0]);
av_frame_free(&frame);
av_frame_free(&video_frame);
av_frame_free(&audio_frame);
av_packet_free(&packet);
}
// Returns true when a frame was decoded and written into video_buf.
// Returns false when g_running goes false.
bool get_next_frame(uint8_t* video_buf, uint32_t mxl_stride) {
FrameKind get_next_frame(uint8_t* video_buf, uint32_t mxl_stride, uint8_t* audio_buf, int max_audio_samples, int& out_samples_written) {
while (dmf::g_running.load(std::memory_order_relaxed)) {
// Drain any frames buffered in the decoder first
if (avcodec_receive_frame(codec_context, frame) == 0) {
if (avcodec_receive_frame(video_codec_context, video_frame) == 0) {
if (!video_buf) {
av_frame_unref(video_frame);
continue; // nowhere to write — discard frame
}
sws_scale(
sws_ctx,
frame->data,
frame->linesize,
sws_video_ctx,
video_frame->data,
video_frame->linesize,
0,
source_info.height,
video_info.height,
p10_data,
p10_linesizes
);
@@ -67,15 +93,48 @@ public:
reinterpret_cast<uint16_t*>(p10_data[1]),
reinterpret_cast<uint16_t*>(p10_data[2]),
video_buf,
source_info.width,
source_info.height,
video_info.width,
video_info.height,
p10_linesizes[0],
p10_linesizes[1],
p10_linesizes[2],
mxl_stride
);
av_frame_unref(frame);
return true;
av_frame_unref(video_frame);
return FrameKind::Video;
}
if (avcodec_receive_frame(audio_codec_context, audio_frame) == 0) {
if (!audio_buf) {
av_frame_unref(audio_frame);
continue; // nowhere to write — discard frame
}
int dst_nb_samples = av_rescale_rnd(
swr_get_delay(swr_audio_ctx, audio_codec_context->sample_rate) + audio_frame->nb_samples,
audio_codec_context->sample_rate, audio_codec_context->sample_rate, AV_ROUND_UP
);
// Guard against buffer overflows
if (dst_nb_samples > max_audio_samples) {
dst_nb_samples = max_audio_samples;
}
std::vector<uint8_t*> dst(audio_info.channels);
for (int ch = 0; ch < audio_info.channels; ch++) {
dst[ch] = audio_buf + ch * max_audio_samples * sizeof(float);
}
// Convert/Resample the audio layout and sample format
int converted_samples = swr_convert(
swr_audio_ctx,
dst.data(), dst_nb_samples,
(const uint8_t**)audio_frame->data, audio_frame->nb_samples
);
out_samples_written = converted_samples;
av_frame_unref(audio_frame);
return FrameKind::Audio;
}
// No buffered frame — read next packet
@@ -83,27 +142,38 @@ public:
if (av_read_frame(format_context, packet) < 0) {
// EOF — loop back to start
avformat_seek_file(format_context, -1, 0, 0, 0, AVSEEK_FLAG_BACKWARD);
avcodec_flush_buffers(codec_context);
avcodec_flush_buffers(video_codec_context);
swr_close(swr_audio_ctx);
swr_init(swr_audio_ctx);
continue;
}
if (packet->stream_index != video_stream_index) continue;
avcodec_send_packet(codec_context, packet);
if (packet->stream_index == video_stream_index) {
avcodec_send_packet(video_codec_context, packet);
} else if (packet->stream_index == audio_stream_index) {
avcodec_send_packet(audio_codec_context, packet);
}
}
return false;
return FrameKind::None;
}
private:
AVFormatContext* format_context = nullptr;
AVCodecContext* codec_context = nullptr;
AVPacket* packet = av_packet_alloc();
AVFrame* frame = av_frame_alloc();
int video_stream_index = -1;
int audio_stream_index = -1;
SwsContext* sws_ctx = nullptr;
AVFormatContext* format_context = nullptr;
AVPacket* packet = av_packet_alloc();
AVCodecContext* video_codec_context = nullptr;
AVFrame* video_frame = av_frame_alloc();
SwsContext* sws_video_ctx = nullptr;
int p10_linesizes[4] = {0, 0, 0, 0};
uint8_t* p10_data[4] = {nullptr, nullptr, nullptr, nullptr};
AVCodecContext* audio_codec_context = nullptr;
AVFrame* audio_frame = av_frame_alloc();
SwrContext* swr_audio_ctx = nullptr;
bool open_file(const std::string& filename) {
if (avformat_open_input(&format_context, filename.c_str(), nullptr, nullptr) != 0)
throw std::runtime_error("Could not open file: " + filename);
@@ -133,45 +203,83 @@ private:
}
void get_source_info() {
AVCodecParameters* codec_params = format_context->streams[video_stream_index]->codecpar;
const AVCodec* codec = avcodec_find_decoder(codec_params->codec_id);
if (!codec)
throw std::runtime_error("Unsupported codec");
AVCodecParameters* video_codec_params = format_context->streams[video_stream_index]->codecpar;
const AVCodec* video_codec = avcodec_find_decoder(video_codec_params->codec_id);
if (!video_codec)
throw std::runtime_error("Unsupported video codec");
codec_context = avcodec_alloc_context3(codec);
if (avcodec_parameters_to_context(codec_context, codec_params) < 0)
throw std::runtime_error("Could not copy codec parameters");
video_codec_context = avcodec_alloc_context3(video_codec);
if (avcodec_parameters_to_context(video_codec_context, video_codec_params) < 0)
throw std::runtime_error("Could not copy video codec parameters");
if (avcodec_open2(codec_context, codec, nullptr) < 0) {
avcodec_free_context(&codec_context);
throw std::runtime_error("Could not open codec");
if (avcodec_open2(video_codec_context, video_codec, nullptr) < 0) {
avcodec_free_context(&video_codec_context);
throw std::runtime_error("Could not open video codec");
}
AVRational fps = codec_context->framerate;
AVRational fps = video_codec_context->framerate;
if (fps.num == 0 || fps.den == 0)
fps = format_context->streams[video_stream_index]->avg_frame_rate;
source_info.width = codec_context->width;
source_info.height = codec_context->height;
source_info.fps_num = fps.num;
source_info.fps_den = fps.den;
source_info.pix_fmt = codec_context->pix_fmt;
video_info.width = video_codec_context->width;
video_info.height = video_codec_context->height;
video_info.fps_num = fps.num;
video_info.fps_den = fps.den;
video_info.pix_fmt = video_codec_context->pix_fmt;
// audio part
if (audio_stream_index == -1) return;
AVCodecParameters* audio_codec_params = format_context->streams[audio_stream_index]->codecpar;
const AVCodec* audio_codec = avcodec_find_decoder(audio_codec_params->codec_id);
if (!audio_codec)
throw std::runtime_error("Unsupported audio codec");
audio_codec_context = avcodec_alloc_context3(audio_codec);
if (avcodec_parameters_to_context(audio_codec_context, audio_codec_params) < 0)
throw std::runtime_error("Could not copy audio codec parameters");
if (avcodec_open2(audio_codec_context, audio_codec, nullptr) < 0) {
avcodec_free_context(&audio_codec_context);
throw std::runtime_error("Could not open audio codec");
}
audio_info.sample_rate = audio_codec_context->sample_rate;
audio_info.channels = audio_codec_context->ch_layout.nb_channels;
}
void allocate_conversion_buffers() {
sws_ctx = sws_getContext(
source_info.width, source_info.height, source_info.pix_fmt,
source_info.width, source_info.height, AV_PIX_FMT_YUV422P10LE,
void allocate_video_conversion_buffers() {
sws_video_ctx = sws_getContext(
video_info.width, video_info.height, video_info.pix_fmt,
video_info.width, video_info.height, AV_PIX_FMT_YUV422P10LE,
SWS_BILINEAR, nullptr, nullptr, nullptr
);
if (!sws_ctx)
if (!sws_video_ctx)
throw std::runtime_error("Failed to create SwsContext");
if (av_image_alloc(p10_data, p10_linesizes,
source_info.width, source_info.height,
video_info.width, video_info.height,
AV_PIX_FMT_YUV422P10LE, 64) < 0)
throw std::runtime_error("Failed to allocate YUV422P10 buffer");
}
void allocate_audio_conversion_buffers() {
swr_audio_ctx = swr_alloc();
// Set input options
av_opt_set_chlayout(swr_audio_ctx, "in_chlayout", &audio_codec_context->ch_layout, 0);
av_opt_set_int(swr_audio_ctx, "in_sample_rate", audio_info.sample_rate, 0);
av_opt_set_sample_fmt(swr_audio_ctx, "in_sample_fmt", audio_codec_context->sample_fmt, 0);
// Set output options
av_opt_set_chlayout(swr_audio_ctx, "out_chlayout", &audio_codec_context->ch_layout, 0);
av_opt_set_int(swr_audio_ctx, "out_sample_rate", audio_info.sample_rate, 0);
av_opt_set_sample_fmt(swr_audio_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
// Initialize the context
if (swr_init(swr_audio_ctx) < 0) {
throw std::runtime_error("Failed to create SwrContext");
}
}
};
} // namespace dmf