fixed the synchronized-audio corruption

This commit is contained in:
Dmitry Sergeev
2026-09-01 01:04:40 +03:00
parent 19e2e076e9
commit e830da4a32
5 changed files with 153 additions and 642 deletions
+104 -578
View File
@@ -6,16 +6,12 @@ import (
"fmt"
"io"
"log"
mxladapter "mxl-player/internal/adapter/mxl"
"mxl-player/internal/imgui"
"mxl-player/internal/output"
"mxl-player/internal/playback"
"mxl-player/internal/renderer"
"mxl-player/internal/sdl"
"mxl-player/internal/source"
"os"
"runtime"
"sync"
"time"
"unsafe"
@@ -170,22 +166,6 @@ func main() {
if !args.ListAudio && !args.ListGPU {
checkMXLargs(args)
}
if args.SyncRequested &&
args.VideoFlowId != "" &&
args.AudioFlowId != "" &&
args.VideoDomain != args.AudioDomain {
fmt.Fprintln(
os.Stderr,
"--sync currently requires video and audio to use the same MXL domain",
)
os.Exit(2)
}
// path selection
useLegacySync := args.SyncRequested &&
args.VideoFlowId != "" &&
args.AudioFlowId != ""
useIndependentSlots := !useLegacySync
runtime.LockOSThread()
if err := sdl.Load(); err != nil {
panic(err)
@@ -289,77 +269,6 @@ func main() {
}
defer vkDevice.Destroy()
var (
syncSrc *source.SyncSource
videoSrc *source.Source
audioSrc *source.AudioSource
audioStream uintptr
audioBatch uint64
aChans uint64
)
interleaveAudio := func(samples [][]byte) []byte {
frameBytes := int(audioBatch) * int(aChans) * 4
out := make([]byte, frameBytes)
for ch := uint64(0); ch < aChans; ch++ {
srcBytes := samples[ch]
for i := uint64(0); i < audioBatch; i++ {
srcOff := i * 4
dstOff := (i*aChans + ch) * 4
if srcOff+4 <= uint64(len(srcBytes)) {
copy(out[dstOff:dstOff+4], srcBytes[srcOff:srcOff+4])
}
}
}
return out
}
switch {
case useLegacySync:
syncSrc, err = source.OpenSameDomainSync(
args.VideoDomain,
args.VideoFlowId,
args.AudioFlowId,
)
if err != nil {
log.Fatalf("sync source: %v", err)
}
aChans = syncSrc.Channels()
audioBatch = uint64(syncSrc.AudioRate().Num) / uint64(syncSrc.Rate().Num)
if audioBatch == 0 {
audioBatch = 1
}
audioStream = sdl.OpenAudioDeviceStream(sdlAudioDevice, sdl.AudioSpec{
Format: sdl.AudioF32,
Channels: int32(aChans),
Freq: int32(syncSrc.AudioRate().Num / syncSrc.AudioRate().Den),
})
if audioStream == 0 {
log.Fatalf("audio: %s", sdl.GetError())
}
sdl.ResumeAudioStreamDevice(audioStream)
fmt.Printf("sync: video %dx%d audio %dch batch=%d\n",
syncSrc.Width(), syncSrc.Height(), aChans, audioBatch)
default:
// Independent slots own their readers.
// Empty startup opens nothing.
}
defer func() {
if syncSrc != nil {
_ = syncSrc.Close()
}
if videoSrc != nil {
_ = videoSrc.Close()
}
if audioSrc != nil {
_ = audioSrc.Close()
}
}()
if audioStream != 0 {
defer sdl.DestroyAudioStream(audioStream)
}
// Create renderer
r, err := renderer.New(renderer.Config{
PhysDevice: vkPhysDevice,
@@ -414,419 +323,58 @@ func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
type reconnectParams struct {
domain string
video string
audio string
}
// Reconnect requests from GUI or automatic retry
control := make(chan reconnectParams, 1)
videoBridge := playback.NewVideoBridge()
statusStore := playback.NewStatusStore()
videoWorker, err := playback.NewVideoWorker(
mxladapter.VideoFactory{},
videoBridge,
retryPolicy,
mxladapter.ShouldRetry,
func(status playback.Status) {
statusStore.Observe(status)
if status.Err != nil {
log.Printf(
"video: state=%v attempt=%d failed=%d: %v",
status.State,
status.Attempt,
status.FailedAttempts,
status.Err,
)
return
}
log.Printf(
"video: state=%v attempt=%d failed=%d",
status.State,
status.Attempt,
status.FailedAttempts,
)
},
)
player, err := newPlayerPlayback(sdlAudioDevice, retryPolicy)
if err != nil {
panic(err)
}
videoSlot, err := playback.NewVideoSlot(videoWorker)
if err != nil {
panic(err)
}
videoCommands := make(chan playback.FeedConfig, 1)
videoBridge := player.Video
statusStore := player.Status
syncRequested := args.SyncRequested
audioOutput := output.NewSDLAudioSink(sdlAudioDevice)
defer audioOutput.Close()
audioWorker, err := playback.NewAudioWorker(
mxladapter.AudioFactory{},
audioOutput,
retryPolicy,
mxladapter.ShouldRetry,
func(status playback.Status) {
statusStore.Observe(status)
if status.Err != nil {
log.Printf(
"audio: state=%v attempt=%d failed=%d: %v",
status.State,
status.Attempt,
status.FailedAttempts,
status.Err,
)
return
}
log.Printf(
"audio: state=%v attempt=%d failed=%d",
status.State,
status.Attempt,
status.FailedAttempts,
)
},
)
if err != nil {
panic(err)
}
audioSlot, err := playback.NewAudioSlot(audioWorker)
if err != nil {
panic(err)
}
audioCommands := make(chan playback.FeedConfig, 1)
reopen := func(params reconnectParams) error {
// Close current sources
if syncSrc != nil {
_ = syncSrc.Close()
syncSrc = nil
}
if videoSrc != nil {
_ = videoSrc.Close()
videoSrc = nil
}
if audioSrc != nil {
_ = audioSrc.Close()
audioSrc = nil
}
// Try once. Return error if fails — caller loops back to select
// and can pick up newer reconnect request.
if params.video != "" && params.audio != "" {
s, e := source.OpenSameDomainSync(params.domain, params.video, params.audio)
if e == nil {
syncSrc = s
aChans = s.Channels()
audioBatch = uint64(s.AudioRate().Num) / uint64(s.Rate().Num)
if audioBatch == 0 {
audioBatch = 1
}
return nil
}
return e
} else if params.video != "" {
s, e := source.Open(params.domain, params.video)
if e == nil {
videoSrc = s
return nil
}
return e
} else if params.audio != "" {
s, e := source.OpenAudio(params.domain, params.audio)
if e == nil {
audioSrc = s
aChans = s.Channels()
audioBatch = uint64(s.Rate().Num) / (100 * uint64(s.Rate().Den))
if audioBatch == 0 {
audioBatch = 1
}
return nil
}
return e
}
return fmt.Errorf("reopen: no flow specified")
}
enqueueVideoConfig := func(config playback.FeedConfig) {
enqueueCommand := func(command playback.SessionCommand) {
select {
case <-videoCommands:
default:
}
select {
case videoCommands <- config:
case player.Commands <- command:
default:
log.Printf("playback command queue is full; ignoring command %d", command.Kind)
}
}
enqueueAudioConfig := func(config playback.FeedConfig) {
select {
case <-audioCommands:
default:
}
select {
case audioCommands <- config:
default:
}
}
doReconnect := func() {
if useIndependentSlots {
videoActive = videoStr != ""
audioActive = audioStr != ""
videoActive = videoStr != ""
audioActive = audioStr != ""
videoConfig := playback.FeedConfig{}
if videoStr != "" {
videoConfig = playback.FeedConfig{
if videoStr == "" {
enqueueCommand(playback.SessionCommand{Kind: playback.CommandRemoveVideo})
} else {
enqueueCommand(playback.SessionCommand{
Kind: playback.CommandSetVideo,
Config: playback.FeedConfig{
Domain: videoDomainStr,
UUID: videoStr,
Active: true,
}
}
audioConfig := playback.FeedConfig{}
if audioStr != "" {
audioConfig = playback.FeedConfig{
},
})
}
if audioStr == "" {
enqueueCommand(playback.SessionCommand{Kind: playback.CommandRemoveAudio})
} else {
enqueueCommand(playback.SessionCommand{
Kind: playback.CommandSetAudio,
Config: playback.FeedConfig{
Domain: audioDomainStr,
UUID: audioStr,
Active: true,
}
}
enqueueVideoConfig(videoConfig)
enqueueAudioConfig(audioConfig)
return
},
})
}
// legacy
if videoDomainStr != audioDomainStr {
log.Printf(
"sync reconnect rejected: video domain %q differs from audio domain %q",
videoDomainStr,
audioDomainStr,
)
return
}
select {
case <-control:
default:
}
control <- reconnectParams{domain: videoDomainStr, video: videoStr, audio: audioStr}
}
playbackDone := make(chan struct{})
playbackDone := make(chan error, 1)
go func() {
defer close(playbackDone)
if useIndependentSlots {
var slots sync.WaitGroup
slots.Add(2)
go func() {
defer slots.Done()
err := videoSlot.Run(
ctx,
playback.FeedConfig{
Domain: args.VideoDomain,
UUID: args.VideoFlowId,
Active: args.VideoFlowId != "",
},
videoCommands,
)
if err != nil && !errors.Is(err, context.Canceled) {
log.Printf("video slot: %v", err)
}
}()
go func() {
defer slots.Done()
err := audioSlot.Run(
ctx,
playback.FeedConfig{
Domain: args.AudioDomain,
UUID: args.AudioFlowId,
Active: args.AudioFlowId != "",
},
audioCommands,
)
if err != nil && !errors.Is(err, context.Canceled) {
log.Printf("audio slot: %v", err)
}
}()
<-ctx.Done()
slots.Wait()
return
}
// Audio-only mode: independent loop.
if audioSrc != nil && syncSrc == nil && videoSrc == nil {
for {
select {
case <-ctx.Done():
return
case params := <-control:
if params.video != "" || params.audio != "" {
if rerr := reopen(params); rerr != nil {
if errors.Is(rerr, context.Canceled) {
return
}
log.Printf("source: reopen failed: %v, retrying", rerr)
select {
case <-time.After(500 * time.Millisecond):
case <-ctx.Done():
return
}
select {
case control <- params:
default:
}
}
}
continue
default:
}
queued := sdl.GetAudioStreamQueued(audioStream)
maxQueued := int32(audioBatch) * int32(aChans) * 4 * 20
if queued > maxQueued {
select {
case <-time.After(10 * time.Millisecond):
case <-ctx.Done():
return
}
continue
}
f, err := audioSrc.NextAudio(ctx, audioBatch, 20*time.Millisecond)
if err != nil {
if errors.Is(err, context.Canceled) {
return
}
log.Printf("source: %v", err)
params := reconnectParams{domain: videoDomainStr, video: videoStr, audio: audioStr}
select {
case <-control:
default:
}
select {
case <-time.After(500 * time.Millisecond):
case <-ctx.Done():
return
}
select {
case control <- params:
default:
}
continue
}
if f.Samples != nil && audioStream != 0 {
sdl.PutAudioStreamData(audioStream, interleaveAudio(f.Samples))
}
}
}
// Video bridge provides backpressure: only one borrowed frame is in flight.
for {
select {
case <-ctx.Done():
return
case params := <-control:
if rerr := reopen(params); rerr != nil {
if errors.Is(rerr, context.Canceled) {
return
}
log.Printf("source: reopen failed: %v, retrying", rerr)
select {
case <-time.After(500 * time.Millisecond):
case <-ctx.Done():
return
}
select {
case control <- params:
default:
// Preserve an already queued, potentially newer request.
}
}
continue
default:
}
var videoFrame playback.VideoFrame
if syncSrc != nil {
vFrame, aFrame, err := syncSrc.NextSync(ctx, audioBatch, 200*time.Millisecond)
if err != nil {
if errors.Is(err, context.Canceled) {
return
}
log.Printf("source: %v", err)
// Request a reconnect after the read failure.
params := reconnectParams{
domain: videoDomainStr,
video: videoStr,
audio: audioStr,
}
select {
case control <- params:
default:
// Preserve an already queued, potentially newer request.
}
continue
}
videoFrame = playback.VideoFrame{
Index: vFrame.Index,
Width: vFrame.Width,
Height: vFrame.Height,
Stride: vFrame.Stride,
Size: vFrame.Size,
Invalid: vFrame.Invalid,
Payload: vFrame.Payload,
}
if aFrame.Samples != nil && audioStream != 0 {
sdl.PutAudioStreamData(audioStream, interleaveAudio(aFrame.Samples))
}
} else if videoSrc != nil {
f, err := videoSrc.NextCtx(ctx, 200*time.Millisecond)
if err != nil {
if errors.Is(err, context.Canceled) {
return
}
log.Printf("source: %v", err)
params := reconnectParams{
domain: videoDomainStr,
video: videoStr,
audio: audioStr,
}
select {
case control <- params:
default:
// Preserve an already queued, potentially newer request.
}
continue
}
videoFrame = playback.VideoFrame{
Index: f.Index,
Width: f.Width,
Height: f.Height,
Stride: f.Stride,
Size: f.Size,
Invalid: f.Invalid,
Payload: f.Payload,
}
}
if err := videoBridge.ConsumeVideo(ctx, videoFrame); err != nil {
if errors.Is(err, context.Canceled) {
return
}
log.Printf("video output: %v", err)
return
}
}
playbackDone <- player.Controller.Run(
ctx,
args.playbackConfig(),
player.Commands,
)
}()
running := true
@@ -979,136 +527,109 @@ func main() {
cimgui.InputTextWithHint("Audio domain", "/dev/shm/mxl", &audioDomainStr, 0, nil)
cimgui.InputTextWithHint("Video UUID", "", &videoStr, 0, nil)
cimgui.InputTextWithHint("Audio UUID", "", &audioStr, 0, nil)
if snapshot, ok := player.Controller.Snapshot(); ok {
videoActive = snapshot.Desired.Video.Active
audioActive = snapshot.Desired.Audio.Active
syncRequested = snapshot.Desired.SyncRequested
}
if cimgui.Button("Connect") {
doReconnect()
}
cimgui.SameLine()
cimgui.Checkbox("Show stats", &showStats)
if useIndependentSlots && videoActive {
if cimgui.Checkbox("Synchronize", &syncRequested) {
kind := playback.CommandDisableSync
if syncRequested {
kind = playback.CommandEnableSync
}
enqueueCommand(playback.SessionCommand{Kind: kind})
}
if videoActive {
if cimgui.Button("Stop video") {
videoActive = false
enqueueVideoConfig(
playback.FeedConfig{
Domain: videoDomainStr,
UUID: videoStr,
Active: false,
})
enqueueCommand(playback.SessionCommand{Kind: playback.CommandStopVideo})
}
}
if useIndependentSlots && !videoActive && videoStr != "" {
if !videoActive && videoStr != "" {
cimgui.SameLine()
if cimgui.Button("Resume video") {
videoActive = true
enqueueVideoConfig(playback.FeedConfig{
Domain: videoDomainStr,
UUID: videoStr,
Active: true,
})
enqueueCommand(playback.SessionCommand{Kind: playback.CommandResumeVideo})
}
}
if useIndependentSlots && videoStr != "" {
if videoStr != "" {
if cimgui.Button("Remove video") {
videoActive = false
videoStr = ""
enqueueVideoConfig(playback.FeedConfig{})
enqueueCommand(playback.SessionCommand{Kind: playback.CommandRemoveVideo})
}
}
if useIndependentSlots {
if videoActive {
cimgui.Text("Video desired: active")
} else if videoStr != "" {
cimgui.Text("Video desired: stopped")
} else {
cimgui.Text("Video desired: not configured")
}
if status, ok := statusStore.Snapshot(playback.UnitVideo); ok {
cimgui.Text(fmt.Sprintf(
"Video actual: %s",
status.State,
))
cimgui.Text(fmt.Sprintf(
"Attempt: %d, failed: %d",
status.Attempt,
status.FailedAttempts,
))
if status.RetryIn > 0 {
cimgui.Text(fmt.Sprintf(
"Retry in: %s",
status.RetryIn.Round(time.Millisecond),
))
}
if status.Err != nil {
cimgui.TextWrapped(status.Err.Error())
}
} else {
cimgui.Text("Video actual: not started")
}
if videoActive {
cimgui.Text("Video desired: active")
} else if videoStr != "" {
cimgui.Text("Video desired: stopped")
} else {
cimgui.Text("Video desired: not configured")
}
if useIndependentSlots && audioActive {
if status, ok := statusStore.Snapshot(playback.UnitVideo); ok {
cimgui.Text(fmt.Sprintf("Video actual: %s", status.State))
cimgui.Text(fmt.Sprintf("Attempt: %d, failed: %d", status.Attempt, status.FailedAttempts))
if status.RetryIn > 0 {
cimgui.Text(fmt.Sprintf("Retry in: %s", status.RetryIn.Round(time.Millisecond)))
}
if status.Err != nil {
cimgui.TextWrapped(status.Err.Error())
}
} else {
cimgui.Text("Video actual: not started")
}
if audioActive {
if cimgui.Button("Stop audio") {
audioActive = false
enqueueAudioConfig(playback.FeedConfig{
Domain: audioDomainStr,
UUID: audioStr,
Active: false,
})
enqueueCommand(playback.SessionCommand{Kind: playback.CommandStopAudio})
}
}
if useIndependentSlots && !audioActive && audioStr != "" {
if !audioActive && audioStr != "" {
if cimgui.Button("Resume audio") {
audioActive = true
enqueueAudioConfig(playback.FeedConfig{
Domain: audioDomainStr,
UUID: audioStr,
Active: true,
})
enqueueCommand(playback.SessionCommand{Kind: playback.CommandResumeAudio})
}
}
if useIndependentSlots && audioStr != "" {
if audioStr != "" {
if cimgui.Button("Remove audio") {
audioActive = false
audioStr = ""
enqueueAudioConfig(playback.FeedConfig{})
enqueueCommand(playback.SessionCommand{Kind: playback.CommandRemoveAudio})
}
}
if useIndependentSlots {
if audioActive {
cimgui.Text("Audio desired: active")
} else if audioStr != "" {
cimgui.Text("Audio desired: stopped")
} else {
cimgui.Text("Audio desired: not configured")
if audioActive {
cimgui.Text("Audio desired: active")
} else if audioStr != "" {
cimgui.Text("Audio desired: stopped")
} else {
cimgui.Text("Audio desired: not configured")
}
if status, ok := statusStore.Snapshot(playback.UnitAudio); ok {
cimgui.Text(fmt.Sprintf("Audio actual: %s", status.State))
cimgui.Text(fmt.Sprintf("Attempt: %d, failed: %d", status.Attempt, status.FailedAttempts))
if status.RetryIn > 0 {
cimgui.Text(fmt.Sprintf("Retry in: %s", status.RetryIn.Round(time.Millisecond)))
}
if status, ok := statusStore.Snapshot(playback.UnitAudio); ok {
cimgui.Text(fmt.Sprintf(
"Audio actual: %s",
status.State,
))
cimgui.Text(fmt.Sprintf(
"Attempt: %d, failed: %d",
status.Attempt,
status.FailedAttempts,
))
if status.RetryIn > 0 {
cimgui.Text(fmt.Sprintf(
"Retry in: %s",
status.RetryIn.Round(time.Millisecond),
))
}
if status.Err != nil {
cimgui.TextWrapped(status.Err.Error())
}
} else {
cimgui.Text("Audio actual: not started")
if status.Err != nil {
cimgui.TextWrapped(status.Err.Error())
}
} else {
cimgui.Text("Audio actual: not started")
}
if status, ok := statusStore.Snapshot(playback.UnitSync); ok {
cimgui.Text(fmt.Sprintf("Sync actual: %s", status.State))
if status.Err != nil {
cimgui.TextWrapped(status.Err.Error())
}
}
@@ -1141,5 +662,10 @@ func main() {
}
cancel()
<-playbackDone
if err := <-playbackDone; err != nil && !errors.Is(err, context.Canceled) {
log.Printf("playback controller: %v", err)
}
if err := player.Close(); err != nil {
log.Printf("close playback: %v", err)
}
}