Audio #1

Merged
itten merged 11 commits from audio into main 2026-09-16 23:50:47 +03:00
3 changed files with 126 additions and 82 deletions
Showing only changes of commit 1a16436688 - Show all commits
+3 -3
View File
@@ -80,8 +80,8 @@ For `audio/float32`, the audio package may use `[]float32` internally and keep b
## Additional implementation priorities
1. Fix the existing tests or update incorrect expectations after confirming the intended color values and overlay positioning.
2. Introduce a `run(...) error` orchestration function using shared cancellation and error propagation instead of calling `log.Fatalf` throughout the media loop.
3. Add a typed audio flow-definition constructor, CPU audio generator, and continuous-flow writer loop using `OpenSamples`, `ChannelFragments`, and `Commit`.
2. Add a typed audio flow-definition constructor, CPU audio generator, and continuous-flow writer loop using `OpenSamples`, `ChannelFragments`, and `Commit`.
3. Run video and audio as sibling goroutines with shared cancellation and error propagation.
4. After audio is complete, measure end-to-end frame time and missed deadlines at 1080p50/60 and UHD. The current wgpu path may be adequate, but it is neither zero-copy nor asynchronous. Treat GPU readback optimization as a separate, potentially large task.
## Suggested package layout
@@ -127,4 +127,4 @@ Package compilation succeeds, but the generator package fails these tests:
## Conclusion
Keep the chosen Go + go-mxl + wgpu stack. Implement audio on the CPU in its own goroutine and give video and audio separate writers, indices, and pacing loops. Coordinate them through a shared context and the common MXL timebase, not through per-frame messages. Typed external-video configuration is now implemented; the next architectural work is extracting the video runner and adding typed audio construction and generation. The three rendering-test failures remain a separate correctness task.
Keep the chosen Go + go-mxl + wgpu stack. Implement audio on the CPU in its own goroutine and give video and audio separate writers, indices, and pacing loops. Coordinate them through a shared context and the common MXL timebase, not through per-frame messages. Typed external-video configuration and the context-aware video runner are now implemented; the next architectural work is typed audio construction and generation. The three rendering-test failures remain a separate correctness task.
+115 -79
View File
@@ -4,7 +4,9 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
@@ -407,6 +409,12 @@ func buildVideoConfig(args appArgs) (video.Config, error) {
return video.Config{
Definition: definition,
Pattern: args.pattern,
Overlay: video.OverlayConfig{
Text: args.textOverlay,
X: args.overlayX,
Y: args.overlayY,
Position: args.overlayPos,
},
}, nil
}
@@ -420,129 +428,157 @@ func main() {
if !parsed.shouldRun {
return
}
args := parsed.args
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, parsed.args); err != nil {
log.Printf("%s: %v", APP_NAME, err)
os.Exit(1)
}
}
func run(ctx context.Context, args appArgs) (runErr error) {
videoCfg, err := buildVideoConfig(args)
if err != nil {
log.Fatalf("video configuration: %v", err)
return fmt.Errorf("video configuration: %w", err)
}
log.Printf("%s %s", APP_NAME, APP_VER)
log.Printf("Domain: %s", args.domain)
log.Printf("Video: %dx%d %d/%d",
videoCfg.Width(),
videoCfg.Height(),
videoCfg.Rate().Num,
videoCfg.Rate().Den,
)
videoCfg.Width(), videoCfg.Height(), videoCfg.Rate().Num, videoCfg.Rate().Den)
log.Printf("Video ID: %s", videoCfg.ID())
// TODO: if init failed -> CPU generator
videoPattern := patterns[videoCfg.Pattern]
gen, err := generator.NewWGPUGenerator(
videoCfg.Width(),
videoCfg.Height(),
videoPattern.kernelPath,
)
if err != nil {
log.Fatalf("wgpu init failed: %v", err)
}
defer gen.Close()
// Static text overlay: rasterized + pre-packed once, stamped on each
// frame after the GPU render (microseconds per frame).
var overlay *generator.TextOverlay
if args.textOverlay != "" {
face, err := generator.LoadFace("assets/fonts/JetBrainsMonoNLNerdFontMono-Regular.ttf", 48)
if err != nil {
log.Fatalf("text overlay init failed: %v", err)
}
defer face.Close()
if args.overlayX < 0 ||
args.overlayX > int(videoCfg.Width()) ||
args.overlayY < 0 ||
args.overlayY > int(videoCfg.Height()) {
log.Fatalf("text overlay position (%d, %d) is outside the %dx%d video frame",
args.overlayX, args.overlayY, videoCfg.Width(), videoCfg.Height())
}
overlay, err = generator.NewTextOverlay(
args.textOverlay,
int(videoCfg.Width()),
int(videoCfg.Height()),
args.overlayX,
args.overlayY,
args.overlayPos,
face,
)
if err != nil {
log.Fatalf("text overlay init failed: %v", err)
}
}
inst, err := mxl.NewInstance(args.domain, "")
if err != nil {
log.Fatalf("initialize MXL domain %q: %v", args.domain, err)
return fmt.Errorf("initialize MXL domain %q: %w", args.domain, err)
}
defer inst.Close()
defer func() {
if err := inst.Close(); err != nil {
runErr = errors.Join(runErr, fmt.Errorf("close MXL instance: %w", err))
}
}()
flowJSON, err := json.Marshal(videoCfg.Definition)
return runVideo(ctx, inst, videoCfg)
}
func runVideo(ctx context.Context, inst *mxl.Instance, cfg video.Config) (runErr error) {
pattern, ok := patterns[cfg.Pattern]
if !ok {
return fmt.Errorf("unknown video pattern %q", cfg.Pattern)
}
// TODO: fall back to a CPU generator if GPU initialization fails.
gen, err := generator.NewWGPUGenerator(cfg.Width(), cfg.Height(), pattern.kernelPath)
if err != nil {
log.Fatalf("marshal video flow definition: %v", err)
return fmt.Errorf("initialize wgpu video generator: %w", err)
}
defer func() {
if err := gen.Close(); err != nil {
runErr = errors.Join(runErr, fmt.Errorf("close video generator: %w", err))
}
}()
overlay, err := buildTextOverlay(cfg)
if err != nil {
return err
}
flowJSON, err := json.Marshal(cfg.Definition)
if err != nil {
return fmt.Errorf("marshal video flow definition: %w", err)
}
writer, isCreated, err := inst.NewWriter(string(flowJSON))
if err != nil {
log.Fatalf("create video writer: %v", err)
return fmt.Errorf("create video writer: %w", err)
}
defer func() {
if err := writer.Close(); err != nil {
runErr = errors.Join(runErr, fmt.Errorf("close video writer: %w", err))
}
}()
if !isCreated {
log.Printf("reusing existing flow: %s, domain: %s", videoCfg.ID(), args.domain)
log.Printf("reusing existing video flow: %s", cfg.ID())
}
defer writer.Close()
flowCfg := writer.Config()
rate := flowCfg.Common.GrainRate
rate := writer.Config().Common.GrainRate
idx := mxl.CurrentIndex(rate)
log.Printf("writing flow grainRate=%d/%d starting at idx=%d", rate.Num, rate.Den, idx)
log.Printf("writing video flow grainRate=%d/%d starting at idx=%d", rate.Num, rate.Den, idx)
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
// core loop
var grainsWritten int64
// animation clock: small counter, not the huge grain index.
// Reason: current wgpu shaders limitations
var tick uint32
var tick uint32 // Small animation counter; shaders cannot use the full grain index yet.
for {
select {
case <-stop:
log.Printf("stopping after %d grains", grainsWritten)
return
case <-ctx.Done():
log.Printf("stopping video after %d grains", grainsWritten)
return nil
default:
}
gwa, err := writer.OpenGrain(idx)
grain, err := writer.OpenGrain(idx)
if err != nil {
log.Fatalf("open video grain %d: %v", idx, err)
return fmt.Errorf("open video grain %d: %w", idx, err)
}
if err := gen.GenerateFrame(gwa.Payload, int(tick)); err != nil {
log.Fatalf("generate frame for grain %d: %v", idx, err)
if err := gen.GenerateFrame(grain.Payload, int(tick)); err != nil {
return cancelVideoGrain(grain, fmt.Errorf("generate frame for grain %d: %w", idx, err))
}
if overlay != nil {
if err := overlay.ApplyV210(gwa.Payload); err != nil {
log.Fatalf("apply text overlay to grain %d: %v", idx, err)
if err := overlay.ApplyV210(grain.Payload); err != nil {
return cancelVideoGrain(grain, fmt.Errorf("apply text overlay to grain %d: %w", idx, err))
}
}
if err := gwa.Commit(gwa.TotalSlices, 0); err != nil {
log.Fatalf("commit video grain %d: %v", idx, err)
if err := grain.Commit(grain.TotalSlices, 0); err != nil {
return fmt.Errorf("commit video grain %d: %w", idx, err)
}
grainsWritten++
idx++
tick++
if grainsWritten%100 == 0 {
log.Printf("grains written=%d, index=%d", grainsWritten, idx)
log.Printf("video grains written=%d, next index=%d", grainsWritten, idx)
}
// Pace ourselves to roughly the grain rate
mxl.SleepNs(mxl.NsUntilIndex(idx, rate))
}
}
func buildTextOverlay(cfg video.Config) (overlay *generator.TextOverlay, resultErr error) {
if cfg.Overlay.Text == "" {
return nil, nil
}
if cfg.Overlay.X < 0 || cfg.Overlay.X > int(cfg.Width()) ||
cfg.Overlay.Y < 0 || cfg.Overlay.Y > int(cfg.Height()) {
return nil, fmt.Errorf("text overlay position (%d, %d) is outside the %dx%d video frame",
cfg.Overlay.X, cfg.Overlay.Y, cfg.Width(), cfg.Height())
}
face, err := generator.LoadFace("assets/fonts/JetBrainsMonoNLNerdFontMono-Regular.ttf", 48)
if err != nil {
return nil, fmt.Errorf("load text overlay font: %w", err)
}
defer func() {
if err := face.Close(); err != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("close text overlay font: %w", err))
}
}()
overlay, err = generator.NewTextOverlay(
cfg.Overlay.Text,
int(cfg.Width()),
int(cfg.Height()),
cfg.Overlay.X,
cfg.Overlay.Y,
cfg.Overlay.Position,
face,
)
if err != nil {
return nil, fmt.Errorf("create text overlay: %w", err)
}
return overlay, nil
}
func cancelVideoGrain(grain *mxl.GrainWriteAccess, cause error) error {
if err := grain.Cancel(); err != nil {
return errors.Join(cause, fmt.Errorf("cancel video grain: %w", err))
}
return cause
}
+8
View File
@@ -9,6 +9,14 @@ import (
type Config struct {
Definition flowdef.Video
Pattern string
Overlay OverlayConfig
}
type OverlayConfig struct {
Text string
X int
Y int
Position string
}
func (c Config) ID() string {