Audio #1

Merged
itten merged 11 commits from audio into main 2026-09-16 23:50:47 +03:00
3 changed files with 126 additions and 82 deletions
Showing only changes of commit 1a16436688 - Show all commits
+3 -3
View File
@@ -80,8 +80,8 @@ For `audio/float32`, the audio package may use `[]float32` internally and keep b
## Additional implementation priorities ## Additional implementation priorities
1. Fix the existing tests or update incorrect expectations after confirming the intended color values and overlay positioning. 1. Fix the existing tests or update incorrect expectations after confirming the intended color values and overlay positioning.
2. Introduce a `run(...) error` orchestration function using shared cancellation and error propagation instead of calling `log.Fatalf` throughout the media loop. 2. Add a typed audio flow-definition constructor, CPU audio generator, and continuous-flow writer loop using `OpenSamples`, `ChannelFragments`, and `Commit`.
3. Add a typed audio flow-definition constructor, CPU audio generator, and continuous-flow writer loop using `OpenSamples`, `ChannelFragments`, and `Commit`. 3. Run video and audio as sibling goroutines with shared cancellation and error propagation.
4. After audio is complete, measure end-to-end frame time and missed deadlines at 1080p50/60 and UHD. The current wgpu path may be adequate, but it is neither zero-copy nor asynchronous. Treat GPU readback optimization as a separate, potentially large task. 4. After audio is complete, measure end-to-end frame time and missed deadlines at 1080p50/60 and UHD. The current wgpu path may be adequate, but it is neither zero-copy nor asynchronous. Treat GPU readback optimization as a separate, potentially large task.
## Suggested package layout ## Suggested package layout
@@ -127,4 +127,4 @@ Package compilation succeeds, but the generator package fails these tests:
## Conclusion ## Conclusion
Keep the chosen Go + go-mxl + wgpu stack. Implement audio on the CPU in its own goroutine and give video and audio separate writers, indices, and pacing loops. Coordinate them through a shared context and the common MXL timebase, not through per-frame messages. Typed external-video configuration is now implemented; the next architectural work is extracting the video runner and adding typed audio construction and generation. The three rendering-test failures remain a separate correctness task. Keep the chosen Go + go-mxl + wgpu stack. Implement audio on the CPU in its own goroutine and give video and audio separate writers, indices, and pacing loops. Coordinate them through a shared context and the common MXL timebase, not through per-frame messages. Typed external-video configuration and the context-aware video runner are now implemented; the next architectural work is typed audio construction and generation. The three rendering-test failures remain a separate correctness task.
+115 -79
View File
@@ -4,7 +4,9 @@
package main package main
import ( import (
"context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"log" "log"
@@ -407,6 +409,12 @@ func buildVideoConfig(args appArgs) (video.Config, error) {
return video.Config{ return video.Config{
Definition: definition, Definition: definition,
Pattern: args.pattern, Pattern: args.pattern,
Overlay: video.OverlayConfig{
Text: args.textOverlay,
X: args.overlayX,
Y: args.overlayY,
Position: args.overlayPos,
},
}, nil }, nil
} }
@@ -420,129 +428,157 @@ func main() {
if !parsed.shouldRun { if !parsed.shouldRun {
return return
} }
args := parsed.args
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, parsed.args); err != nil {
log.Printf("%s: %v", APP_NAME, err)
os.Exit(1)
}
}
func run(ctx context.Context, args appArgs) (runErr error) {
videoCfg, err := buildVideoConfig(args) videoCfg, err := buildVideoConfig(args)
if err != nil { if err != nil {
log.Fatalf("video configuration: %v", err) return fmt.Errorf("video configuration: %w", err)
} }
log.Printf("%s %s", APP_NAME, APP_VER) log.Printf("%s %s", APP_NAME, APP_VER)
log.Printf("Domain: %s", args.domain) log.Printf("Domain: %s", args.domain)
log.Printf("Video: %dx%d %d/%d", log.Printf("Video: %dx%d %d/%d",
videoCfg.Width(), videoCfg.Width(), videoCfg.Height(), videoCfg.Rate().Num, videoCfg.Rate().Den)
videoCfg.Height(),
videoCfg.Rate().Num,
videoCfg.Rate().Den,
)
log.Printf("Video ID: %s", videoCfg.ID()) log.Printf("Video ID: %s", videoCfg.ID())
// TODO: if init failed -> CPU generator
videoPattern := patterns[videoCfg.Pattern]
gen, err := generator.NewWGPUGenerator(
videoCfg.Width(),
videoCfg.Height(),
videoPattern.kernelPath,
)
if err != nil {
log.Fatalf("wgpu init failed: %v", err)
}
defer gen.Close()
// Static text overlay: rasterized + pre-packed once, stamped on each
// frame after the GPU render (microseconds per frame).
var overlay *generator.TextOverlay
if args.textOverlay != "" {
face, err := generator.LoadFace("assets/fonts/JetBrainsMonoNLNerdFontMono-Regular.ttf", 48)
if err != nil {
log.Fatalf("text overlay init failed: %v", err)
}
defer face.Close()
if args.overlayX < 0 ||
args.overlayX > int(videoCfg.Width()) ||
args.overlayY < 0 ||
args.overlayY > int(videoCfg.Height()) {
log.Fatalf("text overlay position (%d, %d) is outside the %dx%d video frame",
args.overlayX, args.overlayY, videoCfg.Width(), videoCfg.Height())
}
overlay, err = generator.NewTextOverlay(
args.textOverlay,
int(videoCfg.Width()),
int(videoCfg.Height()),
args.overlayX,
args.overlayY,
args.overlayPos,
face,
)
if err != nil {
log.Fatalf("text overlay init failed: %v", err)
}
}
inst, err := mxl.NewInstance(args.domain, "") inst, err := mxl.NewInstance(args.domain, "")
if err != nil { if err != nil {
log.Fatalf("initialize MXL domain %q: %v", args.domain, err) return fmt.Errorf("initialize MXL domain %q: %w", args.domain, err)
} }
defer inst.Close() defer func() {
if err := inst.Close(); err != nil {
runErr = errors.Join(runErr, fmt.Errorf("close MXL instance: %w", err))
}
}()
flowJSON, err := json.Marshal(videoCfg.Definition) return runVideo(ctx, inst, videoCfg)
}
func runVideo(ctx context.Context, inst *mxl.Instance, cfg video.Config) (runErr error) {
pattern, ok := patterns[cfg.Pattern]
if !ok {
return fmt.Errorf("unknown video pattern %q", cfg.Pattern)
}
// TODO: fall back to a CPU generator if GPU initialization fails.
gen, err := generator.NewWGPUGenerator(cfg.Width(), cfg.Height(), pattern.kernelPath)
if err != nil { if err != nil {
log.Fatalf("marshal video flow definition: %v", err) return fmt.Errorf("initialize wgpu video generator: %w", err)
}
defer func() {
if err := gen.Close(); err != nil {
runErr = errors.Join(runErr, fmt.Errorf("close video generator: %w", err))
}
}()
overlay, err := buildTextOverlay(cfg)
if err != nil {
return err
} }
flowJSON, err := json.Marshal(cfg.Definition)
if err != nil {
return fmt.Errorf("marshal video flow definition: %w", err)
}
writer, isCreated, err := inst.NewWriter(string(flowJSON)) writer, isCreated, err := inst.NewWriter(string(flowJSON))
if err != nil { if err != nil {
log.Fatalf("create video writer: %v", err) return fmt.Errorf("create video writer: %w", err)
} }
defer func() {
if err := writer.Close(); err != nil {
runErr = errors.Join(runErr, fmt.Errorf("close video writer: %w", err))
}
}()
if !isCreated { if !isCreated {
log.Printf("reusing existing flow: %s, domain: %s", videoCfg.ID(), args.domain) log.Printf("reusing existing video flow: %s", cfg.ID())
} }
defer writer.Close()
flowCfg := writer.Config() rate := writer.Config().Common.GrainRate
rate := flowCfg.Common.GrainRate
idx := mxl.CurrentIndex(rate) idx := mxl.CurrentIndex(rate)
log.Printf("writing flow grainRate=%d/%d starting at idx=%d", rate.Num, rate.Den, idx) log.Printf("writing video flow grainRate=%d/%d starting at idx=%d", rate.Num, rate.Den, idx)
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
// core loop
var grainsWritten int64 var grainsWritten int64
// animation clock: small counter, not the huge grain index. var tick uint32 // Small animation counter; shaders cannot use the full grain index yet.
// Reason: current wgpu shaders limitations
var tick uint32
for { for {
select { select {
case <-stop: case <-ctx.Done():
log.Printf("stopping after %d grains", grainsWritten) log.Printf("stopping video after %d grains", grainsWritten)
return return nil
default: default:
} }
gwa, err := writer.OpenGrain(idx) grain, err := writer.OpenGrain(idx)
if err != nil { if err != nil {
log.Fatalf("open video grain %d: %v", idx, err) return fmt.Errorf("open video grain %d: %w", idx, err)
} }
if err := gen.GenerateFrame(gwa.Payload, int(tick)); err != nil { if err := gen.GenerateFrame(grain.Payload, int(tick)); err != nil {
log.Fatalf("generate frame for grain %d: %v", idx, err) return cancelVideoGrain(grain, fmt.Errorf("generate frame for grain %d: %w", idx, err))
} }
if overlay != nil { if overlay != nil {
if err := overlay.ApplyV210(gwa.Payload); err != nil { if err := overlay.ApplyV210(grain.Payload); err != nil {
log.Fatalf("apply text overlay to grain %d: %v", idx, err) return cancelVideoGrain(grain, fmt.Errorf("apply text overlay to grain %d: %w", idx, err))
} }
} }
if err := gwa.Commit(gwa.TotalSlices, 0); err != nil { if err := grain.Commit(grain.TotalSlices, 0); err != nil {
log.Fatalf("commit video grain %d: %v", idx, err) return fmt.Errorf("commit video grain %d: %w", idx, err)
} }
grainsWritten++ grainsWritten++
idx++ idx++
tick++ tick++
if grainsWritten%100 == 0 { if grainsWritten%100 == 0 {
log.Printf("grains written=%d, index=%d", grainsWritten, idx) log.Printf("video grains written=%d, next index=%d", grainsWritten, idx)
} }
// Pace ourselves to roughly the grain rate
mxl.SleepNs(mxl.NsUntilIndex(idx, rate)) mxl.SleepNs(mxl.NsUntilIndex(idx, rate))
} }
} }
func buildTextOverlay(cfg video.Config) (overlay *generator.TextOverlay, resultErr error) {
if cfg.Overlay.Text == "" {
return nil, nil
}
if cfg.Overlay.X < 0 || cfg.Overlay.X > int(cfg.Width()) ||
cfg.Overlay.Y < 0 || cfg.Overlay.Y > int(cfg.Height()) {
return nil, fmt.Errorf("text overlay position (%d, %d) is outside the %dx%d video frame",
cfg.Overlay.X, cfg.Overlay.Y, cfg.Width(), cfg.Height())
}
face, err := generator.LoadFace("assets/fonts/JetBrainsMonoNLNerdFontMono-Regular.ttf", 48)
if err != nil {
return nil, fmt.Errorf("load text overlay font: %w", err)
}
defer func() {
if err := face.Close(); err != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("close text overlay font: %w", err))
}
}()
overlay, err = generator.NewTextOverlay(
cfg.Overlay.Text,
int(cfg.Width()),
int(cfg.Height()),
cfg.Overlay.X,
cfg.Overlay.Y,
cfg.Overlay.Position,
face,
)
if err != nil {
return nil, fmt.Errorf("create text overlay: %w", err)
}
return overlay, nil
}
func cancelVideoGrain(grain *mxl.GrainWriteAccess, cause error) error {
if err := grain.Cancel(); err != nil {
return errors.Join(cause, fmt.Errorf("cancel video grain: %w", err))
}
return cause
}
+8
View File
@@ -9,6 +9,14 @@ import (
type Config struct { type Config struct {
Definition flowdef.Video Definition flowdef.Video
Pattern string Pattern string
Overlay OverlayConfig
}
type OverlayConfig struct {
Text string
X int
Y int
Position string
} }
func (c Config) ID() string { func (c Config) ID() string {