Dmitry Sergeev
464 lines
11 KiB
Go
464 lines
11 KiB
Go
package source
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
mxl "github.com/qvest-digital/go-mxl/mxl"
|
|
)
|
|
|
|
type flowDef struct {
|
|
FrameWidth int `json:"frame_width"`
|
|
FrameHeight int `json:"frame_height"`
|
|
MediaType string `json:"media_type"`
|
|
Colorspace string `json:"colorspace"`
|
|
GrainRate struct {
|
|
Numerator int64 `json:"numerator"`
|
|
Denominator int64 `json:"denominator"`
|
|
} `json:"grain_rate"`
|
|
}
|
|
|
|
type Frame struct {
|
|
Index uint64
|
|
Width uint32
|
|
Height uint32
|
|
Stride uint32
|
|
Size uint32
|
|
Invalid bool
|
|
Payload []byte
|
|
}
|
|
|
|
type Source struct {
|
|
inst *mxl.Instance
|
|
reader *mxl.Reader
|
|
info mxl.FlowInfo
|
|
def string
|
|
rate mxl.Rational
|
|
stride uint32
|
|
width uint32
|
|
height uint32
|
|
idx uint64
|
|
}
|
|
|
|
func Open(domain, flowID string) (*Source, error) {
|
|
inst, err := mxl.NewInstance(domain, "")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("NewInstance: %w", err)
|
|
}
|
|
r, err := inst.NewReader(flowID)
|
|
if err != nil {
|
|
inst.Close()
|
|
return nil, fmt.Errorf("NewReader: %w", err)
|
|
}
|
|
info, err := r.Info()
|
|
if err != nil {
|
|
r.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("Info: %w", err)
|
|
}
|
|
def, err := inst.FlowDef(flowID)
|
|
if err != nil {
|
|
r.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("FlowDef: %w", err)
|
|
}
|
|
var fd flowDef
|
|
if err := json.Unmarshal([]byte(def), &fd); err != nil {
|
|
r.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("parse flow def: %w", err)
|
|
}
|
|
if fd.FrameWidth == 0 || fd.FrameHeight == 0 {
|
|
r.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("flow has no video dimensions (not a video flow?)")
|
|
}
|
|
rate := info.Config.Common.GrainRate
|
|
idx := mxl.CurrentIndex(rate)
|
|
if idx == mxl.UndefinedIndex {
|
|
r.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("invalid grain rate: %d/%d", rate.Num, rate.Den)
|
|
}
|
|
return &Source{
|
|
inst: inst,
|
|
reader: r,
|
|
info: info,
|
|
def: def,
|
|
rate: info.Config.Common.GrainRate,
|
|
stride: info.Config.Discrete.SliceSizes[0],
|
|
width: uint32(fd.FrameWidth),
|
|
height: uint32(fd.FrameHeight),
|
|
idx: idx,
|
|
}, nil
|
|
}
|
|
|
|
func (s *Source) Close() error {
|
|
_ = s.reader.Close()
|
|
return s.inst.Close()
|
|
}
|
|
|
|
func (s *Source) NextCtx(ctx context.Context, timeout time.Duration) (Frame, error) {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return Frame{}, ctx.Err()
|
|
default:
|
|
}
|
|
g, err := s.reader.GetGrain(s.idx, timeout)
|
|
switch {
|
|
case err == nil:
|
|
f := Frame{
|
|
Index: g.Index,
|
|
Width: s.width,
|
|
Height: s.height,
|
|
Stride: s.stride,
|
|
Size: g.GrainSize,
|
|
Invalid: g.Invalid(),
|
|
Payload: g.Payload,
|
|
}
|
|
s.idx++
|
|
return f, nil
|
|
case errors.Is(err, mxl.ErrTimeout):
|
|
s.idx = mxl.CurrentIndex(s.rate)
|
|
case errors.Is(err, mxl.ErrOutOfRangeEarly):
|
|
time.Sleep(10 * time.Millisecond)
|
|
case errors.Is(err, mxl.ErrOutOfRangeLate):
|
|
s.idx = mxl.CurrentIndex(s.rate)
|
|
default:
|
|
return Frame{}, fmt.Errorf("GetGrain: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Source) FlowDef() string { return s.def }
|
|
func (s *Source) Rate() mxl.Rational { return s.rate }
|
|
func (s *Source) Stride() uint32 { return s.stride }
|
|
func (s *Source) Width() uint32 { return s.width }
|
|
func (s *Source) Height() uint32 { return s.height }
|
|
func (s *Source) GrainCount() uint32 { return s.info.Config.Discrete.GrainCount }
|
|
func (s *Source) Format() mxl.DataFormat { return s.info.Config.Common.Format }
|
|
|
|
type AudioSource struct {
|
|
inst *mxl.Instance
|
|
r *mxl.Reader
|
|
info mxl.FlowInfo
|
|
rate mxl.Rational
|
|
chans uint64
|
|
idx uint64
|
|
}
|
|
|
|
type AudioFrame struct {
|
|
Index uint64
|
|
SampleCount uint64
|
|
Channels uint64
|
|
Samples [][]byte // per-channel byte slices (F32, deinterleaved)
|
|
}
|
|
|
|
func OpenAudio(domain, flowID string) (*AudioSource, error) {
|
|
inst, err := mxl.NewInstance(domain, "")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("NewInstance: %w", err)
|
|
}
|
|
r, err := inst.NewReader(flowID)
|
|
if err != nil {
|
|
inst.Close()
|
|
return nil, fmt.Errorf("NewReader: %w", err)
|
|
}
|
|
info, err := r.Info()
|
|
if err != nil {
|
|
r.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("Info: %w", err)
|
|
}
|
|
if info.Config.Common.Format.IsDiscrete() {
|
|
r.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("flow is discrete (not audio)")
|
|
}
|
|
idx := info.Runtime.HeadIndex
|
|
if idx == 0 {
|
|
r.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("flow has no head yet (no producer?)")
|
|
}
|
|
return &AudioSource{
|
|
inst: inst,
|
|
r: r,
|
|
info: info,
|
|
rate: info.Config.Common.GrainRate,
|
|
chans: uint64(info.Config.Continuous.ChannelCount),
|
|
idx: idx,
|
|
}, nil
|
|
}
|
|
|
|
func (s *AudioSource) NextAudio(ctx context.Context, batch uint64, timeout time.Duration) (AudioFrame, error) {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return AudioFrame{}, ctx.Err()
|
|
default:
|
|
}
|
|
v, err := s.r.GetSamples(s.idx, int(batch), timeout)
|
|
switch {
|
|
case err == nil:
|
|
samples := make([][]byte, s.chans)
|
|
for ch := uint64(0); ch < s.chans; ch++ {
|
|
f1, f2, _ := v.ChannelFragments(ch)
|
|
if len(f2) > 0 {
|
|
samples[ch] = append(f1, f2...)
|
|
} else {
|
|
samples[ch] = f1
|
|
}
|
|
}
|
|
f := AudioFrame{
|
|
Index: s.idx,
|
|
SampleCount: batch,
|
|
Channels: s.chans,
|
|
Samples: samples,
|
|
}
|
|
s.idx += batch
|
|
return f, nil
|
|
case errors.Is(err, mxl.ErrOutOfRangeEarly):
|
|
select {
|
|
case <-time.After(10 * time.Millisecond):
|
|
case <-ctx.Done():
|
|
return AudioFrame{}, ctx.Err()
|
|
}
|
|
case errors.Is(err, mxl.ErrOutOfRangeLate):
|
|
rt, rerr := s.r.Runtime()
|
|
if rerr != nil {
|
|
return AudioFrame{}, fmt.Errorf("Runtime: %w", rerr)
|
|
}
|
|
s.idx = rt.HeadIndex
|
|
default:
|
|
return AudioFrame{}, fmt.Errorf("GetSamples: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *AudioSource) Close() error {
|
|
_ = s.r.Close()
|
|
return s.inst.Close()
|
|
}
|
|
|
|
func (s *AudioSource) Rate() mxl.Rational { return s.rate }
|
|
func (s *AudioSource) Channels() uint64 { return s.chans }
|
|
|
|
type SyncSource struct {
|
|
inst *mxl.Instance
|
|
vr *mxl.Reader
|
|
ar *mxl.Reader
|
|
group *mxl.SyncGroup
|
|
rate mxl.Rational // video rate
|
|
aRate mxl.Rational
|
|
chans uint64
|
|
idx uint64
|
|
width, height, stride uint32
|
|
}
|
|
|
|
func OpenSync(domain, videoFlow, audioFlow string) (*SyncSource, error) {
|
|
inst, err := mxl.NewInstance(domain, "")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("NewInstance: %w", err)
|
|
}
|
|
|
|
vr, err := inst.NewReader(videoFlow)
|
|
if err != nil {
|
|
inst.Close()
|
|
return nil, fmt.Errorf("NewReader(video): %w", err)
|
|
}
|
|
ar, err := inst.NewReader(audioFlow)
|
|
if err != nil {
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("NewReader(audio): %w", err)
|
|
}
|
|
|
|
vInfo, err := vr.Info()
|
|
if err != nil {
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("Info(video): %w", err)
|
|
}
|
|
if !vInfo.Config.Common.Format.IsDiscrete() {
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("video flow is not discrete")
|
|
}
|
|
aInfo, err := ar.Info()
|
|
if err != nil {
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("Info(audio): %w", err)
|
|
}
|
|
if aInfo.Config.Common.Format.IsDiscrete() {
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("audio flow is not continuous")
|
|
}
|
|
|
|
def, err := inst.FlowDef(videoFlow)
|
|
if err != nil {
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("FlowDef: %w", err)
|
|
}
|
|
var fd flowDef
|
|
if err := json.Unmarshal([]byte(def), &fd); err != nil {
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("parse flow def: %w", err)
|
|
}
|
|
|
|
vRate := vInfo.Config.Common.GrainRate
|
|
idx := mxl.CurrentIndex(vRate)
|
|
if idx == mxl.UndefinedIndex {
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("invalid video rate %d/%d", vRate.Num, vRate.Den)
|
|
}
|
|
|
|
group, err := inst.NewSyncGroup()
|
|
if err != nil {
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("NewSyncGroup: %w", err)
|
|
}
|
|
if err := group.AddReader(vr); err != nil {
|
|
group.Close()
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("AddReader(video): %w", err)
|
|
}
|
|
if err := group.AddReader(ar); err != nil {
|
|
group.Close()
|
|
ar.Close()
|
|
vr.Close()
|
|
inst.Close()
|
|
return nil, fmt.Errorf("AddReader(audio): %w", err)
|
|
}
|
|
|
|
return &SyncSource{
|
|
inst: inst,
|
|
vr: vr,
|
|
ar: ar,
|
|
group: group,
|
|
rate: vRate,
|
|
aRate: aInfo.Config.Common.GrainRate,
|
|
chans: uint64(aInfo.Config.Continuous.ChannelCount),
|
|
idx: idx,
|
|
width: uint32(fd.FrameWidth),
|
|
height: uint32(fd.FrameHeight),
|
|
stride: vInfo.Config.Discrete.SliceSizes[0],
|
|
}, nil
|
|
}
|
|
|
|
func (s *SyncSource) Close() error {
|
|
_ = s.group.Close()
|
|
_ = s.ar.Close()
|
|
_ = s.vr.Close()
|
|
return s.inst.Close()
|
|
}
|
|
|
|
// NextSync reads both at a synced timestamp. Returns video Frame + audio AudioFrame
|
|
func (s *SyncSource) NextSync(ctx context.Context, audioBatch uint64, timeout time.Duration) (Frame, AudioFrame, error) {
|
|
var timeouts int
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return Frame{}, AudioFrame{}, ctx.Err()
|
|
default:
|
|
}
|
|
|
|
ts := mxl.IndexToTimestamp(s.rate, s.idx)
|
|
err := s.group.WaitForDataAt(ts, timeout)
|
|
switch {
|
|
case err == nil:
|
|
// read video
|
|
g, gerr := s.vr.GetGrain(s.idx, 50*time.Millisecond)
|
|
if gerr != nil {
|
|
if errors.Is(gerr, mxl.ErrFlowInvalid) {
|
|
s.idx = mxl.CurrentIndex(s.rate)
|
|
continue
|
|
}
|
|
if errors.Is(gerr, mxl.ErrOutOfRangeLate) {
|
|
s.idx = mxl.CurrentIndex(s.rate)
|
|
continue
|
|
}
|
|
if errors.Is(gerr, mxl.ErrOutOfRangeEarly) {
|
|
select {
|
|
case <-time.After(5 * time.Millisecond):
|
|
case <-ctx.Done():
|
|
return Frame{}, AudioFrame{}, ctx.Err()
|
|
}
|
|
continue
|
|
}
|
|
return Frame{}, AudioFrame{}, fmt.Errorf("GetGrain: %w", gerr)
|
|
}
|
|
// read audio at the same timestamp
|
|
aIdx := mxl.TimestampToIndex(s.aRate, ts)
|
|
av, aerr := s.ar.GetSamples(aIdx, int(audioBatch), 50*time.Millisecond)
|
|
vFrame := Frame{
|
|
Index: g.Index, Width: s.width, Height: s.height,
|
|
Stride: s.stride, Size: g.GrainSize,
|
|
Invalid: g.Invalid(), Payload: g.Payload,
|
|
}
|
|
s.idx++
|
|
|
|
var aFrame AudioFrame
|
|
if aerr == nil {
|
|
samples := make([][]byte, s.chans)
|
|
for ch := uint64(0); ch < s.chans; ch++ {
|
|
f1, f2, _ := av.ChannelFragments(ch)
|
|
if len(f2) > 0 {
|
|
samples[ch] = append(f1, f2...)
|
|
} else {
|
|
samples[ch] = f1
|
|
}
|
|
}
|
|
aFrame = AudioFrame{
|
|
Index: aIdx, SampleCount: audioBatch,
|
|
Channels: s.chans, Samples: samples,
|
|
}
|
|
}
|
|
// even if audio failed, video returns
|
|
return vFrame, aFrame, nil
|
|
case errors.Is(err, mxl.ErrTimeout), errors.Is(err, mxl.ErrOutOfRangeEarly), errors.Is(err, mxl.ErrOutOfRangeLate):
|
|
timeouts++
|
|
if timeouts > 10 {
|
|
timeouts = 0
|
|
s.idx = mxl.CurrentIndex(s.rate)
|
|
return Frame{}, AudioFrame{}, fmt.Errorf("sync: feeds not responding")
|
|
}
|
|
s.idx = mxl.CurrentIndex(s.rate)
|
|
select {
|
|
case <-time.After(5 * time.Millisecond):
|
|
case <-ctx.Done():
|
|
return Frame{}, AudioFrame{}, ctx.Err()
|
|
}
|
|
default:
|
|
return Frame{}, AudioFrame{}, fmt.Errorf("WaitForDataAt: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *SyncSource) Width() uint32 { return s.width }
|
|
func (s *SyncSource) Height() uint32 { return s.height }
|
|
func (s *SyncSource) Stride() uint32 { return s.stride }
|
|
func (s *SyncSource) Rate() mxl.Rational { return s.rate }
|
|
func (s *SyncSource) AudioRate() mxl.Rational { return s.aRate }
|
|
func (s *SyncSource) Channels() uint64 { return s.chans }
|