From 5926e606e7987bf95a17387e3f9b9e1412454023 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Wed, 26 Jun 2024 11:44:22 +0200 Subject: [PATCH 01/12] Update writer.go --- writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writer.go b/writer.go index ed76dc9..88973c0 100644 --- a/writer.go +++ b/writer.go @@ -310,7 +310,7 @@ func compareNextPts(a, b *encoder) bool { // Return OUTPUT and combination of DEVICE and STREAM func (w *writer) Type() MediaType { - return OUTPUT + return w.t } // Return the metadata for the media. From a031755d2a71613c84c38f61f84913bc70676164 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sat, 29 Jun 2024 10:33:37 +0200 Subject: [PATCH 02/12] Added generators --- frame.go | 2 +- pkg/ffmpeg/frame.go | 245 ++++++++++++++++++++++++++++++++ pkg/generator/generator.go | 19 +++ pkg/generator/sine.go | 121 ++++++++++++++++ pkg/generator/sine_test.go | 75 ++++++++++ pkg/generator/yuv420p.go | 111 +++++++++++++++ pkg/generator/yuv420p_test.go | 69 +++++++++ sys/ffmpeg61/avutil_frame.go | 30 ++-- sys/ffmpeg61/avutil_rational.go | 6 +- 9 files changed, 664 insertions(+), 14 deletions(-) create mode 100644 pkg/ffmpeg/frame.go create mode 100644 pkg/generator/generator.go create mode 100644 pkg/generator/sine.go create mode 100644 pkg/generator/sine_test.go create mode 100644 pkg/generator/yuv420p.go create mode 100644 pkg/generator/yuv420p_test.go diff --git a/frame.go b/frame.go index 512be63..20db849 100644 --- a/frame.go +++ b/frame.go @@ -82,7 +82,7 @@ func (frame *frame) Time() time.Duration { if pts == ff.AV_NOPTS_VALUE { return -1 } - return secondsToDuration(float64(pts) * ff.AVUtil_q2d(frame.ctx.TimeBase())) + return secondsToDuration(float64(pts) * ff.AVUtil_rational_q2d(frame.ctx.TimeBase())) } // Return the number of planes for a specific PixelFormat diff --git a/pkg/ffmpeg/frame.go b/pkg/ffmpeg/frame.go new file mode 100644 index 0000000..9de504e --- /dev/null +++ b/pkg/ffmpeg/frame.go @@ -0,0 +1,245 @@ +package media + +import ( + "encoding/json" + "image" + "time" + + // Packages + media "github.com/mutablelogic/go-media" + imagex "github.com/mutablelogic/go-media/pkg/image" + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + + // Namespace imports + . "github.com/djthorpe/go-errors" +) + +//////////////////////////////////////////////////////////////////////////////// +// TYPES + +type frame struct { + ctx *ff.AVFrame +} + +var ( + yuvSubsampleRatio = map[ff.AVPixelFormat]image.YCbCrSubsampleRatio{ + ff.AV_PIX_FMT_YUV410P: image.YCbCrSubsampleRatio410, + ff.AV_PIX_FMT_YUV411P: image.YCbCrSubsampleRatio410, + ff.AV_PIX_FMT_YUV420P: image.YCbCrSubsampleRatio420, + ff.AV_PIX_FMT_YUV422P: image.YCbCrSubsampleRatio422, + ff.AV_PIX_FMT_YUV440P: image.YCbCrSubsampleRatio420, + ff.AV_PIX_FMT_YUV444P: image.YCbCrSubsampleRatio444, + } + yuvaSubsampleRatio = map[ff.AVPixelFormat]image.YCbCrSubsampleRatio{ + ff.AV_PIX_FMT_YUVA420P: image.YCbCrSubsampleRatio420, + ff.AV_PIX_FMT_YUVA422P: image.YCbCrSubsampleRatio422, + ff.AV_PIX_FMT_YUVA444P: image.YCbCrSubsampleRatio444, + } +) + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func NewFrame(ctx *ff.AVFrame) *frame { + return &frame{ctx} +} + +//////////////////////////////////////////////////////////////////////////////// +// STRINGIFY + +func (frame *frame) MarshalJSON() ([]byte, error) { + return json.Marshal(frame.ctx) +} + +func (frame *frame) String() string { + data, _ := json.MarshalIndent(frame, "", " ") + return string(data) +} + +//////////////////////////////////////////////////////////////////////////////// +// PARAMETERS + +// Return the media type (AUDIO, VIDEO) +func (frame *frame) Type() media.MediaType { + if frame.ctx.NumSamples() > 0 { + return media.AUDIO + } + if frame.ctx.Width() != 0 && frame.ctx.Height() != 0 { + return media.VIDEO + } + return media.NONE +} + +// Id is unused +func (frame *frame) Id() int { + return 0 +} + +// Return the timestamp as a duration, or minus one if not set +func (frame *frame) Time() time.Duration { + pts := frame.ctx.Pts() + if pts == ff.AV_NOPTS_VALUE { + return -1 + } + if frame.ctx.TimeBase().Den() == 0 { + return -1 + } + return secondsToDuration(float64(pts) * ff.AVUtil_rational_q2d(frame.ctx.TimeBase())) +} + +// Return the number of planes for a specific PixelFormat +// or SampleFormat and ChannelLayout combination +func (frame *frame) NumPlanes() int { + return ff.AVUtil_frame_get_num_planes(frame.ctx) +} + +// Return the byte data for a plane +func (frame *frame) Bytes(plane int) []byte { + return frame.ctx.Bytes(plane)[:frame.ctx.Planesize(plane)] +} + +// Return the int16 data for a plane +func (frame *frame) Int16(plane int) []int16 { + sz := frame.ctx.Planesize(plane) >> 1 + return frame.ctx.Int16(plane)[:sz] +} + +//////////////////////////////////////////////////////////////////////////////// +// AUDIO PARAMETERS + +// Return number of samples +func (frame *frame) NumSamples() int { + if frame.Type() != media.AUDIO { + return 0 + } + return frame.ctx.NumSamples() +} + +// Return channel layout +func (frame *frame) ChannelLayout() string { + if frame.Type() != media.AUDIO { + return "" + } + ch := frame.ctx.ChannelLayout() + if name, err := ff.AVUtil_channel_layout_describe(&ch); err != nil { + return "" + } else { + return name + } +} + +// Return the sample format +func (frame *frame) SampleFormat() string { + if frame.Type() != media.AUDIO { + return "" + } + return ff.AVUtil_get_sample_fmt_name(frame.ctx.SampleFormat()) +} + +// Return the sample rate (Hz) +func (frame *frame) Samplerate() int { + if frame.Type() != media.AUDIO { + return 0 + } + return frame.ctx.SampleRate() + +} + +//////////////////////////////////////////////////////////////////////////////// +// VIDEO PARAMETERS + +// Convert a frame into an image +func (frame *frame) Image() (image.Image, error) { + if t := frame.Type(); t != media.VIDEO { + return nil, ErrBadParameter.With("unsupported frame type", t) + } + pixel_format := frame.ctx.PixFmt() + switch pixel_format { + case ff.AV_PIX_FMT_GRAY8: + return &image.Gray{ + Pix: frame.Bytes(0), + Stride: frame.Stride(0), + Rect: image.Rect(0, 0, frame.Width(), frame.Height()), + }, nil + case ff.AV_PIX_FMT_RGBA: + return &image.RGBA{ + Pix: frame.Bytes(0), + Stride: frame.Stride(0), + Rect: image.Rect(0, 0, frame.Width(), frame.Height()), + }, nil + case ff.AV_PIX_FMT_RGB24: + return &imagex.RGB24{ + Pix: frame.Bytes(0), + Stride: frame.Stride(0), + Rect: image.Rect(0, 0, frame.Width(), frame.Height()), + }, nil + default: + if ratio, exists := yuvSubsampleRatio[pixel_format]; exists { + return &image.YCbCr{ + Y: frame.Bytes(0), + Cb: frame.Bytes(1), + Cr: frame.Bytes(2), + YStride: frame.Stride(0), + CStride: frame.Stride(1), + SubsampleRatio: ratio, + Rect: image.Rect(0, 0, frame.Width(), frame.Height()), + }, nil + } + if ratio, exists := yuvaSubsampleRatio[pixel_format]; exists { + return &image.NYCbCrA{ + YCbCr: image.YCbCr{ + Y: frame.Bytes(0), + Cb: frame.Bytes(1), + Cr: frame.Bytes(2), + YStride: frame.Stride(0), + CStride: frame.Stride(1), + SubsampleRatio: ratio, + Rect: image.Rect(0, 0, frame.Width(), frame.Height()), + }, + A: frame.Bytes(3), + AStride: frame.Stride(3), + }, nil + } + } + return nil, ErrNotImplemented.With("unsupported pixel format", frame.ctx.PixFmt()) +} + +// Return the number of bytes in a single row of the video frame +func (frame *frame) Stride(plane int) int { + if frame.Type() == media.VIDEO { + return frame.ctx.Linesize(plane) + } else { + return 0 + } +} + +// Return the width of the video frame +func (frame *frame) Width() int { + if frame.Type() != media.VIDEO { + return 0 + } + return frame.ctx.Width() +} + +// Return the height of the video frame +func (frame *frame) Height() int { + if frame.Type() != media.VIDEO { + return 0 + } + return frame.ctx.Height() +} + +// Return the pixel format +func (frame *frame) PixelFormat() string { + if frame.Type() != media.VIDEO { + return "" + } + return ff.AVUtil_get_pix_fmt_name(frame.ctx.PixFmt()) +} + +//////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +func secondsToDuration(seconds float64) time.Duration { + return time.Duration(seconds * float64(time.Second)) +} diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go new file mode 100644 index 0000000..80831b7 --- /dev/null +++ b/pkg/generator/generator.go @@ -0,0 +1,19 @@ +package generator + +import ( + "io" + + // Packages + media "github.com/mutablelogic/go-media" +) + +//////////////////////////////////////////////////////////////////////////// +// INTERFACE + +// Generator is an interface for generating frames of audio or video +type Generator interface { + io.Closer + + // Return a generated frame + Frame() media.Frame +} diff --git a/pkg/generator/sine.go b/pkg/generator/sine.go new file mode 100644 index 0000000..8936f4f --- /dev/null +++ b/pkg/generator/sine.go @@ -0,0 +1,121 @@ +package generator + +import ( + "encoding/json" + "errors" + "math" + "time" + + // Packages + media "github.com/mutablelogic/go-media" + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" +) + +//////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +type sine struct { + frame *ff.AVFrame + frequency float64 // in Hz + volume float64 // in decibels +} + +var _ Generator = (*sine)(nil) + +//////////////////////////////////////////////////////////////////////////// +// GLOBALS + +const ( + frameDuration = 20 * time.Millisecond // Each frame is 20ms of audio +) + +//////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +// Sine wave generator - mono float32 +func NewSine(freq float64, volume float64, samplerate int) (*sine, error) { + sine := new(sine) + + // Check parameters + if freq <= 0 { + return nil, errors.New("invalid frequency") + } + if volume <= -100 { + return nil, errors.New("invalid volume") + } + if samplerate <= 0 { + return nil, errors.New("invalid samplerate") + } + + // Create a frame + frame := ff.AVUtil_frame_alloc() + if frame == nil { + return nil, errors.New("failed to allocate frame") + } + + // Set frame parameters + numSamples := int(float64(samplerate) * frameDuration.Seconds()) + + frame.SetSampleFormat(ff.AV_SAMPLE_FMT_FLT) // float32 + if err := frame.SetChannelLayout(ff.AV_CHANNEL_LAYOUT_MONO); err != nil { + return nil, err + } + frame.SetSampleRate(samplerate) + frame.SetNumSamples(numSamples) + frame.SetTimeBase(ff.AVUtil_rational(1, samplerate)) + frame.SetPts(ff.AV_NOPTS_VALUE) + + // Allocate buffer + if err := ff.AVUtil_frame_get_buffer(frame, false); err != nil { + return nil, err + } else { + sine.frame = frame + sine.frequency = freq + sine.volume = volume + } + + // Return success + return sine, nil +} + +// Free resources +func (s *sine) Close() error { + ff.AVUtil_frame_free(s.frame) + s.frame = nil + return nil +} + +//////////////////////////////////////////////////////////////////////////// +// STRINGIFY + +func (s *sine) String() string { + data, _ := json.MarshalIndent(s.frame, "", " ") + return string(data) +} + +//////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +func (s *sine) Frame() media.Frame { + // Set the Pts + if s.frame.Pts() == ff.AV_NOPTS_VALUE { + s.frame.SetPts(0) + } else { + s.frame.SetPts(s.frame.Pts() + int64(s.frame.NumSamples())) + } + + // Calculate current phase and volume + t := ff.AVUtil_rational_q2d(s.frame.TimeBase()) * float64(s.frame.Pts()) + volume := math.Pow(10, s.volume/20.0) + data := s.frame.Float32(0) + + // Generate sine wave + for n := 0; n < s.frame.NumSamples(); n++ { + sampleTime := t + float64(n)/float64(s.frame.SampleRate()) + data[n] = float32(math.Sin(2.0*math.Pi*s.frequency*sampleTime) * volume) + } + + // Return the frame + return ffmpeg.NewFrame(s.frame) +} diff --git a/pkg/generator/sine_test.go b/pkg/generator/sine_test.go new file mode 100644 index 0000000..95d15ad --- /dev/null +++ b/pkg/generator/sine_test.go @@ -0,0 +1,75 @@ +package generator_test + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/mutablelogic/go-media/pkg/generator" + "github.com/stretchr/testify/assert" +) + +func Test_sine_001(t *testing.T) { + assert := assert.New(t) + sine, err := generator.NewSine(2000, 10, 44100) + if !assert.NoError(err) { + t.SkipNow() + } + defer sine.Close() + + t.Log(sine) +} + +func Test_sine_002(t *testing.T) { + assert := assert.New(t) + sine, err := generator.NewSine(2000, 10, 44100) + if !assert.NoError(err) { + t.SkipNow() + } + defer sine.Close() + + for i := 0; i < 10; i++ { + frame := sine.Frame() + t.Log(frame) + } +} + +func Test_sine_003(t *testing.T) { + assert := assert.New(t) + + const sampleRate = 10000 + const frequency = 440 + const volume = -10.0 + + sine, err := generator.NewSine(frequency, volume, sampleRate) + if !assert.NoError(err) { + t.SkipNow() + } + defer sine.Close() + + tmpdir, err := os.MkdirTemp("", t.Name()) + if !assert.NoError(err) { + t.SkipNow() + } + tmpfile := filepath.Join(tmpdir, "sine.f32le") + fh, err := os.Create(tmpfile) + if !assert.NoError(err) { + t.SkipNow() + } + defer fh.Close() + + var bytes_written int + for { + frame := sine.Frame() + if frame.Time() > 10*time.Second { + break + } + n, err := fh.Write(frame.Bytes(0)) + assert.NoError(err) + bytes_written += n + } + + t.Log("Wrote", bytes_written, " bytes to", tmpfile) + t.Log(" play with: ffplay -f f32le -ar", sampleRate, "-ac 1", tmpfile) +} diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go new file mode 100644 index 0000000..2401356 --- /dev/null +++ b/pkg/generator/yuv420p.go @@ -0,0 +1,111 @@ +package generator + +import ( + "encoding/json" + "errors" + + // Packages + media "github.com/mutablelogic/go-media" + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" +) + +//////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +type yuv420p struct { + frame *ff.AVFrame +} + +var _ Generator = (*yuv420p)(nil) + +//////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +// Video generator - YUV420P +func NewYUV420P(size string, framerate int) (*yuv420p, error) { + yuv420p := new(yuv420p) + + // Check parameters + if framerate <= 0 { + return nil, errors.New("invalid framerate") + } + w, h, err := ff.AVUtil_parse_video_size(size) + if err != nil { + return nil, err + } + + // Create a frame + frame := ff.AVUtil_frame_alloc() + if frame == nil { + return nil, errors.New("failed to allocate frame") + } + + frame.SetPixFmt(ff.AV_PIX_FMT_YUV420P) + frame.SetWidth(w) + frame.SetHeight(h) + frame.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + frame.SetTimeBase(ff.AVUtil_rational(1, framerate)) + frame.SetPts(ff.AV_NOPTS_VALUE) + + // Allocate buffer + if err := ff.AVUtil_frame_get_buffer(frame, false); err != nil { + return nil, err + } else { + yuv420p.frame = frame + } + + // Return success + return yuv420p, nil +} + +// Free resources +func (yuv420p *yuv420p) Close() error { + ff.AVUtil_frame_free(yuv420p.frame) + yuv420p.frame = nil + return nil +} + +//////////////////////////////////////////////////////////////////////////// +// STRINGIFY + +func (yuv420p *yuv420p) String() string { + data, _ := json.MarshalIndent(yuv420p.frame, "", " ") + return string(data) +} + +//////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +func (yuv420p *yuv420p) Frame() media.Frame { + // Set the Pts + if yuv420p.frame.Pts() == ff.AV_NOPTS_VALUE { + yuv420p.frame.SetPts(0) + } else { + yuv420p.frame.SetPts(yuv420p.frame.Pts() + 1) + } + + /* Y */ + n := int(yuv420p.frame.Pts()) + yplane := yuv420p.frame.Bytes(0) + ystride := yuv420p.frame.Linesize(0) + for y := 0; y < yuv420p.frame.Height(); y++ { + for x := 0; x < yuv420p.frame.Width(); x++ { + yplane[y*ystride+x] = byte(x + y + n*3) + } + } + + /* Cb and Cr */ + cbplane := yuv420p.frame.Bytes(1) + crplane := yuv420p.frame.Bytes(2) + cstride := yuv420p.frame.Linesize(1) + for y := 0; y < yuv420p.frame.Height()>>1; y++ { + for x := 0; x < yuv420p.frame.Width()>>1; x++ { + cbplane[y*cstride+x] = byte(128 + y + n*2) + crplane[y*cstride+x] = byte(64 + x + n*5) + } + } + + // Return the frame + return ffmpeg.NewFrame(yuv420p.frame) +} diff --git a/pkg/generator/yuv420p_test.go b/pkg/generator/yuv420p_test.go new file mode 100644 index 0000000..c76a590 --- /dev/null +++ b/pkg/generator/yuv420p_test.go @@ -0,0 +1,69 @@ +package generator_test + +import ( + "fmt" + "image/png" + "os" + "path/filepath" + "testing" + + "github.com/mutablelogic/go-media/pkg/generator" + "github.com/stretchr/testify/assert" +) + +func Test_yuv420p_001(t *testing.T) { + assert := assert.New(t) + image, err := generator.NewYUV420P("1024x768", 25) + if !assert.NoError(err) { + t.FailNow() + } + defer image.Close() + + t.Log(image) +} + +func Test_yuv420p_002(t *testing.T) { + assert := assert.New(t) + image, err := generator.NewYUV420P("vga", 25) + if !assert.NoError(err) { + t.FailNow() + } + defer image.Close() + + for i := 0; i < 10; i++ { + frame := image.Frame() + t.Log(frame) + } +} + +func Test_yuv420p_003(t *testing.T) { + assert := assert.New(t) + image, err := generator.NewYUV420P("vga", 25) + if !assert.NoError(err) { + t.FailNow() + } + defer image.Close() + + tmpdir, err := os.MkdirTemp("", t.Name()) + if !assert.NoError(err) { + t.SkipNow() + } + + for i := 0; i < 10; i++ { + img, err := image.Frame().Image() + if !assert.NoError(err) { + t.FailNow() + } + tmpfile := filepath.Join(tmpdir, fmt.Sprintf("image_%03d", i)+".png") + fh, err := os.Create(tmpfile) + if !assert.NoError(err) { + t.SkipNow() + } + defer fh.Close() + err = png.Encode(fh, img) + if !assert.NoError(err) { + t.FailNow() + } + t.Logf("Wrote %s", tmpfile) + } +} diff --git a/sys/ffmpeg61/avutil_frame.go b/sys/ffmpeg61/avutil_frame.go index f1a480c..c6b5fce 100644 --- a/sys/ffmpeg61/avutil_frame.go +++ b/sys/ffmpeg61/avutil_frame.go @@ -29,11 +29,12 @@ type jsonAVAudioFrame struct { } type jsonAVVideoFrame struct { - PixelFormat AVPixelFormat `json:"pixel_format"` - Width int `json:"width"` - Height int `json:"height"` - PictureType AVPictureType `json:"picture_type,omitempty"` - Stride []int `json:"plane_stride,omitempty"` + PixelFormat AVPixelFormat `json:"pixel_format"` + Width int `json:"width"` + Height int `json:"height"` + SampleAspect AVRational `json:"sample_aspect_ratio,omitempty"` + PictureType AVPictureType `json:"picture_type,omitempty"` + Stride []int `json:"plane_stride,omitempty"` } type jsonAVFrame struct { @@ -67,11 +68,12 @@ func (ctx *AVFrame) MarshalJSON() ([]byte, error) { // Video return json.Marshal(jsonAVFrame{ jsonAVVideoFrame: &jsonAVVideoFrame{ - PixelFormat: AVPixelFormat(ctx.format), - Width: int(ctx.width), - Height: int(ctx.height), - PictureType: AVPictureType(ctx.pict_type), - Stride: ctx.linesizes(), + PixelFormat: AVPixelFormat(ctx.format), + Width: int(ctx.width), + Height: int(ctx.height), + SampleAspect: AVRational(ctx.sample_aspect_ratio), + PictureType: AVPictureType(ctx.pict_type), + Stride: ctx.linesizes(), }, Pts: AVTimestamp(ctx.pts), TimeBase: AVRational(ctx.time_base), @@ -187,6 +189,14 @@ func (ctx *AVFrame) SetSampleRate(sample_rate int) { ctx.sample_rate = C.int(sample_rate) } +func (ctx *AVFrame) SampleAspectRatio() AVRational { + return AVRational(ctx.sample_aspect_ratio) +} + +func (ctx *AVFrame) SetSampleAspectRatio(aspect_ratio AVRational) { + ctx.sample_aspect_ratio = C.struct_AVRational(aspect_ratio) +} + func (ctx *AVFrame) ChannelLayout() AVChannelLayout { return AVChannelLayout(ctx.ch_layout) } diff --git a/sys/ffmpeg61/avutil_rational.go b/sys/ffmpeg61/avutil_rational.go index 679ac09..e5dae3d 100644 --- a/sys/ffmpeg61/avutil_rational.go +++ b/sys/ffmpeg61/avutil_rational.go @@ -60,7 +60,7 @@ func (r AVRational) Float(multiplier int64) float64 { //////////////////////////////////////////////////////////////////////////////// // BINDINGS -// Convert a double precision floating point number to a rational. +// Convert a float64 to a rational. func AVUtil_rational_d2q(d float64, max int) AVRational { if max == 0 { max = C.INT_MAX @@ -68,7 +68,7 @@ func AVUtil_rational_d2q(d float64, max int) AVRational { return AVRational(C.av_d2q(C.double(d), C.int(max))) } -// Convert an AVRational to a double. -func AVUtil_q2d(a AVRational) float64 { +// Convert an AVRational to a float64. +func AVUtil_rational_q2d(a AVRational) float64 { return float64(C.av_q2d(C.AVRational(a))) } From 2c733b20285c7e57441c43156cdad407ed1b3af5 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sat, 29 Jun 2024 10:39:53 +0200 Subject: [PATCH 03/12] Added comments --- pkg/generator/sine.go | 7 +++++-- pkg/generator/yuv420p.go | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/generator/sine.go b/pkg/generator/sine.go index 8936f4f..c5e6c82 100644 --- a/pkg/generator/sine.go +++ b/pkg/generator/sine.go @@ -33,7 +33,9 @@ const ( //////////////////////////////////////////////////////////////////////////// // LIFECYCLE -// Sine wave generator - mono float32 +// Create a new sine wave generator with one channel using float32 +// for samples. The frequency in Hz, volume in decibels and samplerate +// (ie, 44100) for the audio stream are passed as arguments. func NewSine(freq float64, volume float64, samplerate int) (*sine, error) { sine := new(sine) @@ -79,7 +81,7 @@ func NewSine(freq float64, volume float64, samplerate int) (*sine, error) { return sine, nil } -// Free resources +// Free resources for the generator func (s *sine) Close() error { ff.AVUtil_frame_free(s.frame) s.frame = nil @@ -97,6 +99,7 @@ func (s *sine) String() string { //////////////////////////////////////////////////////////////////////////// // PUBLIC METHODS +// Return the first and subsequent frames of raw audio data func (s *sine) Frame() media.Frame { // Set the Pts if s.frame.Pts() == ff.AV_NOPTS_VALUE { diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go index 2401356..2d114e6 100644 --- a/pkg/generator/yuv420p.go +++ b/pkg/generator/yuv420p.go @@ -22,7 +22,8 @@ var _ Generator = (*yuv420p)(nil) //////////////////////////////////////////////////////////////////////////// // LIFECYCLE -// Video generator - YUV420P +// Create a new video generator which generates YUV420P frames +// of the specified size and framerate (in frames per second) func NewYUV420P(size string, framerate int) (*yuv420p, error) { yuv420p := new(yuv420p) @@ -77,6 +78,7 @@ func (yuv420p *yuv420p) String() string { //////////////////////////////////////////////////////////////////////////// // PUBLIC METHODS +// Return the first and subsequent frames of raw video data func (yuv420p *yuv420p) Frame() media.Frame { // Set the Pts if yuv420p.frame.Pts() == ff.AV_NOPTS_VALUE { From fe0b071fb3ae90c58f3ebd5a572e063e92be37c1 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sat, 29 Jun 2024 19:46:04 +0200 Subject: [PATCH 04/12] Changes for writer --- pkg/ffmpeg/frame.go | 54 +++---- pkg/ffmpeg/opts.go | 133 +++++++++++++++++ pkg/ffmpeg/packet_decoder.go | 93 ++++++++++++ pkg/ffmpeg/packet_encoder.go | 105 ++++++++++++++ pkg/ffmpeg/resampler.go | 200 ++++++++++++++++++++++++++ pkg/ffmpeg/resampler_test.go | 52 +++++++ pkg/ffmpeg/rescaler.go | 146 +++++++++++++++++++ pkg/ffmpeg/rescaler_test.go | 126 ++++++++++++++++ pkg/ffmpeg/writer.go | 115 +++++++++++++++ pkg/ffmpeg/writer_test.go | 34 +++++ pkg/generator/sine.go | 2 +- pkg/generator/yuv420p.go | 2 +- sys/ffmpeg61/avformat.go | 4 + sys/ffmpeg61/avformat_mux.go | 25 +++- sys/ffmpeg61/avformat_output.go | 4 + sys/ffmpeg61/avutil_channel_layout.go | 9 ++ 16 files changed, 1073 insertions(+), 31 deletions(-) create mode 100644 pkg/ffmpeg/opts.go create mode 100644 pkg/ffmpeg/packet_decoder.go create mode 100644 pkg/ffmpeg/packet_encoder.go create mode 100644 pkg/ffmpeg/resampler.go create mode 100644 pkg/ffmpeg/resampler_test.go create mode 100644 pkg/ffmpeg/rescaler.go create mode 100644 pkg/ffmpeg/rescaler_test.go create mode 100644 pkg/ffmpeg/writer.go create mode 100644 pkg/ffmpeg/writer_test.go diff --git a/pkg/ffmpeg/frame.go b/pkg/ffmpeg/frame.go index 9de504e..06913d0 100644 --- a/pkg/ffmpeg/frame.go +++ b/pkg/ffmpeg/frame.go @@ -1,4 +1,4 @@ -package media +package ffmpeg import ( "encoding/json" @@ -17,8 +17,9 @@ import ( //////////////////////////////////////////////////////////////////////////////// // TYPES -type frame struct { - ctx *ff.AVFrame +type Frame struct { + ctx *ff.AVFrame + stream int } var ( @@ -40,18 +41,18 @@ var ( //////////////////////////////////////////////////////////////////////////////// // LIFECYCLE -func NewFrame(ctx *ff.AVFrame) *frame { - return &frame{ctx} +func NewFrame(ctx *ff.AVFrame, stream int) *Frame { + return &Frame{ctx, stream} } //////////////////////////////////////////////////////////////////////////////// // STRINGIFY -func (frame *frame) MarshalJSON() ([]byte, error) { +func (frame *Frame) MarshalJSON() ([]byte, error) { return json.Marshal(frame.ctx) } -func (frame *frame) String() string { +func (frame *Frame) String() string { data, _ := json.MarshalIndent(frame, "", " ") return string(data) } @@ -59,8 +60,13 @@ func (frame *frame) String() string { //////////////////////////////////////////////////////////////////////////////// // PARAMETERS +// Return the context +func (frame *Frame) AVFrame() *ff.AVFrame { + return frame.ctx +} + // Return the media type (AUDIO, VIDEO) -func (frame *frame) Type() media.MediaType { +func (frame *Frame) Type() media.MediaType { if frame.ctx.NumSamples() > 0 { return media.AUDIO } @@ -70,13 +76,13 @@ func (frame *frame) Type() media.MediaType { return media.NONE } -// Id is unused -func (frame *frame) Id() int { - return 0 +// Return the stream +func (frame *Frame) Id() int { + return frame.stream } // Return the timestamp as a duration, or minus one if not set -func (frame *frame) Time() time.Duration { +func (frame *Frame) Time() time.Duration { pts := frame.ctx.Pts() if pts == ff.AV_NOPTS_VALUE { return -1 @@ -89,17 +95,17 @@ func (frame *frame) Time() time.Duration { // Return the number of planes for a specific PixelFormat // or SampleFormat and ChannelLayout combination -func (frame *frame) NumPlanes() int { +func (frame *Frame) NumPlanes() int { return ff.AVUtil_frame_get_num_planes(frame.ctx) } // Return the byte data for a plane -func (frame *frame) Bytes(plane int) []byte { +func (frame *Frame) Bytes(plane int) []byte { return frame.ctx.Bytes(plane)[:frame.ctx.Planesize(plane)] } // Return the int16 data for a plane -func (frame *frame) Int16(plane int) []int16 { +func (frame *Frame) Int16(plane int) []int16 { sz := frame.ctx.Planesize(plane) >> 1 return frame.ctx.Int16(plane)[:sz] } @@ -108,7 +114,7 @@ func (frame *frame) Int16(plane int) []int16 { // AUDIO PARAMETERS // Return number of samples -func (frame *frame) NumSamples() int { +func (frame *Frame) NumSamples() int { if frame.Type() != media.AUDIO { return 0 } @@ -116,7 +122,7 @@ func (frame *frame) NumSamples() int { } // Return channel layout -func (frame *frame) ChannelLayout() string { +func (frame *Frame) ChannelLayout() string { if frame.Type() != media.AUDIO { return "" } @@ -129,7 +135,7 @@ func (frame *frame) ChannelLayout() string { } // Return the sample format -func (frame *frame) SampleFormat() string { +func (frame *Frame) SampleFormat() string { if frame.Type() != media.AUDIO { return "" } @@ -137,7 +143,7 @@ func (frame *frame) SampleFormat() string { } // Return the sample rate (Hz) -func (frame *frame) Samplerate() int { +func (frame *Frame) Samplerate() int { if frame.Type() != media.AUDIO { return 0 } @@ -149,7 +155,7 @@ func (frame *frame) Samplerate() int { // VIDEO PARAMETERS // Convert a frame into an image -func (frame *frame) Image() (image.Image, error) { +func (frame *Frame) Image() (image.Image, error) { if t := frame.Type(); t != media.VIDEO { return nil, ErrBadParameter.With("unsupported frame type", t) } @@ -205,7 +211,7 @@ func (frame *frame) Image() (image.Image, error) { } // Return the number of bytes in a single row of the video frame -func (frame *frame) Stride(plane int) int { +func (frame *Frame) Stride(plane int) int { if frame.Type() == media.VIDEO { return frame.ctx.Linesize(plane) } else { @@ -214,7 +220,7 @@ func (frame *frame) Stride(plane int) int { } // Return the width of the video frame -func (frame *frame) Width() int { +func (frame *Frame) Width() int { if frame.Type() != media.VIDEO { return 0 } @@ -222,7 +228,7 @@ func (frame *frame) Width() int { } // Return the height of the video frame -func (frame *frame) Height() int { +func (frame *Frame) Height() int { if frame.Type() != media.VIDEO { return 0 } @@ -230,7 +236,7 @@ func (frame *frame) Height() int { } // Return the pixel format -func (frame *frame) PixelFormat() string { +func (frame *Frame) PixelFormat() string { if frame.Type() != media.VIDEO { return "" } diff --git a/pkg/ffmpeg/opts.go b/pkg/ffmpeg/opts.go new file mode 100644 index 0000000..990e2f5 --- /dev/null +++ b/pkg/ffmpeg/opts.go @@ -0,0 +1,133 @@ +package ffmpeg + +import ( + // Namespace imports + + . "github.com/djthorpe/go-errors" + ffmpeg "github.com/mutablelogic/go-media/sys/ffmpeg61" +) + +//////////////////////////////////////////////////////////////////////////////// +// TYPES + +type Opt func(*opts) error + +type opts struct { + // Resample/resize options + force bool + + // Format options + oformat *ffmpeg.AVOutputFormat + + // Audio options + sample_fmt ffmpeg.AVSampleFormat + ch ffmpeg.AVChannelLayout + samplerate int + + // Video options + pix_fmt ffmpeg.AVPixelFormat + width, height int +} + +//////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +// Output format from name or url +func OptOutputFormat(name string) Opt { + return func(o *opts) error { + // By name + if oformat := ffmpeg.AVFormat_guess_format(name, name, name); oformat != nil { + o.oformat = oformat + } else { + return ErrBadParameter.Withf("invalid output format %q", name) + } + return nil + } +} + +// Force resampling and resizing on decode, even if the input and output +// parameters are the same +func OptForce() Opt { + return func(o *opts) error { + o.force = true + return nil + } +} + +// Pixel format of the output frame +func OptPixFormat(format string) Opt { + return func(o *opts) error { + fmt := ffmpeg.AVUtil_get_pix_fmt(format) + if fmt == ffmpeg.AV_PIX_FMT_NONE { + return ErrBadParameter.Withf("invalid pixel format %q", format) + } + o.pix_fmt = fmt + return nil + } +} + +// Width and height of the output frame +func OptWidthHeight(w, h int) Opt { + return func(o *opts) error { + if w <= 0 || h <= 0 { + return ErrBadParameter.Withf("invalid width %v or height %v", w, h) + } + o.width = w + o.height = h + return nil + } +} + +// Frame size +func OptFrameSize(size string) Opt { + return func(o *opts) error { + w, h, err := ffmpeg.AVUtil_parse_video_size(size) + if err != nil { + return ErrBadParameter.Withf("invalid frame size %q", size) + } + o.width = w + o.height = h + return nil + } +} + +// Channel layout +func OptChannelLayout(layout string) Opt { + return func(o *opts) error { + return ffmpeg.AVUtil_channel_layout_from_string(&o.ch, layout) + } +} + +// Nuumber of channels +func OptChannels(ch int) Opt { + return func(o *opts) error { + if ch <= 0 || ch > 64 { + return ErrBadParameter.Withf("invalid number of channels %v", ch) + } + ffmpeg.AVUtil_channel_layout_default(&o.ch, ch) + return nil + } +} + +// Sample Rate +func OptSampleRate(rate int) Opt { + return func(o *opts) error { + if rate <= 0 { + return ErrBadParameter.Withf("invalid sample rate %v", rate) + } + o.samplerate = rate + return nil + } +} + +// Sample format +func OptSampleFormat(format string) Opt { + return func(o *opts) error { + fmt := ffmpeg.AVUtil_get_sample_fmt(format) + if fmt == ffmpeg.AV_SAMPLE_FMT_NONE { + return ErrBadParameter.Withf("invalid sample format %q", format) + } + o.sample_fmt = fmt + return nil + } +} diff --git a/pkg/ffmpeg/packet_decoder.go b/pkg/ffmpeg/packet_decoder.go new file mode 100644 index 0000000..076462f --- /dev/null +++ b/pkg/ffmpeg/packet_decoder.go @@ -0,0 +1,93 @@ +package ffmpeg + +import ( + "errors" + "io" + "syscall" + + // Packages + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" +) + +type decoder struct { + codec *ff.AVCodecContext + frame *ff.AVFrame + stream int +} + +// DecodeFn is a function which is called for each frame decoded +// with the stream id of the packet. It should return nil to continue +// decoding or io.EOF to stop decoding. +type DecodeFn func(int, *ff.AVFrame) error + +////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func NewPacketDecoder(codec *ff.AVCodecContext, stream int) (*decoder, error) { + decoder := new(decoder) + decoder.codec = codec + + // Create a frame + frame := ff.AVUtil_frame_alloc() + if frame == nil { + return nil, errors.New("failed to allocate frame") + } else { + decoder.frame = frame + decoder.stream = stream + } + + // Return success + return decoder, nil +} + +func (d *decoder) Close() error { + if d.frame != nil { + ff.AVUtil_frame_free(d.frame) + d.frame = nil + } + return nil +} + +////////////////////////////////////////////////////////////////////////////// +// PUBLIC FUNCTIONS + +// Decode a packet using the decoder and pass frames onto the FrameFn +// function. The FrameFn function should return nil to continue decoding +// or io.EOF to stop decoding. +func (d *decoder) Decode(packet *ff.AVPacket, fn DecodeFn) error { + // Submit the packet to the decoder (nil packet will flush the decoder) + if err := ff.AVCodec_send_packet(d.codec, packet); err != nil { + return err + } + + // get all the available frames from the decoder + var result error + for { + if err := ff.AVCodec_receive_frame(d.codec, d.frame); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { + // Finished decoding packet or EOF + break + } else if err != nil { + return err + } + + // Pass back to the caller + if err := fn(d.stream, d.frame); errors.Is(err, io.EOF) { + // End early, return EOF + result = io.EOF + break + } else if err != nil { + return err + } + + // Re-allocate frames for next iteration + ff.AVUtil_frame_unref(d.frame) + } + + // Flush + if result == nil { + result = fn(d.stream, nil) + } + + // Return success or EOF + return result +} diff --git a/pkg/ffmpeg/packet_encoder.go b/pkg/ffmpeg/packet_encoder.go new file mode 100644 index 0000000..0ce5ad5 --- /dev/null +++ b/pkg/ffmpeg/packet_encoder.go @@ -0,0 +1,105 @@ +package ffmpeg + +import ( + "errors" + "io" + "syscall" + + // Packages + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" +) + +////////////////////////////////////////////////////////////////////////////// +// TYPES + +type encoder struct { + codec *ff.AVCodecContext + packet *ff.AVPacket + stream int +} + +// EncodeFn is a function which is called for each packet encoded. It should +// return nil to continue encoding or io.EOF to stop decoding. +type EncodeFn func(*ff.AVPacket) error + +////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func NewPacketEncoder(codec *ff.AVCodecContext, stream int) (*encoder, error) { + encoder := new(encoder) + encoder.codec = codec + + // Create a packet + packet := ff.AVCodec_packet_alloc() + if packet == nil { + return nil, errors.New("failed to allocate packet") + } else { + encoder.packet = packet + encoder.stream = stream + } + + // Return success + return encoder, nil +} + +func (e *encoder) Close() error { + if e.packet != nil { + ff.AVCodec_packet_free(e.packet) + e.packet = nil + } + return nil +} + +////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func (e *encoder) Encode(frame *ff.AVFrame, fn EncodeFn) error { + // Encode a frame + if err := e.encode(frame, fn); err != nil { + return err + } + // Flush + return e.encode(nil, fn) +} + +////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +func (e *encoder) encode(frame *ff.AVFrame, fn EncodeFn) error { + // Send the frame to the encoder + if err := ff.AVCodec_send_frame(e.codec, frame); err != nil { + return err + } + + // Write out the packets + var result error + for { + // Receive the packet + if err := ff.AVCodec_receive_packet(e.codec, e.packet); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { + // Finished receiving packet or EOF + break + } else if err != nil { + return err + } + + // Pass back to the caller + if err := fn(e.packet); errors.Is(err, io.EOF) { + // End early, return EOF + result = io.EOF + break + } else if err != nil { + return err + } + + // Re-allocate frames for next iteration + ff.AVCodec_packet_unref(e.packet) + } + + // Flush + if result == nil { + result = fn(nil) + } + + // Return success or EOF + return result +} diff --git a/pkg/ffmpeg/resampler.go b/pkg/ffmpeg/resampler.go new file mode 100644 index 0000000..84b0aeb --- /dev/null +++ b/pkg/ffmpeg/resampler.go @@ -0,0 +1,200 @@ +package ffmpeg + +import ( + "errors" + "fmt" + + // Packages + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" +) + +//////////////////////////////////////////////////////////////////////////////// +// TYPES + +type resampler struct { + opts + ctx *ff.SWRContext + dest *ff.AVFrame +} + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +// Create a new audio resampler which will resample the input frame to the +// specified channel layout, sample rate and sample format. +func NewResampler(format ff.AVSampleFormat, opt ...Opt) (*resampler, error) { + resampler := new(resampler) + + // Apply options + resampler.sample_fmt = format + resampler.ch = ff.AV_CHANNEL_LAYOUT_MONO + resampler.samplerate = 44100 + for _, o := range opt { + if err := o(&resampler.opts); err != nil { + return nil, err + } + } + + // Check parameters + if resampler.sample_fmt == ff.AV_SAMPLE_FMT_NONE { + return nil, errors.New("invalid sample format parameters") + } + if !ff.AVUtil_channel_layout_check(&resampler.ch) { + return nil, errors.New("invalid channel layout parameters") + } + + // Create a destimation frame + dest := ff.AVUtil_frame_alloc() + if dest == nil { + return nil, errors.New("failed to allocate frame") + } + + // Set parameters - we don't allocate the buffer here, + // we do that when we have a source frame and know how + // large the destination frame should be + dest.SetSampleFormat(resampler.sample_fmt) + dest.SetSampleRate(resampler.samplerate) + if err := dest.SetChannelLayout(resampler.ch); err != nil { + ff.AVUtil_frame_free(dest) + return nil, err + } else { + resampler.dest = dest + } + + // Return success + return resampler, nil +} + +// Release resources +func (r *resampler) Close() error { + if r.ctx != nil { + ff.SWResample_free(r.ctx) + r.ctx = nil + } + if r.dest != nil { + ff.AVUtil_frame_free(r.dest) + r.dest = nil + } + return nil +} + +//////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +// Resample the source and return the destination +func (r *resampler) Frame(src *ff.AVFrame) (*ff.AVFrame, error) { + // Simply return the frame if it matches the destination format + if src != nil { + if matchesAudioFormat(src, r.dest) && !r.force { + return src, nil + } + } + + // Create a resampling context + if r.ctx == nil { + if src == nil { + return nil, nil + } + ctx, err := newResampler(r.dest, src) + if err != nil { + return nil, err + } else { + r.ctx = ctx + } + } + + // Copy parameters from the source frame + if src != nil { + if err := ff.AVUtil_frame_copy_props(r.dest, src); err != nil { + return nil, err + } + } + + // Get remaining samples + if src == nil { + samples := ff.SWResample_get_delay(r.ctx, int64(r.dest.SampleRate())) + fmt.Println("TODO: remaining samples=", samples) + } + + // Perform resampling + if err := ff.SWResample_convert_frame(r.ctx, src, r.dest); err != nil { + return nil, fmt.Errorf("SWResample_convert_frame: %w", err) + } + + // Return the destination frame or nil + if r.dest.NumSamples() == 0 { + return nil, nil + } else { + return r.dest, nil + } +} + +//////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +// Returns true if the sample format, channels and sample rate of the source +// and destination frames match. +func matchesAudioFormat(src, dest *ff.AVFrame) bool { + if src.SampleFormat() != dest.SampleFormat() { + return false + } + if src.SampleRate() != dest.SampleRate() { + return false + } + a := src.ChannelLayout() + b := dest.ChannelLayout() + return ff.AVUtil_channel_layout_compare(&a, &b) +} + +func newResampler(dest, src *ff.AVFrame) (*ff.SWRContext, error) { + // Create a new resampler + ctx := ff.SWResample_alloc() + if ctx == nil { + return nil, errors.New("failed to allocate resampler") + } + + // Set options to covert from the codec frame to the decoder frame + if err := ff.SWResample_set_opts(ctx, + dest.ChannelLayout(), dest.SampleFormat(), dest.SampleRate(), // destination + src.ChannelLayout(), src.SampleFormat(), src.SampleRate(), // source + ); err != nil { + ff.SWResample_free(ctx) + return nil, fmt.Errorf("SWResample_set_opts: %w", err) + } + + // Initialize the resampling context + if err := ff.SWResample_init(ctx); err != nil { + ff.SWResample_free(ctx) + return nil, fmt.Errorf("SWResample_init: %w", err) + } + + sample_fmt := dest.SampleFormat() + sample_rate := dest.SampleRate() + sample_ch := dest.ChannelLayout() + + // Unreference the current frame + ff.AVUtil_frame_unref(dest) + + // Set the number of samples + if dest_samples, err := ff.SWResample_get_out_samples(ctx, src.NumSamples()); err != nil { + ff.SWResample_free(ctx) + return nil, fmt.Errorf("SWResample_get_out_samples: %w", err) + } else if dest_samples == 0 { + ff.SWResample_free(ctx) + return nil, fmt.Errorf("SWResample_get_out_samples: number of samples is zero") + } else { + dest.SetSampleFormat(sample_fmt) + dest.SetSampleRate(sample_rate) + dest.SetChannelLayout(sample_ch) + dest.SetNumSamples(dest_samples) + } + + // Create buffers + if err := ff.AVUtil_frame_get_buffer(dest, false); err != nil { + ff.SWResample_free(ctx) + return nil, fmt.Errorf("AVUtil_frame_get_buffer: %w", err) + } + + // Return success + return ctx, nil +} diff --git a/pkg/ffmpeg/resampler_test.go b/pkg/ffmpeg/resampler_test.go new file mode 100644 index 0000000..e84f54d --- /dev/null +++ b/pkg/ffmpeg/resampler_test.go @@ -0,0 +1,52 @@ +package ffmpeg_test + +import ( + "testing" + + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + generator "github.com/mutablelogic/go-media/pkg/generator" + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + assert "github.com/stretchr/testify/assert" +) + +func Test_resampler_001(t *testing.T) { + assert := assert.New(t) + + // Sine wave generator + audio, err := generator.NewSine(2000, 10, 44100) + if !assert.NoError(err) { + t.FailNow() + } + defer audio.Close() + + // Create a rescaler + resampler, err := ffmpeg.NewResampler(ff.AV_SAMPLE_FMT_S16P, ffmpeg.OptChannels(2)) + if !assert.NoError(err) { + t.FailNow() + } + defer resampler.Close() + + // Rescale ten frames + for i := 0; i < 10; i++ { + src := audio.Frame().(*ffmpeg.Frame) + if !assert.NotNil(src) { + t.FailNow() + } + + // Rescale the frame + dest, err := resampler.Frame(src.AVFrame()) + if !assert.NoError(err) { + t.FailNow() + } + + // Display information + t.Log(src, "=>", dest) + } + + // Flush + dest, err := resampler.Frame(nil) + if !assert.NoError(err) { + t.FailNow() + } + t.Log("FLUSH =>", dest) +} diff --git a/pkg/ffmpeg/rescaler.go b/pkg/ffmpeg/rescaler.go new file mode 100644 index 0000000..87deafa --- /dev/null +++ b/pkg/ffmpeg/rescaler.go @@ -0,0 +1,146 @@ +package ffmpeg + +import ( + "errors" + + // Packages + + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" +) + +//////////////////////////////////////////////////////////////////////////////// +// TYPES + +type rescaler struct { + opts + + src_pix_fmt ff.AVPixelFormat + src_width int + src_height int + ctx *ff.SWSContext + flags ff.SWSFlag + dest *ff.AVFrame +} + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +// Create a new rescaler which will rescale the input frame to the +// specified format, width and height. +func NewRescaler(format ff.AVPixelFormat, opt ...Opt) (*rescaler, error) { + rescaler := new(rescaler) + + // Apply options + rescaler.pix_fmt = format + rescaler.width = 640 + rescaler.height = 480 + for _, o := range opt { + if err := o(&rescaler.opts); err != nil { + return nil, err + } + } + + // Check parameters + if rescaler.pix_fmt == ff.AV_PIX_FMT_NONE { + return nil, errors.New("invalid parameters") + } + + // Create a destimation frame + dest := ff.AVUtil_frame_alloc() + if dest == nil { + return nil, errors.New("failed to allocate frame") + } + + // Set parameters + dest.SetPixFmt(rescaler.pix_fmt) + dest.SetWidth(rescaler.width) + dest.SetHeight(rescaler.height) + + // Allocate buffer + if err := ff.AVUtil_frame_get_buffer(dest, false); err != nil { + ff.AVUtil_frame_free(dest) + return nil, err + } else { + rescaler.dest = dest + rescaler.flags = ff.SWS_POINT + } + + // Return success + return rescaler, nil +} + +// Release resources +func (r *rescaler) Close() error { + if r.ctx != nil { + ff.SWScale_free_context(r.ctx) + r.ctx = nil + } + if r.dest != nil { + ff.AVUtil_frame_free(r.dest) + r.dest = nil + } + return nil +} + +//////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +// Scale the source image and return the destination image +func (r *rescaler) Frame(src *ff.AVFrame) (*ff.AVFrame, error) { + // If source is null then return null (no flushing) + if src == nil { + return nil, nil + } + + // Simply return the frame if it matches the destination format + if matchesVideoFormat(src, r.dest) && !r.force { + return src, nil + } + + // Allocate a context + if r.ctx == nil || r.src_pix_fmt != src.PixFmt() || r.src_width != src.Width() || r.src_height != src.Height() { + // Release existing scaling context, if any + if r.ctx != nil { + ff.SWScale_free_context(r.ctx) + } + // Create a new scaling context + ctx := ff.SWScale_get_context( + src.Width(), src.Height(), src.PixFmt(), // source + r.dest.Width(), r.dest.Height(), r.dest.PixFmt(), // destination + r.flags, nil, nil, nil, + ) + if ctx == nil { + return nil, errors.New("failed to allocate swscale context") + } else { + r.ctx = ctx + r.src_pix_fmt = src.PixFmt() + r.src_width = src.Width() + r.src_height = src.Height() + } + } + + // Rescale the image + if err := ff.SWScale_scale_frame(r.ctx, r.dest, src, false); err != nil { + return nil, err + } + + // Copy parameters from the source frame + if err := ff.AVUtil_frame_copy_props(r.dest, src); err != nil { + return nil, err + } + + // Return the destination frame + return r.dest, nil +} + +//////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +// Returns true if the pixel format, width and height of the source +// and destination frames match +func matchesVideoFormat(src, dest *ff.AVFrame) bool { + if src.PixFmt() == dest.PixFmt() && src.Width() == dest.Width() && src.Height() == dest.Height() { + return true + } + return false +} diff --git a/pkg/ffmpeg/rescaler_test.go b/pkg/ffmpeg/rescaler_test.go new file mode 100644 index 0000000..d7e4074 --- /dev/null +++ b/pkg/ffmpeg/rescaler_test.go @@ -0,0 +1,126 @@ +package ffmpeg_test + +import ( + "fmt" + "image/png" + "os" + "path/filepath" + "testing" + + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + generator "github.com/mutablelogic/go-media/pkg/generator" + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + assert "github.com/stretchr/testify/assert" +) + +func Test_rescaler_001(t *testing.T) { + assert := assert.New(t) + + // Create an image generator + image, err := generator.NewYUV420P("vga", 25) + if !assert.NoError(err) { + t.FailNow() + } + defer image.Close() + + // Create a rescaler + rescaler, err := ffmpeg.NewRescaler(ff.AV_PIX_FMT_RGB24, ffmpeg.OptFrameSize("1024x768")) + if !assert.NoError(err) { + t.FailNow() + } + defer rescaler.Close() + + // Rescale ten frames + for i := 0; i < 10; i++ { + src := image.Frame().(*ffmpeg.Frame) + if !assert.NotNil(src) { + t.FailNow() + } + + // Rescale the frame + dest, err := rescaler.Frame(src.AVFrame()) + if !assert.NoError(err) { + t.FailNow() + } + + // Display information + t.Log(src, "=>", dest) + } + +} + +func Test_rescaler_002(t *testing.T) { + assert := assert.New(t) + + // Create an image generator + image, err := generator.NewYUV420P("vga", 25) + if !assert.NoError(err) { + t.FailNow() + } + defer image.Close() + + // Create a rescaler + rescaler, err := ffmpeg.NewRescaler(ff.AV_PIX_FMT_RGB24, ffmpeg.OptFrameSize("1024x768")) + if !assert.NoError(err) { + t.FailNow() + } + + // Temp output + tmpdir, err := os.MkdirTemp("", t.Name()) + if !assert.NoError(err) { + t.FailNow() + } + + // Rescale ten frames + for i := 0; i < 10; i++ { + f := image.Frame().(*ffmpeg.Frame) + if !assert.NotNil(f) { + t.FailNow() + } + src_image, err := f.Image() + if !assert.NoError(err) { + t.FailNow() + } + + // Output as PNG + tmpfile := filepath.Join(tmpdir, fmt.Sprintf("src_image_%03d", i)+".png") + fsrc, err := os.Create(tmpfile) + if !assert.NoError(err) { + t.SkipNow() + } + defer fsrc.Close() + err = png.Encode(fsrc, src_image) + if !assert.NoError(err) { + t.FailNow() + } + t.Logf("Wrote %s", tmpfile) + + // Rescale the frame + dest, err := rescaler.Frame(f.AVFrame()) + if !assert.NoError(err) { + t.FailNow() + } + + // Make a naive image + dest_frame := ffmpeg.NewFrame(dest, 0) + dest_image, err := dest_frame.Image() + if !assert.NoError(err) { + t.FailNow() + } + + // Output as PNG + tmpfile = filepath.Join(tmpdir, fmt.Sprintf("dest_image_%03d", i)+".png") + fh, err := os.Create(tmpfile) + if !assert.NoError(err) { + t.SkipNow() + } + defer fh.Close() + err = png.Encode(fh, dest_image) + if !assert.NoError(err) { + t.FailNow() + } + t.Logf("Wrote %s", tmpfile) + + } + +} diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go new file mode 100644 index 0000000..ef0f781 --- /dev/null +++ b/pkg/ffmpeg/writer.go @@ -0,0 +1,115 @@ +package ffmpeg + +import ( + "errors" + "fmt" + "io" + + // Packages + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + + // Namespace imports + . "github.com/djthorpe/go-errors" +) + +////////////////////////////////////////////////////////////////////////////// +// TYPES + +// Create media from io.Writer +type Writer struct { + output *ff.AVFormatContext +} + +type writer_callback struct { + w io.Writer +} + +////////////////////////////////////////////////////////////////////////////// +// GLOBALS + +const ( + bufSize = 4096 +) + +////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { + var o opts + + writer := new(Writer) + + // Apply options + for _, opt := range opt { + if err := opt(&o); err != nil { + return nil, err + } + } + + // Check output + if o.oformat == nil { + return nil, ErrBadParameter.Withf("invalid output format") + } + + // Allocate the AVIO context + avio := ff.AVFormat_avio_alloc_context(bufSize, false, &writer_callback{w}) + if avio == nil { + return nil, errors.New("failed to allocate avio context") + } else if ctx, err := ff.AVFormat_open_writer(avio, o.oformat, ""); err != nil { + return nil, err + } else { + writer.output = ctx + } + + fmt.Println("WRITER", writer.output) + + // Return success + return writer, nil +} + +func (w *Writer) Close() error { + var result error + + // Free output resources + if w.output != nil { + // This calls avio_close(w.avio) + fmt.Println("TODO: AVFormat_close_writer") + //result = errors.Join(result, ff.AVFormat_close_writer(w.output)) + w.output = nil + } + + // Return any errors + return result +} + +//////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +func (w *writer_callback) Reader(buf []byte) int { + return 0 +} + +func (w *writer_callback) Seeker(offset int64, whence int) int64 { + whence = whence & ^ff.AVSEEK_FORCE + seeker, ok := w.w.(io.ReadSeeker) + if !ok { + return -1 + } + switch whence { + case io.SeekStart, io.SeekCurrent, io.SeekEnd: + n, err := seeker.Seek(offset, whence) + if err != nil { + return -1 + } + return n + } + return -1 +} + +func (w *writer_callback) Writer(buf []byte) int { + if n, err := w.w.Write(buf); err != nil { + return -1 + } else { + return n + } +} diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go new file mode 100644 index 0000000..027c005 --- /dev/null +++ b/pkg/ffmpeg/writer_test.go @@ -0,0 +1,34 @@ +package ffmpeg_test + +import ( + "fmt" + "os" + "testing" + + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + assert "github.com/stretchr/testify/assert" +) + +func Test_writer_001(t *testing.T) { + assert := assert.New(t) + + // Write to a file + w, err := os.CreateTemp("", t.Name()+"_*.mp3") + if !assert.NoError(err) { + t.FailNow() + } + defer w.Close() + + // Create a writer with an audio stream + writer, err := ffmpeg.NewWriter(w, + ffmpeg.OptOutputFormat(w.Name()), + ffmpeg.OptAudioStream(), + ffmpeg.OptVideoStream("1280x720"), + ) + if !assert.NoError(err) { + t.FailNow() + } + defer writer.Close() + + fmt.Println("Written to", w.Name()) +} diff --git a/pkg/generator/sine.go b/pkg/generator/sine.go index c5e6c82..32b1e0e 100644 --- a/pkg/generator/sine.go +++ b/pkg/generator/sine.go @@ -120,5 +120,5 @@ func (s *sine) Frame() media.Frame { } // Return the frame - return ffmpeg.NewFrame(s.frame) + return ffmpeg.NewFrame(s.frame, 0) } diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go index 2d114e6..cb3133c 100644 --- a/pkg/generator/yuv420p.go +++ b/pkg/generator/yuv420p.go @@ -109,5 +109,5 @@ func (yuv420p *yuv420p) Frame() media.Frame { } // Return the frame - return ffmpeg.NewFrame(yuv420p.frame) + return ffmpeg.NewFrame(yuv420p.frame, 0) } diff --git a/sys/ffmpeg61/avformat.go b/sys/ffmpeg61/avformat.go index 40fb89b..a3b7237 100644 --- a/sys/ffmpeg61/avformat.go +++ b/sys/ffmpeg61/avformat.go @@ -258,6 +258,10 @@ func (ctx *AVFormatContext) Flags() AVFormat { return AVFormat(ctx.flags) } +func (ctx *AVFormatContext) SetFlags(flag AVFormat) { + ctx.flags = C.int(flag) +} + func (ctx *AVFormatContext) Duration() int64 { return int64(ctx.duration) } diff --git a/sys/ffmpeg61/avformat_mux.go b/sys/ffmpeg61/avformat_mux.go index 25bf5f9..6f0ef5e 100644 --- a/sys/ffmpeg61/avformat_mux.go +++ b/sys/ffmpeg61/avformat_mux.go @@ -17,18 +17,33 @@ import "C" //////////////////////////////////////////////////////////////////////////////// // PUBLIC METHODS -// Open an output stream. +// Open an output stream without managing a file. func AVFormat_open_writer(writer *AVIOContextEx, format *AVOutputFormat, filename string) (*AVFormatContext, error) { - // TODO - return nil, errors.New("not implemented") + var ctx *AVFormatContext + + cFilename := C.CString(filename) + defer C.free(unsafe.Pointer(cFilename)) + if err := AVError(C.avformat_alloc_output_context2((**C.struct_AVFormatContext)(unsafe.Pointer(&ctx)), (*C.struct_AVOutputFormat)(format), nil, cFilename)); err != 0 { + return nil, err + } else { + ctx.SetPb(writer) + } + + // TODO: Mark AVFMT_NOFILE + + // Return success + return ctx, nil } // Open an output file. func AVFormat_create_file(filename string, format *AVOutputFormat) (*AVFormatContext, error) { var ctx *AVFormatContext - if err := AVError(C.avformat_alloc_output_context2((**C.struct_AVFormatContext)(unsafe.Pointer(&ctx)), (*C.struct_AVOutputFormat)(format), nil, C.CString(filename))); err != 0 { + + cFilename := C.CString(filename) + defer C.free(unsafe.Pointer(cFilename)) + if err := AVError(C.avformat_alloc_output_context2((**C.struct_AVFormatContext)(unsafe.Pointer(&ctx)), (*C.struct_AVOutputFormat)(format), nil, cFilename)); err != 0 { return nil, err - } else if !ctx.Flags().Is(AVFMT_NOFILE) { + } else if !ctx.Output().Flags().Is(AVFMT_NOFILE) { if ioctx, err := AVFormat_avio_open(filename, AVIO_FLAG_WRITE); err != nil { return nil, err } else { diff --git a/sys/ffmpeg61/avformat_output.go b/sys/ffmpeg61/avformat_output.go index 1aeb213..1500fe0 100644 --- a/sys/ffmpeg61/avformat_output.go +++ b/sys/ffmpeg61/avformat_output.go @@ -64,6 +64,10 @@ func (ctx *AVOutputFormat) Flags() AVFormat { return AVFormat(ctx.flags) } +func (ctx *AVOutputFormat) SetFlags(flags AVFormat) { + ctx.flags = C.int(flags) +} + func (ctx *AVOutputFormat) MimeTypes() string { return C.GoString(ctx.mime_type) } diff --git a/sys/ffmpeg61/avutil_channel_layout.go b/sys/ffmpeg61/avutil_channel_layout.go index e97d0e6..90ff709 100644 --- a/sys/ffmpeg61/avutil_channel_layout.go +++ b/sys/ffmpeg61/avutil_channel_layout.go @@ -122,6 +122,15 @@ func AVUtil_channel_layout_check(ch_layout *AVChannelLayout) bool { return C.av_channel_layout_check((*C.struct_AVChannelLayout)(ch_layout)) != 0 } +// Check whether two channel layouts are semantically the same +func AVUtil_channel_layout_compare(a *AVChannelLayout, b *AVChannelLayout) bool { + if ret := C.av_channel_layout_compare((*C.struct_AVChannelLayout)(a), (*C.struct_AVChannelLayout)(b)); ret == 0 { + return false + } else { + return true + } +} + //////////////////////////////////////////////////////////////////////////////// // PROPERTIES From ab3159d12978d8f1dbee142857dbe4e472a91cac Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sat, 29 Jun 2024 20:13:08 +0200 Subject: [PATCH 05/12] Updated writer --- encoder.go | 2 +- pkg/ffmpeg/writer.go | 2 +- pkg/ffmpeg/writer_test.go | 2 -- sys/ffmpeg61/avformat.go | 6 +++--- sys/ffmpeg61/avformat_mux.go | 4 +++- writer.go | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/encoder.go b/encoder.go index 16d1254..9229a6e 100644 --- a/encoder.go +++ b/encoder.go @@ -135,7 +135,7 @@ func newEncoder(ctx *ff.AVFormatContext, stream_id int, param Parameters) (*enco } // Some formats want stream headers to be separate. - if ctx.Flags().Is(ff.AVFMT_GLOBALHEADER) { + if ctx.Output().Flags().Is(ff.AVFMT_GLOBALHEADER) { codecctx.SetFlags(codecctx.Flags() | ff.AV_CODEC_FLAG_GLOBAL_HEADER) } diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index ef0f781..82f7377 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -74,7 +74,7 @@ func (w *Writer) Close() error { if w.output != nil { // This calls avio_close(w.avio) fmt.Println("TODO: AVFormat_close_writer") - //result = errors.Join(result, ff.AVFormat_close_writer(w.output)) + result = errors.Join(result, ff.AVFormat_close_writer(w.output)) w.output = nil } diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index 027c005..2836309 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -22,8 +22,6 @@ func Test_writer_001(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.NewWriter(w, ffmpeg.OptOutputFormat(w.Name()), - ffmpeg.OptAudioStream(), - ffmpeg.OptVideoStream("1280x720"), ) if !assert.NoError(err) { t.FailNow() diff --git a/sys/ffmpeg61/avformat.go b/sys/ffmpeg61/avformat.go index a3b7237..ce8cada 100644 --- a/sys/ffmpeg61/avformat.go +++ b/sys/ffmpeg61/avformat.go @@ -254,11 +254,11 @@ func (ctx *AVFormatContext) Stream(stream int) *AVStream { } } -func (ctx *AVFormatContext) Flags() AVFormat { - return AVFormat(ctx.flags) +func (ctx *AVFormatContext) Flags() AVFormatFlag { + return AVFormatFlag(ctx.flags) } -func (ctx *AVFormatContext) SetFlags(flag AVFormat) { +func (ctx *AVFormatContext) SetFlags(flag AVFormatFlag) { ctx.flags = C.int(flag) } diff --git a/sys/ffmpeg61/avformat_mux.go b/sys/ffmpeg61/avformat_mux.go index 6f0ef5e..8c89dc6 100644 --- a/sys/ffmpeg61/avformat_mux.go +++ b/sys/ffmpeg61/avformat_mux.go @@ -29,6 +29,8 @@ func AVFormat_open_writer(writer *AVIOContextEx, format *AVOutputFormat, filenam ctx.SetPb(writer) } + ctx.SetFlags(ctx.Flags() | AVFMT_FLAG_CUSTOM_IO) + // TODO: Mark AVFMT_NOFILE // Return success @@ -59,7 +61,7 @@ func AVFormat_close_writer(ctx *AVFormatContext) error { var result error octx := (*C.struct_AVFormatContext)(ctx) - if octx.oformat.flags&C.int(AVFMT_NOFILE) == 0 && octx.pb != nil { + if octx.oformat.flags&C.int(AVFMT_NOFILE) == 0 && octx.flags&C.int(AVFMT_FLAG_CUSTOM_IO) == 0 { if err := AVError(C.avio_closep(&octx.pb)); err != 0 { result = errors.Join(result, err) } diff --git a/writer.go b/writer.go index 88973c0..4589824 100644 --- a/writer.go +++ b/writer.go @@ -89,7 +89,7 @@ func createMedia(url string, format Format, metadata []Metadata, params ...Param } // Open the output file, if needed - if !ctx.Flags().Is(ff.AVFMT_NOFILE) { + if !ctx.Output().Flags().Is(ff.AVFMT_NOFILE) { w, err := ff.AVFormat_avio_open(url, ff.AVIO_FLAG_WRITE) if err != nil { return nil, errors.Join(err, writer.Close()) From c932e1777cb961c12ef23b925f459a8dd3bd5569 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 12:11:15 +0200 Subject: [PATCH 06/12] Updates for encoder --- pkg/ffmpeg/encoder.go | 222 +++++++++++++++++++++++++++++ pkg/ffmpeg/opts.go | 101 ++++++++++--- pkg/ffmpeg/packet_encoder.go | 105 -------------- pkg/ffmpeg/par.go | 209 +++++++++++++++++++++++++++ pkg/ffmpeg/par_test.go | 28 ++++ pkg/ffmpeg/resampler.go | 30 ++-- pkg/ffmpeg/resampler_test.go | 2 +- pkg/ffmpeg/rescaler.go | 24 ++-- pkg/ffmpeg/writer.go | 146 +++++++++++++++++-- pkg/ffmpeg/writer_test.go | 32 ++++- pkg/generator/sine.go | 29 ++-- pkg/generator/sine_test.go | 7 +- pkg/generator/yuv420p.go | 4 + sys/ffmpeg61/avcodec.go | 2 + sys/ffmpeg61/avcodec_parameters.go | 51 ++++++- 15 files changed, 814 insertions(+), 178 deletions(-) create mode 100644 pkg/ffmpeg/encoder.go delete mode 100644 pkg/ffmpeg/packet_encoder.go create mode 100644 pkg/ffmpeg/par.go create mode 100644 pkg/ffmpeg/par_test.go diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go new file mode 100644 index 0000000..142ab39 --- /dev/null +++ b/pkg/ffmpeg/encoder.go @@ -0,0 +1,222 @@ +package ffmpeg + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "syscall" + + // Packages + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + + // Namespace imports + . "github.com/djthorpe/go-errors" +) + +//////////////////////////////////////////////////////////////////////////////// +// TYPES + +type Encoder struct { + ctx *ff.AVCodecContext + stream *ff.AVStream + packet *ff.AVPacket + //next_pts int64 +} + +// EncoderFrameFn is a function which is called to receive a frame to encode. It should +// return nil to continue encoding or io.EOF to stop encoding. +type EncoderFrameFn func(int) (*ff.AVFrame, error) + +// EncoderPacketFn is a function which is called for each packet encoded. It should +// return nil to continue encoding or io.EOF to stop encoding immediately. +type EncoderPacketFn func(*ff.AVPacket) error + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +// Create an encoder with the given parameters +func NewEncoder(ctx *ff.AVFormatContext, stream int, par *Par) (*Encoder, error) { + encoder := new(Encoder) + + // Get codec + codec_id := ff.AV_CODEC_ID_NONE + switch par.CodecType() { + case ff.AVMEDIA_TYPE_AUDIO: + codec_id = ctx.Output().AudioCodec() + case ff.AVMEDIA_TYPE_VIDEO: + codec_id = ctx.Output().VideoCodec() + case ff.AVMEDIA_TYPE_SUBTITLE: + codec_id = ctx.Output().SubtitleCodec() + } + if codec_id == ff.AV_CODEC_ID_NONE { + return nil, ErrBadParameter.Withf("no codec specified for stream %v", stream) + } + + // Allocate codec + codec := ff.AVCodec_find_encoder(codec_id) + if codec == nil { + return nil, ErrBadParameter.Withf("codec %q cannot encode", codec_id) + } + if codecctx := ff.AVCodec_alloc_context(codec); codecctx == nil { + return nil, ErrInternalAppError.With("could not allocate audio codec context") + } else { + encoder.ctx = codecctx + } + + // Check codec against parameters and set defaults as needed, then + // copy back to codec + if err := par.ValidateFromCodec(encoder.ctx); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, err + } else if err := par.CopyToCodec(encoder.ctx); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, err + } + + // Create the stream + if streamctx := ff.AVFormat_new_stream(ctx, nil); streamctx == nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, ErrInternalAppError.With("could not allocate stream") + } else { + streamctx.SetId(stream) + encoder.stream = streamctx + } + + // Copy parameters to stream + if err := ff.AVCodec_parameters_from_context(encoder.stream.CodecPar(), encoder.ctx); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, err + } + + // Some formats want stream headers to be separate. + if ctx.Output().Flags().Is(ff.AVFMT_GLOBALHEADER) { + encoder.ctx.SetFlags(encoder.ctx.Flags() | ff.AV_CODEC_FLAG_GLOBAL_HEADER) + } + + // Open it + if err := ff.AVCodec_open(encoder.ctx, codec, nil); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, ErrInternalAppError.Withf("codec_open: %v", err) + } + + // Create a packet + packet := ff.AVCodec_packet_alloc() + if packet == nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, errors.New("failed to allocate packet") + } else { + encoder.packet = packet + } + + // Return it + return encoder, nil +} + +func (encoder *Encoder) Close() error { + // Free respurces + if encoder.ctx != nil { + ff.AVCodec_free_context(encoder.ctx) + } + if encoder.packet != nil { + ff.AVCodec_packet_free(encoder.packet) + } + + // Release resources + encoder.packet = nil + encoder.stream = nil + encoder.ctx = nil + + // Return success + return nil +} + +//////////////////////////////////////////////////////////////////////////////// +// STRINGIFY + +func (e *Encoder) MarshalJSON() ([]byte, error) { + type jsonEncoder struct { + Codec *ff.AVCodecContext `json:"codec"` + Stream *ff.AVStream `json:"stream"` + } + return json.Marshal(&jsonEncoder{ + Codec: e.ctx, + Stream: e.stream, + }) +} + +func (e *Encoder) String() string { + data, _ := json.MarshalIndent(e, "", " ") + return string(data) +} + +////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +// Encode a frame and pass packets to the EncoderPacketFn. If the frame is nil, then +// the encoder will flush any remaining packets. If io.EOF is returned then +// it indicates that the encoder has ended prematurely. +func (e *Encoder) Encode(frame *ff.AVFrame, fn EncoderPacketFn) error { + if fn == nil { + return ErrBadParameter.With("nil fn") + } + // Encode a frame (or flush the encoder) + return e.encode(frame, fn) +} + +// Return the codec parameters +func (e *Encoder) Par() *Par { + par := new(Par) + if err := ff.AVCodec_parameters_from_context(&par.AVCodecParameters, e.ctx); err != nil { + return nil + } else { + return par + } +} + +////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { + // Send the frame to the encoder + fmt.Println("Sending frame", frame) + if err := ff.AVCodec_send_frame(e.ctx, frame); err != nil { + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { + return nil + } + return err + } + + // Write out the packets + var result error + for { + // Receive the packet + fmt.Println("Receiving packet") + if err := ff.AVCodec_receive_packet(e.ctx, e.packet); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { + // Finished receiving packet or EOF + break + } else if err != nil { + return err + } + + // Pass back to the caller + if err := fn(e.packet); errors.Is(err, io.EOF) { + // End early, return EOF + result = io.EOF + break + } else if err != nil { + return err + } + + // Re-allocate frames for next iteration + ff.AVCodec_packet_unref(e.packet) + } + + // Flush + if result == nil { + result = fn(nil) + } + + // Return success or EOF + return result +} diff --git a/pkg/ffmpeg/opts.go b/pkg/ffmpeg/opts.go index 990e2f5..90b8c63 100644 --- a/pkg/ffmpeg/opts.go +++ b/pkg/ffmpeg/opts.go @@ -15,18 +15,23 @@ type Opt func(*opts) error type opts struct { // Resample/resize options force bool + par *Par // Format options oformat *ffmpeg.AVOutputFormat - // Audio options - sample_fmt ffmpeg.AVSampleFormat - ch ffmpeg.AVChannelLayout - samplerate int + // Stream options + streams map[int]*Par +} + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE - // Video options - pix_fmt ffmpeg.AVPixelFormat - width, height int +func newOpts() *opts { + return &opts{ + par: new(Par), + streams: make(map[int]*Par), + } } //////////////////////////////////////////////////////////////////////////////// @@ -45,6 +50,50 @@ func OptOutputFormat(name string) Opt { } } +// New audio stream with parameters +func OptAudioStream(stream int, par *Par) Opt { + return func(o *opts) error { + if par == nil || par.CodecType() != ffmpeg.AVMEDIA_TYPE_AUDIO { + return ErrBadParameter.With("invalid audio parameters") + } + if stream == 0 { + stream = len(o.streams) + 1 + } + if _, exists := o.streams[stream]; exists { + return ErrDuplicateEntry.Withf("stream %v", stream) + } + if stream < 0 { + return ErrBadParameter.Withf("invalid stream %v", stream) + } + o.streams[stream] = par + + // Return success + return nil + } +} + +// New video stream with parameters +func OptVideoStream(stream int, par *Par) Opt { + return func(o *opts) error { + if par == nil || par.CodecType() != ffmpeg.AVMEDIA_TYPE_VIDEO { + return ErrBadParameter.With("invalid video parameters") + } + if stream == 0 { + stream = len(o.streams) + 1 + } + if _, exists := o.streams[stream]; exists { + return ErrDuplicateEntry.Withf("stream %v", stream) + } + if stream < 0 { + return ErrBadParameter.Withf("invalid stream %v", stream) + } + o.streams[stream] = par + + // Return success + return nil + } +} + // Force resampling and resizing on decode, even if the input and output // parameters are the same func OptForce() Opt { @@ -61,7 +110,8 @@ func OptPixFormat(format string) Opt { if fmt == ffmpeg.AV_PIX_FMT_NONE { return ErrBadParameter.Withf("invalid pixel format %q", format) } - o.pix_fmt = fmt + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_VIDEO) + o.par.SetPixelFormat(fmt) return nil } } @@ -72,8 +122,9 @@ func OptWidthHeight(w, h int) Opt { if w <= 0 || h <= 0 { return ErrBadParameter.Withf("invalid width %v or height %v", w, h) } - o.width = w - o.height = h + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_VIDEO) + o.par.SetWidth(w) + o.par.SetHeight(h) return nil } } @@ -85,8 +136,9 @@ func OptFrameSize(size string) Opt { if err != nil { return ErrBadParameter.Withf("invalid frame size %q", size) } - o.width = w - o.height = h + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_VIDEO) + o.par.SetWidth(w) + o.par.SetHeight(h) return nil } } @@ -94,18 +146,25 @@ func OptFrameSize(size string) Opt { // Channel layout func OptChannelLayout(layout string) Opt { return func(o *opts) error { - return ffmpeg.AVUtil_channel_layout_from_string(&o.ch, layout) + var ch ffmpeg.AVChannelLayout + if err := ffmpeg.AVUtil_channel_layout_from_string(&ch, layout); err != nil { + return ErrBadParameter.Withf("invalid channel layout %q", layout) + } + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_AUDIO) + return o.par.SetChannelLayout(ch) } } // Nuumber of channels -func OptChannels(ch int) Opt { +func OptChannels(num int) Opt { return func(o *opts) error { - if ch <= 0 || ch > 64 { - return ErrBadParameter.Withf("invalid number of channels %v", ch) + var ch ffmpeg.AVChannelLayout + if num <= 0 || num > 64 { + return ErrBadParameter.Withf("invalid number of channels %v", num) } - ffmpeg.AVUtil_channel_layout_default(&o.ch, ch) - return nil + ffmpeg.AVUtil_channel_layout_default(&ch, num) + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_AUDIO) + return o.par.SetChannelLayout(ch) } } @@ -115,7 +174,8 @@ func OptSampleRate(rate int) Opt { if rate <= 0 { return ErrBadParameter.Withf("invalid sample rate %v", rate) } - o.samplerate = rate + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_AUDIO) + o.par.SetSamplerate(rate) return nil } } @@ -127,7 +187,8 @@ func OptSampleFormat(format string) Opt { if fmt == ffmpeg.AV_SAMPLE_FMT_NONE { return ErrBadParameter.Withf("invalid sample format %q", format) } - o.sample_fmt = fmt + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_AUDIO) + o.par.SetSampleFormat(fmt) return nil } } diff --git a/pkg/ffmpeg/packet_encoder.go b/pkg/ffmpeg/packet_encoder.go deleted file mode 100644 index 0ce5ad5..0000000 --- a/pkg/ffmpeg/packet_encoder.go +++ /dev/null @@ -1,105 +0,0 @@ -package ffmpeg - -import ( - "errors" - "io" - "syscall" - - // Packages - ff "github.com/mutablelogic/go-media/sys/ffmpeg61" -) - -////////////////////////////////////////////////////////////////////////////// -// TYPES - -type encoder struct { - codec *ff.AVCodecContext - packet *ff.AVPacket - stream int -} - -// EncodeFn is a function which is called for each packet encoded. It should -// return nil to continue encoding or io.EOF to stop decoding. -type EncodeFn func(*ff.AVPacket) error - -////////////////////////////////////////////////////////////////////////////// -// LIFECYCLE - -func NewPacketEncoder(codec *ff.AVCodecContext, stream int) (*encoder, error) { - encoder := new(encoder) - encoder.codec = codec - - // Create a packet - packet := ff.AVCodec_packet_alloc() - if packet == nil { - return nil, errors.New("failed to allocate packet") - } else { - encoder.packet = packet - encoder.stream = stream - } - - // Return success - return encoder, nil -} - -func (e *encoder) Close() error { - if e.packet != nil { - ff.AVCodec_packet_free(e.packet) - e.packet = nil - } - return nil -} - -////////////////////////////////////////////////////////////////////////////// -// LIFECYCLE - -func (e *encoder) Encode(frame *ff.AVFrame, fn EncodeFn) error { - // Encode a frame - if err := e.encode(frame, fn); err != nil { - return err - } - // Flush - return e.encode(nil, fn) -} - -////////////////////////////////////////////////////////////////////////////// -// PRIVATE METHODS - -func (e *encoder) encode(frame *ff.AVFrame, fn EncodeFn) error { - // Send the frame to the encoder - if err := ff.AVCodec_send_frame(e.codec, frame); err != nil { - return err - } - - // Write out the packets - var result error - for { - // Receive the packet - if err := ff.AVCodec_receive_packet(e.codec, e.packet); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { - // Finished receiving packet or EOF - break - } else if err != nil { - return err - } - - // Pass back to the caller - if err := fn(e.packet); errors.Is(err, io.EOF) { - // End early, return EOF - result = io.EOF - break - } else if err != nil { - return err - } - - // Re-allocate frames for next iteration - ff.AVCodec_packet_unref(e.packet) - } - - // Flush - if result == nil { - result = fn(nil) - } - - // Return success or EOF - return result -} diff --git a/pkg/ffmpeg/par.go b/pkg/ffmpeg/par.go new file mode 100644 index 0000000..3149feb --- /dev/null +++ b/pkg/ffmpeg/par.go @@ -0,0 +1,209 @@ +package ffmpeg + +import ( + "encoding/json" + "fmt" + "slices" + + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + + // Namespace imports + . "github.com/djthorpe/go-errors" +) + +/////////////////////////////////////////////////////////////////////////////// +// TYPES + +type Par struct { + ff.AVCodecParameters +} + +/////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func NewAudioPar(samplefmt string, channellayout string, samplerate int) (*Par, error) { + par := new(Par) + par.SetCodecType(ff.AVMEDIA_TYPE_AUDIO) + + // Sample Format + if samplefmt_ := ff.AVUtil_get_sample_fmt(samplefmt); samplefmt_ == ff.AV_SAMPLE_FMT_NONE { + return nil, ErrBadParameter.Withf("unknown sample format %q", samplefmt) + } else { + par.SetSampleFormat(samplefmt_) + } + + // Channel layout + var ch ff.AVChannelLayout + if err := ff.AVUtil_channel_layout_from_string(&ch, channellayout); err != nil { + return nil, ErrBadParameter.Withf("channel layout %q", channellayout) + } else if err := par.SetChannelLayout(ch); err != nil { + return nil, err + } + + // Sample rate + if samplerate <= 0 { + return nil, ErrBadParameter.Withf("negative or zero samplerate %v", samplerate) + } else { + par.SetSamplerate(samplerate) + } + + // Return success + return par, nil +} + +func NewVideoPar(pixfmt string, size string) (*Par, error) { + par := new(Par) + par.SetCodecType(ff.AVMEDIA_TYPE_VIDEO) + + // Pixel Format + if pixfmt_ := ff.AVUtil_get_pix_fmt(pixfmt); pixfmt_ == ff.AV_PIX_FMT_NONE { + return nil, ErrBadParameter.Withf("unknown pixel format %q", pixfmt) + } else { + par.SetPixelFormat(pixfmt_) + } + + // Frame size + if w, h, err := ff.AVUtil_parse_video_size(size); err != nil { + return nil, ErrBadParameter.Withf("size %q", size) + } else { + par.SetWidth(w) + par.SetHeight(h) + } + + // Set default sample aspect ratio + par.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + + // Return success + return par, nil +} + +func AudioPar(samplefmt string, channellayout string, samplerate int) *Par { + if par, err := NewAudioPar(samplefmt, channellayout, samplerate); err != nil { + panic(err) + } else { + return par + } +} + +func VideoPar(pixfmt string, size string) *Par { + if par, err := NewVideoPar(pixfmt, size); err != nil { + panic(err) + } else { + return par + } +} + +/////////////////////////////////////////////////////////////////////////////// +// STRINGIFY + +func (ctx *Par) MarshalJSON() ([]byte, error) { + return json.Marshal(ctx.AVCodecParameters) +} + +func (ctx *Par) String() string { + data, _ := json.MarshalIndent(ctx, "", " ") + return string(data) +} + +/////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +func (ctx *Par) ValidateFromCodec(codec *ff.AVCodecContext) error { + switch codec.Codec().Type() { + case ff.AVMEDIA_TYPE_AUDIO: + return ctx.validateAudioCodec(codec) + case ff.AVMEDIA_TYPE_VIDEO: + return ctx.validateVideoCodec(codec) + } + return nil +} + +func (ctx *Par) CopyToCodec(codec *ff.AVCodecContext) error { + switch codec.Codec().Type() { + case ff.AVMEDIA_TYPE_AUDIO: + return ctx.copyAudioCodec(codec) + case ff.AVMEDIA_TYPE_VIDEO: + return ctx.copyVideoCodec(codec) + } + return nil +} + +/////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +func (ctx *Par) copyAudioCodec(codec *ff.AVCodecContext) error { + codec.SetSampleFormat(ctx.SampleFormat()) + codec.SetSampleRate(ctx.Samplerate()) + if err := codec.SetChannelLayout(ctx.ChannelLayout()); err != nil { + return err + } + return nil +} + +func (ctx *Par) validateAudioCodec(codec *ff.AVCodecContext) error { + sampleformats := codec.Codec().SampleFormats() + samplerates := codec.Codec().SupportedSamplerates() + channellayouts := codec.Codec().ChannelLayouts() + + // First we set params from the codec which are not already set + if ctx.SampleFormat() == ff.AV_SAMPLE_FMT_NONE { + if len(sampleformats) > 0 { + ctx.SetSampleFormat(sampleformats[0]) + } + } + if ctx.Samplerate() == 0 { + if len(samplerates) > 0 { + ctx.SetSamplerate(samplerates[0]) + } + } + if ctx.ChannelLayout().NumChannels() == 0 { + if len(channellayouts) > 0 { + ctx.SetChannelLayout(channellayouts[0]) + } + } + + // Then we check to make sure the parameters are compatible with + // the codec + if len(sampleformats) > 0 { + if !slices.Contains(sampleformats, ctx.SampleFormat()) { + return ErrBadParameter.Withf("unsupported sample format %v", ctx.SampleFormat()) + } + } else if ctx.SampleFormat() == ff.AV_SAMPLE_FMT_NONE { + return ErrBadParameter.With("sample format not set") + } + if len(samplerates) > 0 { + if !slices.Contains(samplerates, ctx.Samplerate()) { + return ErrBadParameter.Withf("unsupported samplerate %v", ctx.Samplerate()) + } + } else if ctx.Samplerate() == 0 { + return ErrBadParameter.With("samplerate not set") + } + if len(channellayouts) > 0 { + valid := false + for _, ch := range channellayouts { + chctx := ctx.ChannelLayout() + if ff.AVUtil_channel_layout_compare(&ch, &chctx) { + valid = true + break + } + } + if !valid { + return ErrBadParameter.Withf("unsupported channel layout %v", ctx.ChannelLayout()) + } + } else if ctx.ChannelLayout().NumChannels() == 0 { + return ErrBadParameter.With("channel layout not set") + } + + // Validated + return nil +} + +func (ctx *Par) copyVideoCodec(codec *ff.AVCodecContext) error { + fmt.Println("TODO: copyVideoCodec") + return nil +} + +func (ctx *Par) validateVideoCodec(codec *ff.AVCodecContext) error { + fmt.Println("TODO: validateVideoCodec") + return nil +} diff --git a/pkg/ffmpeg/par_test.go b/pkg/ffmpeg/par_test.go new file mode 100644 index 0000000..4f8d88a --- /dev/null +++ b/pkg/ffmpeg/par_test.go @@ -0,0 +1,28 @@ +package ffmpeg_test + +import ( + "testing" + + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + assert "github.com/stretchr/testify/assert" +) + +func Test_par_001(t *testing.T) { + assert := assert.New(t) + + par, err := ffmpeg.NewAudioPar("fltp", "mono", 22050) + if !assert.NoError(err) { + t.FailNow() + } + t.Log(par) +} + +func Test_par_002(t *testing.T) { + assert := assert.New(t) + + par, err := ffmpeg.NewVideoPar("yuv420p", "1280x720") + if !assert.NoError(err) { + t.FailNow() + } + t.Log(par) +} diff --git a/pkg/ffmpeg/resampler.go b/pkg/ffmpeg/resampler.go index 84b0aeb..e56d2f8 100644 --- a/pkg/ffmpeg/resampler.go +++ b/pkg/ffmpeg/resampler.go @@ -12,9 +12,9 @@ import ( // TYPES type resampler struct { - opts - ctx *ff.SWRContext - dest *ff.AVFrame + ctx *ff.SWRContext + dest *ff.AVFrame + force bool } //////////////////////////////////////////////////////////////////////////////// @@ -23,23 +23,26 @@ type resampler struct { // Create a new audio resampler which will resample the input frame to the // specified channel layout, sample rate and sample format. func NewResampler(format ff.AVSampleFormat, opt ...Opt) (*resampler, error) { + options := newOpts() resampler := new(resampler) // Apply options - resampler.sample_fmt = format - resampler.ch = ff.AV_CHANNEL_LAYOUT_MONO - resampler.samplerate = 44100 + options.par.SetCodecType(ff.AVMEDIA_TYPE_AUDIO) + options.par.SetSampleFormat(format) + options.par.SetChannelLayout(ff.AV_CHANNEL_LAYOUT_MONO) + options.par.SetSamplerate(44100) for _, o := range opt { - if err := o(&resampler.opts); err != nil { + if err := o(options); err != nil { return nil, err } } // Check parameters - if resampler.sample_fmt == ff.AV_SAMPLE_FMT_NONE { + if options.par.SampleFormat() == ff.AV_SAMPLE_FMT_NONE { return nil, errors.New("invalid sample format parameters") } - if !ff.AVUtil_channel_layout_check(&resampler.ch) { + ch := options.par.ChannelLayout() + if !ff.AVUtil_channel_layout_check(&ch) { return nil, errors.New("invalid channel layout parameters") } @@ -52,15 +55,18 @@ func NewResampler(format ff.AVSampleFormat, opt ...Opt) (*resampler, error) { // Set parameters - we don't allocate the buffer here, // we do that when we have a source frame and know how // large the destination frame should be - dest.SetSampleFormat(resampler.sample_fmt) - dest.SetSampleRate(resampler.samplerate) - if err := dest.SetChannelLayout(resampler.ch); err != nil { + dest.SetSampleFormat(options.par.SampleFormat()) + dest.SetSampleRate(options.par.Samplerate()) + if err := dest.SetChannelLayout(options.par.ChannelLayout()); err != nil { ff.AVUtil_frame_free(dest) return nil, err } else { resampler.dest = dest } + // Set force flag + resampler.force = options.force + // Return success return resampler, nil } diff --git a/pkg/ffmpeg/resampler_test.go b/pkg/ffmpeg/resampler_test.go index e84f54d..dc3adc5 100644 --- a/pkg/ffmpeg/resampler_test.go +++ b/pkg/ffmpeg/resampler_test.go @@ -13,7 +13,7 @@ func Test_resampler_001(t *testing.T) { assert := assert.New(t) // Sine wave generator - audio, err := generator.NewSine(2000, 10, 44100) + audio, err := generator.NewSine(2000, 10, ffmpeg.AudioPar("fltp", "mono", 44100)) if !assert.NoError(err) { t.FailNow() } diff --git a/pkg/ffmpeg/rescaler.go b/pkg/ffmpeg/rescaler.go index 87deafa..f6cfb6e 100644 --- a/pkg/ffmpeg/rescaler.go +++ b/pkg/ffmpeg/rescaler.go @@ -12,13 +12,12 @@ import ( // TYPES type rescaler struct { - opts - src_pix_fmt ff.AVPixelFormat src_width int src_height int ctx *ff.SWSContext flags ff.SWSFlag + force bool dest *ff.AVFrame } @@ -28,20 +27,22 @@ type rescaler struct { // Create a new rescaler which will rescale the input frame to the // specified format, width and height. func NewRescaler(format ff.AVPixelFormat, opt ...Opt) (*rescaler, error) { + options := newOpts() rescaler := new(rescaler) // Apply options - rescaler.pix_fmt = format - rescaler.width = 640 - rescaler.height = 480 + options.par.SetCodecType(ff.AVMEDIA_TYPE_VIDEO) + options.par.SetPixelFormat(format) + options.par.SetWidth(640) + options.par.SetHeight(480) for _, o := range opt { - if err := o(&rescaler.opts); err != nil { + if err := o(options); err != nil { return nil, err } } // Check parameters - if rescaler.pix_fmt == ff.AV_PIX_FMT_NONE { + if options.par.PixelFormat() == ff.AV_PIX_FMT_NONE { return nil, errors.New("invalid parameters") } @@ -51,10 +52,13 @@ func NewRescaler(format ff.AVPixelFormat, opt ...Opt) (*rescaler, error) { return nil, errors.New("failed to allocate frame") } + // Set force flag + rescaler.force = options.force + // Set parameters - dest.SetPixFmt(rescaler.pix_fmt) - dest.SetWidth(rescaler.width) - dest.SetHeight(rescaler.height) + dest.SetPixFmt(options.par.PixelFormat()) + dest.SetWidth(options.par.Width()) + dest.SetHeight(options.par.Height()) // Allocate buffer if err := ff.AVUtil_frame_get_buffer(dest, false); err != nil { diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index 82f7377..2987c28 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -4,9 +4,12 @@ import ( "errors" "fmt" "io" + "os" + "sort" // Packages ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + maps "golang.org/x/exp/maps" // Namespace imports . "github.com/djthorpe/go-errors" @@ -17,7 +20,9 @@ import ( // Create media from io.Writer type Writer struct { - output *ff.AVFormatContext + output *ff.AVFormatContext + header bool + encoders []*Encoder } type writer_callback struct { @@ -35,33 +40,58 @@ const ( // LIFECYCLE func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { - var o opts - + options := newOpts() writer := new(Writer) // Apply options for _, opt := range opt { - if err := opt(&o); err != nil { + if err := opt(options); err != nil { return nil, err } } // Check output - if o.oformat == nil { + var filename string + if options.oformat == nil { return nil, ErrBadParameter.Withf("invalid output format") + } else if w_, ok := w.(*os.File); ok { + filename = w_.Name() } // Allocate the AVIO context - avio := ff.AVFormat_avio_alloc_context(bufSize, false, &writer_callback{w}) + avio := ff.AVFormat_avio_alloc_context(bufSize, true, &writer_callback{w}) if avio == nil { return nil, errors.New("failed to allocate avio context") - } else if ctx, err := ff.AVFormat_open_writer(avio, o.oformat, ""); err != nil { + } else if ctx, err := ff.AVFormat_open_writer(avio, options.oformat, filename); err != nil { return nil, err } else { writer.output = ctx } - fmt.Println("WRITER", writer.output) + // Create codec contexts for each stream + var result error + keys := sort.IntSlice(maps.Keys(options.streams)) + for _, stream := range keys { + encoder, err := NewEncoder(writer.output, stream, options.streams[stream]) + if err != nil { + result = errors.Join(result, err) + continue + } else { + writer.encoders = append(writer.encoders, encoder) + } + } + + // Return any errors + if result != nil { + return nil, errors.Join(result, writer.Close()) + } + + // Write the header + if err := ff.AVFormat_write_header(writer.output, nil); err != nil { + return nil, errors.Join(err, writer.Close()) + } else { + writer.header = true + } // Return success return writer, nil @@ -70,23 +100,113 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { func (w *Writer) Close() error { var result error + // Write the trailer if the header was written + if w.header { + if err := ff.AVFormat_write_trailer(w.output); err != nil { + result = errors.Join(result, err) + } + } + + // Close encoders + for _, encoder := range w.encoders { + result = errors.Join(result, encoder.Close()) + } + // Free output resources if w.output != nil { - // This calls avio_close(w.avio) - fmt.Println("TODO: AVFormat_close_writer") result = errors.Join(result, ff.AVFormat_close_writer(w.output)) - w.output = nil } + // Free resources + w.output = nil + w.encoders = nil + // Return any errors return result } +////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +// Return a "stream" for encoding +func (w *Writer) Stream(stream int) *Encoder { + for _, encoder := range w.encoders { + if encoder.stream.Id() == stream { + return encoder + } + } + return nil +} + +// Encode frames from all encoders, calling the callback function to encode +// the frame. If the callback function returns io.EOF then the encoding for +// that encoder is stopped after flushing. If the second callback is nil, +// then packets are written to the output. +func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { + if in == nil { + return ErrBadParameter.With("nil in or out") + } + if out == nil { + // By default, write packet to output + out = w.Write + } + + // Initialise encoders + encoders := make(map[int]*Encoder, len(w.encoders)) + for _, encoder := range w.encoders { + stream := encoder.stream.Id() + if _, exists := encoders[stream]; exists { + return ErrBadParameter.Withf("duplicate stream %v", stream) + } + encoders[stream] = encoder + } + + // Continue until all encoders have returned io.EOF + for { + // No more encoding to do + if len(encoders) == 0 { + break + } + for stream, encoder := range encoders { + // Receive a frame for the encoder + frame, err := in(stream) + if errors.Is(err, io.EOF) { + fmt.Println("EOF for frame on stream", stream) + delete(encoders, stream) + } else if err != nil { + return fmt.Errorf("stream %v: %w", stream, err) + } else if frame == nil { + return fmt.Errorf("stream %v: nil frame received", stream) + } else if err := encoder.Encode(frame, out); errors.Is(err, io.EOF) { + fmt.Println("EOF for packet on stream", stream) + delete(encoders, stream) + } else if err != nil { + return fmt.Errorf("stream %v: %w", stream, err) + } + } + } + + // Return success + return nil +} + +// Write a packet to the output +func (w *Writer) Write(packet *ff.AVPacket) error { + return ff.AVCodec_interleaved_write_frame(w.output, packet) +} + //////////////////////////////////////////////////////////////////////////////// -// PRIVATE METHODS +// PRIVATE METHODS - Writer func (w *writer_callback) Reader(buf []byte) int { - return 0 + if r, ok := w.w.(io.Reader); ok { + if n, err := r.Read(buf); err != nil { + return -1 + } else { + return n + } + } + return -1 } func (w *writer_callback) Seeker(offset int64, whence int) int64 { diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index 2836309..e421574 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -1,11 +1,14 @@ package ffmpeg_test import ( - "fmt" + "io" "os" "testing" + "time" ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + generator "github.com/mutablelogic/go-media/pkg/generator" + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" assert "github.com/stretchr/testify/assert" ) @@ -22,11 +25,36 @@ func Test_writer_001(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.NewWriter(w, ffmpeg.OptOutputFormat(w.Name()), + ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), ) if !assert.NoError(err) { t.FailNow() } defer writer.Close() - fmt.Println("Written to", w.Name()) + // Make an audio generator + audio, err := generator.NewSine(440, -5, writer.Stream(1).Par()) + if !assert.NoError(err) { + t.FailNow() + } + defer audio.Close() + + // Write frames + n := 0 + assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { + frame := audio.Frame() + t.Log("Frame s", frame.Time().Truncate(time.Millisecond)) + if frame.Time() > 10*time.Second { + return nil, io.EOF + } else { + return frame.(*ffmpeg.Frame).AVFrame(), nil + } + }, func(packet *ff.AVPacket) error { + if packet != nil { + t.Log("Packet ts", packet.Pts()) + n += packet.Size() + } + return writer.Write(packet) + })) + t.Log("Written", n, "bytes to", w.Name()) } diff --git a/pkg/generator/sine.go b/pkg/generator/sine.go index 32b1e0e..2763ff1 100644 --- a/pkg/generator/sine.go +++ b/pkg/generator/sine.go @@ -27,7 +27,7 @@ var _ Generator = (*sine)(nil) // GLOBALS const ( - frameDuration = 20 * time.Millisecond // Each frame is 20ms of audio + frameDuration = 10 * time.Millisecond // Each frame is 10ms of audio ) //////////////////////////////////////////////////////////////////////////// @@ -36,19 +36,29 @@ const ( // Create a new sine wave generator with one channel using float32 // for samples. The frequency in Hz, volume in decibels and samplerate // (ie, 44100) for the audio stream are passed as arguments. -func NewSine(freq float64, volume float64, samplerate int) (*sine, error) { +func NewSine(freq, volume float64, par *ffmpeg.Par) (*sine, error) { sine := new(sine) // Check parameters + if par.CodecType() != ff.AVMEDIA_TYPE_AUDIO { + return nil, errors.New("invalid codec type") + } else if par.ChannelLayout().NumChannels() != 1 { + return nil, errors.New("invalid channel layout, only mono is supported") + } else if par.SampleFormat() != ff.AV_SAMPLE_FMT_FLT && par.SampleFormat() != ff.AV_SAMPLE_FMT_FLTP { + return nil, errors.New("invalid sample format, only float32 is supported") + } if freq <= 0 { return nil, errors.New("invalid frequency") } if volume <= -100 { return nil, errors.New("invalid volume") } - if samplerate <= 0 { + if par.Samplerate() <= 0 { return nil, errors.New("invalid samplerate") } + if par.FrameSize() <= 0 { + par.SetFrameSize(int(float64(par.Samplerate()) * frameDuration.Seconds())) + } // Create a frame frame := ff.AVUtil_frame_alloc() @@ -56,16 +66,13 @@ func NewSine(freq float64, volume float64, samplerate int) (*sine, error) { return nil, errors.New("failed to allocate frame") } - // Set frame parameters - numSamples := int(float64(samplerate) * frameDuration.Seconds()) - frame.SetSampleFormat(ff.AV_SAMPLE_FMT_FLT) // float32 if err := frame.SetChannelLayout(ff.AV_CHANNEL_LAYOUT_MONO); err != nil { return nil, err } - frame.SetSampleRate(samplerate) - frame.SetNumSamples(numSamples) - frame.SetTimeBase(ff.AVUtil_rational(1, samplerate)) + frame.SetSampleRate(par.Samplerate()) + frame.SetNumSamples(par.FrameSize()) + frame.SetTimeBase(ff.AVUtil_rational(1, par.Samplerate())) frame.SetPts(ff.AV_NOPTS_VALUE) // Allocate buffer @@ -101,6 +108,10 @@ func (s *sine) String() string { // Return the first and subsequent frames of raw audio data func (s *sine) Frame() media.Frame { + if err := ff.AVUtil_frame_make_writable(s.frame); err != nil { + return nil + } + // Set the Pts if s.frame.Pts() == ff.AV_NOPTS_VALUE { s.frame.SetPts(0) diff --git a/pkg/generator/sine_test.go b/pkg/generator/sine_test.go index 95d15ad..7781aee 100644 --- a/pkg/generator/sine_test.go +++ b/pkg/generator/sine_test.go @@ -6,13 +6,14 @@ import ( "testing" "time" + "github.com/mutablelogic/go-media/pkg/ffmpeg" "github.com/mutablelogic/go-media/pkg/generator" "github.com/stretchr/testify/assert" ) func Test_sine_001(t *testing.T) { assert := assert.New(t) - sine, err := generator.NewSine(2000, 10, 44100) + sine, err := generator.NewSine(2000, 10, ffmpeg.AudioPar("fltp", "mono", 44100)) if !assert.NoError(err) { t.SkipNow() } @@ -23,7 +24,7 @@ func Test_sine_001(t *testing.T) { func Test_sine_002(t *testing.T) { assert := assert.New(t) - sine, err := generator.NewSine(2000, 10, 44100) + sine, err := generator.NewSine(2000, 10, ffmpeg.AudioPar("fltp", "mono", 44100)) if !assert.NoError(err) { t.SkipNow() } @@ -42,7 +43,7 @@ func Test_sine_003(t *testing.T) { const frequency = 440 const volume = -10.0 - sine, err := generator.NewSine(frequency, volume, sampleRate) + sine, err := generator.NewSine(frequency, volume, ffmpeg.AudioPar("fltp", "mono", sampleRate)) if !assert.NoError(err) { t.SkipNow() } diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go index cb3133c..3ad5556 100644 --- a/pkg/generator/yuv420p.go +++ b/pkg/generator/yuv420p.go @@ -80,6 +80,10 @@ func (yuv420p *yuv420p) String() string { // Return the first and subsequent frames of raw video data func (yuv420p *yuv420p) Frame() media.Frame { + if err := ff.AVUtil_frame_make_writable(yuv420p.frame); err != nil { + return nil + } + // Set the Pts if yuv420p.frame.Pts() == ff.AV_NOPTS_VALUE { yuv420p.frame.SetPts(0) diff --git a/sys/ffmpeg61/avcodec.go b/sys/ffmpeg61/avcodec.go index e7bd26e..d4e6def 100644 --- a/sys/ffmpeg61/avcodec.go +++ b/sys/ffmpeg61/avcodec.go @@ -59,6 +59,7 @@ type jsonAVCodecContext struct { SampleFormat AVSampleFormat `json:"sample_fmt,omitempty"` SampleRate int `json:"sample_rate,omitempty"` ChannelLayout AVChannelLayout `json:"channel_layout,omitempty"` + FrameSize int `json:"frame_size,omitempty"` TimeBase AVRational `json:"time_base,omitempty"` } @@ -192,6 +193,7 @@ func (ctx *AVCodecContext) MarshalJSON() ([]byte, error) { SampleFormat: AVSampleFormat(ctx.sample_fmt), SampleRate: int(ctx.sample_rate), ChannelLayout: AVChannelLayout(ctx.ch_layout), + FrameSize: int(ctx.frame_size), }) default: return json.Marshal(jsonAVCodecContext{ diff --git a/sys/ffmpeg61/avcodec_parameters.go b/sys/ffmpeg61/avcodec_parameters.go index 8166bea..833c303 100644 --- a/sys/ffmpeg61/avcodec_parameters.go +++ b/sys/ffmpeg61/avcodec_parameters.go @@ -2,6 +2,7 @@ package ffmpeg import ( "encoding/json" + "errors" ) //////////////////////////////////////////////////////////////////////////////// @@ -18,14 +19,14 @@ import "C" // TYPES type jsonAVCodecParametersAudio struct { - SampleFormat AVSampleFormat `json:"format"` + SampleFormat AVSampleFormat `json:"sample_format"` SampleRate int `json:"sample_rate"` ChannelLayout AVChannelLayout `json:"channel_layout"` FrameSize int `json:"frame_size,omitempty"` } type jsonAVCodecParameterVideo struct { - PixelFormat AVPixelFormat `json:"format"` + PixelFormat AVPixelFormat `json:"pixel_format"` Width int `json:"width"` Height int `json:"height"` SampleAspectRatio AVRational `json:"sample_aspect_ratio,omitempty"` @@ -43,7 +44,7 @@ type jsonAVCodecParameters struct { //////////////////////////////////////////////////////////////////////////////// // STRINGIFY -func (ctx *AVCodecParameters) MarshalJSON() ([]byte, error) { +func (ctx AVCodecParameters) MarshalJSON() ([]byte, error) { par := jsonAVCodecParameters{ CodecType: AVMediaType(ctx.codec_type), CodecID: AVCodecID(ctx.codec_id), @@ -82,6 +83,10 @@ func (ctx *AVCodecParameters) CodecType() AVMediaType { return AVMediaType(ctx.codec_type) } +func (ctx *AVCodecParameters) SetCodecType(t AVMediaType) { + ctx.codec_type = C.enum_AVMediaType(t) +} + func (ctx *AVCodecParameters) CodecID() AVCodecID { return AVCodecID(ctx.codec_id) } @@ -104,6 +109,10 @@ func (ctx *AVCodecParameters) BitRate() int64 { return int64(ctx.bit_rate) } +func (ctx *AVCodecParameters) SetBitRate(rate int64) { + ctx.bit_rate = C.int64_t(rate) +} + // Audio func (ctx *AVCodecParameters) SampleFormat() AVSampleFormat { if AVMediaType(ctx.codec_type) == AVMEDIA_TYPE_AUDIO { @@ -113,21 +122,41 @@ func (ctx *AVCodecParameters) SampleFormat() AVSampleFormat { } } +func (ctx *AVCodecParameters) SetSampleFormat(format AVSampleFormat) { + ctx.format = C.int(format) +} + // Audio func (ctx *AVCodecParameters) Samplerate() int { return int(ctx.sample_rate) } +func (ctx *AVCodecParameters) SetSamplerate(rate int) { + ctx.sample_rate = C.int(rate) +} + // Audio func (ctx *AVCodecParameters) ChannelLayout() AVChannelLayout { return AVChannelLayout(ctx.ch_layout) } +func (ctx *AVCodecParameters) SetChannelLayout(layout AVChannelLayout) error { + if !AVUtil_channel_layout_check(&layout) { + return errors.New("invalid channel layout") + } + ctx.ch_layout = C.AVChannelLayout(layout) + return nil +} + // Audio func (ctx *AVCodecParameters) FrameSize() int { return int(ctx.frame_size) } +func (ctx *AVCodecParameters) SetFrameSize(size int) { + ctx.frame_size = C.int(size) +} + // Video func (ctx *AVCodecParameters) PixelFormat() AVPixelFormat { if AVMediaType(ctx.codec_type) == AVMEDIA_TYPE_VIDEO { @@ -137,17 +166,33 @@ func (ctx *AVCodecParameters) PixelFormat() AVPixelFormat { } } +func (ctx *AVCodecParameters) SetPixelFormat(format AVPixelFormat) { + ctx.format = C.int(format) +} + // Video func (ctx *AVCodecParameters) SampleAspectRatio() AVRational { return AVRational(ctx.sample_aspect_ratio) } +func (ctx *AVCodecParameters) SetSampleAspectRatio(aspect AVRational) { + ctx.sample_aspect_ratio = C.AVRational(aspect) +} + // Video func (ctx *AVCodecParameters) Width() int { return int(ctx.width) } +func (ctx *AVCodecParameters) SetWidth(width int) { + ctx.width = C.int(width) +} + // Video func (ctx *AVCodecParameters) Height() int { return int(ctx.height) } + +func (ctx *AVCodecParameters) SetHeight(height int) { + ctx.height = C.int(height) +} From 80bf31d7e09ab12718f4a3ad336a6cfb3af2b5f3 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 14:20:00 +0200 Subject: [PATCH 07/12] Updated writer --- pkg/ffmpeg/encoder.go | 18 +++--- pkg/ffmpeg/metadata.go | 35 ++++++++++++ pkg/ffmpeg/opts.go | 22 ++++--- pkg/ffmpeg/writer.go | 105 ++++++++++++++++++++++++++++------ pkg/ffmpeg/writer_test.go | 60 ++++++++++++++++--- sys/ffmpeg61/avformat.go | 10 ++++ sys/ffmpeg61/avformat_avio.go | 49 ++++++++++------ 7 files changed, 244 insertions(+), 55 deletions(-) create mode 100644 pkg/ffmpeg/metadata.go diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go index 142ab39..36e332e 100644 --- a/pkg/ffmpeg/encoder.go +++ b/pkg/ffmpeg/encoder.go @@ -3,7 +3,6 @@ package ffmpeg import ( "encoding/json" "errors" - "fmt" "io" "syscall" @@ -21,6 +20,9 @@ type Encoder struct { ctx *ff.AVCodecContext stream *ff.AVStream packet *ff.AVPacket + + // We are flushing the encoder + eof bool //next_pts int64 } @@ -28,9 +30,9 @@ type Encoder struct { // return nil to continue encoding or io.EOF to stop encoding. type EncoderFrameFn func(int) (*ff.AVFrame, error) -// EncoderPacketFn is a function which is called for each packet encoded. It should -// return nil to continue encoding or io.EOF to stop encoding immediately. -type EncoderPacketFn func(*ff.AVPacket) error +// EncoderPacketFn is a function which is called for each packet encoded, with +// the stream timebase. +type EncoderPacketFn func(*ff.AVPacket, *ff.AVRational) error //////////////////////////////////////////////////////////////////////////////// // LIFECYCLE @@ -178,8 +180,9 @@ func (e *Encoder) Par() *Par { // PRIVATE METHODS func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { + timebase := e.stream.TimeBase() + // Send the frame to the encoder - fmt.Println("Sending frame", frame) if err := ff.AVCodec_send_frame(e.ctx, frame); err != nil { if errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { return nil @@ -191,7 +194,6 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { var result error for { // Receive the packet - fmt.Println("Receiving packet") if err := ff.AVCodec_receive_packet(e.ctx, e.packet); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { // Finished receiving packet or EOF break @@ -200,7 +202,7 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { } // Pass back to the caller - if err := fn(e.packet); errors.Is(err, io.EOF) { + if err := fn(e.packet, &timebase); errors.Is(err, io.EOF) { // End early, return EOF result = io.EOF break @@ -214,7 +216,7 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { // Flush if result == nil { - result = fn(nil) + result = fn(nil, &timebase) } // Return success or EOF diff --git a/pkg/ffmpeg/metadata.go b/pkg/ffmpeg/metadata.go new file mode 100644 index 0000000..6964b74 --- /dev/null +++ b/pkg/ffmpeg/metadata.go @@ -0,0 +1,35 @@ +package ffmpeg + +import ( + "encoding/json" +) + +//////////////////////////////////////////////////////////////////////////////// +// TYPES + +type Metadata struct { + Key string `json:"key" writer:",width:30"` + Value any `json:"value,omitempty" writer:",wrap,width:50"` +} + +const ( + MetaArtwork = "artwork" // Metadata key for artwork, set the value as []byte +) + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func NewMetadata(key string, value any) *Metadata { + return &Metadata{ + Key: key, + Value: value, + } +} + +//////////////////////////////////////////////////////////////////////////////// +// STRINIGY + +func (m *Metadata) String() string { + data, _ := json.MarshalIndent(m, "", " ") + return string(data) +} diff --git a/pkg/ffmpeg/opts.go b/pkg/ffmpeg/opts.go index 90b8c63..cfb27fd 100644 --- a/pkg/ffmpeg/opts.go +++ b/pkg/ffmpeg/opts.go @@ -1,10 +1,11 @@ package ffmpeg import ( - // Namespace imports + // Package imports + ffmpeg "github.com/mutablelogic/go-media/sys/ffmpeg61" + // Namespace imports . "github.com/djthorpe/go-errors" - ffmpeg "github.com/mutablelogic/go-media/sys/ffmpeg61" ) //////////////////////////////////////////////////////////////////////////////// @@ -17,11 +18,10 @@ type opts struct { force bool par *Par - // Format options - oformat *ffmpeg.AVOutputFormat - - // Stream options - streams map[int]*Par + // Writer options + oformat *ffmpeg.AVOutputFormat + streams map[int]*Par + metadata []*Metadata } //////////////////////////////////////////////////////////////////////////////// @@ -103,6 +103,14 @@ func OptForce() Opt { } } +// Append metadata to the output file, including artwork +func OptMetadata(entry ...*Metadata) Opt { + return func(o *opts) error { + o.metadata = append(o.metadata, entry...) + return nil + } +} + // Pixel format of the output frame func OptPixFormat(format string) Opt { return func(o *opts) error { diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index 2987c28..02c8b4e 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -39,6 +39,40 @@ const ( ////////////////////////////////////////////////////////////////////////////// // LIFECYCLE +// Create a new writer with a URL and options +func Create(url string, opt ...Opt) (*Writer, error) { + options := newOpts() + writer := new(Writer) + + // Apply options + for _, opt := range opt { + if err := opt(options); err != nil { + return nil, err + } + } + + // Guess the output format + var ofmt *ff.AVOutputFormat + if options.oformat == nil && url != "" { + options.oformat = ff.AVFormat_guess_format("", url, "") + } + if options.oformat == nil { + return nil, ErrBadParameter.With("unable to guess the output format") + } + + // Allocate the output media context + ctx, err := ff.AVFormat_create_file(url, ofmt) + if err != nil { + return nil, err + } else { + writer.output = ctx + } + + // Continue with open + return writer.open(options) +} + +// Create a new writer with an io.Writer and options func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { options := newOpts() writer := new(Writer) @@ -68,6 +102,11 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { writer.output = ctx } + // Continue with open + return writer.open(options) +} + +func (writer *Writer) open(options *opts) (*Writer, error) { // Create codec contexts for each stream var result error keys := sort.IntSlice(maps.Keys(options.streams)) @@ -86,7 +125,25 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { return nil, errors.Join(result, writer.Close()) } - // Write the header + // Add metadata + metadata := ff.AVUtil_dict_alloc() + if metadata == nil { + return nil, errors.Join(errors.New("unable to allocate metadata dictionary"), writer.Close()) + } + for _, entry := range options.metadata { + // Ignore artwork fields + if entry.Key == MetaArtwork || entry.Key == "" || entry.Value == nil { + continue + } + // Set dictionary entry + if err := ff.AVUtil_dict_set(metadata, entry.Key, fmt.Sprint(entry.Value), ff.AV_DICT_APPEND); err != nil { + return nil, errors.Join(err, writer.Close()) + } + } + + // Set metadata, write the header + // Metadata ownership is transferred to the output context + writer.output.SetMetadata(metadata) if err := ff.AVFormat_write_header(writer.output, nil); err != nil { return nil, errors.Join(err, writer.Close()) } else { @@ -97,6 +154,7 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { return writer, nil } +// Close a writer and release resources func (w *Writer) Close() error { var result error @@ -148,7 +206,9 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { } if out == nil { // By default, write packet to output - out = w.Write + out = func(pkt *ff.AVPacket, tb *ff.AVRational) error { + return w.Write(pkt) + } } // Initialise encoders @@ -159,29 +219,41 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { return ErrBadParameter.Withf("duplicate stream %v", stream) } encoders[stream] = encoder + + // Initialize the encoder + encoder.eof = false } - // Continue until all encoders have returned io.EOF + // Continue until all encoders have returned io.EOF and have been flushed for { // No more encoding to do if len(encoders) == 0 { break } + + // TODO: We get the encoder with the lowest timestamp for stream, encoder := range encoders { - // Receive a frame for the encoder - frame, err := in(stream) - if errors.Is(err, io.EOF) { - fmt.Println("EOF for frame on stream", stream) - delete(encoders, stream) - } else if err != nil { + var frame *ff.AVFrame + var err error + + // Receive a frame if not EOF + if !encoder.eof { + frame, err = in(stream) + if errors.Is(err, io.EOF) { + encoder.eof = true + } else if err != nil { + return fmt.Errorf("stream %v: %w", stream, err) + } + } + + // Send a frame for encoding + if err := encoder.Encode(frame, out); err != nil { return fmt.Errorf("stream %v: %w", stream, err) - } else if frame == nil { - return fmt.Errorf("stream %v: nil frame received", stream) - } else if err := encoder.Encode(frame, out); errors.Is(err, io.EOF) { - fmt.Println("EOF for packet on stream", stream) + } + + // If eof then delete the encoder + if encoder.eof { delete(encoders, stream) - } else if err != nil { - return fmt.Errorf("stream %v: %w", stream, err) } } } @@ -190,7 +262,8 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { return nil } -// Write a packet to the output +// Write a packet to the output. If you intercept the packets in the +// Encode method, then you can use this method to write packets to the output. func (w *Writer) Write(packet *ff.AVPacket) error { return ff.AVCodec_interleaved_write_frame(w.output, packet) } diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index e421574..730631e 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -25,6 +25,7 @@ func Test_writer_001(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.NewWriter(w, ffmpeg.OptOutputFormat(w.Name()), + ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), ) if !assert.NoError(err) { @@ -40,21 +41,66 @@ func Test_writer_001(t *testing.T) { defer audio.Close() // Write frames - n := 0 + duration := 1000 * time.Minute assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { frame := audio.Frame() - t.Log("Frame s", frame.Time().Truncate(time.Millisecond)) - if frame.Time() > 10*time.Second { + if frame.Time() >= duration { return nil, io.EOF } else { + t.Log("Frame s", frame.Time().Truncate(time.Millisecond)) return frame.(*ffmpeg.Frame).AVFrame(), nil } - }, func(packet *ff.AVPacket) error { + }, func(packet *ff.AVPacket, timebase *ff.AVRational) error { if packet != nil { - t.Log("Packet ts", packet.Pts()) - n += packet.Size() + t.Log("Packet", packet) } return writer.Write(packet) })) - t.Log("Written", n, "bytes to", w.Name()) + t.Log("Written to", w.Name()) +} + +func Test_writer_002(t *testing.T) { + assert := assert.New(t) + + // Write to a file + w, err := os.CreateTemp("", t.Name()+"_*.mp3") + if !assert.NoError(err) { + t.FailNow() + } + defer w.Close() + + // Create a writer with an audio stream + writer, err := ffmpeg.Create(w.Name(), + ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), + ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), + ) + if !assert.NoError(err) { + t.FailNow() + } + defer writer.Close() + + // Make an audio generator + audio, err := generator.NewSine(440, -5, writer.Stream(1).Par()) + if !assert.NoError(err) { + t.FailNow() + } + defer audio.Close() + + // Write frames + duration := 1000 * time.Minute + assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { + frame := audio.Frame() + if frame.Time() >= duration { + return nil, io.EOF + } else { + t.Log("Frame s", frame.Time().Truncate(time.Millisecond)) + return frame.(*ffmpeg.Frame).AVFrame(), nil + } + }, func(packet *ff.AVPacket, timebase *ff.AVRational) error { + if packet != nil { + t.Log("Packet", packet) + } + return writer.Write(packet) + })) + t.Log("Written to", w.Name()) } diff --git a/sys/ffmpeg61/avformat.go b/sys/ffmpeg61/avformat.go index ce8cada..911015a 100644 --- a/sys/ffmpeg61/avformat.go +++ b/sys/ffmpeg61/avformat.go @@ -119,6 +119,7 @@ type jsonAVFormatContext struct { BitRate int64 `json:"bit_rate,omitempty"` PacketSize uint `json:"packet_size,omitempty"` Flags AVFormatFlag `json:"flags,omitempty"` + Metadata *AVDictionary `json:"metadata,omitempty"` } func (ctx *AVFormatContext) MarshalJSON() ([]byte, error) { @@ -134,6 +135,7 @@ func (ctx *AVFormatContext) MarshalJSON() ([]byte, error) { BitRate: int64(ctx.bit_rate), PacketSize: uint(ctx.packet_size), Flags: AVFormatFlag(ctx.flags), + Metadata: ctx.Metadata(), }) } @@ -229,6 +231,14 @@ func (ctx *AVFormatContext) Metadata() *AVDictionary { return &AVDictionary{ctx.metadata} } +func (ctx *AVFormatContext) SetMetadata(dict *AVDictionary) { + if dict == nil { + ctx.metadata = nil + } else { + ctx.metadata = dict.ctx + } +} + func (ctx *AVFormatContext) SetPb(pb *AVIOContextEx) { if pb == nil { ctx.pb = nil diff --git a/sys/ffmpeg61/avformat_avio.go b/sys/ffmpeg61/avformat_avio.go index d2dbe42..f1a8e6c 100644 --- a/sys/ffmpeg61/avformat_avio.go +++ b/sys/ffmpeg61/avformat_avio.go @@ -1,7 +1,7 @@ package ffmpeg import ( - "runtime" + "fmt" "unsafe" ) @@ -21,7 +21,7 @@ static AVIOContext* avio_alloc_context_(int sz, int writeable, void* userInfo) { if (!buf) { return NULL; } - return avio_alloc_context(buf, sz, writeable, userInfo,avio_read_callback,avio_write_callback,avio_seek_callback); + return avio_alloc_context(buf, sz, writeable, userInfo, avio_read_callback, avio_write_callback, avio_seek_callback); } */ import "C" @@ -32,8 +32,6 @@ import "C" // Wrapper around AVIOContext with callbacks type AVIOContextEx struct { *AVIOContext - cb AVIOContextCallback - pin *runtime.Pinner } // Callbacks for AVIOContextEx @@ -43,6 +41,10 @@ type AVIOContextCallback interface { Seeker(offset int64, whence int) int64 } +var ( + callbacks = make(map[uintptr]AVIOContextCallback) +) + //////////////////////////////////////////////////////////////////////////////// // FUNCTIONS @@ -50,10 +52,10 @@ type AVIOContextCallback interface { func AVFormat_avio_alloc_context(sz int, writeable bool, callback AVIOContextCallback) *AVIOContextEx { // Create a context ctx := new(AVIOContextEx) - ctx.cb = callback - ctx.pin = new(runtime.Pinner) - ctx.pin.Pin(ctx.cb) - ctx.pin.Pin(ctx.pin) + + // Set the callback + ptr := uintptr(unsafe.Pointer(ctx)) + callbacks[ptr] = callback // Allocate the context ctx.AVIOContext = (*AVIOContext)(C.avio_alloc_context_( @@ -71,8 +73,6 @@ func AVFormat_avio_alloc_context(sz int, writeable bool, callback AVIOContextCal // Create and initialize a AVIOContext for accessing the resource indicated by url. func AVFormat_avio_open(url string, flags AVIOFlag) (*AVIOContextEx, error) { ctx := new(AVIOContextEx) - ctx.pin = new(runtime.Pinner) - ctx.pin.Pin(ctx.pin) cUrl := C.CString(url) defer C.free(unsafe.Pointer(cUrl)) if err := AVError(C.avio_open((**C.struct_AVIOContext)(unsafe.Pointer(&ctx.AVIOContext)), cUrl, C.int(flags))); err != 0 { @@ -99,7 +99,10 @@ func AVFormat_avio_close(ctx *AVIOContextEx) error { func AVFormat_avio_context_free(ctx *AVIOContextEx) { C.av_free(unsafe.Pointer(ctx.buffer)) C.avio_context_free((**C.struct_AVIOContext)(unsafe.Pointer(&ctx.AVIOContext))) - ctx.pin.Unpin() + + // Remove the callback + ptr := uintptr(unsafe.Pointer(ctx)) + delete(callbacks, ptr) } // avio_w8 @@ -145,18 +148,30 @@ func AVFormat_avio_read(ctx *AVIOContextEx, buf []byte) int { //export avio_read_callback func avio_read_callback(userInfo unsafe.Pointer, buf *C.uint8_t, size C.int) C.int { - ctx := (*AVIOContextEx)(userInfo) - return C.int(ctx.cb.Reader(cByteSlice(unsafe.Pointer(buf), size))) + ptr := uintptr(userInfo) + callback, ok := callbacks[ptr] + if !ok { + panic("avio_read_callback: callback not found") + } + return C.int(callback.Reader(cByteSlice(unsafe.Pointer(buf), size))) } //export avio_write_callback func avio_write_callback(userInfo unsafe.Pointer, buf *C.uint8_t, size C.int) C.int { - ctx := (*AVIOContextEx)(userInfo) - return C.int(ctx.cb.Writer(cByteSlice(unsafe.Pointer(buf), size))) + ptr := uintptr(userInfo) + callback, ok := callbacks[ptr] + if !ok { + panic("avio_write_callback: callback not found " + fmt.Sprint(ptr)) + } + return C.int(callback.Writer(cByteSlice(unsafe.Pointer(buf), size))) } //export avio_seek_callback func avio_seek_callback(userInfo unsafe.Pointer, offset C.int64_t, whence C.int) C.int64_t { - ctx := (*AVIOContextEx)(userInfo) - return C.int64_t(ctx.cb.Seeker(int64(offset), int(whence))) + ptr := uintptr(userInfo) + callback, ok := callbacks[ptr] + if !ok { + panic("avio_seek_callback: callback not found") + } + return C.int64_t(callback.Seeker(int64(offset), int(whence))) } From 80efb245875ac10761f4d8df1086098dbd80a17d Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 15:11:44 +0200 Subject: [PATCH 08/12] Updated --- pkg/ffmpeg/opts.go | 30 ++----------- pkg/ffmpeg/par.go | 69 +++++++++++++++++++++++++++--- pkg/ffmpeg/par_test.go | 2 +- pkg/ffmpeg/rescaler_test.go | 6 +-- pkg/ffmpeg/writer.go | 14 ++++-- pkg/ffmpeg/writer_test.go | 58 ++++++++++++++++++++++--- pkg/generator/yuv420p.go | 18 ++++---- pkg/generator/yuv420p_test.go | 7 +-- sys/ffmpeg61/avcodec.go | 58 +++++++++++++++---------- sys/ffmpeg61/avcodec_parameters.go | 10 +++++ sys/ffmpeg61/avutil_rational.go | 10 +++++ 11 files changed, 202 insertions(+), 80 deletions(-) diff --git a/pkg/ffmpeg/opts.go b/pkg/ffmpeg/opts.go index cfb27fd..dbe1660 100644 --- a/pkg/ffmpeg/opts.go +++ b/pkg/ffmpeg/opts.go @@ -50,33 +50,11 @@ func OptOutputFormat(name string) Opt { } } -// New audio stream with parameters -func OptAudioStream(stream int, par *Par) Opt { +// New stream with parameters +func OptStream(stream int, par *Par) Opt { return func(o *opts) error { - if par == nil || par.CodecType() != ffmpeg.AVMEDIA_TYPE_AUDIO { - return ErrBadParameter.With("invalid audio parameters") - } - if stream == 0 { - stream = len(o.streams) + 1 - } - if _, exists := o.streams[stream]; exists { - return ErrDuplicateEntry.Withf("stream %v", stream) - } - if stream < 0 { - return ErrBadParameter.Withf("invalid stream %v", stream) - } - o.streams[stream] = par - - // Return success - return nil - } -} - -// New video stream with parameters -func OptVideoStream(stream int, par *Par) Opt { - return func(o *opts) error { - if par == nil || par.CodecType() != ffmpeg.AVMEDIA_TYPE_VIDEO { - return ErrBadParameter.With("invalid video parameters") + if par == nil { + return ErrBadParameter.With("invalid parameters") } if stream == 0 { stream = len(o.streams) + 1 diff --git a/pkg/ffmpeg/par.go b/pkg/ffmpeg/par.go index 3149feb..99ed6a1 100644 --- a/pkg/ffmpeg/par.go +++ b/pkg/ffmpeg/par.go @@ -2,9 +2,9 @@ package ffmpeg import ( "encoding/json" - "fmt" "slices" + // Packages ff "github.com/mutablelogic/go-media/sys/ffmpeg61" // Namespace imports @@ -51,7 +51,7 @@ func NewAudioPar(samplefmt string, channellayout string, samplerate int) (*Par, return par, nil } -func NewVideoPar(pixfmt string, size string) (*Par, error) { +func NewVideoPar(pixfmt string, size string, framerate float64) (*Par, error) { par := new(Par) par.SetCodecType(ff.AVMEDIA_TYPE_VIDEO) @@ -70,6 +70,13 @@ func NewVideoPar(pixfmt string, size string) (*Par, error) { par.SetHeight(h) } + // Frame rate + if framerate <= 0 { + return nil, ErrBadParameter.Withf("negative or zero framerate %v", framerate) + } else { + par.SetFramerate(ff.AVUtil_rational_d2q(framerate, 1<<24)) + } + // Set default sample aspect ratio par.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) @@ -85,8 +92,8 @@ func AudioPar(samplefmt string, channellayout string, samplerate int) *Par { } } -func VideoPar(pixfmt string, size string) *Par { - if par, err := NewVideoPar(pixfmt, size); err != nil { +func VideoPar(pixfmt string, size string, framerate float64) *Par { + if par, err := NewVideoPar(pixfmt, size, framerate); err != nil { panic(err) } else { return par @@ -199,11 +206,61 @@ func (ctx *Par) validateAudioCodec(codec *ff.AVCodecContext) error { } func (ctx *Par) copyVideoCodec(codec *ff.AVCodecContext) error { - fmt.Println("TODO: copyVideoCodec") + codec.SetPixFmt(ctx.PixelFormat()) + codec.SetWidth(ctx.Width()) + codec.SetHeight(ctx.Height()) + codec.SetSampleAspectRatio(ctx.SampleAspectRatio()) + codec.SetFramerate(ctx.Framerate()) + codec.SetTimeBase(ff.AVUtil_rational_invert(ctx.Framerate())) return nil } func (ctx *Par) validateVideoCodec(codec *ff.AVCodecContext) error { - fmt.Println("TODO: validateVideoCodec") + pixelformats := codec.Codec().PixelFormats() + framerates := codec.Codec().SupportedFramerates() + + // First we set params from the codec which are not already set + if ctx.PixelFormat() == ff.AV_PIX_FMT_NONE { + if len(pixelformats) > 0 { + ctx.SetPixelFormat(pixelformats[0]) + } + } + if ctx.Framerate().Num() == 0 || ctx.Framerate().Den() == 0 { + if len(framerates) > 0 { + ctx.SetFramerate(framerates[0]) + } + } + + // Then we check to make sure the parameters are compatible with + // the codec + if len(pixelformats) > 0 { + if !slices.Contains(pixelformats, ctx.PixelFormat()) { + return ErrBadParameter.Withf("unsupported pixel format %v", ctx.PixelFormat()) + } + } else if ctx.PixelFormat() == ff.AV_PIX_FMT_NONE { + return ErrBadParameter.With("pixel format not set") + } + if ctx.Width() == 0 || ctx.Height() == 0 { + return ErrBadParameter.Withf("invalid width %v or height %v", ctx.Width(), ctx.Height()) + } + if ctx.SampleAspectRatio().Num() == 0 || ctx.SampleAspectRatio().Den() == 0 { + ctx.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + } + if ctx.Framerate().Num() == 0 || ctx.Framerate().Den() == 0 { + return ErrBadParameter.With("framerate not set") + } else if len(framerates) > 0 { + valid := false + for _, fr := range framerates { + if ff.AVUtil_rational_equal(fr, ctx.Framerate()) { + valid = true + break + } + } + if !valid { + return ErrBadParameter.Withf("unsupported framerate %v", ctx.Framerate()) + } + } + + // Return success return nil } diff --git a/pkg/ffmpeg/par_test.go b/pkg/ffmpeg/par_test.go index 4f8d88a..52a06aa 100644 --- a/pkg/ffmpeg/par_test.go +++ b/pkg/ffmpeg/par_test.go @@ -20,7 +20,7 @@ func Test_par_001(t *testing.T) { func Test_par_002(t *testing.T) { assert := assert.New(t) - par, err := ffmpeg.NewVideoPar("yuv420p", "1280x720") + par, err := ffmpeg.NewVideoPar("yuv420p", "1280x720", 25) if !assert.NoError(err) { t.FailNow() } diff --git a/pkg/ffmpeg/rescaler_test.go b/pkg/ffmpeg/rescaler_test.go index d7e4074..577836f 100644 --- a/pkg/ffmpeg/rescaler_test.go +++ b/pkg/ffmpeg/rescaler_test.go @@ -17,7 +17,7 @@ func Test_rescaler_001(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P("vga", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuv420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } @@ -53,7 +53,7 @@ func Test_rescaler_002(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P("vga", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } @@ -120,7 +120,5 @@ func Test_rescaler_002(t *testing.T) { t.FailNow() } t.Logf("Wrote %s", tmpfile) - } - } diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index 02c8b4e..3eaf0d3 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -84,12 +84,20 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { } } - // Check output + // Try once more to get the output format var filename string + if options.oformat == nil { + if w_, ok := w.(*os.File); ok { + filename = w_.Name() + if err := OptOutputFormat(filename)(options); err != nil { + return nil, err + } + } + } + + // Bail out if options.oformat == nil { return nil, ErrBadParameter.Withf("invalid output format") - } else if w_, ok := w.(*os.File); ok { - filename = w_.Name() } // Allocate the AVIO context diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index 730631e..2197bbf 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -26,7 +26,7 @@ func Test_writer_001(t *testing.T) { writer, err := ffmpeg.NewWriter(w, ffmpeg.OptOutputFormat(w.Name()), ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), - ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), + ffmpeg.OptStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), ) if !assert.NoError(err) { t.FailNow() @@ -40,8 +40,8 @@ func Test_writer_001(t *testing.T) { } defer audio.Close() - // Write frames - duration := 1000 * time.Minute + // Write 15 mins of frames + duration := 60 * time.Minute assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { frame := audio.Frame() if frame.Time() >= duration { @@ -72,7 +72,7 @@ func Test_writer_002(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.Create(w.Name(), ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), - ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), + ffmpeg.OptStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), ) if !assert.NoError(err) { t.FailNow() @@ -86,8 +86,8 @@ func Test_writer_002(t *testing.T) { } defer audio.Close() - // Write frames - duration := 1000 * time.Minute + // Write 15 mins of frames + duration := 15 * time.Minute assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { frame := audio.Frame() if frame.Time() >= duration { @@ -104,3 +104,49 @@ func Test_writer_002(t *testing.T) { })) t.Log("Written to", w.Name()) } + +func Test_writer_003(t *testing.T) { + assert := assert.New(t) + + // Write to a file + w, err := os.CreateTemp("", t.Name()+"_*.ts") + if !assert.NoError(err) { + t.FailNow() + } + defer w.Close() + + // Create a writer with an audio stream + writer, err := ffmpeg.NewWriter(w, + ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), + ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "1280x720", 25)), + ) + if !assert.NoError(err) { + t.FailNow() + } + defer writer.Close() + + // Make an video generator + video, err := generator.NewYUV420P(25, writer.Stream(1).Par()) + if !assert.NoError(err) { + t.FailNow() + } + defer video.Close() + + // Write 1 min of frames + duration := time.Minute + assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { + frame := video.Frame() + if frame.Time() >= duration { + return nil, io.EOF + } else { + t.Log("Frame", frame.Time().Truncate(time.Millisecond)) + return frame.(*ffmpeg.Frame).AVFrame(), nil + } + }, func(packet *ff.AVPacket, timebase *ff.AVRational) error { + if packet != nil { + t.Log("Packet", packet) + } + return writer.Write(packet) + })) + t.Log("Written to", w.Name()) +} diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go index 3ad5556..16a9d26 100644 --- a/pkg/generator/yuv420p.go +++ b/pkg/generator/yuv420p.go @@ -24,16 +24,18 @@ var _ Generator = (*yuv420p)(nil) // Create a new video generator which generates YUV420P frames // of the specified size and framerate (in frames per second) -func NewYUV420P(size string, framerate int) (*yuv420p, error) { +func NewYUV420P(framerate int, par *ffmpeg.Par) (*yuv420p, error) { yuv420p := new(yuv420p) // Check parameters if framerate <= 0 { return nil, errors.New("invalid framerate") } - w, h, err := ff.AVUtil_parse_video_size(size) - if err != nil { - return nil, err + // Check parameters + if par.CodecType() != ff.AVMEDIA_TYPE_VIDEO { + return nil, errors.New("invalid codec type") + } else if par.PixelFormat() != ff.AV_PIX_FMT_YUV420P { + return nil, errors.New("invalid pixel format, only yuv420p is supported") } // Create a frame @@ -42,10 +44,10 @@ func NewYUV420P(size string, framerate int) (*yuv420p, error) { return nil, errors.New("failed to allocate frame") } - frame.SetPixFmt(ff.AV_PIX_FMT_YUV420P) - frame.SetWidth(w) - frame.SetHeight(h) - frame.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + frame.SetPixFmt(par.PixelFormat()) + frame.SetWidth(par.Width()) + frame.SetHeight(par.Height()) + frame.SetSampleAspectRatio(par.SampleAspectRatio()) frame.SetTimeBase(ff.AVUtil_rational(1, framerate)) frame.SetPts(ff.AV_NOPTS_VALUE) diff --git a/pkg/generator/yuv420p_test.go b/pkg/generator/yuv420p_test.go index c76a590..8d037d1 100644 --- a/pkg/generator/yuv420p_test.go +++ b/pkg/generator/yuv420p_test.go @@ -7,13 +7,14 @@ import ( "path/filepath" "testing" + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" "github.com/mutablelogic/go-media/pkg/generator" "github.com/stretchr/testify/assert" ) func Test_yuv420p_001(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P("1024x768", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) if !assert.NoError(err) { t.FailNow() } @@ -24,7 +25,7 @@ func Test_yuv420p_001(t *testing.T) { func Test_yuv420p_002(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P("vga", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) if !assert.NoError(err) { t.FailNow() } @@ -38,7 +39,7 @@ func Test_yuv420p_002(t *testing.T) { func Test_yuv420p_003(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P("vga", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) if !assert.NoError(err) { t.FailNow() } diff --git a/sys/ffmpeg61/avcodec.go b/sys/ffmpeg61/avcodec.go index d4e6def..3e7f2c9 100644 --- a/sys/ffmpeg61/avcodec.go +++ b/sys/ffmpeg61/avcodec.go @@ -49,18 +49,20 @@ type jsonAVCodec struct { } type jsonAVCodecContext struct { - CodecType AVMediaType `json:"codec_type,omitempty"` - Codec *AVCodec `json:"codec,omitempty"` - BitRate int64 `json:"bit_rate,omitempty"` - BitRateTolerance int `json:"bit_rate_tolerance,omitempty"` - PixelFormat AVPixelFormat `json:"pix_fmt,omitempty"` - Width int `json:"width,omitempty"` - Height int `json:"height,omitempty"` - SampleFormat AVSampleFormat `json:"sample_fmt,omitempty"` - SampleRate int `json:"sample_rate,omitempty"` - ChannelLayout AVChannelLayout `json:"channel_layout,omitempty"` - FrameSize int `json:"frame_size,omitempty"` - TimeBase AVRational `json:"time_base,omitempty"` + CodecType AVMediaType `json:"codec_type,omitempty"` + Codec *AVCodec `json:"codec,omitempty"` + BitRate int64 `json:"bit_rate,omitempty"` + BitRateTolerance int `json:"bit_rate_tolerance,omitempty"` + PixelFormat AVPixelFormat `json:"pix_fmt,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + SampleAspectRatio AVRational `json:"sample_aspect_ratio,omitempty"` + Framerate AVRational `json:"framerate,omitempty"` + SampleFormat AVSampleFormat `json:"sample_fmt,omitempty"` + SampleRate int `json:"sample_rate,omitempty"` + ChannelLayout AVChannelLayout `json:"channel_layout,omitempty"` + FrameSize int `json:"frame_size,omitempty"` + TimeBase AVRational `json:"time_base,omitempty"` } //////////////////////////////////////////////////////////////////////////////// @@ -175,13 +177,15 @@ func (ctx *AVCodecContext) MarshalJSON() ([]byte, error) { switch ctx.codec_type { case C.AVMEDIA_TYPE_VIDEO: return json.Marshal(jsonAVCodecContext{ - CodecType: AVMediaType(ctx.codec_type), - Codec: (*AVCodec)(ctx.codec), - BitRate: int64(ctx.bit_rate), - BitRateTolerance: int(ctx.bit_rate_tolerance), - PixelFormat: AVPixelFormat(ctx.pix_fmt), - Width: int(ctx.width), - Height: int(ctx.height), + CodecType: AVMediaType(ctx.codec_type), + Codec: (*AVCodec)(ctx.codec), + BitRate: int64(ctx.bit_rate), + BitRateTolerance: int(ctx.bit_rate_tolerance), + PixelFormat: AVPixelFormat(ctx.pix_fmt), + Width: int(ctx.width), + Height: int(ctx.height), + SampleAspectRatio: AVRational(ctx.sample_aspect_ratio), + Framerate: AVRational(ctx.framerate), }) case C.AVMEDIA_TYPE_AUDIO: return json.Marshal(jsonAVCodecContext{ @@ -405,12 +409,12 @@ func (ctx *AVCodecContext) SetHeight(height int) { ctx.height = C.int(height) } -func (ctx *AVCodecContext) TimeBase() AVRational { - return (AVRational)(ctx.time_base) +func (ctx *AVCodecContext) SampleAspectRatio() AVRational { + return (AVRational)(ctx.sample_aspect_ratio) } -func (ctx *AVCodecContext) SetTimeBase(time_base AVRational) { - ctx.time_base = C.struct_AVRational(time_base) +func (ctx *AVCodecContext) SetSampleAspectRatio(sample_aspect_ratio AVRational) { + ctx.sample_aspect_ratio = C.struct_AVRational(sample_aspect_ratio) } func (ctx *AVCodecContext) Framerate() AVRational { @@ -421,6 +425,14 @@ func (ctx *AVCodecContext) SetFramerate(framerate AVRational) { ctx.framerate = C.struct_AVRational(framerate) } +func (ctx *AVCodecContext) TimeBase() AVRational { + return (AVRational)(ctx.time_base) +} + +func (ctx *AVCodecContext) SetTimeBase(time_base AVRational) { + ctx.time_base = C.struct_AVRational(time_base) +} + // Audio sample format. func (ctx *AVCodecContext) SampleFormat() AVSampleFormat { return AVSampleFormat(ctx.sample_fmt) diff --git a/sys/ffmpeg61/avcodec_parameters.go b/sys/ffmpeg61/avcodec_parameters.go index 833c303..917fcc3 100644 --- a/sys/ffmpeg61/avcodec_parameters.go +++ b/sys/ffmpeg61/avcodec_parameters.go @@ -30,6 +30,7 @@ type jsonAVCodecParameterVideo struct { Width int `json:"width"` Height int `json:"height"` SampleAspectRatio AVRational `json:"sample_aspect_ratio,omitempty"` + Framerate AVRational `json:"framerate,omitempty"` } type jsonAVCodecParameters struct { @@ -65,6 +66,7 @@ func (ctx AVCodecParameters) MarshalJSON() ([]byte, error) { Width: int(ctx.width), Height: int(ctx.height), SampleAspectRatio: AVRational(ctx.sample_aspect_ratio), + Framerate: AVRational(ctx.framerate), } } @@ -179,6 +181,14 @@ func (ctx *AVCodecParameters) SetSampleAspectRatio(aspect AVRational) { ctx.sample_aspect_ratio = C.AVRational(aspect) } +func (ctx *AVCodecParameters) Framerate() AVRational { + return AVRational(ctx.framerate) +} + +func (ctx *AVCodecParameters) SetFramerate(rate AVRational) { + ctx.framerate = C.AVRational(rate) +} + // Video func (ctx *AVCodecParameters) Width() int { return int(ctx.width) diff --git a/sys/ffmpeg61/avutil_rational.go b/sys/ffmpeg61/avutil_rational.go index e5dae3d..590b2fa 100644 --- a/sys/ffmpeg61/avutil_rational.go +++ b/sys/ffmpeg61/avutil_rational.go @@ -72,3 +72,13 @@ func AVUtil_rational_d2q(d float64, max int) AVRational { func AVUtil_rational_q2d(a AVRational) float64 { return float64(C.av_q2d(C.AVRational(a))) } + +// Compare two rationals. +func AVUtil_rational_equal(a, b AVRational) bool { + return C.av_cmp_q(C.AVRational(a), C.AVRational(b)) == 0 +} + +// Invert a rational. +func AVUtil_rational_invert(q AVRational) AVRational { + return AVRational(C.av_inv_q(C.AVRational(q))) +} From 61ee5594dda1dbfb14188d5fb04612eabd09c479 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 15:48:43 +0200 Subject: [PATCH 09/12] Updated encoder --- pkg/ffmpeg/encoder.go | 7 ++++--- pkg/ffmpeg/par.go | 15 +++++++++++++++ sys/ffmpeg61/avcodec_packet.go | 4 ++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go index 36e332e..39637ac 100644 --- a/pkg/ffmpeg/encoder.go +++ b/pkg/ffmpeg/encoder.go @@ -184,9 +184,6 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { // Send the frame to the encoder if err := ff.AVCodec_send_frame(e.ctx, frame); err != nil { - if errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { - return nil - } return err } @@ -201,6 +198,10 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { return err } + // rescale output packet timestamp values from codec to stream timebase + ff.AVCodec_packet_rescale_ts(e.packet, e.ctx.TimeBase(), e.stream.TimeBase()) + e.packet.SetStreamIndex(e.stream.Index()) + // Pass back to the caller if err := fn(e.packet, &timebase); errors.Is(err, io.EOF) { // End early, return EOF diff --git a/pkg/ffmpeg/par.go b/pkg/ffmpeg/par.go index 99ed6a1..203c443 100644 --- a/pkg/ffmpeg/par.go +++ b/pkg/ffmpeg/par.go @@ -80,6 +80,21 @@ func NewVideoPar(pixfmt string, size string, framerate float64) (*Par, error) { // Set default sample aspect ratio par.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + /* TODO + c->gop_size = 12; // emit one intra frame every twelve frames at most + c->pix_fmt = STREAM_PIX_FMT; + if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) { + // just for testing, we also add B-frames + c->max_b_frames = 2; + } + if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) { + // Needed to avoid using macroblocks in which some coeffs overflow. + // This does not happen with normal video, it just happens here as + // the motion of the chroma plane does not match the luma plane. + c->mb_decision = 2; + } + */ + // Return success return par, nil } diff --git a/sys/ffmpeg61/avcodec_packet.go b/sys/ffmpeg61/avcodec_packet.go index c49f734..b7d013f 100644 --- a/sys/ffmpeg61/avcodec_packet.go +++ b/sys/ffmpeg61/avcodec_packet.go @@ -110,6 +110,10 @@ func (ctx *AVPacket) StreamIndex() int { return int(ctx.stream_index) } +func (ctx *AVPacket) SetStreamIndex(index int) { + ctx.stream_index = C.int(index) +} + func (ctx *AVPacket) Pts() int64 { return int64(ctx.pts) } From 9c2694f9c6ecc0c2bc2960120c0c7ebfed31fee1 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 17:57:10 +0200 Subject: [PATCH 10/12] Added encoding --- pkg/ffmpeg/encoder.go | 34 +++++++++++++---- pkg/ffmpeg/rescaler_test.go | 4 +- pkg/ffmpeg/writer.go | 55 ++++++++++++++++++---------- pkg/ffmpeg/writer_test.go | 65 +++++++++++++++++++++++++++++++-- pkg/generator/yuv420p.go | 12 +++--- sys/ffmpeg61/avcodec_packet.go | 26 +++++++++---- sys/ffmpeg61/avutil_math.go | 22 ----------- sys/ffmpeg61/avutil_rational.go | 16 ++++++++ 8 files changed, 166 insertions(+), 68 deletions(-) delete mode 100644 sys/ffmpeg61/avutil_math.go diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go index 39637ac..521c1e8 100644 --- a/pkg/ffmpeg/encoder.go +++ b/pkg/ffmpeg/encoder.go @@ -3,6 +3,7 @@ package ffmpeg import ( "encoding/json" "errors" + "fmt" "io" "syscall" @@ -23,7 +24,9 @@ type Encoder struct { // We are flushing the encoder eof bool - //next_pts int64 + + // The next presentation timestamp + next_pts int64 } // EncoderFrameFn is a function which is called to receive a frame to encode. It should @@ -85,12 +88,6 @@ func NewEncoder(ctx *ff.AVFormatContext, stream int, par *Par) (*Encoder, error) encoder.stream = streamctx } - // Copy parameters to stream - if err := ff.AVCodec_parameters_from_context(encoder.stream.CodecPar(), encoder.ctx); err != nil { - ff.AVCodec_free_context(encoder.ctx) - return nil, err - } - // Some formats want stream headers to be separate. if ctx.Output().Flags().Is(ff.AVFMT_GLOBALHEADER) { encoder.ctx.SetFlags(encoder.ctx.Flags() | ff.AV_CODEC_FLAG_GLOBAL_HEADER) @@ -102,6 +99,12 @@ func NewEncoder(ctx *ff.AVFormatContext, stream int, par *Par) (*Encoder, error) return nil, ErrInternalAppError.Withf("codec_open: %v", err) } + // Copy parameters to stream + if err := ff.AVCodec_parameters_from_context(encoder.stream.CodecPar(), encoder.ctx); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, err + } + // Create a packet packet := ff.AVCodec_packet_alloc() if packet == nil { @@ -176,6 +179,22 @@ func (e *Encoder) Par() *Par { } } +// Return the codec type +func (e *Encoder) nextPts(frame *ff.AVFrame) int64 { + next_pts := int64(0) + switch e.ctx.Codec().Type() { + case ff.AVMEDIA_TYPE_AUDIO: + next_pts = ff.AVUtil_rational_rescale_q(int64(frame.NumSamples()), ff.AVUtil_rational(1, frame.SampleRate()), e.stream.TimeBase()) + case ff.AVMEDIA_TYPE_VIDEO: + next_pts = ff.AVUtil_rational_rescale_q(1, ff.AVUtil_rational_invert(e.ctx.Framerate()), e.stream.TimeBase()) + default: + // Dunno what to do with subtitle and data streams yet + fmt.Println("TODO: next_pts for subtitle and data streams") + return 0 + } + return next_pts +} + ////////////////////////////////////////////////////////////////////////////// // PRIVATE METHODS @@ -201,6 +220,7 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { // rescale output packet timestamp values from codec to stream timebase ff.AVCodec_packet_rescale_ts(e.packet, e.ctx.TimeBase(), e.stream.TimeBase()) e.packet.SetStreamIndex(e.stream.Index()) + e.packet.SetTimeBase(e.stream.TimeBase()) // Pass back to the caller if err := fn(e.packet, &timebase); errors.Is(err, io.EOF) { diff --git a/pkg/ffmpeg/rescaler_test.go b/pkg/ffmpeg/rescaler_test.go index 577836f..46930e3 100644 --- a/pkg/ffmpeg/rescaler_test.go +++ b/pkg/ffmpeg/rescaler_test.go @@ -17,7 +17,7 @@ func Test_rescaler_001(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuv420p", "1280x720", 25)) + image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuv420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } @@ -53,7 +53,7 @@ func Test_rescaler_002(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720", 25)) + image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuva420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index 3eaf0d3..a1c9eaa 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -230,6 +230,7 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { // Initialize the encoder encoder.eof = false + encoder.next_pts = 0 } // Continue until all encoders have returned io.EOF and have been flushed @@ -239,31 +240,42 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { break } - // TODO: We get the encoder with the lowest timestamp + // Find encoder with the lowest timestamp, based on next_pts and timebase + next_stream := -1 + var next_encoder *Encoder for stream, encoder := range encoders { - var frame *ff.AVFrame - var err error - - // Receive a frame if not EOF - if !encoder.eof { - frame, err = in(stream) - if errors.Is(err, io.EOF) { - encoder.eof = true - } else if err != nil { - return fmt.Errorf("stream %v: %w", stream, err) - } + if next_encoder == nil || compareNextPts(encoder, next_encoder) < 0 { + next_encoder = encoder + next_stream = stream } + } - // Send a frame for encoding - if err := encoder.Encode(frame, out); err != nil { - return fmt.Errorf("stream %v: %w", stream, err) - } + var frame *ff.AVFrame + var err error - // If eof then delete the encoder - if encoder.eof { - delete(encoders, stream) + // Receive a frame if not EOF + if !next_encoder.eof { + frame, err = in(next_stream) + if errors.Is(err, io.EOF) { + next_encoder.eof = true + } else if err != nil { + return fmt.Errorf("stream %v: %w", next_stream, err) } } + + // Send a frame for encoding + if err := next_encoder.Encode(frame, out); err != nil { + return fmt.Errorf("stream %v: %w", next_stream, err) + } + + // If eof then delete the encoder + if next_encoder.eof { + delete(encoders, next_stream) + continue + } + + // Calculate the next PTS + next_encoder.next_pts = next_encoder.next_pts + next_encoder.nextPts(frame) } // Return success @@ -276,6 +288,11 @@ func (w *Writer) Write(packet *ff.AVPacket) error { return ff.AVCodec_interleaved_write_frame(w.output, packet) } +// Returns -1 if a is before v +func compareNextPts(a, b *Encoder) int { + return ff.AVUtil_compare_ts(a.next_pts, a.stream.TimeBase(), b.next_pts, b.stream.TimeBase()) +} + //////////////////////////////////////////////////////////////////////////////// // PRIVATE METHODS - Writer diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index 2197bbf..ed7bc6e 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + media "github.com/mutablelogic/go-media" ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" generator "github.com/mutablelogic/go-media/pkg/generator" ff "github.com/mutablelogic/go-media/sys/ffmpeg61" @@ -118,7 +119,7 @@ func Test_writer_003(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.NewWriter(w, ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), - ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "1280x720", 25)), + ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "640x480", 30)), ) if !assert.NoError(err) { t.FailNow() @@ -126,7 +127,7 @@ func Test_writer_003(t *testing.T) { defer writer.Close() // Make an video generator - video, err := generator.NewYUV420P(25, writer.Stream(1).Par()) + video, err := generator.NewYUV420P(writer.Stream(1).Par()) if !assert.NoError(err) { t.FailNow() } @@ -139,14 +140,70 @@ func Test_writer_003(t *testing.T) { if frame.Time() >= duration { return nil, io.EOF } else { - t.Log("Frame", frame.Time().Truncate(time.Millisecond)) + t.Log("Frame", stream, "=>", frame.Time().Truncate(time.Millisecond)) return frame.(*ffmpeg.Frame).AVFrame(), nil } }, func(packet *ff.AVPacket, timebase *ff.AVRational) error { if packet != nil { - t.Log("Packet", packet) + d := time.Duration(ff.AVUtil_rational_q2d(packet.TimeBase()) * float64(packet.Pts()) * float64(time.Second)) + t.Log("Packet", d.Truncate(time.Millisecond)) } return writer.Write(packet) })) t.Log("Written to", w.Name()) } + +func Test_writer_004(t *testing.T) { + assert := assert.New(t) + + // Write to a file + w, err := os.CreateTemp("", t.Name()+"_*.m4v") + if !assert.NoError(err) { + t.FailNow() + } + defer w.Close() + + // Create a writer with an audio stream + writer, err := ffmpeg.Create(w.Name(), + ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), + ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "640x480", 30)), + ffmpeg.OptStream(2, ffmpeg.AudioPar("fltp", "mono", 22050)), + ) + if !assert.NoError(err) { + t.FailNow() + } + defer writer.Close() + + // Make an video generator + video, err := generator.NewYUV420P(writer.Stream(1).Par()) + if !assert.NoError(err) { + t.FailNow() + } + defer video.Close() + + // Make an audio generator + audio, err := generator.NewSine(440, -5, writer.Stream(2).Par()) + if !assert.NoError(err) { + t.FailNow() + } + + // Write 10 secs of frames + duration := time.Minute * 10 + assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { + var frame media.Frame + switch stream { + case 1: + frame = video.Frame() + case 2: + frame = audio.Frame() + } + if frame.Time() >= duration { + t.Log("Frame time is EOF", frame.Time()) + return nil, io.EOF + } else { + t.Log("Frame", stream, "=>", frame.Time().Truncate(time.Millisecond)) + return frame.(*ffmpeg.Frame).AVFrame(), nil + } + }, nil)) + t.Log("Written to", w.Name()) +} diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go index 16a9d26..b06f87a 100644 --- a/pkg/generator/yuv420p.go +++ b/pkg/generator/yuv420p.go @@ -24,19 +24,19 @@ var _ Generator = (*yuv420p)(nil) // Create a new video generator which generates YUV420P frames // of the specified size and framerate (in frames per second) -func NewYUV420P(framerate int, par *ffmpeg.Par) (*yuv420p, error) { +func NewYUV420P(par *ffmpeg.Par) (*yuv420p, error) { yuv420p := new(yuv420p) - // Check parameters - if framerate <= 0 { - return nil, errors.New("invalid framerate") - } // Check parameters if par.CodecType() != ff.AVMEDIA_TYPE_VIDEO { return nil, errors.New("invalid codec type") } else if par.PixelFormat() != ff.AV_PIX_FMT_YUV420P { return nil, errors.New("invalid pixel format, only yuv420p is supported") } + framerate := ff.AVUtil_rational_q2d(par.Framerate()) + if framerate <= 0 { + return nil, errors.New("invalid framerate") + } // Create a frame frame := ff.AVUtil_frame_alloc() @@ -48,7 +48,7 @@ func NewYUV420P(framerate int, par *ffmpeg.Par) (*yuv420p, error) { frame.SetWidth(par.Width()) frame.SetHeight(par.Height()) frame.SetSampleAspectRatio(par.SampleAspectRatio()) - frame.SetTimeBase(ff.AVUtil_rational(1, framerate)) + frame.SetTimeBase(ff.AVUtil_rational_invert(par.Framerate())) frame.SetPts(ff.AV_NOPTS_VALUE) // Allocate buffer diff --git a/sys/ffmpeg61/avcodec_packet.go b/sys/ffmpeg61/avcodec_packet.go index b7d013f..dcf3536 100644 --- a/sys/ffmpeg61/avcodec_packet.go +++ b/sys/ffmpeg61/avcodec_packet.go @@ -18,14 +18,15 @@ import "C" // TYPES type jsonAVPacket struct { - Pts int64 `json:"pts,omitempty"` - Dts int64 `json:"dts,omitempty"` - Size int `json:"size,omitempty"` - StreamIndex int `json:"stream_index"` // Stream index starts at 0 - Flags int `json:"flags,omitempty"` - SideDataElems int `json:"side_data_elems,omitempty"` - Duration int64 `json:"duration,omitempty"` - Pos int64 `json:"pos,omitempty"` + Pts int64 `json:"pts,omitempty"` + Dts int64 `json:"dts,omitempty"` + Size int `json:"size,omitempty"` + StreamIndex int `json:"stream_index"` // Stream index starts at 0 + Flags int `json:"flags,omitempty"` + SideDataElems int `json:"side_data_elems,omitempty"` + Duration int64 `json:"duration,omitempty"` + TimeBase AVRational `json:"time_base,omitempty"` + Pos int64 `json:"pos,omitempty"` } //////////////////////////////////////////////////////////////////////////////// @@ -40,6 +41,7 @@ func (ctx *AVPacket) MarshalJSON() ([]byte, error) { Flags: int(ctx.flags), SideDataElems: int(ctx.side_data_elems), Duration: int64(ctx.duration), + TimeBase: AVRational(ctx.time_base), Pos: int64(ctx.pos), }) } @@ -114,6 +116,14 @@ func (ctx *AVPacket) SetStreamIndex(index int) { ctx.stream_index = C.int(index) } +func (ctx *AVPacket) TimeBase() AVRational { + return AVRational(ctx.time_base) +} + +func (ctx *AVPacket) SetTimeBase(tb AVRational) { + ctx.time_base = C.AVRational(tb) +} + func (ctx *AVPacket) Pts() int64 { return int64(ctx.pts) } diff --git a/sys/ffmpeg61/avutil_math.go b/sys/ffmpeg61/avutil_math.go deleted file mode 100644 index cfae419..0000000 --- a/sys/ffmpeg61/avutil_math.go +++ /dev/null @@ -1,22 +0,0 @@ -package ffmpeg - -//////////////////////////////////////////////////////////////////////////////// -// CGO - -/* -#cgo pkg-config: libavutil -#include -*/ -import "C" - -//////////////////////////////////////////////////////////////////////////////// - -// Compare two timestamps each in its own time base. Returns -1 if a is before b, 1 if a is after b, or 0 if they are equal. -func AVUtil_compare_ts(a int64, a_tb AVRational, b int64, b_tb AVRational) int { - return int(C.av_compare_ts(C.int64_t(a), C.AVRational(a_tb), C.int64_t(b), C.AVRational(b_tb))) -} - -// Rescale a value from one range to another. -func AVUtil_rescale_rnd(a, b, c int64, rnd AVRounding) int64 { - return int64(C.av_rescale_rnd(C.int64_t(a), C.int64_t(b), C.int64_t(c), C.enum_AVRounding(rnd))) -} diff --git a/sys/ffmpeg61/avutil_rational.go b/sys/ffmpeg61/avutil_rational.go index 590b2fa..c28485d 100644 --- a/sys/ffmpeg61/avutil_rational.go +++ b/sys/ffmpeg61/avutil_rational.go @@ -11,6 +11,7 @@ import ( /* #cgo pkg-config: libavutil #include +#include */ import "C" @@ -82,3 +83,18 @@ func AVUtil_rational_equal(a, b AVRational) bool { func AVUtil_rational_invert(q AVRational) AVRational { return AVRational(C.av_inv_q(C.AVRational(q))) } + +// Resacale a rational +func AVUtil_rational_rescale_q(a int64, bq AVRational, cq AVRational) int64 { + return int64(C.av_rescale_q(C.int64_t(a), C.AVRational(bq), C.AVRational(cq))) +} + +// Rescale a value from one range to another. +func AVUtil_rescale_rnd(a, b, c int64, rnd AVRounding) int64 { + return int64(C.av_rescale_rnd(C.int64_t(a), C.int64_t(b), C.int64_t(c), C.enum_AVRounding(rnd))) +} + +// Compare two timestamps each in its own time base. Returns -1 if a is before b, 1 if a is after b, or 0 if they are equal. +func AVUtil_compare_ts(a int64, a_tb AVRational, b int64, b_tb AVRational) int { + return int(C.av_compare_ts(C.int64_t(a), C.AVRational(a_tb), C.int64_t(b), C.AVRational(b_tb))) +} From cbc6fc2b24bdefef57724ddd1151db9e27de110e Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 19:34:29 +0200 Subject: [PATCH 11/12] Updated --- cmd/ffmpeg/mux/generate.go_old | 113 ------------ cmd/ffmpeg/mux/main.go | 129 ------------- cmd/ffmpeg/mux/stream.go | 256 -------------------------- manager.go | 5 +- writer.go | 320 --------------------------------- writer_test.go | 84 --------- 6 files changed, 3 insertions(+), 904 deletions(-) delete mode 100644 cmd/ffmpeg/mux/generate.go_old delete mode 100644 cmd/ffmpeg/mux/main.go delete mode 100644 cmd/ffmpeg/mux/stream.go delete mode 100644 writer.go delete mode 100644 writer_test.go diff --git a/cmd/ffmpeg/mux/generate.go_old b/cmd/ffmpeg/mux/generate.go_old deleted file mode 100644 index 02657a9..0000000 --- a/cmd/ffmpeg/mux/generate.go_old +++ /dev/null @@ -1,113 +0,0 @@ -package main - -import ( - ff "github.com/mutablelogic/go-media/sys/ffmpeg61" -) - -//////////////////////////////////////////////////////////////////////////////// - - -// Prepare a 16 bit dummy audio frame of 'frame_size' samples and 'nb_channels' channels -func get_audio_frame(stream *Stream) *AVFrame { - AVFrame *frame = stream.tmp_frame - - int j, i, v; - int16_t *q = (int16_t*)frame->data[0]; - - /* check if we want to generate more frames */ - if (av_compare_ts(ost->next_pts, ost->enc->time_base, - STREAM_DURATION, (AVRational){ 1, 1 }) > 0) - return NULL; - - for (j = 0; j nb_samples; j++) { - v = (int)(sin(ost->t) * 10000); - for (i = 0; i < ost->enc->ch_layout.nb_channels; i++) - *q++ = v; - ost->t += ost->tincr; - ost->tincr += ost->tincr2; - } - - frame->pts = ost->next_pts; - ost->next_pts += frame->nb_samples; - - return frame; - } - -/* - * encode one audio frame and send it to the muxer - * return true when encoding is finished - */ -func write_audio_frame(ctx *ff.AVFormatContext, stream *Stream) bool { - frame := get_audio_frame(stream) - if frame != nil { - // convert samples from native format to destination codec format, using the resampler - // compute destination number of samples - delay := ff.SWResample_get_delay(stream.SWRContext(), stream.Encoder().SampleRate()) + frame.NumSamples() - dst_nb_samples := ff.AVUtil_rescale_rnd(delay,ctx.SampleRate(),ctx.SampleRate(),ff.AV_ROUND_UP); - - // When we pass a frame to the encoder, it may keep a reference to it internally; make sure we do not overwrite it here - ff.AVUtil_frame_make_writable(stream.Frame()) - - // Convert to destination format - ff.SWResample_convert_frame(stream.swr_ctx,frame,stream.frame) - - return false - } - frame = ost->frame; - - frame->pts = av_rescale_q(ost->samples_count, (AVRational){1, c->sample_rate}, c->time_base); - ost->samples_count += dst_nb_samples; - } - return write_frame(ctx, stream, frame) -} - -func write_video_frame(ctx *ff.AVFormatContext, stream *Stream) bool { - return true -} - - - -/* - static int write_audio_frame(AVFormatContext *oc, OutputStream *ost) - { - AVCodecContext *c; - AVFrame *frame; - int ret; - int dst_nb_samples; - - c = ost->enc; - - frame = get_audio_frame(ost); - - if (frame) { - /* convert samples from native format to destination codec format, using the resampler - /* compute destination number of samples - dst_nb_samples = av_rescale_rnd(swr_get_delay(ost->swr_ctx, c->sample_rate) + frame->nb_samples, - c->sample_rate, c->sample_rate, AV_ROUND_UP); - av_assert0(dst_nb_samples == frame->nb_samples); - - /* when we pass a frame to the encoder, it may keep a reference to it - * internally; - * make sure we do not overwrite it here - */ - ret = av_frame_make_writable(ost->frame); - if (ret < 0) - exit(1); - - /* convert to destination format - ret = swr_convert(ost->swr_ctx, - ost->frame->data, dst_nb_samples, - (const uint8_t **)frame->data, frame->nb_samples); - if (ret < 0) { - fprintf(stderr, "Error while converting\n"); - exit(1); - } - frame = ost->frame; - - frame->pts = av_rescale_q(ost->samples_count, (AVRational){1, c->sample_rate}, c->time_base); - ost->samples_count += dst_nb_samples; - } - - return write_frame(oc, c, ost->st, frame, ost->tmp_pkt); - } - */ diff --git a/cmd/ffmpeg/mux/main.go b/cmd/ffmpeg/mux/main.go deleted file mode 100644 index 1c10c3b..0000000 --- a/cmd/ffmpeg/mux/main.go +++ /dev/null @@ -1,129 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - - // Packages - ff "github.com/mutablelogic/go-media/sys/ffmpeg61" -) - -const ( - STREAM_DURATION = 10.0 -) - -func main() { - out := flag.String("out", "", "output file") - flag.Parse() - - // Check in and out - if *out == "" { - log.Fatal("-out flag must be specified") - } - - // Allocate the output media context - ctx, err := ff.AVFormat_create_file(*out, nil) - if err != nil { - log.Fatal(err) - } - defer ff.AVFormat_close_writer(ctx) - - // Add the audio and video streams using the default format codecs and initialize the codecs. - var video, audio *Stream - if codec := ctx.Output().VideoCodec(); codec != ff.AV_CODEC_ID_NONE { - if stream, err := NewStream(ctx, codec); err != nil { - log.Fatalf("could not add video stream: %v", err) - } else { - video = stream - } - defer video.Close() - } - if codec := ctx.Output().AudioCodec(); codec != ff.AV_CODEC_ID_NONE { - if stream, err := NewStream(ctx, codec); err != nil { - log.Fatalf("could not add audio stream: %v", err) - } else { - audio = stream - } - defer audio.Close() - } - - // Now that all the parameters are set, we can open the audio - // and video codecs and allocate the necessary encode buffers. - if video != nil { - // TODO: AVDictionary of options - if err := video.Open(nil); err != nil { - log.Fatalf("could not open video codec: %v", err) - } - } - if audio != nil { - // TODO: AVDictionary of options - if err := audio.Open(nil); err != nil { - log.Fatalf("could not open audio codec: %v", err) - } - } - - fmt.Println(ctx) - - // Dump the output format - ff.AVFormat_dump_format(ctx, 0, *out) - - // Open the output file, if needed - if !ctx.Flags().Is(ff.AVFMT_NOFILE) { - w, err := ff.AVFormat_avio_open(*out, ff.AVIO_FLAG_WRITE) - if err != nil { - log.Fatalf("could not open output file: %v", err) - } else { - ctx.SetPb(w) - } - defer ff.AVFormat_avio_close(w) - } - - // Write the stream header, if any - // TODO: AVDictionary of options - if err := ff.AVFormat_write_header(ctx, nil); err != nil { - log.Fatalf("could not write header: %v", err) - } - - // TODO Write data - encode_audio, encode_video := true, true - for encode_audio || encode_video { - // Choose video if both are available, and video is earlier than audio - if (encode_video && !encode_audio) || (encode_video && ff.AVUtil_compare_ts(video.next_pts, video.Encoder.TimeBase(), audio.next_pts, audio.Encoder.TimeBase()) <= 0) { - fmt.Println("TODO: Write video frame") - encode_video = false - // encode_video = !write_video_frame(ctx, video) - } else { - fmt.Println("TODO: Write audio frame") - encode_audio = false - // encode_audio = !write_audio_frame(ctx, audio) - } - } - - // Write the trailer - if err := ff.AVFormat_write_trailer(ctx); err != nil { - log.Fatalf("could not write trailer: %v", err) - } -} - -/* - * Copyright (c) 2003 Fabrice Bellard - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ diff --git a/cmd/ffmpeg/mux/stream.go b/cmd/ffmpeg/mux/stream.go deleted file mode 100644 index 63ea0c9..0000000 --- a/cmd/ffmpeg/mux/stream.go +++ /dev/null @@ -1,256 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "math" - - // Packages - ff "github.com/mutablelogic/go-media/sys/ffmpeg61" -) - -//////////////////////////////////////////////////////////////////////////////// - -// a wrapper around an output AVStream -type Stream struct { - // Main parameters - Codec *ff.AVCodec - Encoder *ff.AVCodecContext - Stream *ff.AVStream - - tmp_packet *ff.AVPacket - next_pts int64 // pts of the next frame that will be generated - samples_count int - frame *ff.AVFrame - tmp_frame *ff.AVFrame - packet *ff.AVPacket - t, tincr, tincr2 float64 - sws_ctx *ff.SWSContext - swr_ctx *ff.SWRContext -} - -func (stream *Stream) String() string { - data, _ := json.MarshalIndent(stream, "", " ") - return string(data) -} - -//////////////////////////////////////////////////////////////////////////////// - -// Create a new output stream, add it to the media context and initialize the codec. -func NewStream(ctx *ff.AVFormatContext, codec_id ff.AVCodecID) (*Stream, error) { - stream := &Stream{} - - // Codec - codec := ff.AVCodec_find_encoder(codec_id) - if codec == nil { - return nil, errors.New("could not find codec") - } else { - stream.Codec = codec - } - - // Packet - if packet := ff.AVCodec_packet_alloc(); packet == nil { - return nil, errors.New("could not allocate packet") - } else { - stream.tmp_packet = packet - } - - // Stream - if str := ff.AVFormat_new_stream(ctx, nil); str == nil { - ff.AVCodec_packet_free(stream.tmp_packet) - return nil, errors.New("could not allocate stream") - } else { - stream_id := int(ctx.NumStreams()) - stream.Stream = str - stream.Stream.SetId(stream_id) - } - - // Codec context - if encoder := ff.AVCodec_alloc_context(codec); encoder == nil { - ff.AVCodec_packet_free(stream.tmp_packet) - return nil, errors.New("could not allocate codec context") - } else { - stream.Encoder = encoder - } - - // Set parameters for the encoder - switch stream.Codec.Type() { - case ff.AVMEDIA_TYPE_AUDIO: - if fmts := stream.Codec.SampleFormats(); len(fmts) > 0 { - stream.Encoder.SetSampleFormat(fmts[0]) - } else { - stream.Encoder.SetSampleFormat(ff.AV_SAMPLE_FMT_FLTP) - } - if rates := stream.Codec.SupportedSamplerates(); len(rates) > 0 { - stream.Encoder.SetSampleRate(rates[0]) - } else { - stream.Encoder.SetSampleRate(44100) - } - stream.Encoder.SetBitRate(64000) - if err := stream.Encoder.SetChannelLayout(ff.AV_CHANNEL_LAYOUT_STEREO); err != nil { - ff.AVCodec_packet_free(stream.tmp_packet) - return nil, err - } - stream.Stream.SetTimeBase(ff.AVUtil_rational(1, stream.Encoder.SampleRate())) - case ff.AVMEDIA_TYPE_VIDEO: - stream.Encoder.SetBitRate(400000) - // Resolution must be a multiple of two. - stream.Encoder.SetWidth(352) - stream.Encoder.SetHeight(288) - /* timebase: This is the fundamental unit of time (in seconds) in terms - * of which frame timestamps are represented. For fixed-fps content, - * timebase should be 1/framerate and timestamp increments should be - * identical to 1. */ - stream.Stream.SetTimeBase(ff.AVUtil_rational(1, 25)) - stream.Encoder.SetTimeBase(stream.Stream.TimeBase()) - stream.Encoder.SetGopSize(12) /* emit one intra frame every twelve frames at most */ - stream.Encoder.SetPixFmt(ff.AV_PIX_FMT_YUV420P) - - if stream.Codec.ID() == ff.AV_CODEC_ID_MPEG2VIDEO { - /* just for testing, we also add B frames */ - stream.Encoder.SetMaxBFrames(2) - } - if stream.Codec.ID() == ff.AV_CODEC_ID_MPEG1VIDEO { - /* Needed to avoid using macroblocks in which some coeffs overflow. - * This does not happen with normal video, it just happens here as - * the motion of the chroma plane does not match the luma plane. */ - stream.Encoder.SetMbDecision(ff.FF_MB_DECISION_SIMPLE) - } - } - - // Some formats want stream headers to be separate - if ctx.Output().Flags().Is(ff.AVFMT_GLOBALHEADER) { - stream.Encoder.SetFlags(stream.Encoder.Flags() | ff.AV_CODEC_FLAG_GLOBAL_HEADER) - } - - // Return success - return stream, nil -} - -func (stream *Stream) Close() { - ff.AVCodec_packet_free(stream.tmp_packet) - ff.AVCodec_free_context(stream.Encoder) - ff.AVUtil_frame_free(stream.frame) - ff.AVUtil_frame_free(stream.tmp_frame) - ff.SWResample_free(stream.swr_ctx) -} - -func (stream *Stream) Open(opts *ff.AVDictionary) error { - // Create a copy of the opts - opt, err := ff.AVUtil_dict_copy(opts, 0) - if err != nil { - return err - } - defer ff.AVUtil_dict_free(opt) - - // Open the codec - if err := ff.AVCodec_open(stream.Encoder, stream.Codec, opt); err != nil { - return err - } - - switch stream.Codec.Type() { - case ff.AVMEDIA_TYPE_AUDIO: - stream.t = 0 - // increment frequency by 110 Hz per second - stream.tincr = 2 * math.Pi * 110.0 / float64(stream.Encoder.SampleRate()) - stream.tincr2 = 2 * math.Pi * 110.0 / float64(stream.Encoder.SampleRate()) / float64(stream.Encoder.SampleRate()) - - // Number of samples in a frame - nb_samples := stream.Encoder.FrameSize() - if stream.Codec.Capabilities().Is(ff.AV_CODEC_CAP_VARIABLE_FRAME_SIZE) { - nb_samples = 10000 - } - - if frame, err := alloc_audio_frame(stream.Encoder.SampleFormat(), stream.Encoder.ChannelLayout(), stream.Encoder.SampleRate(), nb_samples); err != nil { - return err - } else { - stream.frame = frame - } - if frame, err := alloc_audio_frame(ff.AV_SAMPLE_FMT_S16, stream.Encoder.ChannelLayout(), stream.Encoder.SampleRate(), nb_samples); err != nil { - return err - } else { - stream.tmp_frame = frame - } - // create resampler context - if swr_ctx := ff.SWResample_alloc(); swr_ctx == nil { - return errors.New("could not allocate resample context") - } else if err := ff.SWResample_set_opts(swr_ctx, - stream.Encoder.ChannelLayout(), stream.Encoder.SampleFormat(), stream.Encoder.SampleRate(), // out - stream.Encoder.ChannelLayout(), ff.AV_SAMPLE_FMT_S16, stream.Encoder.SampleRate(), // in - ); err != nil { - ff.SWResample_free(swr_ctx) - return err - } else if err := ff.SWResample_init(swr_ctx); err != nil { - ff.SWResample_free(swr_ctx) - return err - } else { - stream.swr_ctx = swr_ctx - } - case ff.AVMEDIA_TYPE_VIDEO: - // Allocate a re-usable frame - if frame, err := alloc_video_frame(stream.Encoder.PixFmt(), stream.Encoder.Width(), stream.Encoder.Height()); err != nil { - return err - } else { - stream.frame = frame - } - // If the output format is not YUV420P, then a temporary YUV420P picture is needed too. It is then converted to the required - // output format. - if stream.Encoder.PixFmt() != ff.AV_PIX_FMT_YUV420P { - if frame, err := alloc_video_frame(ff.AV_PIX_FMT_YUV420P, stream.Encoder.Width(), stream.Encoder.Height()); err != nil { - return err - } else { - stream.tmp_frame = frame - } - } - } - - // copy the stream parameters to the muxer - if err := ff.AVCodec_parameters_from_context(stream.Stream.CodecPar(), stream.Encoder); err != nil { - return err - } - - // Return success - return nil -} - -func alloc_video_frame(pix_fmt ff.AVPixelFormat, width, height int) (*ff.AVFrame, error) { - frame := ff.AVUtil_frame_alloc() - if frame == nil { - return nil, errors.New("could not allocate video frame") - } - frame.SetWidth(width) - frame.SetHeight(height) - frame.SetPixFmt(pix_fmt) - - // allocate the buffers for the frame data - if err := ff.AVUtil_frame_get_buffer(frame, false); err != nil { - ff.AVUtil_frame_free(frame) - return nil, err - } - - // Return success - return frame, nil -} - -func alloc_audio_frame(sample_fmt ff.AVSampleFormat, channel_layout ff.AVChannelLayout, sample_rate, nb_samples int) (*ff.AVFrame, error) { - frame := ff.AVUtil_frame_alloc() - if frame == nil { - return nil, errors.New("could not allocate audio frame") - } - frame.SetSampleFormat(sample_fmt) - frame.SetSampleRate(sample_rate) - frame.SetNumSamples(nb_samples) - if err := frame.SetChannelLayout(channel_layout); err != nil { - ff.AVUtil_frame_free(frame) - return nil, err - } - - // allocate the buffers for the frame data - if err := ff.AVUtil_frame_get_buffer(frame, false); err != nil { - ff.AVUtil_frame_free(frame) - return nil, err - } - - // Return success - return frame, nil -} diff --git a/manager.go b/manager.go index b0a9c2c..10af4e6 100644 --- a/manager.go +++ b/manager.go @@ -1,6 +1,7 @@ package media import ( + "errors" "fmt" "io" "runtime" @@ -301,12 +302,12 @@ func (manager *manager) Read(r io.Reader, format Format, opts ...string) (Media, // Create a media file for writing, from a url, path, or device. func (manager *manager) Create(url string, format Format, metadata []Metadata, params ...Parameters) (Media, error) { - return createMedia(url, format, metadata, params...) + return nil, errors.New("not implemented") } // Create a media stream for writing. func (manager *manager) Write(w io.Writer, format Format, metadata []Metadata, params ...Parameters) (Media, error) { - return createWriter(w, format, metadata, params...) + return nil, errors.New("not implemented") } // Return version information for the media manager as a set of metadata diff --git a/writer.go b/writer.go deleted file mode 100644 index 4589824..0000000 --- a/writer.go +++ /dev/null @@ -1,320 +0,0 @@ -package media - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - - // Packages - ff "github.com/mutablelogic/go-media/sys/ffmpeg61" - - // Namespace imports - . "github.com/djthorpe/go-errors" -) - -//////////////////////////////////////////////////////////////////////////////// -// TYPES - -type writer struct { - t MediaType - output *ff.AVFormatContext - avio *ff.AVIOContextEx - metadata *ff.AVDictionary - header bool - encoder map[int]*encoder -} - -type writer_callback struct { - w io.Writer -} - -var _ Media = (*writer)(nil) - -//////////////////////////////////////////////////////////////////////////////// -// LIFECYCLE - -// Create media from a url or device -func createMedia(url string, format Format, metadata []Metadata, params ...Parameters) (*writer, error) { - writer := new(writer) - writer.t = OUTPUT - writer.encoder = make(map[int]*encoder, len(params)) - - // If there are no streams, then return an error - if len(params) == 0 { - return nil, ErrBadParameter.With("no streams specified for encoder") - } - - // Guess the output format - var ofmt *ff.AVOutputFormat - if format == nil && url != "" { - ofmt = ff.AVFormat_guess_format("", url, "") - } else if format != nil { - ofmt = format.(*outputformat).ctx - } - if ofmt == nil { - return nil, ErrBadParameter.With("unable to guess the output format") - } - - // Allocate the output media context - ctx, err := ff.AVFormat_create_file(url, ofmt) - if err != nil { - return nil, err - } else { - writer.output = ctx - } - - // Add encoders and streams - var result error - for i, param := range params { - // Stream Id from codec parameters, or use the index - stream_id := param.Id() - if stream_id <= 0 { - stream_id = i + 1 - } - encoder, err := newEncoder(ctx, stream_id, param) - if err != nil { - result = errors.Join(result, err) - } else if _, exists := writer.encoder[stream_id]; exists { - - } else { - writer.encoder[stream_id] = encoder - } - } - - // Return any errors from creating the streams - if result != nil { - return nil, errors.Join(result, writer.Close()) - } - - // Open the output file, if needed - if !ctx.Output().Flags().Is(ff.AVFMT_NOFILE) { - w, err := ff.AVFormat_avio_open(url, ff.AVIO_FLAG_WRITE) - if err != nil { - return nil, errors.Join(err, writer.Close()) - } else { - ctx.SetPb(w) - writer.avio = w - } - } - - // Set metadata - if len(metadata) > 0 { - writer.metadata = ff.AVUtil_dict_alloc() - if writer.metadata == nil { - return nil, errors.Join(errors.New("unable to allocate metadata dictionary"), writer.Close()) - } - for _, m := range metadata { - // Ignore duration and artwork fields - key := m.Key() - if key == MetaArtwork || key == MetaDuration { - continue - } - // Set dictionary entry - if err := ff.AVUtil_dict_set(writer.metadata, key, fmt.Sprint(m.Value()), ff.AV_DICT_APPEND); err != nil { - return nil, errors.Join(err, writer.Close()) - } - } - // TODO: Create artwork streams - } - - // Write the header - if err := ff.AVFormat_write_header(ctx, nil); err != nil { - return nil, errors.Join(err, writer.Close()) - } else { - writer.header = true - } - - // Return success - return writer, nil -} - -// Create media from io.Writer -// TODO -func createWriter(w io.Writer, format Format, metadata []Metadata, params ...Parameters) (*writer, error) { - return nil, ErrNotImplemented -} - -func (w *writer) Close() error { - var result error - - // Write the trailer if the header was written - if w.header { - if err := ff.AVFormat_write_trailer(w.output); err != nil { - result = errors.Join(result, err) - } - } - - // Close encoders - for _, encoder := range w.encoder { - result = errors.Join(result, encoder.Close()) - } - - // Free output resources - if w.output != nil { - // This calls avio_close(w.avio) - result = errors.Join(result, ff.AVFormat_close_writer(w.output)) - } - - // Free resources - if w.metadata != nil { - ff.AVUtil_dict_free(w.metadata) - } - - // Release resources - w.encoder = nil - w.metadata = nil - w.avio = nil - w.output = nil - - // Return any errors - return result -} - -//////////////////////////////////////////////////////////////////////////////// -// STRINGIFY - -// Display the reader as a string -func (w *writer) MarshalJSON() ([]byte, error) { - return json.Marshal(w.output) -} - -// Display the reader as a string -func (w *writer) String() string { - data, _ := json.MarshalIndent(w, "", " ") - return string(data) -} - -//////////////////////////////////////////////////////////////////////////////// -// PUBLIC METHODS - -func (w *writer) Decoder(DecoderMapFunc) (Decoder, error) { - return nil, ErrOutOfOrder.With("not an input stream") -} - -func (w *writer) Mux(ctx context.Context, fn MuxFunc) error { - // Check fn - if fn == nil { - return ErrBadParameter.With("nil mux function") - } - - // Create a new map of encoders - encoders := make(map[int]*encoder, len(w.encoder)) - for k, v := range w.encoder { - encoders[k] = v - } - -FOR_LOOP: - for { - select { - case <-ctx.Done(): - break FOR_LOOP - default: - // Loop until no more encoders are available to send packets - if len(encoders) == 0 { - break FOR_LOOP - } - - // Find the first encoder which should return a packet - var next_encoder *encoder - var next_stream int - for stream, encoder := range encoders { - // Initialise the next encoder - if next_encoder == nil { - next_encoder = encoder - next_stream = stream - continue - } - // Compare - if !compareNextPts(next_encoder, encoder) { - next_encoder = encoder - next_stream = stream - } - } - - // Get a packet from the encoder - packet, err := next_encoder.encode(fn) - if errors.Is(err, io.EOF) { - break FOR_LOOP - } else if err != nil { - return err - } else if packet == nil { - // Remove the encoder from the map - delete(encoders, next_stream) - continue FOR_LOOP - } - - // Send the packet to the muxer - //av_packet_rescale_ts(pkt, in_stream->time_base, out_stream->time_base); - // Packet's stream_index field must be set to the index of the corresponding stream in s->streams. - // The timestamps (pts, dts) must be set to correct values in the stream's timebase - // (unless the output format is flagged with the AVFMT_NOTIMESTAMPS flag, then they can be set - // to AV_NOPTS_VALUE). The dts for subsequent packets in one stream must be strictly increasing - // (unless the output format is flagged with the AVFMT_TS_NONSTRICT, then they merely have to - // be nondecreasing). duration should also be set if known. - if err := ff.AVCodec_interleaved_write_frame(w.output, packet); err != nil { - return err - } - } - } - - // Flush - if err := ff.AVCodec_interleaved_write_frame(w.output, nil); err != nil { - return err - } - - // Return the context error, which will be nil if the loop ended normally - return ctx.Err() -} - -// Returns true if a.next_pts is greater than b.next_pts -func compareNextPts(a, b *encoder) bool { - return ff.AVUtil_compare_ts(a.next_pts, a.stream.TimeBase(), b.next_pts, b.stream.TimeBase()) > 0 -} - -/* - while (1) { - AVStream *in_stream, *out_stream; - - ret = av_read_frame(ifmt_ctx, pkt); - if (ret < 0) - break; - - in_stream = ifmt_ctx->streams[pkt->stream_index]; - if (pkt->stream_index >= stream_mapping_size || - stream_mapping[pkt->stream_index] < 0) { - av_packet_unref(pkt); - continue; - } - - pkt->stream_index = stream_mapping[pkt->stream_index]; - out_stream = ofmt_ctx->streams[pkt->stream_index]; - log_packet(ifmt_ctx, pkt, "in"); - - // copy packet - av_packet_rescale_ts(pkt, in_stream->time_base, out_stream->time_base); - pkt->pos = -1; - log_packet(ofmt_ctx, pkt, "out"); - - ret = av_interleaved_write_frame(ofmt_ctx, pkt); - // pkt is now blank (av_interleaved_write_frame() takes ownership of - // its contents and resets pkt), so that no unreferencing is necessary. - // This would be different if one used av_write_frame(). - if (ret < 0) { - fprintf(stderr, "Error muxing packet\n"); - break; - } - } -*/ - -// Return OUTPUT and combination of DEVICE and STREAM -func (w *writer) Type() MediaType { - return w.t -} - -// Return the metadata for the media. -func (w *writer) Metadata(...string) []Metadata { - // Not yet implemented - return nil -} diff --git a/writer_test.go b/writer_test.go deleted file mode 100644 index 5e21d43..0000000 --- a/writer_test.go +++ /dev/null @@ -1,84 +0,0 @@ -//go:build !container - -package media_test - -// TODO: Allow this test to run in containers - -import ( - "context" - "path/filepath" - "strings" - "testing" - - // Package imports - "github.com/stretchr/testify/assert" - - // Namespace imports - . "github.com/mutablelogic/go-media" -) - -func Test_writer_001(t *testing.T) { - assert := assert.New(t) - manager, err := NewManager(OptLog(true, func(v string) { - t.Log(strings.TrimSpace(v)) - })) - if !assert.NoError(err) { - t.SkipNow() - } - - // Write audio file - filename := filepath.Join(t.TempDir(), t.Name()+".mp3") - stream, err := manager.AudioParameters("mono", "fltp", 22050) - if !assert.NoError(err) { - t.SkipNow() - } - - writer, err := manager.Create(filename, nil, nil, stream) - if !assert.NoError(err) { - t.SkipNow() - } - defer writer.Close() - - t.Log(writer, "=>", filename) - - // Perform muxing of packets - writer.Mux(context.Background(), func(stream int) (Packet, error) { - t.Log("Muxing packet for stream", stream) - return nil, nil - }) -} - -func Test_writer_002(t *testing.T) { - assert := assert.New(t) - manager, err := NewManager(OptLog(true, func(v string) { - t.Log(strings.TrimSpace(v)) - })) - if !assert.NoError(err) { - t.SkipNow() - } - - // Write file with both audio and video - filename := filepath.Join(t.TempDir(), t.Name()+".mp4") - audio, err := manager.AudioParameters("mono", "fltp", 22050) - if !assert.NoError(err) { - t.SkipNow() - } - video, err := manager.VideoParameters(1280, 720, "yuv420p") - if !assert.NoError(err) { - t.SkipNow() - } - - writer, err := manager.Create(filename, nil, nil, audio, video) - if !assert.NoError(err) { - t.SkipNow() - } - defer writer.Close() - - t.Log(writer, "=>", filename) - - // Perform muxing of packets - writer.Mux(context.Background(), func(stream int) (Packet, error) { - t.Log("Muxing packet for stream", stream) - return nil, nil - }) -} From 252ef43f6202bec44ff46fc4c0a2510938cfa457 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 19:50:20 +0200 Subject: [PATCH 12/12] Some more fixes --- go.mod | 1 + manager.go | 29 +--------------------- manager_test.go | 14 ----------- pkg/ffmpeg/rescaler_test.go | 2 +- pkg/generator/yuv420p_test.go | 6 ++--- pkg/version/version.go | 45 +++++++++++++++++++++++++++++++++++ 6 files changed, 51 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index f0e0ff2..56f2997 100755 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/djthorpe/go-tablewriter v0.0.8 github.com/mutablelogic/go-client v1.0.8 github.com/stretchr/testify v1.9.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 ) require ( diff --git a/manager.go b/manager.go index 10af4e6..bf1373f 100644 --- a/manager.go +++ b/manager.go @@ -4,10 +4,8 @@ import ( "errors" "fmt" "io" - "runtime" // Package imports - version "github.com/mutablelogic/go-media/pkg/version" ff "github.com/mutablelogic/go-media/sys/ffmpeg61" ) @@ -312,32 +310,7 @@ func (manager *manager) Write(w io.Writer, format Format, metadata []Metadata, p // Return version information for the media manager as a set of metadata func (manager *manager) Version() []Metadata { - metadata := []Metadata{ - newMetadata("libavcodec_version", ffVersionAsString(ff.AVCodec_version())), - newMetadata("libavformat_version", ffVersionAsString(ff.AVFormat_version())), - newMetadata("libavutil_version", ffVersionAsString(ff.AVUtil_version())), - newMetadata("libavdevice_version", ffVersionAsString(ff.AVDevice_version())), - // newMetadata("libavfilter_version", ff.AVFilter_version()), - newMetadata("libswscale_version", ffVersionAsString(ff.SWScale_version())), - newMetadata("libswresample_version", ffVersionAsString(ff.SWResample_version())), - } - if version.GitSource != "" { - metadata = append(metadata, newMetadata("git_source", version.GitSource)) - } - if version.GitBranch != "" { - metadata = append(metadata, newMetadata("git_branch", version.GitBranch)) - } - if version.GitTag != "" { - metadata = append(metadata, newMetadata("git_tag", version.GitTag)) - } - if version.GoBuildTime != "" { - metadata = append(metadata, newMetadata("go_build_time", version.GoBuildTime)) - } - if runtime.Version() != "" { - metadata = append(metadata, newMetadata("go_version", runtime.Version())) - metadata = append(metadata, newMetadata("go_arch", runtime.GOOS+"/"+runtime.GOARCH)) - } - return metadata + return nil } // Log error messages diff --git a/manager_test.go b/manager_test.go index 37f1994..9bf3a8e 100644 --- a/manager_test.go +++ b/manager_test.go @@ -39,20 +39,6 @@ func Test_manager_002(t *testing.T) { t.Log(formats) } -func Test_manager_003(t *testing.T) { - assert := assert.New(t) - - manager, err := NewManager() - if !assert.NoError(err) { - t.SkipNow() - } - - version := manager.Version() - assert.NotNil(version) - - tablewriter.New(os.Stderr, tablewriter.OptHeader(), tablewriter.OptOutputText()).Write(version) -} - func Test_manager_004(t *testing.T) { assert := assert.New(t) diff --git a/pkg/ffmpeg/rescaler_test.go b/pkg/ffmpeg/rescaler_test.go index 46930e3..e90307e 100644 --- a/pkg/ffmpeg/rescaler_test.go +++ b/pkg/ffmpeg/rescaler_test.go @@ -53,7 +53,7 @@ func Test_rescaler_002(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuva420p", "1280x720", 25)) + image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuv420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } diff --git a/pkg/generator/yuv420p_test.go b/pkg/generator/yuv420p_test.go index 8d037d1..32b46da 100644 --- a/pkg/generator/yuv420p_test.go +++ b/pkg/generator/yuv420p_test.go @@ -14,7 +14,7 @@ import ( func Test_yuv420p_001(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) + image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuv420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } @@ -25,7 +25,7 @@ func Test_yuv420p_001(t *testing.T) { func Test_yuv420p_002(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) + image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuv420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } @@ -39,7 +39,7 @@ func Test_yuv420p_002(t *testing.T) { func Test_yuv420p_003(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) + image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuv420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } diff --git a/pkg/version/version.go b/pkg/version/version.go index cc3e270..cb16017 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,8 +1,53 @@ package version +import ( + "fmt" + "runtime" + + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" +) + var ( GitSource string GitTag string GitBranch string GoBuildTime string ) + +// Return version information as a set of metadata +func Version() []*ffmpeg.Metadata { + metadata := []*ffmpeg.Metadata{ + ffmpeg.NewMetadata("libavcodec_version", ffVersionAsString(ff.AVCodec_version())), + ffmpeg.NewMetadata("libavformat_version", ffVersionAsString(ff.AVFormat_version())), + ffmpeg.NewMetadata("libavutil_version", ffVersionAsString(ff.AVUtil_version())), + ffmpeg.NewMetadata("libavdevice_version", ffVersionAsString(ff.AVDevice_version())), + // newMetadata("libavfilter_version", ff.AVFilter_version()), + ffmpeg.NewMetadata("libswscale_version", ffVersionAsString(ff.SWScale_version())), + ffmpeg.NewMetadata("libswresample_version", ffVersionAsString(ff.SWResample_version())), + } + if GitSource != "" { + metadata = append(metadata, ffmpeg.NewMetadata("git_source", GitSource)) + } + if GitBranch != "" { + metadata = append(metadata, ffmpeg.NewMetadata("git_branch", GitBranch)) + } + if GitTag != "" { + metadata = append(metadata, ffmpeg.NewMetadata("git_tag", GitTag)) + } + if GoBuildTime != "" { + metadata = append(metadata, ffmpeg.NewMetadata("go_build_time", GoBuildTime)) + } + if runtime.Version() != "" { + metadata = append(metadata, ffmpeg.NewMetadata("go_version", runtime.Version())) + metadata = append(metadata, ffmpeg.NewMetadata("go_arch", runtime.GOOS+"/"+runtime.GOARCH)) + } + return metadata +} + +//////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +func ffVersionAsString(version uint) string { + return fmt.Sprintf("%d.%d.%d", version&0xFF0000>>16, version&0xFF00>>8, version&0xFF) +}