From c932e1777cb961c12ef23b925f459a8dd3bd5569 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 12:11:15 +0200 Subject: [PATCH 1/5] Updates for encoder --- pkg/ffmpeg/encoder.go | 222 +++++++++++++++++++++++++++++ pkg/ffmpeg/opts.go | 101 ++++++++++--- pkg/ffmpeg/packet_encoder.go | 105 -------------- pkg/ffmpeg/par.go | 209 +++++++++++++++++++++++++++ pkg/ffmpeg/par_test.go | 28 ++++ pkg/ffmpeg/resampler.go | 30 ++-- pkg/ffmpeg/resampler_test.go | 2 +- pkg/ffmpeg/rescaler.go | 24 ++-- pkg/ffmpeg/writer.go | 146 +++++++++++++++++-- pkg/ffmpeg/writer_test.go | 32 ++++- pkg/generator/sine.go | 29 ++-- pkg/generator/sine_test.go | 7 +- pkg/generator/yuv420p.go | 4 + sys/ffmpeg61/avcodec.go | 2 + sys/ffmpeg61/avcodec_parameters.go | 51 ++++++- 15 files changed, 814 insertions(+), 178 deletions(-) create mode 100644 pkg/ffmpeg/encoder.go delete mode 100644 pkg/ffmpeg/packet_encoder.go create mode 100644 pkg/ffmpeg/par.go create mode 100644 pkg/ffmpeg/par_test.go diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go new file mode 100644 index 0000000..142ab39 --- /dev/null +++ b/pkg/ffmpeg/encoder.go @@ -0,0 +1,222 @@ +package ffmpeg + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "syscall" + + // Packages + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + + // Namespace imports + . "github.com/djthorpe/go-errors" +) + +//////////////////////////////////////////////////////////////////////////////// +// TYPES + +type Encoder struct { + ctx *ff.AVCodecContext + stream *ff.AVStream + packet *ff.AVPacket + //next_pts int64 +} + +// EncoderFrameFn is a function which is called to receive a frame to encode. It should +// return nil to continue encoding or io.EOF to stop encoding. +type EncoderFrameFn func(int) (*ff.AVFrame, error) + +// EncoderPacketFn is a function which is called for each packet encoded. It should +// return nil to continue encoding or io.EOF to stop encoding immediately. +type EncoderPacketFn func(*ff.AVPacket) error + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +// Create an encoder with the given parameters +func NewEncoder(ctx *ff.AVFormatContext, stream int, par *Par) (*Encoder, error) { + encoder := new(Encoder) + + // Get codec + codec_id := ff.AV_CODEC_ID_NONE + switch par.CodecType() { + case ff.AVMEDIA_TYPE_AUDIO: + codec_id = ctx.Output().AudioCodec() + case ff.AVMEDIA_TYPE_VIDEO: + codec_id = ctx.Output().VideoCodec() + case ff.AVMEDIA_TYPE_SUBTITLE: + codec_id = ctx.Output().SubtitleCodec() + } + if codec_id == ff.AV_CODEC_ID_NONE { + return nil, ErrBadParameter.Withf("no codec specified for stream %v", stream) + } + + // Allocate codec + codec := ff.AVCodec_find_encoder(codec_id) + if codec == nil { + return nil, ErrBadParameter.Withf("codec %q cannot encode", codec_id) + } + if codecctx := ff.AVCodec_alloc_context(codec); codecctx == nil { + return nil, ErrInternalAppError.With("could not allocate audio codec context") + } else { + encoder.ctx = codecctx + } + + // Check codec against parameters and set defaults as needed, then + // copy back to codec + if err := par.ValidateFromCodec(encoder.ctx); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, err + } else if err := par.CopyToCodec(encoder.ctx); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, err + } + + // Create the stream + if streamctx := ff.AVFormat_new_stream(ctx, nil); streamctx == nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, ErrInternalAppError.With("could not allocate stream") + } else { + streamctx.SetId(stream) + encoder.stream = streamctx + } + + // Copy parameters to stream + if err := ff.AVCodec_parameters_from_context(encoder.stream.CodecPar(), encoder.ctx); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, err + } + + // Some formats want stream headers to be separate. + if ctx.Output().Flags().Is(ff.AVFMT_GLOBALHEADER) { + encoder.ctx.SetFlags(encoder.ctx.Flags() | ff.AV_CODEC_FLAG_GLOBAL_HEADER) + } + + // Open it + if err := ff.AVCodec_open(encoder.ctx, codec, nil); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, ErrInternalAppError.Withf("codec_open: %v", err) + } + + // Create a packet + packet := ff.AVCodec_packet_alloc() + if packet == nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, errors.New("failed to allocate packet") + } else { + encoder.packet = packet + } + + // Return it + return encoder, nil +} + +func (encoder *Encoder) Close() error { + // Free respurces + if encoder.ctx != nil { + ff.AVCodec_free_context(encoder.ctx) + } + if encoder.packet != nil { + ff.AVCodec_packet_free(encoder.packet) + } + + // Release resources + encoder.packet = nil + encoder.stream = nil + encoder.ctx = nil + + // Return success + return nil +} + +//////////////////////////////////////////////////////////////////////////////// +// STRINGIFY + +func (e *Encoder) MarshalJSON() ([]byte, error) { + type jsonEncoder struct { + Codec *ff.AVCodecContext `json:"codec"` + Stream *ff.AVStream `json:"stream"` + } + return json.Marshal(&jsonEncoder{ + Codec: e.ctx, + Stream: e.stream, + }) +} + +func (e *Encoder) String() string { + data, _ := json.MarshalIndent(e, "", " ") + return string(data) +} + +////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +// Encode a frame and pass packets to the EncoderPacketFn. If the frame is nil, then +// the encoder will flush any remaining packets. If io.EOF is returned then +// it indicates that the encoder has ended prematurely. +func (e *Encoder) Encode(frame *ff.AVFrame, fn EncoderPacketFn) error { + if fn == nil { + return ErrBadParameter.With("nil fn") + } + // Encode a frame (or flush the encoder) + return e.encode(frame, fn) +} + +// Return the codec parameters +func (e *Encoder) Par() *Par { + par := new(Par) + if err := ff.AVCodec_parameters_from_context(&par.AVCodecParameters, e.ctx); err != nil { + return nil + } else { + return par + } +} + +////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { + // Send the frame to the encoder + fmt.Println("Sending frame", frame) + if err := ff.AVCodec_send_frame(e.ctx, frame); err != nil { + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { + return nil + } + return err + } + + // Write out the packets + var result error + for { + // Receive the packet + fmt.Println("Receiving packet") + if err := ff.AVCodec_receive_packet(e.ctx, e.packet); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { + // Finished receiving packet or EOF + break + } else if err != nil { + return err + } + + // Pass back to the caller + if err := fn(e.packet); errors.Is(err, io.EOF) { + // End early, return EOF + result = io.EOF + break + } else if err != nil { + return err + } + + // Re-allocate frames for next iteration + ff.AVCodec_packet_unref(e.packet) + } + + // Flush + if result == nil { + result = fn(nil) + } + + // Return success or EOF + return result +} diff --git a/pkg/ffmpeg/opts.go b/pkg/ffmpeg/opts.go index 990e2f5..90b8c63 100644 --- a/pkg/ffmpeg/opts.go +++ b/pkg/ffmpeg/opts.go @@ -15,18 +15,23 @@ type Opt func(*opts) error type opts struct { // Resample/resize options force bool + par *Par // Format options oformat *ffmpeg.AVOutputFormat - // Audio options - sample_fmt ffmpeg.AVSampleFormat - ch ffmpeg.AVChannelLayout - samplerate int + // Stream options + streams map[int]*Par +} + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE - // Video options - pix_fmt ffmpeg.AVPixelFormat - width, height int +func newOpts() *opts { + return &opts{ + par: new(Par), + streams: make(map[int]*Par), + } } //////////////////////////////////////////////////////////////////////////////// @@ -45,6 +50,50 @@ func OptOutputFormat(name string) Opt { } } +// New audio stream with parameters +func OptAudioStream(stream int, par *Par) Opt { + return func(o *opts) error { + if par == nil || par.CodecType() != ffmpeg.AVMEDIA_TYPE_AUDIO { + return ErrBadParameter.With("invalid audio parameters") + } + if stream == 0 { + stream = len(o.streams) + 1 + } + if _, exists := o.streams[stream]; exists { + return ErrDuplicateEntry.Withf("stream %v", stream) + } + if stream < 0 { + return ErrBadParameter.Withf("invalid stream %v", stream) + } + o.streams[stream] = par + + // Return success + return nil + } +} + +// New video stream with parameters +func OptVideoStream(stream int, par *Par) Opt { + return func(o *opts) error { + if par == nil || par.CodecType() != ffmpeg.AVMEDIA_TYPE_VIDEO { + return ErrBadParameter.With("invalid video parameters") + } + if stream == 0 { + stream = len(o.streams) + 1 + } + if _, exists := o.streams[stream]; exists { + return ErrDuplicateEntry.Withf("stream %v", stream) + } + if stream < 0 { + return ErrBadParameter.Withf("invalid stream %v", stream) + } + o.streams[stream] = par + + // Return success + return nil + } +} + // Force resampling and resizing on decode, even if the input and output // parameters are the same func OptForce() Opt { @@ -61,7 +110,8 @@ func OptPixFormat(format string) Opt { if fmt == ffmpeg.AV_PIX_FMT_NONE { return ErrBadParameter.Withf("invalid pixel format %q", format) } - o.pix_fmt = fmt + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_VIDEO) + o.par.SetPixelFormat(fmt) return nil } } @@ -72,8 +122,9 @@ func OptWidthHeight(w, h int) Opt { if w <= 0 || h <= 0 { return ErrBadParameter.Withf("invalid width %v or height %v", w, h) } - o.width = w - o.height = h + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_VIDEO) + o.par.SetWidth(w) + o.par.SetHeight(h) return nil } } @@ -85,8 +136,9 @@ func OptFrameSize(size string) Opt { if err != nil { return ErrBadParameter.Withf("invalid frame size %q", size) } - o.width = w - o.height = h + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_VIDEO) + o.par.SetWidth(w) + o.par.SetHeight(h) return nil } } @@ -94,18 +146,25 @@ func OptFrameSize(size string) Opt { // Channel layout func OptChannelLayout(layout string) Opt { return func(o *opts) error { - return ffmpeg.AVUtil_channel_layout_from_string(&o.ch, layout) + var ch ffmpeg.AVChannelLayout + if err := ffmpeg.AVUtil_channel_layout_from_string(&ch, layout); err != nil { + return ErrBadParameter.Withf("invalid channel layout %q", layout) + } + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_AUDIO) + return o.par.SetChannelLayout(ch) } } // Nuumber of channels -func OptChannels(ch int) Opt { +func OptChannels(num int) Opt { return func(o *opts) error { - if ch <= 0 || ch > 64 { - return ErrBadParameter.Withf("invalid number of channels %v", ch) + var ch ffmpeg.AVChannelLayout + if num <= 0 || num > 64 { + return ErrBadParameter.Withf("invalid number of channels %v", num) } - ffmpeg.AVUtil_channel_layout_default(&o.ch, ch) - return nil + ffmpeg.AVUtil_channel_layout_default(&ch, num) + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_AUDIO) + return o.par.SetChannelLayout(ch) } } @@ -115,7 +174,8 @@ func OptSampleRate(rate int) Opt { if rate <= 0 { return ErrBadParameter.Withf("invalid sample rate %v", rate) } - o.samplerate = rate + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_AUDIO) + o.par.SetSamplerate(rate) return nil } } @@ -127,7 +187,8 @@ func OptSampleFormat(format string) Opt { if fmt == ffmpeg.AV_SAMPLE_FMT_NONE { return ErrBadParameter.Withf("invalid sample format %q", format) } - o.sample_fmt = fmt + o.par.SetCodecType(ffmpeg.AVMEDIA_TYPE_AUDIO) + o.par.SetSampleFormat(fmt) return nil } } diff --git a/pkg/ffmpeg/packet_encoder.go b/pkg/ffmpeg/packet_encoder.go deleted file mode 100644 index 0ce5ad5..0000000 --- a/pkg/ffmpeg/packet_encoder.go +++ /dev/null @@ -1,105 +0,0 @@ -package ffmpeg - -import ( - "errors" - "io" - "syscall" - - // Packages - ff "github.com/mutablelogic/go-media/sys/ffmpeg61" -) - -////////////////////////////////////////////////////////////////////////////// -// TYPES - -type encoder struct { - codec *ff.AVCodecContext - packet *ff.AVPacket - stream int -} - -// EncodeFn is a function which is called for each packet encoded. It should -// return nil to continue encoding or io.EOF to stop decoding. -type EncodeFn func(*ff.AVPacket) error - -////////////////////////////////////////////////////////////////////////////// -// LIFECYCLE - -func NewPacketEncoder(codec *ff.AVCodecContext, stream int) (*encoder, error) { - encoder := new(encoder) - encoder.codec = codec - - // Create a packet - packet := ff.AVCodec_packet_alloc() - if packet == nil { - return nil, errors.New("failed to allocate packet") - } else { - encoder.packet = packet - encoder.stream = stream - } - - // Return success - return encoder, nil -} - -func (e *encoder) Close() error { - if e.packet != nil { - ff.AVCodec_packet_free(e.packet) - e.packet = nil - } - return nil -} - -////////////////////////////////////////////////////////////////////////////// -// LIFECYCLE - -func (e *encoder) Encode(frame *ff.AVFrame, fn EncodeFn) error { - // Encode a frame - if err := e.encode(frame, fn); err != nil { - return err - } - // Flush - return e.encode(nil, fn) -} - -////////////////////////////////////////////////////////////////////////////// -// PRIVATE METHODS - -func (e *encoder) encode(frame *ff.AVFrame, fn EncodeFn) error { - // Send the frame to the encoder - if err := ff.AVCodec_send_frame(e.codec, frame); err != nil { - return err - } - - // Write out the packets - var result error - for { - // Receive the packet - if err := ff.AVCodec_receive_packet(e.codec, e.packet); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { - // Finished receiving packet or EOF - break - } else if err != nil { - return err - } - - // Pass back to the caller - if err := fn(e.packet); errors.Is(err, io.EOF) { - // End early, return EOF - result = io.EOF - break - } else if err != nil { - return err - } - - // Re-allocate frames for next iteration - ff.AVCodec_packet_unref(e.packet) - } - - // Flush - if result == nil { - result = fn(nil) - } - - // Return success or EOF - return result -} diff --git a/pkg/ffmpeg/par.go b/pkg/ffmpeg/par.go new file mode 100644 index 0000000..3149feb --- /dev/null +++ b/pkg/ffmpeg/par.go @@ -0,0 +1,209 @@ +package ffmpeg + +import ( + "encoding/json" + "fmt" + "slices" + + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + + // Namespace imports + . "github.com/djthorpe/go-errors" +) + +/////////////////////////////////////////////////////////////////////////////// +// TYPES + +type Par struct { + ff.AVCodecParameters +} + +/////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func NewAudioPar(samplefmt string, channellayout string, samplerate int) (*Par, error) { + par := new(Par) + par.SetCodecType(ff.AVMEDIA_TYPE_AUDIO) + + // Sample Format + if samplefmt_ := ff.AVUtil_get_sample_fmt(samplefmt); samplefmt_ == ff.AV_SAMPLE_FMT_NONE { + return nil, ErrBadParameter.Withf("unknown sample format %q", samplefmt) + } else { + par.SetSampleFormat(samplefmt_) + } + + // Channel layout + var ch ff.AVChannelLayout + if err := ff.AVUtil_channel_layout_from_string(&ch, channellayout); err != nil { + return nil, ErrBadParameter.Withf("channel layout %q", channellayout) + } else if err := par.SetChannelLayout(ch); err != nil { + return nil, err + } + + // Sample rate + if samplerate <= 0 { + return nil, ErrBadParameter.Withf("negative or zero samplerate %v", samplerate) + } else { + par.SetSamplerate(samplerate) + } + + // Return success + return par, nil +} + +func NewVideoPar(pixfmt string, size string) (*Par, error) { + par := new(Par) + par.SetCodecType(ff.AVMEDIA_TYPE_VIDEO) + + // Pixel Format + if pixfmt_ := ff.AVUtil_get_pix_fmt(pixfmt); pixfmt_ == ff.AV_PIX_FMT_NONE { + return nil, ErrBadParameter.Withf("unknown pixel format %q", pixfmt) + } else { + par.SetPixelFormat(pixfmt_) + } + + // Frame size + if w, h, err := ff.AVUtil_parse_video_size(size); err != nil { + return nil, ErrBadParameter.Withf("size %q", size) + } else { + par.SetWidth(w) + par.SetHeight(h) + } + + // Set default sample aspect ratio + par.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + + // Return success + return par, nil +} + +func AudioPar(samplefmt string, channellayout string, samplerate int) *Par { + if par, err := NewAudioPar(samplefmt, channellayout, samplerate); err != nil { + panic(err) + } else { + return par + } +} + +func VideoPar(pixfmt string, size string) *Par { + if par, err := NewVideoPar(pixfmt, size); err != nil { + panic(err) + } else { + return par + } +} + +/////////////////////////////////////////////////////////////////////////////// +// STRINGIFY + +func (ctx *Par) MarshalJSON() ([]byte, error) { + return json.Marshal(ctx.AVCodecParameters) +} + +func (ctx *Par) String() string { + data, _ := json.MarshalIndent(ctx, "", " ") + return string(data) +} + +/////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +func (ctx *Par) ValidateFromCodec(codec *ff.AVCodecContext) error { + switch codec.Codec().Type() { + case ff.AVMEDIA_TYPE_AUDIO: + return ctx.validateAudioCodec(codec) + case ff.AVMEDIA_TYPE_VIDEO: + return ctx.validateVideoCodec(codec) + } + return nil +} + +func (ctx *Par) CopyToCodec(codec *ff.AVCodecContext) error { + switch codec.Codec().Type() { + case ff.AVMEDIA_TYPE_AUDIO: + return ctx.copyAudioCodec(codec) + case ff.AVMEDIA_TYPE_VIDEO: + return ctx.copyVideoCodec(codec) + } + return nil +} + +/////////////////////////////////////////////////////////////////////////////// +// PRIVATE METHODS + +func (ctx *Par) copyAudioCodec(codec *ff.AVCodecContext) error { + codec.SetSampleFormat(ctx.SampleFormat()) + codec.SetSampleRate(ctx.Samplerate()) + if err := codec.SetChannelLayout(ctx.ChannelLayout()); err != nil { + return err + } + return nil +} + +func (ctx *Par) validateAudioCodec(codec *ff.AVCodecContext) error { + sampleformats := codec.Codec().SampleFormats() + samplerates := codec.Codec().SupportedSamplerates() + channellayouts := codec.Codec().ChannelLayouts() + + // First we set params from the codec which are not already set + if ctx.SampleFormat() == ff.AV_SAMPLE_FMT_NONE { + if len(sampleformats) > 0 { + ctx.SetSampleFormat(sampleformats[0]) + } + } + if ctx.Samplerate() == 0 { + if len(samplerates) > 0 { + ctx.SetSamplerate(samplerates[0]) + } + } + if ctx.ChannelLayout().NumChannels() == 0 { + if len(channellayouts) > 0 { + ctx.SetChannelLayout(channellayouts[0]) + } + } + + // Then we check to make sure the parameters are compatible with + // the codec + if len(sampleformats) > 0 { + if !slices.Contains(sampleformats, ctx.SampleFormat()) { + return ErrBadParameter.Withf("unsupported sample format %v", ctx.SampleFormat()) + } + } else if ctx.SampleFormat() == ff.AV_SAMPLE_FMT_NONE { + return ErrBadParameter.With("sample format not set") + } + if len(samplerates) > 0 { + if !slices.Contains(samplerates, ctx.Samplerate()) { + return ErrBadParameter.Withf("unsupported samplerate %v", ctx.Samplerate()) + } + } else if ctx.Samplerate() == 0 { + return ErrBadParameter.With("samplerate not set") + } + if len(channellayouts) > 0 { + valid := false + for _, ch := range channellayouts { + chctx := ctx.ChannelLayout() + if ff.AVUtil_channel_layout_compare(&ch, &chctx) { + valid = true + break + } + } + if !valid { + return ErrBadParameter.Withf("unsupported channel layout %v", ctx.ChannelLayout()) + } + } else if ctx.ChannelLayout().NumChannels() == 0 { + return ErrBadParameter.With("channel layout not set") + } + + // Validated + return nil +} + +func (ctx *Par) copyVideoCodec(codec *ff.AVCodecContext) error { + fmt.Println("TODO: copyVideoCodec") + return nil +} + +func (ctx *Par) validateVideoCodec(codec *ff.AVCodecContext) error { + fmt.Println("TODO: validateVideoCodec") + return nil +} diff --git a/pkg/ffmpeg/par_test.go b/pkg/ffmpeg/par_test.go new file mode 100644 index 0000000..4f8d88a --- /dev/null +++ b/pkg/ffmpeg/par_test.go @@ -0,0 +1,28 @@ +package ffmpeg_test + +import ( + "testing" + + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + assert "github.com/stretchr/testify/assert" +) + +func Test_par_001(t *testing.T) { + assert := assert.New(t) + + par, err := ffmpeg.NewAudioPar("fltp", "mono", 22050) + if !assert.NoError(err) { + t.FailNow() + } + t.Log(par) +} + +func Test_par_002(t *testing.T) { + assert := assert.New(t) + + par, err := ffmpeg.NewVideoPar("yuv420p", "1280x720") + if !assert.NoError(err) { + t.FailNow() + } + t.Log(par) +} diff --git a/pkg/ffmpeg/resampler.go b/pkg/ffmpeg/resampler.go index 84b0aeb..e56d2f8 100644 --- a/pkg/ffmpeg/resampler.go +++ b/pkg/ffmpeg/resampler.go @@ -12,9 +12,9 @@ import ( // TYPES type resampler struct { - opts - ctx *ff.SWRContext - dest *ff.AVFrame + ctx *ff.SWRContext + dest *ff.AVFrame + force bool } //////////////////////////////////////////////////////////////////////////////// @@ -23,23 +23,26 @@ type resampler struct { // Create a new audio resampler which will resample the input frame to the // specified channel layout, sample rate and sample format. func NewResampler(format ff.AVSampleFormat, opt ...Opt) (*resampler, error) { + options := newOpts() resampler := new(resampler) // Apply options - resampler.sample_fmt = format - resampler.ch = ff.AV_CHANNEL_LAYOUT_MONO - resampler.samplerate = 44100 + options.par.SetCodecType(ff.AVMEDIA_TYPE_AUDIO) + options.par.SetSampleFormat(format) + options.par.SetChannelLayout(ff.AV_CHANNEL_LAYOUT_MONO) + options.par.SetSamplerate(44100) for _, o := range opt { - if err := o(&resampler.opts); err != nil { + if err := o(options); err != nil { return nil, err } } // Check parameters - if resampler.sample_fmt == ff.AV_SAMPLE_FMT_NONE { + if options.par.SampleFormat() == ff.AV_SAMPLE_FMT_NONE { return nil, errors.New("invalid sample format parameters") } - if !ff.AVUtil_channel_layout_check(&resampler.ch) { + ch := options.par.ChannelLayout() + if !ff.AVUtil_channel_layout_check(&ch) { return nil, errors.New("invalid channel layout parameters") } @@ -52,15 +55,18 @@ func NewResampler(format ff.AVSampleFormat, opt ...Opt) (*resampler, error) { // Set parameters - we don't allocate the buffer here, // we do that when we have a source frame and know how // large the destination frame should be - dest.SetSampleFormat(resampler.sample_fmt) - dest.SetSampleRate(resampler.samplerate) - if err := dest.SetChannelLayout(resampler.ch); err != nil { + dest.SetSampleFormat(options.par.SampleFormat()) + dest.SetSampleRate(options.par.Samplerate()) + if err := dest.SetChannelLayout(options.par.ChannelLayout()); err != nil { ff.AVUtil_frame_free(dest) return nil, err } else { resampler.dest = dest } + // Set force flag + resampler.force = options.force + // Return success return resampler, nil } diff --git a/pkg/ffmpeg/resampler_test.go b/pkg/ffmpeg/resampler_test.go index e84f54d..dc3adc5 100644 --- a/pkg/ffmpeg/resampler_test.go +++ b/pkg/ffmpeg/resampler_test.go @@ -13,7 +13,7 @@ func Test_resampler_001(t *testing.T) { assert := assert.New(t) // Sine wave generator - audio, err := generator.NewSine(2000, 10, 44100) + audio, err := generator.NewSine(2000, 10, ffmpeg.AudioPar("fltp", "mono", 44100)) if !assert.NoError(err) { t.FailNow() } diff --git a/pkg/ffmpeg/rescaler.go b/pkg/ffmpeg/rescaler.go index 87deafa..f6cfb6e 100644 --- a/pkg/ffmpeg/rescaler.go +++ b/pkg/ffmpeg/rescaler.go @@ -12,13 +12,12 @@ import ( // TYPES type rescaler struct { - opts - src_pix_fmt ff.AVPixelFormat src_width int src_height int ctx *ff.SWSContext flags ff.SWSFlag + force bool dest *ff.AVFrame } @@ -28,20 +27,22 @@ type rescaler struct { // Create a new rescaler which will rescale the input frame to the // specified format, width and height. func NewRescaler(format ff.AVPixelFormat, opt ...Opt) (*rescaler, error) { + options := newOpts() rescaler := new(rescaler) // Apply options - rescaler.pix_fmt = format - rescaler.width = 640 - rescaler.height = 480 + options.par.SetCodecType(ff.AVMEDIA_TYPE_VIDEO) + options.par.SetPixelFormat(format) + options.par.SetWidth(640) + options.par.SetHeight(480) for _, o := range opt { - if err := o(&rescaler.opts); err != nil { + if err := o(options); err != nil { return nil, err } } // Check parameters - if rescaler.pix_fmt == ff.AV_PIX_FMT_NONE { + if options.par.PixelFormat() == ff.AV_PIX_FMT_NONE { return nil, errors.New("invalid parameters") } @@ -51,10 +52,13 @@ func NewRescaler(format ff.AVPixelFormat, opt ...Opt) (*rescaler, error) { return nil, errors.New("failed to allocate frame") } + // Set force flag + rescaler.force = options.force + // Set parameters - dest.SetPixFmt(rescaler.pix_fmt) - dest.SetWidth(rescaler.width) - dest.SetHeight(rescaler.height) + dest.SetPixFmt(options.par.PixelFormat()) + dest.SetWidth(options.par.Width()) + dest.SetHeight(options.par.Height()) // Allocate buffer if err := ff.AVUtil_frame_get_buffer(dest, false); err != nil { diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index 82f7377..2987c28 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -4,9 +4,12 @@ import ( "errors" "fmt" "io" + "os" + "sort" // Packages ff "github.com/mutablelogic/go-media/sys/ffmpeg61" + maps "golang.org/x/exp/maps" // Namespace imports . "github.com/djthorpe/go-errors" @@ -17,7 +20,9 @@ import ( // Create media from io.Writer type Writer struct { - output *ff.AVFormatContext + output *ff.AVFormatContext + header bool + encoders []*Encoder } type writer_callback struct { @@ -35,33 +40,58 @@ const ( // LIFECYCLE func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { - var o opts - + options := newOpts() writer := new(Writer) // Apply options for _, opt := range opt { - if err := opt(&o); err != nil { + if err := opt(options); err != nil { return nil, err } } // Check output - if o.oformat == nil { + var filename string + if options.oformat == nil { return nil, ErrBadParameter.Withf("invalid output format") + } else if w_, ok := w.(*os.File); ok { + filename = w_.Name() } // Allocate the AVIO context - avio := ff.AVFormat_avio_alloc_context(bufSize, false, &writer_callback{w}) + avio := ff.AVFormat_avio_alloc_context(bufSize, true, &writer_callback{w}) if avio == nil { return nil, errors.New("failed to allocate avio context") - } else if ctx, err := ff.AVFormat_open_writer(avio, o.oformat, ""); err != nil { + } else if ctx, err := ff.AVFormat_open_writer(avio, options.oformat, filename); err != nil { return nil, err } else { writer.output = ctx } - fmt.Println("WRITER", writer.output) + // Create codec contexts for each stream + var result error + keys := sort.IntSlice(maps.Keys(options.streams)) + for _, stream := range keys { + encoder, err := NewEncoder(writer.output, stream, options.streams[stream]) + if err != nil { + result = errors.Join(result, err) + continue + } else { + writer.encoders = append(writer.encoders, encoder) + } + } + + // Return any errors + if result != nil { + return nil, errors.Join(result, writer.Close()) + } + + // Write the header + if err := ff.AVFormat_write_header(writer.output, nil); err != nil { + return nil, errors.Join(err, writer.Close()) + } else { + writer.header = true + } // Return success return writer, nil @@ -70,23 +100,113 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { func (w *Writer) Close() error { var result error + // Write the trailer if the header was written + if w.header { + if err := ff.AVFormat_write_trailer(w.output); err != nil { + result = errors.Join(result, err) + } + } + + // Close encoders + for _, encoder := range w.encoders { + result = errors.Join(result, encoder.Close()) + } + // Free output resources if w.output != nil { - // This calls avio_close(w.avio) - fmt.Println("TODO: AVFormat_close_writer") result = errors.Join(result, ff.AVFormat_close_writer(w.output)) - w.output = nil } + // Free resources + w.output = nil + w.encoders = nil + // Return any errors return result } +////////////////////////////////////////////////////////////////////////////// +// PUBLIC METHODS + +// Return a "stream" for encoding +func (w *Writer) Stream(stream int) *Encoder { + for _, encoder := range w.encoders { + if encoder.stream.Id() == stream { + return encoder + } + } + return nil +} + +// Encode frames from all encoders, calling the callback function to encode +// the frame. If the callback function returns io.EOF then the encoding for +// that encoder is stopped after flushing. If the second callback is nil, +// then packets are written to the output. +func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { + if in == nil { + return ErrBadParameter.With("nil in or out") + } + if out == nil { + // By default, write packet to output + out = w.Write + } + + // Initialise encoders + encoders := make(map[int]*Encoder, len(w.encoders)) + for _, encoder := range w.encoders { + stream := encoder.stream.Id() + if _, exists := encoders[stream]; exists { + return ErrBadParameter.Withf("duplicate stream %v", stream) + } + encoders[stream] = encoder + } + + // Continue until all encoders have returned io.EOF + for { + // No more encoding to do + if len(encoders) == 0 { + break + } + for stream, encoder := range encoders { + // Receive a frame for the encoder + frame, err := in(stream) + if errors.Is(err, io.EOF) { + fmt.Println("EOF for frame on stream", stream) + delete(encoders, stream) + } else if err != nil { + return fmt.Errorf("stream %v: %w", stream, err) + } else if frame == nil { + return fmt.Errorf("stream %v: nil frame received", stream) + } else if err := encoder.Encode(frame, out); errors.Is(err, io.EOF) { + fmt.Println("EOF for packet on stream", stream) + delete(encoders, stream) + } else if err != nil { + return fmt.Errorf("stream %v: %w", stream, err) + } + } + } + + // Return success + return nil +} + +// Write a packet to the output +func (w *Writer) Write(packet *ff.AVPacket) error { + return ff.AVCodec_interleaved_write_frame(w.output, packet) +} + //////////////////////////////////////////////////////////////////////////////// -// PRIVATE METHODS +// PRIVATE METHODS - Writer func (w *writer_callback) Reader(buf []byte) int { - return 0 + if r, ok := w.w.(io.Reader); ok { + if n, err := r.Read(buf); err != nil { + return -1 + } else { + return n + } + } + return -1 } func (w *writer_callback) Seeker(offset int64, whence int) int64 { diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index 2836309..e421574 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -1,11 +1,14 @@ package ffmpeg_test import ( - "fmt" + "io" "os" "testing" + "time" ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" + generator "github.com/mutablelogic/go-media/pkg/generator" + ff "github.com/mutablelogic/go-media/sys/ffmpeg61" assert "github.com/stretchr/testify/assert" ) @@ -22,11 +25,36 @@ func Test_writer_001(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.NewWriter(w, ffmpeg.OptOutputFormat(w.Name()), + ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), ) if !assert.NoError(err) { t.FailNow() } defer writer.Close() - fmt.Println("Written to", w.Name()) + // Make an audio generator + audio, err := generator.NewSine(440, -5, writer.Stream(1).Par()) + if !assert.NoError(err) { + t.FailNow() + } + defer audio.Close() + + // Write frames + n := 0 + assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { + frame := audio.Frame() + t.Log("Frame s", frame.Time().Truncate(time.Millisecond)) + if frame.Time() > 10*time.Second { + return nil, io.EOF + } else { + return frame.(*ffmpeg.Frame).AVFrame(), nil + } + }, func(packet *ff.AVPacket) error { + if packet != nil { + t.Log("Packet ts", packet.Pts()) + n += packet.Size() + } + return writer.Write(packet) + })) + t.Log("Written", n, "bytes to", w.Name()) } diff --git a/pkg/generator/sine.go b/pkg/generator/sine.go index 32b1e0e..2763ff1 100644 --- a/pkg/generator/sine.go +++ b/pkg/generator/sine.go @@ -27,7 +27,7 @@ var _ Generator = (*sine)(nil) // GLOBALS const ( - frameDuration = 20 * time.Millisecond // Each frame is 20ms of audio + frameDuration = 10 * time.Millisecond // Each frame is 10ms of audio ) //////////////////////////////////////////////////////////////////////////// @@ -36,19 +36,29 @@ const ( // Create a new sine wave generator with one channel using float32 // for samples. The frequency in Hz, volume in decibels and samplerate // (ie, 44100) for the audio stream are passed as arguments. -func NewSine(freq float64, volume float64, samplerate int) (*sine, error) { +func NewSine(freq, volume float64, par *ffmpeg.Par) (*sine, error) { sine := new(sine) // Check parameters + if par.CodecType() != ff.AVMEDIA_TYPE_AUDIO { + return nil, errors.New("invalid codec type") + } else if par.ChannelLayout().NumChannels() != 1 { + return nil, errors.New("invalid channel layout, only mono is supported") + } else if par.SampleFormat() != ff.AV_SAMPLE_FMT_FLT && par.SampleFormat() != ff.AV_SAMPLE_FMT_FLTP { + return nil, errors.New("invalid sample format, only float32 is supported") + } if freq <= 0 { return nil, errors.New("invalid frequency") } if volume <= -100 { return nil, errors.New("invalid volume") } - if samplerate <= 0 { + if par.Samplerate() <= 0 { return nil, errors.New("invalid samplerate") } + if par.FrameSize() <= 0 { + par.SetFrameSize(int(float64(par.Samplerate()) * frameDuration.Seconds())) + } // Create a frame frame := ff.AVUtil_frame_alloc() @@ -56,16 +66,13 @@ func NewSine(freq float64, volume float64, samplerate int) (*sine, error) { return nil, errors.New("failed to allocate frame") } - // Set frame parameters - numSamples := int(float64(samplerate) * frameDuration.Seconds()) - frame.SetSampleFormat(ff.AV_SAMPLE_FMT_FLT) // float32 if err := frame.SetChannelLayout(ff.AV_CHANNEL_LAYOUT_MONO); err != nil { return nil, err } - frame.SetSampleRate(samplerate) - frame.SetNumSamples(numSamples) - frame.SetTimeBase(ff.AVUtil_rational(1, samplerate)) + frame.SetSampleRate(par.Samplerate()) + frame.SetNumSamples(par.FrameSize()) + frame.SetTimeBase(ff.AVUtil_rational(1, par.Samplerate())) frame.SetPts(ff.AV_NOPTS_VALUE) // Allocate buffer @@ -101,6 +108,10 @@ func (s *sine) String() string { // Return the first and subsequent frames of raw audio data func (s *sine) Frame() media.Frame { + if err := ff.AVUtil_frame_make_writable(s.frame); err != nil { + return nil + } + // Set the Pts if s.frame.Pts() == ff.AV_NOPTS_VALUE { s.frame.SetPts(0) diff --git a/pkg/generator/sine_test.go b/pkg/generator/sine_test.go index 95d15ad..7781aee 100644 --- a/pkg/generator/sine_test.go +++ b/pkg/generator/sine_test.go @@ -6,13 +6,14 @@ import ( "testing" "time" + "github.com/mutablelogic/go-media/pkg/ffmpeg" "github.com/mutablelogic/go-media/pkg/generator" "github.com/stretchr/testify/assert" ) func Test_sine_001(t *testing.T) { assert := assert.New(t) - sine, err := generator.NewSine(2000, 10, 44100) + sine, err := generator.NewSine(2000, 10, ffmpeg.AudioPar("fltp", "mono", 44100)) if !assert.NoError(err) { t.SkipNow() } @@ -23,7 +24,7 @@ func Test_sine_001(t *testing.T) { func Test_sine_002(t *testing.T) { assert := assert.New(t) - sine, err := generator.NewSine(2000, 10, 44100) + sine, err := generator.NewSine(2000, 10, ffmpeg.AudioPar("fltp", "mono", 44100)) if !assert.NoError(err) { t.SkipNow() } @@ -42,7 +43,7 @@ func Test_sine_003(t *testing.T) { const frequency = 440 const volume = -10.0 - sine, err := generator.NewSine(frequency, volume, sampleRate) + sine, err := generator.NewSine(frequency, volume, ffmpeg.AudioPar("fltp", "mono", sampleRate)) if !assert.NoError(err) { t.SkipNow() } diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go index cb3133c..3ad5556 100644 --- a/pkg/generator/yuv420p.go +++ b/pkg/generator/yuv420p.go @@ -80,6 +80,10 @@ func (yuv420p *yuv420p) String() string { // Return the first and subsequent frames of raw video data func (yuv420p *yuv420p) Frame() media.Frame { + if err := ff.AVUtil_frame_make_writable(yuv420p.frame); err != nil { + return nil + } + // Set the Pts if yuv420p.frame.Pts() == ff.AV_NOPTS_VALUE { yuv420p.frame.SetPts(0) diff --git a/sys/ffmpeg61/avcodec.go b/sys/ffmpeg61/avcodec.go index e7bd26e..d4e6def 100644 --- a/sys/ffmpeg61/avcodec.go +++ b/sys/ffmpeg61/avcodec.go @@ -59,6 +59,7 @@ type jsonAVCodecContext struct { SampleFormat AVSampleFormat `json:"sample_fmt,omitempty"` SampleRate int `json:"sample_rate,omitempty"` ChannelLayout AVChannelLayout `json:"channel_layout,omitempty"` + FrameSize int `json:"frame_size,omitempty"` TimeBase AVRational `json:"time_base,omitempty"` } @@ -192,6 +193,7 @@ func (ctx *AVCodecContext) MarshalJSON() ([]byte, error) { SampleFormat: AVSampleFormat(ctx.sample_fmt), SampleRate: int(ctx.sample_rate), ChannelLayout: AVChannelLayout(ctx.ch_layout), + FrameSize: int(ctx.frame_size), }) default: return json.Marshal(jsonAVCodecContext{ diff --git a/sys/ffmpeg61/avcodec_parameters.go b/sys/ffmpeg61/avcodec_parameters.go index 8166bea..833c303 100644 --- a/sys/ffmpeg61/avcodec_parameters.go +++ b/sys/ffmpeg61/avcodec_parameters.go @@ -2,6 +2,7 @@ package ffmpeg import ( "encoding/json" + "errors" ) //////////////////////////////////////////////////////////////////////////////// @@ -18,14 +19,14 @@ import "C" // TYPES type jsonAVCodecParametersAudio struct { - SampleFormat AVSampleFormat `json:"format"` + SampleFormat AVSampleFormat `json:"sample_format"` SampleRate int `json:"sample_rate"` ChannelLayout AVChannelLayout `json:"channel_layout"` FrameSize int `json:"frame_size,omitempty"` } type jsonAVCodecParameterVideo struct { - PixelFormat AVPixelFormat `json:"format"` + PixelFormat AVPixelFormat `json:"pixel_format"` Width int `json:"width"` Height int `json:"height"` SampleAspectRatio AVRational `json:"sample_aspect_ratio,omitempty"` @@ -43,7 +44,7 @@ type jsonAVCodecParameters struct { //////////////////////////////////////////////////////////////////////////////// // STRINGIFY -func (ctx *AVCodecParameters) MarshalJSON() ([]byte, error) { +func (ctx AVCodecParameters) MarshalJSON() ([]byte, error) { par := jsonAVCodecParameters{ CodecType: AVMediaType(ctx.codec_type), CodecID: AVCodecID(ctx.codec_id), @@ -82,6 +83,10 @@ func (ctx *AVCodecParameters) CodecType() AVMediaType { return AVMediaType(ctx.codec_type) } +func (ctx *AVCodecParameters) SetCodecType(t AVMediaType) { + ctx.codec_type = C.enum_AVMediaType(t) +} + func (ctx *AVCodecParameters) CodecID() AVCodecID { return AVCodecID(ctx.codec_id) } @@ -104,6 +109,10 @@ func (ctx *AVCodecParameters) BitRate() int64 { return int64(ctx.bit_rate) } +func (ctx *AVCodecParameters) SetBitRate(rate int64) { + ctx.bit_rate = C.int64_t(rate) +} + // Audio func (ctx *AVCodecParameters) SampleFormat() AVSampleFormat { if AVMediaType(ctx.codec_type) == AVMEDIA_TYPE_AUDIO { @@ -113,21 +122,41 @@ func (ctx *AVCodecParameters) SampleFormat() AVSampleFormat { } } +func (ctx *AVCodecParameters) SetSampleFormat(format AVSampleFormat) { + ctx.format = C.int(format) +} + // Audio func (ctx *AVCodecParameters) Samplerate() int { return int(ctx.sample_rate) } +func (ctx *AVCodecParameters) SetSamplerate(rate int) { + ctx.sample_rate = C.int(rate) +} + // Audio func (ctx *AVCodecParameters) ChannelLayout() AVChannelLayout { return AVChannelLayout(ctx.ch_layout) } +func (ctx *AVCodecParameters) SetChannelLayout(layout AVChannelLayout) error { + if !AVUtil_channel_layout_check(&layout) { + return errors.New("invalid channel layout") + } + ctx.ch_layout = C.AVChannelLayout(layout) + return nil +} + // Audio func (ctx *AVCodecParameters) FrameSize() int { return int(ctx.frame_size) } +func (ctx *AVCodecParameters) SetFrameSize(size int) { + ctx.frame_size = C.int(size) +} + // Video func (ctx *AVCodecParameters) PixelFormat() AVPixelFormat { if AVMediaType(ctx.codec_type) == AVMEDIA_TYPE_VIDEO { @@ -137,17 +166,33 @@ func (ctx *AVCodecParameters) PixelFormat() AVPixelFormat { } } +func (ctx *AVCodecParameters) SetPixelFormat(format AVPixelFormat) { + ctx.format = C.int(format) +} + // Video func (ctx *AVCodecParameters) SampleAspectRatio() AVRational { return AVRational(ctx.sample_aspect_ratio) } +func (ctx *AVCodecParameters) SetSampleAspectRatio(aspect AVRational) { + ctx.sample_aspect_ratio = C.AVRational(aspect) +} + // Video func (ctx *AVCodecParameters) Width() int { return int(ctx.width) } +func (ctx *AVCodecParameters) SetWidth(width int) { + ctx.width = C.int(width) +} + // Video func (ctx *AVCodecParameters) Height() int { return int(ctx.height) } + +func (ctx *AVCodecParameters) SetHeight(height int) { + ctx.height = C.int(height) +} From 80bf31d7e09ab12718f4a3ad336a6cfb3af2b5f3 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 14:20:00 +0200 Subject: [PATCH 2/5] Updated writer --- pkg/ffmpeg/encoder.go | 18 +++--- pkg/ffmpeg/metadata.go | 35 ++++++++++++ pkg/ffmpeg/opts.go | 22 ++++--- pkg/ffmpeg/writer.go | 105 ++++++++++++++++++++++++++++------ pkg/ffmpeg/writer_test.go | 60 ++++++++++++++++--- sys/ffmpeg61/avformat.go | 10 ++++ sys/ffmpeg61/avformat_avio.go | 49 ++++++++++------ 7 files changed, 244 insertions(+), 55 deletions(-) create mode 100644 pkg/ffmpeg/metadata.go diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go index 142ab39..36e332e 100644 --- a/pkg/ffmpeg/encoder.go +++ b/pkg/ffmpeg/encoder.go @@ -3,7 +3,6 @@ package ffmpeg import ( "encoding/json" "errors" - "fmt" "io" "syscall" @@ -21,6 +20,9 @@ type Encoder struct { ctx *ff.AVCodecContext stream *ff.AVStream packet *ff.AVPacket + + // We are flushing the encoder + eof bool //next_pts int64 } @@ -28,9 +30,9 @@ type Encoder struct { // return nil to continue encoding or io.EOF to stop encoding. type EncoderFrameFn func(int) (*ff.AVFrame, error) -// EncoderPacketFn is a function which is called for each packet encoded. It should -// return nil to continue encoding or io.EOF to stop encoding immediately. -type EncoderPacketFn func(*ff.AVPacket) error +// EncoderPacketFn is a function which is called for each packet encoded, with +// the stream timebase. +type EncoderPacketFn func(*ff.AVPacket, *ff.AVRational) error //////////////////////////////////////////////////////////////////////////////// // LIFECYCLE @@ -178,8 +180,9 @@ func (e *Encoder) Par() *Par { // PRIVATE METHODS func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { + timebase := e.stream.TimeBase() + // Send the frame to the encoder - fmt.Println("Sending frame", frame) if err := ff.AVCodec_send_frame(e.ctx, frame); err != nil { if errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { return nil @@ -191,7 +194,6 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { var result error for { // Receive the packet - fmt.Println("Receiving packet") if err := ff.AVCodec_receive_packet(e.ctx, e.packet); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { // Finished receiving packet or EOF break @@ -200,7 +202,7 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { } // Pass back to the caller - if err := fn(e.packet); errors.Is(err, io.EOF) { + if err := fn(e.packet, &timebase); errors.Is(err, io.EOF) { // End early, return EOF result = io.EOF break @@ -214,7 +216,7 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { // Flush if result == nil { - result = fn(nil) + result = fn(nil, &timebase) } // Return success or EOF diff --git a/pkg/ffmpeg/metadata.go b/pkg/ffmpeg/metadata.go new file mode 100644 index 0000000..6964b74 --- /dev/null +++ b/pkg/ffmpeg/metadata.go @@ -0,0 +1,35 @@ +package ffmpeg + +import ( + "encoding/json" +) + +//////////////////////////////////////////////////////////////////////////////// +// TYPES + +type Metadata struct { + Key string `json:"key" writer:",width:30"` + Value any `json:"value,omitempty" writer:",wrap,width:50"` +} + +const ( + MetaArtwork = "artwork" // Metadata key for artwork, set the value as []byte +) + +//////////////////////////////////////////////////////////////////////////////// +// LIFECYCLE + +func NewMetadata(key string, value any) *Metadata { + return &Metadata{ + Key: key, + Value: value, + } +} + +//////////////////////////////////////////////////////////////////////////////// +// STRINIGY + +func (m *Metadata) String() string { + data, _ := json.MarshalIndent(m, "", " ") + return string(data) +} diff --git a/pkg/ffmpeg/opts.go b/pkg/ffmpeg/opts.go index 90b8c63..cfb27fd 100644 --- a/pkg/ffmpeg/opts.go +++ b/pkg/ffmpeg/opts.go @@ -1,10 +1,11 @@ package ffmpeg import ( - // Namespace imports + // Package imports + ffmpeg "github.com/mutablelogic/go-media/sys/ffmpeg61" + // Namespace imports . "github.com/djthorpe/go-errors" - ffmpeg "github.com/mutablelogic/go-media/sys/ffmpeg61" ) //////////////////////////////////////////////////////////////////////////////// @@ -17,11 +18,10 @@ type opts struct { force bool par *Par - // Format options - oformat *ffmpeg.AVOutputFormat - - // Stream options - streams map[int]*Par + // Writer options + oformat *ffmpeg.AVOutputFormat + streams map[int]*Par + metadata []*Metadata } //////////////////////////////////////////////////////////////////////////////// @@ -103,6 +103,14 @@ func OptForce() Opt { } } +// Append metadata to the output file, including artwork +func OptMetadata(entry ...*Metadata) Opt { + return func(o *opts) error { + o.metadata = append(o.metadata, entry...) + return nil + } +} + // Pixel format of the output frame func OptPixFormat(format string) Opt { return func(o *opts) error { diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index 2987c28..02c8b4e 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -39,6 +39,40 @@ const ( ////////////////////////////////////////////////////////////////////////////// // LIFECYCLE +// Create a new writer with a URL and options +func Create(url string, opt ...Opt) (*Writer, error) { + options := newOpts() + writer := new(Writer) + + // Apply options + for _, opt := range opt { + if err := opt(options); err != nil { + return nil, err + } + } + + // Guess the output format + var ofmt *ff.AVOutputFormat + if options.oformat == nil && url != "" { + options.oformat = ff.AVFormat_guess_format("", url, "") + } + if options.oformat == nil { + return nil, ErrBadParameter.With("unable to guess the output format") + } + + // Allocate the output media context + ctx, err := ff.AVFormat_create_file(url, ofmt) + if err != nil { + return nil, err + } else { + writer.output = ctx + } + + // Continue with open + return writer.open(options) +} + +// Create a new writer with an io.Writer and options func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { options := newOpts() writer := new(Writer) @@ -68,6 +102,11 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { writer.output = ctx } + // Continue with open + return writer.open(options) +} + +func (writer *Writer) open(options *opts) (*Writer, error) { // Create codec contexts for each stream var result error keys := sort.IntSlice(maps.Keys(options.streams)) @@ -86,7 +125,25 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { return nil, errors.Join(result, writer.Close()) } - // Write the header + // Add metadata + metadata := ff.AVUtil_dict_alloc() + if metadata == nil { + return nil, errors.Join(errors.New("unable to allocate metadata dictionary"), writer.Close()) + } + for _, entry := range options.metadata { + // Ignore artwork fields + if entry.Key == MetaArtwork || entry.Key == "" || entry.Value == nil { + continue + } + // Set dictionary entry + if err := ff.AVUtil_dict_set(metadata, entry.Key, fmt.Sprint(entry.Value), ff.AV_DICT_APPEND); err != nil { + return nil, errors.Join(err, writer.Close()) + } + } + + // Set metadata, write the header + // Metadata ownership is transferred to the output context + writer.output.SetMetadata(metadata) if err := ff.AVFormat_write_header(writer.output, nil); err != nil { return nil, errors.Join(err, writer.Close()) } else { @@ -97,6 +154,7 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { return writer, nil } +// Close a writer and release resources func (w *Writer) Close() error { var result error @@ -148,7 +206,9 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { } if out == nil { // By default, write packet to output - out = w.Write + out = func(pkt *ff.AVPacket, tb *ff.AVRational) error { + return w.Write(pkt) + } } // Initialise encoders @@ -159,29 +219,41 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { return ErrBadParameter.Withf("duplicate stream %v", stream) } encoders[stream] = encoder + + // Initialize the encoder + encoder.eof = false } - // Continue until all encoders have returned io.EOF + // Continue until all encoders have returned io.EOF and have been flushed for { // No more encoding to do if len(encoders) == 0 { break } + + // TODO: We get the encoder with the lowest timestamp for stream, encoder := range encoders { - // Receive a frame for the encoder - frame, err := in(stream) - if errors.Is(err, io.EOF) { - fmt.Println("EOF for frame on stream", stream) - delete(encoders, stream) - } else if err != nil { + var frame *ff.AVFrame + var err error + + // Receive a frame if not EOF + if !encoder.eof { + frame, err = in(stream) + if errors.Is(err, io.EOF) { + encoder.eof = true + } else if err != nil { + return fmt.Errorf("stream %v: %w", stream, err) + } + } + + // Send a frame for encoding + if err := encoder.Encode(frame, out); err != nil { return fmt.Errorf("stream %v: %w", stream, err) - } else if frame == nil { - return fmt.Errorf("stream %v: nil frame received", stream) - } else if err := encoder.Encode(frame, out); errors.Is(err, io.EOF) { - fmt.Println("EOF for packet on stream", stream) + } + + // If eof then delete the encoder + if encoder.eof { delete(encoders, stream) - } else if err != nil { - return fmt.Errorf("stream %v: %w", stream, err) } } } @@ -190,7 +262,8 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { return nil } -// Write a packet to the output +// Write a packet to the output. If you intercept the packets in the +// Encode method, then you can use this method to write packets to the output. func (w *Writer) Write(packet *ff.AVPacket) error { return ff.AVCodec_interleaved_write_frame(w.output, packet) } diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index e421574..730631e 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -25,6 +25,7 @@ func Test_writer_001(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.NewWriter(w, ffmpeg.OptOutputFormat(w.Name()), + ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), ) if !assert.NoError(err) { @@ -40,21 +41,66 @@ func Test_writer_001(t *testing.T) { defer audio.Close() // Write frames - n := 0 + duration := 1000 * time.Minute assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { frame := audio.Frame() - t.Log("Frame s", frame.Time().Truncate(time.Millisecond)) - if frame.Time() > 10*time.Second { + if frame.Time() >= duration { return nil, io.EOF } else { + t.Log("Frame s", frame.Time().Truncate(time.Millisecond)) return frame.(*ffmpeg.Frame).AVFrame(), nil } - }, func(packet *ff.AVPacket) error { + }, func(packet *ff.AVPacket, timebase *ff.AVRational) error { if packet != nil { - t.Log("Packet ts", packet.Pts()) - n += packet.Size() + t.Log("Packet", packet) } return writer.Write(packet) })) - t.Log("Written", n, "bytes to", w.Name()) + t.Log("Written to", w.Name()) +} + +func Test_writer_002(t *testing.T) { + assert := assert.New(t) + + // Write to a file + w, err := os.CreateTemp("", t.Name()+"_*.mp3") + if !assert.NoError(err) { + t.FailNow() + } + defer w.Close() + + // Create a writer with an audio stream + writer, err := ffmpeg.Create(w.Name(), + ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), + ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), + ) + if !assert.NoError(err) { + t.FailNow() + } + defer writer.Close() + + // Make an audio generator + audio, err := generator.NewSine(440, -5, writer.Stream(1).Par()) + if !assert.NoError(err) { + t.FailNow() + } + defer audio.Close() + + // Write frames + duration := 1000 * time.Minute + assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { + frame := audio.Frame() + if frame.Time() >= duration { + return nil, io.EOF + } else { + t.Log("Frame s", frame.Time().Truncate(time.Millisecond)) + return frame.(*ffmpeg.Frame).AVFrame(), nil + } + }, func(packet *ff.AVPacket, timebase *ff.AVRational) error { + if packet != nil { + t.Log("Packet", packet) + } + return writer.Write(packet) + })) + t.Log("Written to", w.Name()) } diff --git a/sys/ffmpeg61/avformat.go b/sys/ffmpeg61/avformat.go index ce8cada..911015a 100644 --- a/sys/ffmpeg61/avformat.go +++ b/sys/ffmpeg61/avformat.go @@ -119,6 +119,7 @@ type jsonAVFormatContext struct { BitRate int64 `json:"bit_rate,omitempty"` PacketSize uint `json:"packet_size,omitempty"` Flags AVFormatFlag `json:"flags,omitempty"` + Metadata *AVDictionary `json:"metadata,omitempty"` } func (ctx *AVFormatContext) MarshalJSON() ([]byte, error) { @@ -134,6 +135,7 @@ func (ctx *AVFormatContext) MarshalJSON() ([]byte, error) { BitRate: int64(ctx.bit_rate), PacketSize: uint(ctx.packet_size), Flags: AVFormatFlag(ctx.flags), + Metadata: ctx.Metadata(), }) } @@ -229,6 +231,14 @@ func (ctx *AVFormatContext) Metadata() *AVDictionary { return &AVDictionary{ctx.metadata} } +func (ctx *AVFormatContext) SetMetadata(dict *AVDictionary) { + if dict == nil { + ctx.metadata = nil + } else { + ctx.metadata = dict.ctx + } +} + func (ctx *AVFormatContext) SetPb(pb *AVIOContextEx) { if pb == nil { ctx.pb = nil diff --git a/sys/ffmpeg61/avformat_avio.go b/sys/ffmpeg61/avformat_avio.go index d2dbe42..f1a8e6c 100644 --- a/sys/ffmpeg61/avformat_avio.go +++ b/sys/ffmpeg61/avformat_avio.go @@ -1,7 +1,7 @@ package ffmpeg import ( - "runtime" + "fmt" "unsafe" ) @@ -21,7 +21,7 @@ static AVIOContext* avio_alloc_context_(int sz, int writeable, void* userInfo) { if (!buf) { return NULL; } - return avio_alloc_context(buf, sz, writeable, userInfo,avio_read_callback,avio_write_callback,avio_seek_callback); + return avio_alloc_context(buf, sz, writeable, userInfo, avio_read_callback, avio_write_callback, avio_seek_callback); } */ import "C" @@ -32,8 +32,6 @@ import "C" // Wrapper around AVIOContext with callbacks type AVIOContextEx struct { *AVIOContext - cb AVIOContextCallback - pin *runtime.Pinner } // Callbacks for AVIOContextEx @@ -43,6 +41,10 @@ type AVIOContextCallback interface { Seeker(offset int64, whence int) int64 } +var ( + callbacks = make(map[uintptr]AVIOContextCallback) +) + //////////////////////////////////////////////////////////////////////////////// // FUNCTIONS @@ -50,10 +52,10 @@ type AVIOContextCallback interface { func AVFormat_avio_alloc_context(sz int, writeable bool, callback AVIOContextCallback) *AVIOContextEx { // Create a context ctx := new(AVIOContextEx) - ctx.cb = callback - ctx.pin = new(runtime.Pinner) - ctx.pin.Pin(ctx.cb) - ctx.pin.Pin(ctx.pin) + + // Set the callback + ptr := uintptr(unsafe.Pointer(ctx)) + callbacks[ptr] = callback // Allocate the context ctx.AVIOContext = (*AVIOContext)(C.avio_alloc_context_( @@ -71,8 +73,6 @@ func AVFormat_avio_alloc_context(sz int, writeable bool, callback AVIOContextCal // Create and initialize a AVIOContext for accessing the resource indicated by url. func AVFormat_avio_open(url string, flags AVIOFlag) (*AVIOContextEx, error) { ctx := new(AVIOContextEx) - ctx.pin = new(runtime.Pinner) - ctx.pin.Pin(ctx.pin) cUrl := C.CString(url) defer C.free(unsafe.Pointer(cUrl)) if err := AVError(C.avio_open((**C.struct_AVIOContext)(unsafe.Pointer(&ctx.AVIOContext)), cUrl, C.int(flags))); err != 0 { @@ -99,7 +99,10 @@ func AVFormat_avio_close(ctx *AVIOContextEx) error { func AVFormat_avio_context_free(ctx *AVIOContextEx) { C.av_free(unsafe.Pointer(ctx.buffer)) C.avio_context_free((**C.struct_AVIOContext)(unsafe.Pointer(&ctx.AVIOContext))) - ctx.pin.Unpin() + + // Remove the callback + ptr := uintptr(unsafe.Pointer(ctx)) + delete(callbacks, ptr) } // avio_w8 @@ -145,18 +148,30 @@ func AVFormat_avio_read(ctx *AVIOContextEx, buf []byte) int { //export avio_read_callback func avio_read_callback(userInfo unsafe.Pointer, buf *C.uint8_t, size C.int) C.int { - ctx := (*AVIOContextEx)(userInfo) - return C.int(ctx.cb.Reader(cByteSlice(unsafe.Pointer(buf), size))) + ptr := uintptr(userInfo) + callback, ok := callbacks[ptr] + if !ok { + panic("avio_read_callback: callback not found") + } + return C.int(callback.Reader(cByteSlice(unsafe.Pointer(buf), size))) } //export avio_write_callback func avio_write_callback(userInfo unsafe.Pointer, buf *C.uint8_t, size C.int) C.int { - ctx := (*AVIOContextEx)(userInfo) - return C.int(ctx.cb.Writer(cByteSlice(unsafe.Pointer(buf), size))) + ptr := uintptr(userInfo) + callback, ok := callbacks[ptr] + if !ok { + panic("avio_write_callback: callback not found " + fmt.Sprint(ptr)) + } + return C.int(callback.Writer(cByteSlice(unsafe.Pointer(buf), size))) } //export avio_seek_callback func avio_seek_callback(userInfo unsafe.Pointer, offset C.int64_t, whence C.int) C.int64_t { - ctx := (*AVIOContextEx)(userInfo) - return C.int64_t(ctx.cb.Seeker(int64(offset), int(whence))) + ptr := uintptr(userInfo) + callback, ok := callbacks[ptr] + if !ok { + panic("avio_seek_callback: callback not found") + } + return C.int64_t(callback.Seeker(int64(offset), int(whence))) } From 80efb245875ac10761f4d8df1086098dbd80a17d Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 15:11:44 +0200 Subject: [PATCH 3/5] Updated --- pkg/ffmpeg/opts.go | 30 ++----------- pkg/ffmpeg/par.go | 69 +++++++++++++++++++++++++++--- pkg/ffmpeg/par_test.go | 2 +- pkg/ffmpeg/rescaler_test.go | 6 +-- pkg/ffmpeg/writer.go | 14 ++++-- pkg/ffmpeg/writer_test.go | 58 ++++++++++++++++++++++--- pkg/generator/yuv420p.go | 18 ++++---- pkg/generator/yuv420p_test.go | 7 +-- sys/ffmpeg61/avcodec.go | 58 +++++++++++++++---------- sys/ffmpeg61/avcodec_parameters.go | 10 +++++ sys/ffmpeg61/avutil_rational.go | 10 +++++ 11 files changed, 202 insertions(+), 80 deletions(-) diff --git a/pkg/ffmpeg/opts.go b/pkg/ffmpeg/opts.go index cfb27fd..dbe1660 100644 --- a/pkg/ffmpeg/opts.go +++ b/pkg/ffmpeg/opts.go @@ -50,33 +50,11 @@ func OptOutputFormat(name string) Opt { } } -// New audio stream with parameters -func OptAudioStream(stream int, par *Par) Opt { +// New stream with parameters +func OptStream(stream int, par *Par) Opt { return func(o *opts) error { - if par == nil || par.CodecType() != ffmpeg.AVMEDIA_TYPE_AUDIO { - return ErrBadParameter.With("invalid audio parameters") - } - if stream == 0 { - stream = len(o.streams) + 1 - } - if _, exists := o.streams[stream]; exists { - return ErrDuplicateEntry.Withf("stream %v", stream) - } - if stream < 0 { - return ErrBadParameter.Withf("invalid stream %v", stream) - } - o.streams[stream] = par - - // Return success - return nil - } -} - -// New video stream with parameters -func OptVideoStream(stream int, par *Par) Opt { - return func(o *opts) error { - if par == nil || par.CodecType() != ffmpeg.AVMEDIA_TYPE_VIDEO { - return ErrBadParameter.With("invalid video parameters") + if par == nil { + return ErrBadParameter.With("invalid parameters") } if stream == 0 { stream = len(o.streams) + 1 diff --git a/pkg/ffmpeg/par.go b/pkg/ffmpeg/par.go index 3149feb..99ed6a1 100644 --- a/pkg/ffmpeg/par.go +++ b/pkg/ffmpeg/par.go @@ -2,9 +2,9 @@ package ffmpeg import ( "encoding/json" - "fmt" "slices" + // Packages ff "github.com/mutablelogic/go-media/sys/ffmpeg61" // Namespace imports @@ -51,7 +51,7 @@ func NewAudioPar(samplefmt string, channellayout string, samplerate int) (*Par, return par, nil } -func NewVideoPar(pixfmt string, size string) (*Par, error) { +func NewVideoPar(pixfmt string, size string, framerate float64) (*Par, error) { par := new(Par) par.SetCodecType(ff.AVMEDIA_TYPE_VIDEO) @@ -70,6 +70,13 @@ func NewVideoPar(pixfmt string, size string) (*Par, error) { par.SetHeight(h) } + // Frame rate + if framerate <= 0 { + return nil, ErrBadParameter.Withf("negative or zero framerate %v", framerate) + } else { + par.SetFramerate(ff.AVUtil_rational_d2q(framerate, 1<<24)) + } + // Set default sample aspect ratio par.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) @@ -85,8 +92,8 @@ func AudioPar(samplefmt string, channellayout string, samplerate int) *Par { } } -func VideoPar(pixfmt string, size string) *Par { - if par, err := NewVideoPar(pixfmt, size); err != nil { +func VideoPar(pixfmt string, size string, framerate float64) *Par { + if par, err := NewVideoPar(pixfmt, size, framerate); err != nil { panic(err) } else { return par @@ -199,11 +206,61 @@ func (ctx *Par) validateAudioCodec(codec *ff.AVCodecContext) error { } func (ctx *Par) copyVideoCodec(codec *ff.AVCodecContext) error { - fmt.Println("TODO: copyVideoCodec") + codec.SetPixFmt(ctx.PixelFormat()) + codec.SetWidth(ctx.Width()) + codec.SetHeight(ctx.Height()) + codec.SetSampleAspectRatio(ctx.SampleAspectRatio()) + codec.SetFramerate(ctx.Framerate()) + codec.SetTimeBase(ff.AVUtil_rational_invert(ctx.Framerate())) return nil } func (ctx *Par) validateVideoCodec(codec *ff.AVCodecContext) error { - fmt.Println("TODO: validateVideoCodec") + pixelformats := codec.Codec().PixelFormats() + framerates := codec.Codec().SupportedFramerates() + + // First we set params from the codec which are not already set + if ctx.PixelFormat() == ff.AV_PIX_FMT_NONE { + if len(pixelformats) > 0 { + ctx.SetPixelFormat(pixelformats[0]) + } + } + if ctx.Framerate().Num() == 0 || ctx.Framerate().Den() == 0 { + if len(framerates) > 0 { + ctx.SetFramerate(framerates[0]) + } + } + + // Then we check to make sure the parameters are compatible with + // the codec + if len(pixelformats) > 0 { + if !slices.Contains(pixelformats, ctx.PixelFormat()) { + return ErrBadParameter.Withf("unsupported pixel format %v", ctx.PixelFormat()) + } + } else if ctx.PixelFormat() == ff.AV_PIX_FMT_NONE { + return ErrBadParameter.With("pixel format not set") + } + if ctx.Width() == 0 || ctx.Height() == 0 { + return ErrBadParameter.Withf("invalid width %v or height %v", ctx.Width(), ctx.Height()) + } + if ctx.SampleAspectRatio().Num() == 0 || ctx.SampleAspectRatio().Den() == 0 { + ctx.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + } + if ctx.Framerate().Num() == 0 || ctx.Framerate().Den() == 0 { + return ErrBadParameter.With("framerate not set") + } else if len(framerates) > 0 { + valid := false + for _, fr := range framerates { + if ff.AVUtil_rational_equal(fr, ctx.Framerate()) { + valid = true + break + } + } + if !valid { + return ErrBadParameter.Withf("unsupported framerate %v", ctx.Framerate()) + } + } + + // Return success return nil } diff --git a/pkg/ffmpeg/par_test.go b/pkg/ffmpeg/par_test.go index 4f8d88a..52a06aa 100644 --- a/pkg/ffmpeg/par_test.go +++ b/pkg/ffmpeg/par_test.go @@ -20,7 +20,7 @@ func Test_par_001(t *testing.T) { func Test_par_002(t *testing.T) { assert := assert.New(t) - par, err := ffmpeg.NewVideoPar("yuv420p", "1280x720") + par, err := ffmpeg.NewVideoPar("yuv420p", "1280x720", 25) if !assert.NoError(err) { t.FailNow() } diff --git a/pkg/ffmpeg/rescaler_test.go b/pkg/ffmpeg/rescaler_test.go index d7e4074..577836f 100644 --- a/pkg/ffmpeg/rescaler_test.go +++ b/pkg/ffmpeg/rescaler_test.go @@ -17,7 +17,7 @@ func Test_rescaler_001(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P("vga", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuv420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } @@ -53,7 +53,7 @@ func Test_rescaler_002(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P("vga", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } @@ -120,7 +120,5 @@ func Test_rescaler_002(t *testing.T) { t.FailNow() } t.Logf("Wrote %s", tmpfile) - } - } diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index 02c8b4e..3eaf0d3 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -84,12 +84,20 @@ func NewWriter(w io.Writer, opt ...Opt) (*Writer, error) { } } - // Check output + // Try once more to get the output format var filename string + if options.oformat == nil { + if w_, ok := w.(*os.File); ok { + filename = w_.Name() + if err := OptOutputFormat(filename)(options); err != nil { + return nil, err + } + } + } + + // Bail out if options.oformat == nil { return nil, ErrBadParameter.Withf("invalid output format") - } else if w_, ok := w.(*os.File); ok { - filename = w_.Name() } // Allocate the AVIO context diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index 730631e..2197bbf 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -26,7 +26,7 @@ func Test_writer_001(t *testing.T) { writer, err := ffmpeg.NewWriter(w, ffmpeg.OptOutputFormat(w.Name()), ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), - ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), + ffmpeg.OptStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), ) if !assert.NoError(err) { t.FailNow() @@ -40,8 +40,8 @@ func Test_writer_001(t *testing.T) { } defer audio.Close() - // Write frames - duration := 1000 * time.Minute + // Write 15 mins of frames + duration := 60 * time.Minute assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { frame := audio.Frame() if frame.Time() >= duration { @@ -72,7 +72,7 @@ func Test_writer_002(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.Create(w.Name(), ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), - ffmpeg.OptAudioStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), + ffmpeg.OptStream(1, ffmpeg.AudioPar("fltp", "mono", 22050)), ) if !assert.NoError(err) { t.FailNow() @@ -86,8 +86,8 @@ func Test_writer_002(t *testing.T) { } defer audio.Close() - // Write frames - duration := 1000 * time.Minute + // Write 15 mins of frames + duration := 15 * time.Minute assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { frame := audio.Frame() if frame.Time() >= duration { @@ -104,3 +104,49 @@ func Test_writer_002(t *testing.T) { })) t.Log("Written to", w.Name()) } + +func Test_writer_003(t *testing.T) { + assert := assert.New(t) + + // Write to a file + w, err := os.CreateTemp("", t.Name()+"_*.ts") + if !assert.NoError(err) { + t.FailNow() + } + defer w.Close() + + // Create a writer with an audio stream + writer, err := ffmpeg.NewWriter(w, + ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), + ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "1280x720", 25)), + ) + if !assert.NoError(err) { + t.FailNow() + } + defer writer.Close() + + // Make an video generator + video, err := generator.NewYUV420P(25, writer.Stream(1).Par()) + if !assert.NoError(err) { + t.FailNow() + } + defer video.Close() + + // Write 1 min of frames + duration := time.Minute + assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { + frame := video.Frame() + if frame.Time() >= duration { + return nil, io.EOF + } else { + t.Log("Frame", frame.Time().Truncate(time.Millisecond)) + return frame.(*ffmpeg.Frame).AVFrame(), nil + } + }, func(packet *ff.AVPacket, timebase *ff.AVRational) error { + if packet != nil { + t.Log("Packet", packet) + } + return writer.Write(packet) + })) + t.Log("Written to", w.Name()) +} diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go index 3ad5556..16a9d26 100644 --- a/pkg/generator/yuv420p.go +++ b/pkg/generator/yuv420p.go @@ -24,16 +24,18 @@ var _ Generator = (*yuv420p)(nil) // Create a new video generator which generates YUV420P frames // of the specified size and framerate (in frames per second) -func NewYUV420P(size string, framerate int) (*yuv420p, error) { +func NewYUV420P(framerate int, par *ffmpeg.Par) (*yuv420p, error) { yuv420p := new(yuv420p) // Check parameters if framerate <= 0 { return nil, errors.New("invalid framerate") } - w, h, err := ff.AVUtil_parse_video_size(size) - if err != nil { - return nil, err + // Check parameters + if par.CodecType() != ff.AVMEDIA_TYPE_VIDEO { + return nil, errors.New("invalid codec type") + } else if par.PixelFormat() != ff.AV_PIX_FMT_YUV420P { + return nil, errors.New("invalid pixel format, only yuv420p is supported") } // Create a frame @@ -42,10 +44,10 @@ func NewYUV420P(size string, framerate int) (*yuv420p, error) { return nil, errors.New("failed to allocate frame") } - frame.SetPixFmt(ff.AV_PIX_FMT_YUV420P) - frame.SetWidth(w) - frame.SetHeight(h) - frame.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + frame.SetPixFmt(par.PixelFormat()) + frame.SetWidth(par.Width()) + frame.SetHeight(par.Height()) + frame.SetSampleAspectRatio(par.SampleAspectRatio()) frame.SetTimeBase(ff.AVUtil_rational(1, framerate)) frame.SetPts(ff.AV_NOPTS_VALUE) diff --git a/pkg/generator/yuv420p_test.go b/pkg/generator/yuv420p_test.go index c76a590..8d037d1 100644 --- a/pkg/generator/yuv420p_test.go +++ b/pkg/generator/yuv420p_test.go @@ -7,13 +7,14 @@ import ( "path/filepath" "testing" + ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" "github.com/mutablelogic/go-media/pkg/generator" "github.com/stretchr/testify/assert" ) func Test_yuv420p_001(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P("1024x768", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) if !assert.NoError(err) { t.FailNow() } @@ -24,7 +25,7 @@ func Test_yuv420p_001(t *testing.T) { func Test_yuv420p_002(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P("vga", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) if !assert.NoError(err) { t.FailNow() } @@ -38,7 +39,7 @@ func Test_yuv420p_002(t *testing.T) { func Test_yuv420p_003(t *testing.T) { assert := assert.New(t) - image, err := generator.NewYUV420P("vga", 25) + image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720")) if !assert.NoError(err) { t.FailNow() } diff --git a/sys/ffmpeg61/avcodec.go b/sys/ffmpeg61/avcodec.go index d4e6def..3e7f2c9 100644 --- a/sys/ffmpeg61/avcodec.go +++ b/sys/ffmpeg61/avcodec.go @@ -49,18 +49,20 @@ type jsonAVCodec struct { } type jsonAVCodecContext struct { - CodecType AVMediaType `json:"codec_type,omitempty"` - Codec *AVCodec `json:"codec,omitempty"` - BitRate int64 `json:"bit_rate,omitempty"` - BitRateTolerance int `json:"bit_rate_tolerance,omitempty"` - PixelFormat AVPixelFormat `json:"pix_fmt,omitempty"` - Width int `json:"width,omitempty"` - Height int `json:"height,omitempty"` - SampleFormat AVSampleFormat `json:"sample_fmt,omitempty"` - SampleRate int `json:"sample_rate,omitempty"` - ChannelLayout AVChannelLayout `json:"channel_layout,omitempty"` - FrameSize int `json:"frame_size,omitempty"` - TimeBase AVRational `json:"time_base,omitempty"` + CodecType AVMediaType `json:"codec_type,omitempty"` + Codec *AVCodec `json:"codec,omitempty"` + BitRate int64 `json:"bit_rate,omitempty"` + BitRateTolerance int `json:"bit_rate_tolerance,omitempty"` + PixelFormat AVPixelFormat `json:"pix_fmt,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + SampleAspectRatio AVRational `json:"sample_aspect_ratio,omitempty"` + Framerate AVRational `json:"framerate,omitempty"` + SampleFormat AVSampleFormat `json:"sample_fmt,omitempty"` + SampleRate int `json:"sample_rate,omitempty"` + ChannelLayout AVChannelLayout `json:"channel_layout,omitempty"` + FrameSize int `json:"frame_size,omitempty"` + TimeBase AVRational `json:"time_base,omitempty"` } //////////////////////////////////////////////////////////////////////////////// @@ -175,13 +177,15 @@ func (ctx *AVCodecContext) MarshalJSON() ([]byte, error) { switch ctx.codec_type { case C.AVMEDIA_TYPE_VIDEO: return json.Marshal(jsonAVCodecContext{ - CodecType: AVMediaType(ctx.codec_type), - Codec: (*AVCodec)(ctx.codec), - BitRate: int64(ctx.bit_rate), - BitRateTolerance: int(ctx.bit_rate_tolerance), - PixelFormat: AVPixelFormat(ctx.pix_fmt), - Width: int(ctx.width), - Height: int(ctx.height), + CodecType: AVMediaType(ctx.codec_type), + Codec: (*AVCodec)(ctx.codec), + BitRate: int64(ctx.bit_rate), + BitRateTolerance: int(ctx.bit_rate_tolerance), + PixelFormat: AVPixelFormat(ctx.pix_fmt), + Width: int(ctx.width), + Height: int(ctx.height), + SampleAspectRatio: AVRational(ctx.sample_aspect_ratio), + Framerate: AVRational(ctx.framerate), }) case C.AVMEDIA_TYPE_AUDIO: return json.Marshal(jsonAVCodecContext{ @@ -405,12 +409,12 @@ func (ctx *AVCodecContext) SetHeight(height int) { ctx.height = C.int(height) } -func (ctx *AVCodecContext) TimeBase() AVRational { - return (AVRational)(ctx.time_base) +func (ctx *AVCodecContext) SampleAspectRatio() AVRational { + return (AVRational)(ctx.sample_aspect_ratio) } -func (ctx *AVCodecContext) SetTimeBase(time_base AVRational) { - ctx.time_base = C.struct_AVRational(time_base) +func (ctx *AVCodecContext) SetSampleAspectRatio(sample_aspect_ratio AVRational) { + ctx.sample_aspect_ratio = C.struct_AVRational(sample_aspect_ratio) } func (ctx *AVCodecContext) Framerate() AVRational { @@ -421,6 +425,14 @@ func (ctx *AVCodecContext) SetFramerate(framerate AVRational) { ctx.framerate = C.struct_AVRational(framerate) } +func (ctx *AVCodecContext) TimeBase() AVRational { + return (AVRational)(ctx.time_base) +} + +func (ctx *AVCodecContext) SetTimeBase(time_base AVRational) { + ctx.time_base = C.struct_AVRational(time_base) +} + // Audio sample format. func (ctx *AVCodecContext) SampleFormat() AVSampleFormat { return AVSampleFormat(ctx.sample_fmt) diff --git a/sys/ffmpeg61/avcodec_parameters.go b/sys/ffmpeg61/avcodec_parameters.go index 833c303..917fcc3 100644 --- a/sys/ffmpeg61/avcodec_parameters.go +++ b/sys/ffmpeg61/avcodec_parameters.go @@ -30,6 +30,7 @@ type jsonAVCodecParameterVideo struct { Width int `json:"width"` Height int `json:"height"` SampleAspectRatio AVRational `json:"sample_aspect_ratio,omitempty"` + Framerate AVRational `json:"framerate,omitempty"` } type jsonAVCodecParameters struct { @@ -65,6 +66,7 @@ func (ctx AVCodecParameters) MarshalJSON() ([]byte, error) { Width: int(ctx.width), Height: int(ctx.height), SampleAspectRatio: AVRational(ctx.sample_aspect_ratio), + Framerate: AVRational(ctx.framerate), } } @@ -179,6 +181,14 @@ func (ctx *AVCodecParameters) SetSampleAspectRatio(aspect AVRational) { ctx.sample_aspect_ratio = C.AVRational(aspect) } +func (ctx *AVCodecParameters) Framerate() AVRational { + return AVRational(ctx.framerate) +} + +func (ctx *AVCodecParameters) SetFramerate(rate AVRational) { + ctx.framerate = C.AVRational(rate) +} + // Video func (ctx *AVCodecParameters) Width() int { return int(ctx.width) diff --git a/sys/ffmpeg61/avutil_rational.go b/sys/ffmpeg61/avutil_rational.go index e5dae3d..590b2fa 100644 --- a/sys/ffmpeg61/avutil_rational.go +++ b/sys/ffmpeg61/avutil_rational.go @@ -72,3 +72,13 @@ func AVUtil_rational_d2q(d float64, max int) AVRational { func AVUtil_rational_q2d(a AVRational) float64 { return float64(C.av_q2d(C.AVRational(a))) } + +// Compare two rationals. +func AVUtil_rational_equal(a, b AVRational) bool { + return C.av_cmp_q(C.AVRational(a), C.AVRational(b)) == 0 +} + +// Invert a rational. +func AVUtil_rational_invert(q AVRational) AVRational { + return AVRational(C.av_inv_q(C.AVRational(q))) +} From 61ee5594dda1dbfb14188d5fb04612eabd09c479 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 15:48:43 +0200 Subject: [PATCH 4/5] Updated encoder --- pkg/ffmpeg/encoder.go | 7 ++++--- pkg/ffmpeg/par.go | 15 +++++++++++++++ sys/ffmpeg61/avcodec_packet.go | 4 ++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go index 36e332e..39637ac 100644 --- a/pkg/ffmpeg/encoder.go +++ b/pkg/ffmpeg/encoder.go @@ -184,9 +184,6 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { // Send the frame to the encoder if err := ff.AVCodec_send_frame(e.ctx, frame); err != nil { - if errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) { - return nil - } return err } @@ -201,6 +198,10 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { return err } + // rescale output packet timestamp values from codec to stream timebase + ff.AVCodec_packet_rescale_ts(e.packet, e.ctx.TimeBase(), e.stream.TimeBase()) + e.packet.SetStreamIndex(e.stream.Index()) + // Pass back to the caller if err := fn(e.packet, &timebase); errors.Is(err, io.EOF) { // End early, return EOF diff --git a/pkg/ffmpeg/par.go b/pkg/ffmpeg/par.go index 99ed6a1..203c443 100644 --- a/pkg/ffmpeg/par.go +++ b/pkg/ffmpeg/par.go @@ -80,6 +80,21 @@ func NewVideoPar(pixfmt string, size string, framerate float64) (*Par, error) { // Set default sample aspect ratio par.SetSampleAspectRatio(ff.AVUtil_rational(1, 1)) + /* TODO + c->gop_size = 12; // emit one intra frame every twelve frames at most + c->pix_fmt = STREAM_PIX_FMT; + if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) { + // just for testing, we also add B-frames + c->max_b_frames = 2; + } + if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) { + // Needed to avoid using macroblocks in which some coeffs overflow. + // This does not happen with normal video, it just happens here as + // the motion of the chroma plane does not match the luma plane. + c->mb_decision = 2; + } + */ + // Return success return par, nil } diff --git a/sys/ffmpeg61/avcodec_packet.go b/sys/ffmpeg61/avcodec_packet.go index c49f734..b7d013f 100644 --- a/sys/ffmpeg61/avcodec_packet.go +++ b/sys/ffmpeg61/avcodec_packet.go @@ -110,6 +110,10 @@ func (ctx *AVPacket) StreamIndex() int { return int(ctx.stream_index) } +func (ctx *AVPacket) SetStreamIndex(index int) { + ctx.stream_index = C.int(index) +} + func (ctx *AVPacket) Pts() int64 { return int64(ctx.pts) } From 9c2694f9c6ecc0c2bc2960120c0c7ebfed31fee1 Mon Sep 17 00:00:00 2001 From: David Thorpe Date: Sun, 30 Jun 2024 17:57:10 +0200 Subject: [PATCH 5/5] Added encoding --- pkg/ffmpeg/encoder.go | 34 +++++++++++++---- pkg/ffmpeg/rescaler_test.go | 4 +- pkg/ffmpeg/writer.go | 55 ++++++++++++++++++---------- pkg/ffmpeg/writer_test.go | 65 +++++++++++++++++++++++++++++++-- pkg/generator/yuv420p.go | 12 +++--- sys/ffmpeg61/avcodec_packet.go | 26 +++++++++---- sys/ffmpeg61/avutil_math.go | 22 ----------- sys/ffmpeg61/avutil_rational.go | 16 ++++++++ 8 files changed, 166 insertions(+), 68 deletions(-) delete mode 100644 sys/ffmpeg61/avutil_math.go diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go index 39637ac..521c1e8 100644 --- a/pkg/ffmpeg/encoder.go +++ b/pkg/ffmpeg/encoder.go @@ -3,6 +3,7 @@ package ffmpeg import ( "encoding/json" "errors" + "fmt" "io" "syscall" @@ -23,7 +24,9 @@ type Encoder struct { // We are flushing the encoder eof bool - //next_pts int64 + + // The next presentation timestamp + next_pts int64 } // EncoderFrameFn is a function which is called to receive a frame to encode. It should @@ -85,12 +88,6 @@ func NewEncoder(ctx *ff.AVFormatContext, stream int, par *Par) (*Encoder, error) encoder.stream = streamctx } - // Copy parameters to stream - if err := ff.AVCodec_parameters_from_context(encoder.stream.CodecPar(), encoder.ctx); err != nil { - ff.AVCodec_free_context(encoder.ctx) - return nil, err - } - // Some formats want stream headers to be separate. if ctx.Output().Flags().Is(ff.AVFMT_GLOBALHEADER) { encoder.ctx.SetFlags(encoder.ctx.Flags() | ff.AV_CODEC_FLAG_GLOBAL_HEADER) @@ -102,6 +99,12 @@ func NewEncoder(ctx *ff.AVFormatContext, stream int, par *Par) (*Encoder, error) return nil, ErrInternalAppError.Withf("codec_open: %v", err) } + // Copy parameters to stream + if err := ff.AVCodec_parameters_from_context(encoder.stream.CodecPar(), encoder.ctx); err != nil { + ff.AVCodec_free_context(encoder.ctx) + return nil, err + } + // Create a packet packet := ff.AVCodec_packet_alloc() if packet == nil { @@ -176,6 +179,22 @@ func (e *Encoder) Par() *Par { } } +// Return the codec type +func (e *Encoder) nextPts(frame *ff.AVFrame) int64 { + next_pts := int64(0) + switch e.ctx.Codec().Type() { + case ff.AVMEDIA_TYPE_AUDIO: + next_pts = ff.AVUtil_rational_rescale_q(int64(frame.NumSamples()), ff.AVUtil_rational(1, frame.SampleRate()), e.stream.TimeBase()) + case ff.AVMEDIA_TYPE_VIDEO: + next_pts = ff.AVUtil_rational_rescale_q(1, ff.AVUtil_rational_invert(e.ctx.Framerate()), e.stream.TimeBase()) + default: + // Dunno what to do with subtitle and data streams yet + fmt.Println("TODO: next_pts for subtitle and data streams") + return 0 + } + return next_pts +} + ////////////////////////////////////////////////////////////////////////////// // PRIVATE METHODS @@ -201,6 +220,7 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error { // rescale output packet timestamp values from codec to stream timebase ff.AVCodec_packet_rescale_ts(e.packet, e.ctx.TimeBase(), e.stream.TimeBase()) e.packet.SetStreamIndex(e.stream.Index()) + e.packet.SetTimeBase(e.stream.TimeBase()) // Pass back to the caller if err := fn(e.packet, &timebase); errors.Is(err, io.EOF) { diff --git a/pkg/ffmpeg/rescaler_test.go b/pkg/ffmpeg/rescaler_test.go index 577836f..46930e3 100644 --- a/pkg/ffmpeg/rescaler_test.go +++ b/pkg/ffmpeg/rescaler_test.go @@ -17,7 +17,7 @@ func Test_rescaler_001(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuv420p", "1280x720", 25)) + image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuv420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } @@ -53,7 +53,7 @@ func Test_rescaler_002(t *testing.T) { assert := assert.New(t) // Create an image generator - image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720", 25)) + image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuva420p", "1280x720", 25)) if !assert.NoError(err) { t.FailNow() } diff --git a/pkg/ffmpeg/writer.go b/pkg/ffmpeg/writer.go index 3eaf0d3..a1c9eaa 100644 --- a/pkg/ffmpeg/writer.go +++ b/pkg/ffmpeg/writer.go @@ -230,6 +230,7 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { // Initialize the encoder encoder.eof = false + encoder.next_pts = 0 } // Continue until all encoders have returned io.EOF and have been flushed @@ -239,31 +240,42 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error { break } - // TODO: We get the encoder with the lowest timestamp + // Find encoder with the lowest timestamp, based on next_pts and timebase + next_stream := -1 + var next_encoder *Encoder for stream, encoder := range encoders { - var frame *ff.AVFrame - var err error - - // Receive a frame if not EOF - if !encoder.eof { - frame, err = in(stream) - if errors.Is(err, io.EOF) { - encoder.eof = true - } else if err != nil { - return fmt.Errorf("stream %v: %w", stream, err) - } + if next_encoder == nil || compareNextPts(encoder, next_encoder) < 0 { + next_encoder = encoder + next_stream = stream } + } - // Send a frame for encoding - if err := encoder.Encode(frame, out); err != nil { - return fmt.Errorf("stream %v: %w", stream, err) - } + var frame *ff.AVFrame + var err error - // If eof then delete the encoder - if encoder.eof { - delete(encoders, stream) + // Receive a frame if not EOF + if !next_encoder.eof { + frame, err = in(next_stream) + if errors.Is(err, io.EOF) { + next_encoder.eof = true + } else if err != nil { + return fmt.Errorf("stream %v: %w", next_stream, err) } } + + // Send a frame for encoding + if err := next_encoder.Encode(frame, out); err != nil { + return fmt.Errorf("stream %v: %w", next_stream, err) + } + + // If eof then delete the encoder + if next_encoder.eof { + delete(encoders, next_stream) + continue + } + + // Calculate the next PTS + next_encoder.next_pts = next_encoder.next_pts + next_encoder.nextPts(frame) } // Return success @@ -276,6 +288,11 @@ func (w *Writer) Write(packet *ff.AVPacket) error { return ff.AVCodec_interleaved_write_frame(w.output, packet) } +// Returns -1 if a is before v +func compareNextPts(a, b *Encoder) int { + return ff.AVUtil_compare_ts(a.next_pts, a.stream.TimeBase(), b.next_pts, b.stream.TimeBase()) +} + //////////////////////////////////////////////////////////////////////////////// // PRIVATE METHODS - Writer diff --git a/pkg/ffmpeg/writer_test.go b/pkg/ffmpeg/writer_test.go index 2197bbf..ed7bc6e 100644 --- a/pkg/ffmpeg/writer_test.go +++ b/pkg/ffmpeg/writer_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + media "github.com/mutablelogic/go-media" ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg" generator "github.com/mutablelogic/go-media/pkg/generator" ff "github.com/mutablelogic/go-media/sys/ffmpeg61" @@ -118,7 +119,7 @@ func Test_writer_003(t *testing.T) { // Create a writer with an audio stream writer, err := ffmpeg.NewWriter(w, ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), - ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "1280x720", 25)), + ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "640x480", 30)), ) if !assert.NoError(err) { t.FailNow() @@ -126,7 +127,7 @@ func Test_writer_003(t *testing.T) { defer writer.Close() // Make an video generator - video, err := generator.NewYUV420P(25, writer.Stream(1).Par()) + video, err := generator.NewYUV420P(writer.Stream(1).Par()) if !assert.NoError(err) { t.FailNow() } @@ -139,14 +140,70 @@ func Test_writer_003(t *testing.T) { if frame.Time() >= duration { return nil, io.EOF } else { - t.Log("Frame", frame.Time().Truncate(time.Millisecond)) + t.Log("Frame", stream, "=>", frame.Time().Truncate(time.Millisecond)) return frame.(*ffmpeg.Frame).AVFrame(), nil } }, func(packet *ff.AVPacket, timebase *ff.AVRational) error { if packet != nil { - t.Log("Packet", packet) + d := time.Duration(ff.AVUtil_rational_q2d(packet.TimeBase()) * float64(packet.Pts()) * float64(time.Second)) + t.Log("Packet", d.Truncate(time.Millisecond)) } return writer.Write(packet) })) t.Log("Written to", w.Name()) } + +func Test_writer_004(t *testing.T) { + assert := assert.New(t) + + // Write to a file + w, err := os.CreateTemp("", t.Name()+"_*.m4v") + if !assert.NoError(err) { + t.FailNow() + } + defer w.Close() + + // Create a writer with an audio stream + writer, err := ffmpeg.Create(w.Name(), + ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())), + ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "640x480", 30)), + ffmpeg.OptStream(2, ffmpeg.AudioPar("fltp", "mono", 22050)), + ) + if !assert.NoError(err) { + t.FailNow() + } + defer writer.Close() + + // Make an video generator + video, err := generator.NewYUV420P(writer.Stream(1).Par()) + if !assert.NoError(err) { + t.FailNow() + } + defer video.Close() + + // Make an audio generator + audio, err := generator.NewSine(440, -5, writer.Stream(2).Par()) + if !assert.NoError(err) { + t.FailNow() + } + + // Write 10 secs of frames + duration := time.Minute * 10 + assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) { + var frame media.Frame + switch stream { + case 1: + frame = video.Frame() + case 2: + frame = audio.Frame() + } + if frame.Time() >= duration { + t.Log("Frame time is EOF", frame.Time()) + return nil, io.EOF + } else { + t.Log("Frame", stream, "=>", frame.Time().Truncate(time.Millisecond)) + return frame.(*ffmpeg.Frame).AVFrame(), nil + } + }, nil)) + t.Log("Written to", w.Name()) +} diff --git a/pkg/generator/yuv420p.go b/pkg/generator/yuv420p.go index 16a9d26..b06f87a 100644 --- a/pkg/generator/yuv420p.go +++ b/pkg/generator/yuv420p.go @@ -24,19 +24,19 @@ var _ Generator = (*yuv420p)(nil) // Create a new video generator which generates YUV420P frames // of the specified size and framerate (in frames per second) -func NewYUV420P(framerate int, par *ffmpeg.Par) (*yuv420p, error) { +func NewYUV420P(par *ffmpeg.Par) (*yuv420p, error) { yuv420p := new(yuv420p) - // Check parameters - if framerate <= 0 { - return nil, errors.New("invalid framerate") - } // Check parameters if par.CodecType() != ff.AVMEDIA_TYPE_VIDEO { return nil, errors.New("invalid codec type") } else if par.PixelFormat() != ff.AV_PIX_FMT_YUV420P { return nil, errors.New("invalid pixel format, only yuv420p is supported") } + framerate := ff.AVUtil_rational_q2d(par.Framerate()) + if framerate <= 0 { + return nil, errors.New("invalid framerate") + } // Create a frame frame := ff.AVUtil_frame_alloc() @@ -48,7 +48,7 @@ func NewYUV420P(framerate int, par *ffmpeg.Par) (*yuv420p, error) { frame.SetWidth(par.Width()) frame.SetHeight(par.Height()) frame.SetSampleAspectRatio(par.SampleAspectRatio()) - frame.SetTimeBase(ff.AVUtil_rational(1, framerate)) + frame.SetTimeBase(ff.AVUtil_rational_invert(par.Framerate())) frame.SetPts(ff.AV_NOPTS_VALUE) // Allocate buffer diff --git a/sys/ffmpeg61/avcodec_packet.go b/sys/ffmpeg61/avcodec_packet.go index b7d013f..dcf3536 100644 --- a/sys/ffmpeg61/avcodec_packet.go +++ b/sys/ffmpeg61/avcodec_packet.go @@ -18,14 +18,15 @@ import "C" // TYPES type jsonAVPacket struct { - Pts int64 `json:"pts,omitempty"` - Dts int64 `json:"dts,omitempty"` - Size int `json:"size,omitempty"` - StreamIndex int `json:"stream_index"` // Stream index starts at 0 - Flags int `json:"flags,omitempty"` - SideDataElems int `json:"side_data_elems,omitempty"` - Duration int64 `json:"duration,omitempty"` - Pos int64 `json:"pos,omitempty"` + Pts int64 `json:"pts,omitempty"` + Dts int64 `json:"dts,omitempty"` + Size int `json:"size,omitempty"` + StreamIndex int `json:"stream_index"` // Stream index starts at 0 + Flags int `json:"flags,omitempty"` + SideDataElems int `json:"side_data_elems,omitempty"` + Duration int64 `json:"duration,omitempty"` + TimeBase AVRational `json:"time_base,omitempty"` + Pos int64 `json:"pos,omitempty"` } //////////////////////////////////////////////////////////////////////////////// @@ -40,6 +41,7 @@ func (ctx *AVPacket) MarshalJSON() ([]byte, error) { Flags: int(ctx.flags), SideDataElems: int(ctx.side_data_elems), Duration: int64(ctx.duration), + TimeBase: AVRational(ctx.time_base), Pos: int64(ctx.pos), }) } @@ -114,6 +116,14 @@ func (ctx *AVPacket) SetStreamIndex(index int) { ctx.stream_index = C.int(index) } +func (ctx *AVPacket) TimeBase() AVRational { + return AVRational(ctx.time_base) +} + +func (ctx *AVPacket) SetTimeBase(tb AVRational) { + ctx.time_base = C.AVRational(tb) +} + func (ctx *AVPacket) Pts() int64 { return int64(ctx.pts) } diff --git a/sys/ffmpeg61/avutil_math.go b/sys/ffmpeg61/avutil_math.go deleted file mode 100644 index cfae419..0000000 --- a/sys/ffmpeg61/avutil_math.go +++ /dev/null @@ -1,22 +0,0 @@ -package ffmpeg - -//////////////////////////////////////////////////////////////////////////////// -// CGO - -/* -#cgo pkg-config: libavutil -#include -*/ -import "C" - -//////////////////////////////////////////////////////////////////////////////// - -// Compare two timestamps each in its own time base. Returns -1 if a is before b, 1 if a is after b, or 0 if they are equal. -func AVUtil_compare_ts(a int64, a_tb AVRational, b int64, b_tb AVRational) int { - return int(C.av_compare_ts(C.int64_t(a), C.AVRational(a_tb), C.int64_t(b), C.AVRational(b_tb))) -} - -// Rescale a value from one range to another. -func AVUtil_rescale_rnd(a, b, c int64, rnd AVRounding) int64 { - return int64(C.av_rescale_rnd(C.int64_t(a), C.int64_t(b), C.int64_t(c), C.enum_AVRounding(rnd))) -} diff --git a/sys/ffmpeg61/avutil_rational.go b/sys/ffmpeg61/avutil_rational.go index 590b2fa..c28485d 100644 --- a/sys/ffmpeg61/avutil_rational.go +++ b/sys/ffmpeg61/avutil_rational.go @@ -11,6 +11,7 @@ import ( /* #cgo pkg-config: libavutil #include +#include */ import "C" @@ -82,3 +83,18 @@ func AVUtil_rational_equal(a, b AVRational) bool { func AVUtil_rational_invert(q AVRational) AVRational { return AVRational(C.av_inv_q(C.AVRational(q))) } + +// Resacale a rational +func AVUtil_rational_rescale_q(a int64, bq AVRational, cq AVRational) int64 { + return int64(C.av_rescale_q(C.int64_t(a), C.AVRational(bq), C.AVRational(cq))) +} + +// Rescale a value from one range to another. +func AVUtil_rescale_rnd(a, b, c int64, rnd AVRounding) int64 { + return int64(C.av_rescale_rnd(C.int64_t(a), C.int64_t(b), C.int64_t(c), C.enum_AVRounding(rnd))) +} + +// Compare two timestamps each in its own time base. Returns -1 if a is before b, 1 if a is after b, or 0 if they are equal. +func AVUtil_compare_ts(a int64, a_tb AVRational, b int64, b_tb AVRational) int { + return int(C.av_compare_ts(C.int64_t(a), C.AVRational(a_tb), C.int64_t(b), C.AVRational(b_tb))) +}