Added encoding

pkg/ffmpeg/encoder.go

@@ -3,6 +3,7 @@ package ffmpeg
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"syscall"
 
@@ -23,7 +24,9 @@ type Encoder struct {
 
 	// We are flushing the encoder
 	eof bool
-	//next_pts int64
+
+	// The next presentation timestamp
+	next_pts int64
 }
 
 // EncoderFrameFn is a function which is called to receive a frame to encode. It should
@@ -85,12 +88,6 @@ func NewEncoder(ctx *ff.AVFormatContext, stream int, par *Par) (*Encoder, error)
 		encoder.stream = streamctx
 	}
 
-	// Copy parameters to stream
-	if err := ff.AVCodec_parameters_from_context(encoder.stream.CodecPar(), encoder.ctx); err != nil {
-		ff.AVCodec_free_context(encoder.ctx)
-		return nil, err
-	}
-
 	// Some formats want stream headers to be separate.
 	if ctx.Output().Flags().Is(ff.AVFMT_GLOBALHEADER) {
 		encoder.ctx.SetFlags(encoder.ctx.Flags() | ff.AV_CODEC_FLAG_GLOBAL_HEADER)
@@ -102,6 +99,12 @@ func NewEncoder(ctx *ff.AVFormatContext, stream int, par *Par) (*Encoder, error)
 		return nil, ErrInternalAppError.Withf("codec_open: %v", err)
 	}
 
+	// Copy parameters to stream
+	if err := ff.AVCodec_parameters_from_context(encoder.stream.CodecPar(), encoder.ctx); err != nil {
+		ff.AVCodec_free_context(encoder.ctx)
+		return nil, err
+	}
+
 	// Create a packet
 	packet := ff.AVCodec_packet_alloc()
 	if packet == nil {
@@ -176,6 +179,22 @@ func (e *Encoder) Par() *Par {
 	}
 }
 
+// Return the codec type
+func (e *Encoder) nextPts(frame *ff.AVFrame) int64 {
+	next_pts := int64(0)
+	switch e.ctx.Codec().Type() {
+	case ff.AVMEDIA_TYPE_AUDIO:
+		next_pts = ff.AVUtil_rational_rescale_q(int64(frame.NumSamples()), ff.AVUtil_rational(1, frame.SampleRate()), e.stream.TimeBase())
+	case ff.AVMEDIA_TYPE_VIDEO:
+		next_pts = ff.AVUtil_rational_rescale_q(1, ff.AVUtil_rational_invert(e.ctx.Framerate()), e.stream.TimeBase())
+	default:
+		// Dunno what to do with subtitle and data streams yet
+		fmt.Println("TODO: next_pts for subtitle and data streams")
+		return 0
+	}
+	return next_pts
+}
+
 //////////////////////////////////////////////////////////////////////////////
 // PRIVATE METHODS
 
@@ -201,6 +220,7 @@ func (e *Encoder) encode(frame *ff.AVFrame, fn EncoderPacketFn) error {
 		// rescale output packet timestamp values from codec to stream timebase
 		ff.AVCodec_packet_rescale_ts(e.packet, e.ctx.TimeBase(), e.stream.TimeBase())
 		e.packet.SetStreamIndex(e.stream.Index())
+		e.packet.SetTimeBase(e.stream.TimeBase())
 
 		// Pass back to the caller
 		if err := fn(e.packet, &timebase); errors.Is(err, io.EOF) {

pkg/ffmpeg/rescaler_test.go

@@ -17,7 +17,7 @@ func Test_rescaler_001(t *testing.T) {
 	assert := assert.New(t)
 
 	// Create an image generator
-	image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuv420p", "1280x720", 25))
+	image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuv420p", "1280x720", 25))
 	if !assert.NoError(err) {
 		t.FailNow()
 	}
@@ -53,7 +53,7 @@ func Test_rescaler_002(t *testing.T) {
 	assert := assert.New(t)
 
 	// Create an image generator
-	image, err := generator.NewYUV420P(25, ffmpeg.VideoPar("yuva420p", "1280x720", 25))
+	image, err := generator.NewYUV420P(ffmpeg.VideoPar("yuva420p", "1280x720", 25))
 	if !assert.NoError(err) {
 		t.FailNow()
 	}

pkg/ffmpeg/writer.go

@@ -230,6 +230,7 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error {
 
 		// Initialize the encoder
 		encoder.eof = false
+		encoder.next_pts = 0
 	}
 
 	// Continue until all encoders have returned io.EOF and have been flushed
@@ -239,31 +240,42 @@ func (w *Writer) Encode(in EncoderFrameFn, out EncoderPacketFn) error {
 			break
 		}
 
-		// TODO: We get the encoder with the lowest timestamp
+		// Find encoder with the lowest timestamp, based on next_pts and timebase
+		next_stream := -1
+		var next_encoder *Encoder
 		for stream, encoder := range encoders {
-			var frame *ff.AVFrame
-			var err error
-
-			// Receive a frame if not EOF
-			if !encoder.eof {
-				frame, err = in(stream)
-				if errors.Is(err, io.EOF) {
-					encoder.eof = true
-				} else if err != nil {
-					return fmt.Errorf("stream %v: %w", stream, err)
-				}
+			if next_encoder == nil || compareNextPts(encoder, next_encoder) < 0 {
+				next_encoder = encoder
+				next_stream = stream
 			}
+		}
 
-			// Send a frame for encoding
-			if err := encoder.Encode(frame, out); err != nil {
-				return fmt.Errorf("stream %v: %w", stream, err)
-			}
+		var frame *ff.AVFrame
+		var err error
 
-			// If eof then delete the encoder
-			if encoder.eof {
-				delete(encoders, stream)
+		// Receive a frame if not EOF
+		if !next_encoder.eof {
+			frame, err = in(next_stream)
+			if errors.Is(err, io.EOF) {
+				next_encoder.eof = true
+			} else if err != nil {
+				return fmt.Errorf("stream %v: %w", next_stream, err)
 			}
 		}
+
+		// Send a frame for encoding
+		if err := next_encoder.Encode(frame, out); err != nil {
+			return fmt.Errorf("stream %v: %w", next_stream, err)
+		}
+
+		// If eof then delete the encoder
+		if next_encoder.eof {
+			delete(encoders, next_stream)
+			continue
+		}
+
+		// Calculate the next PTS
+		next_encoder.next_pts = next_encoder.next_pts + next_encoder.nextPts(frame)
 	}
 
 	// Return success
@@ -276,6 +288,11 @@ func (w *Writer) Write(packet *ff.AVPacket) error {
 	return ff.AVCodec_interleaved_write_frame(w.output, packet)
 }
 
+// Returns -1 if a is before v
+func compareNextPts(a, b *Encoder) int {
+	return ff.AVUtil_compare_ts(a.next_pts, a.stream.TimeBase(), b.next_pts, b.stream.TimeBase())
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // PRIVATE METHODS - Writer
 

pkg/ffmpeg/writer_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	media "github.com/mutablelogic/go-media"
 	ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg"
 	generator "github.com/mutablelogic/go-media/pkg/generator"
 	ff "github.com/mutablelogic/go-media/sys/ffmpeg61"
@@ -118,15 +119,15 @@ func Test_writer_003(t *testing.T) {
 	// Create a writer with an audio stream
 	writer, err := ffmpeg.NewWriter(w,
 		ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())),
-		ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "1280x720", 25)),
+		ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "640x480", 30)),
 	)
 	if !assert.NoError(err) {
 		t.FailNow()
 	}
 	defer writer.Close()
 
 	// Make an video generator
-	video, err := generator.NewYUV420P(25, writer.Stream(1).Par())
+	video, err := generator.NewYUV420P(writer.Stream(1).Par())
 	if !assert.NoError(err) {
 		t.FailNow()
 	}
@@ -139,14 +140,70 @@ func Test_writer_003(t *testing.T) {
 		if frame.Time() >= duration {
 			return nil, io.EOF
 		} else {
-			t.Log("Frame", frame.Time().Truncate(time.Millisecond))
+			t.Log("Frame", stream, "=>", frame.Time().Truncate(time.Millisecond))
 			return frame.(*ffmpeg.Frame).AVFrame(), nil
 		}
 	}, func(packet *ff.AVPacket, timebase *ff.AVRational) error {
 		if packet != nil {
-			t.Log("Packet", packet)
+			d := time.Duration(ff.AVUtil_rational_q2d(packet.TimeBase()) * float64(packet.Pts()) * float64(time.Second))
+			t.Log("Packet", d.Truncate(time.Millisecond))
 		}
 		return writer.Write(packet)
 	}))
 	t.Log("Written to", w.Name())
 }
+
+func Test_writer_004(t *testing.T) {
+	assert := assert.New(t)
+
+	// Write to a file
+	w, err := os.CreateTemp("", t.Name()+"_*.m4v")
+	if !assert.NoError(err) {
+		t.FailNow()
+	}
+	defer w.Close()
+
+	// Create a writer with an audio stream
+	writer, err := ffmpeg.Create(w.Name(),
+		ffmpeg.OptMetadata(ffmpeg.NewMetadata("title", t.Name())),
+		ffmpeg.OptStream(1, ffmpeg.VideoPar("yuv420p", "640x480", 30)),
+		ffmpeg.OptStream(2, ffmpeg.AudioPar("fltp", "mono", 22050)),
+	)
+	if !assert.NoError(err) {
+		t.FailNow()
+	}
+	defer writer.Close()
+
+	// Make an video generator
+	video, err := generator.NewYUV420P(writer.Stream(1).Par())
+	if !assert.NoError(err) {
+		t.FailNow()
+	}
+	defer video.Close()
+
+	// Make an audio generator
+	audio, err := generator.NewSine(440, -5, writer.Stream(2).Par())
+	if !assert.NoError(err) {
+		t.FailNow()
+	}
+
+	// Write 10 secs of frames
+	duration := time.Minute * 10
+	assert.NoError(writer.Encode(func(stream int) (*ff.AVFrame, error) {
+		var frame media.Frame
+		switch stream {
+		case 1:
+			frame = video.Frame()
+		case 2:
+			frame = audio.Frame()
+		}
+		if frame.Time() >= duration {
+			t.Log("Frame time is EOF", frame.Time())
+			return nil, io.EOF
+		} else {
+			t.Log("Frame", stream, "=>", frame.Time().Truncate(time.Millisecond))
+			return frame.(*ffmpeg.Frame).AVFrame(), nil
+		}
+	}, nil))
+	t.Log("Written to", w.Name())
+}

pkg/generator/yuv420p.go

@@ -24,19 +24,19 @@ var _ Generator = (*yuv420p)(nil)
 
 // Create a new video generator which generates YUV420P frames
 // of the specified size and framerate (in frames per second)
-func NewYUV420P(framerate int, par *ffmpeg.Par) (*yuv420p, error) {
+func NewYUV420P(par *ffmpeg.Par) (*yuv420p, error) {
 	yuv420p := new(yuv420p)
 
-	// Check parameters
-	if framerate <= 0 {
-		return nil, errors.New("invalid framerate")
-	}
 	// Check parameters
 	if par.CodecType() != ff.AVMEDIA_TYPE_VIDEO {
 		return nil, errors.New("invalid codec type")
 	} else if par.PixelFormat() != ff.AV_PIX_FMT_YUV420P {
 		return nil, errors.New("invalid pixel format, only yuv420p is supported")
 	}
+	framerate := ff.AVUtil_rational_q2d(par.Framerate())
+	if framerate <= 0 {
+		return nil, errors.New("invalid framerate")
+	}
 
 	// Create a frame
 	frame := ff.AVUtil_frame_alloc()
@@ -48,7 +48,7 @@ func NewYUV420P(framerate int, par *ffmpeg.Par) (*yuv420p, error) {
 	frame.SetWidth(par.Width())
 	frame.SetHeight(par.Height())
 	frame.SetSampleAspectRatio(par.SampleAspectRatio())
-	frame.SetTimeBase(ff.AVUtil_rational(1, framerate))
+	frame.SetTimeBase(ff.AVUtil_rational_invert(par.Framerate()))
 	frame.SetPts(ff.AV_NOPTS_VALUE)
 
 	// Allocate buffer

sys/ffmpeg61/avcodec_packet.go

@@ -18,14 +18,15 @@ import "C"
 // TYPES
 
 type jsonAVPacket struct {
-	Pts           int64 `json:"pts,omitempty"`
-	Dts           int64 `json:"dts,omitempty"`
-	Size          int   `json:"size,omitempty"`
-	StreamIndex   int   `json:"stream_index"` // Stream index starts at 0
-	Flags         int   `json:"flags,omitempty"`
-	SideDataElems int   `json:"side_data_elems,omitempty"`
-	Duration      int64 `json:"duration,omitempty"`
-	Pos           int64 `json:"pos,omitempty"`
+	Pts           int64      `json:"pts,omitempty"`
+	Dts           int64      `json:"dts,omitempty"`
+	Size          int        `json:"size,omitempty"`
+	StreamIndex   int        `json:"stream_index"` // Stream index starts at 0
+	Flags         int        `json:"flags,omitempty"`
+	SideDataElems int        `json:"side_data_elems,omitempty"`
+	Duration      int64      `json:"duration,omitempty"`
+	TimeBase      AVRational `json:"time_base,omitempty"`
+	Pos           int64      `json:"pos,omitempty"`
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -40,6 +41,7 @@ func (ctx *AVPacket) MarshalJSON() ([]byte, error) {
 		Flags:         int(ctx.flags),
 		SideDataElems: int(ctx.side_data_elems),
 		Duration:      int64(ctx.duration),
+		TimeBase:      AVRational(ctx.time_base),
 		Pos:           int64(ctx.pos),
 	})
 }
@@ -114,6 +116,14 @@ func (ctx *AVPacket) SetStreamIndex(index int) {
 	ctx.stream_index = C.int(index)
 }
 
+func (ctx *AVPacket) TimeBase() AVRational {
+	return AVRational(ctx.time_base)
+}
+
+func (ctx *AVPacket) SetTimeBase(tb AVRational) {
+	ctx.time_base = C.AVRational(tb)
+}
+
 func (ctx *AVPacket) Pts() int64 {
 	return int64(ctx.pts)
 }