pcap unpack

CHANGELOG.md

@@ -9,12 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Nothing yet
 
+## [0.2.0] - 2024-12-18
+
+- New tool pcap-unpack that that extracts and writes UDP streams to file from one or more PCAP files
+
 ## [0.1.0] - 2024-06-27
 
 ### Added
 
 - New tool pcap-replay that resends all UDP packets carrying UDP/TS or UDP/RTP/TS streams
 - initial version of the repo
 
-[Unreleased]: https://github.com/Eyevinn/mp2ts-tools/releases/tag/v0.1.0...HEAD
+[Unreleased]: https://github.com/Eyevinn/mp2ts-tools/releases/tag/v0.2.0...HEAD
+[0.2.0]: https://github.com/Eyevinn/mp2ts-tools/releases/tag/v0.1.0...v0.2.0
 [0.1.0]: https://github.com/Eyevinn/mp2ts-tools/releases/tag/v0.1.0

Makefile

@@ -2,7 +2,7 @@
 all: lint test coverage build
 
 .PHONY: build
-build: pcap-replay
+build: pcap-replay pcap-unpack
 
 .PHONY: lint
 lint: prepare
@@ -16,6 +16,9 @@ prepare:
 pcap-replay:
 	go build -ldflags "-X github.com/Eyevinn/pcap-tools/internal.commitVersion=$$(git describe --tags HEAD) -X github.com/Eyevinn/pcap-tools/internal.commitDate=$$(git log -1 --format=%ct)" -o out/$@ ./cmd/$@
 
+pcap-unpack:
+	go build -ldflags "-X github.com/Eyevinn/pcap-tools/internal.commitVersion=$$(git describe --tags HEAD) -X github.com/Eyevinn/pcap-tools/internal.commitDate=$$(git log -1 --format=%ct)" -o out/$@ ./cmd/$@
+
 .PHONY: test
 test: prepare
 	go test ./...

README.md

@@ -26,15 +26,15 @@ Tools for investigating and reusing tcpdump/Wireshark captures of TS streams.
 
 The tools available this far are:
 
-* pcap-replay
+* pcap-replay which replaces UDP streams from a pcap file and send to a specified address
+* pcap-unpack unpacks TS (or other UDP) streams from one or more pcap files
 
 ## Requirements
 
 This project uses Go version 1.22 or later.
 
 ## Installation / Usage
 
-
 Use the `Makefile`  to get build artifacts into the out directory,
 or use the standard go build steps:
 
@@ -44,6 +44,11 @@ cd cmd/pcap-replay
 go run .
 ```
 
+### Build on Windows
+
+If Makefile does not work, you can use the line from the Makefile to build the tools.
+To get the version correctly, note that the "-X" flags should be used.
+
 ## Development
 
 Uses standard Go tool chain.

cmd/pcap-replay/main.go

@@ -22,13 +22,14 @@ be mapped to new values. One can alternatively export the streams to files.
 `
 
 type options struct {
-	pcap      string
-	dstAddr   string
-	portMap   string
-	dstDir    string
-	logLevel  string
-	naiveLoop bool
-	version   bool
+	pcap           string
+	dstAddr        string
+	portMap        string
+	dstDir         string
+	logLevel       string
+	gapThresholdMS int
+	naiveLoop      bool
+	version        bool
 }
 
 func parseOptions() (*options, error) {
@@ -39,6 +40,7 @@ func parseOptions() (*options, error) {
 	flag.StringVar(&opts.portMap, "portmap", "", "Port mapping (e.g. 1234:5678,2345:6789)")
 	flag.BoolVar(&opts.naiveLoop, "naiveloop", false, "Loop the PCAP file in a naive way without rewrite of packets")
 	flag.StringVar(&opts.logLevel, "loglevel", "info", "Log level (info, debug, warn)")
+	flag.IntVar(&opts.gapThresholdMS, "gap", -1, "Report gap if capture times between packets exceeds this value in ms")
 	flag.BoolVar(&opts.version, "version", false, "Get version")
 
 	flag.Usage = func() {
@@ -62,11 +64,11 @@ func main() {
 		os.Exit(1)
 	}
 	if opts.version {
-		fmt.Printf("ew-pcap-replay %s\n", internal.GetVersion())
+		fmt.Printf("pcap-replay %s\n", internal.GetVersion())
 		os.Exit(0)
 	}
 	if opts.pcap == "" || (opts.dstAddr == "" && opts.dstDir == "") {
-		fmt.Fprintf(os.Stderr, "pcap and either addr or dir must be specified")
+		fmt.Fprintf(os.Stderr, "Error: pcap and either addr or dir must be specified\n\n")
 		flag.Usage()
 		os.Exit(1)
 	}
@@ -78,7 +80,7 @@ func main() {
 }
 
 func run(ctx context.Context, opts *options) error {
-	hdlr := createUDPHandler(opts.dstAddr, opts.dstDir)
+	hdlr := createUDPHandler(opts.dstAddr, opts.dstDir, opts.gapThresholdMS)
 	nrLoops := 1
 	portMap, err := parsePortMap(opts.portMap)
 	if err != nil {
@@ -93,7 +95,7 @@ func run(ctx context.Context, opts *options) error {
 			break
 		}
 		log.Infof("Loop %d done", nrLoops)
-		hdlr = createUDPHandler(opts.dstAddr, opts.dstDir)
+		hdlr = createUDPHandler(opts.dstAddr, opts.dstDir, opts.gapThresholdMS)
 		nrLoops++
 	}
 	return nil

cmd/pcap-replay/sender.go

@@ -29,12 +29,14 @@ type udpHandler struct {
 	lastPayload    []byte
 	startTimeStamp time.Time
 	startTime      time.Time
+	lastTime       time.Time // Used to detect gaps
 	firstSame      int
 	nrRepeatedPkts int
+	gapThresholdMS int
 	started        bool
 }
 
-func createUDPHandler(dstAddr string, dstDir string) *udpHandler {
+func createUDPHandler(dstAddr string, dstDir string, gapThresholdMS int) *udpHandler {
 	maxNrPackets := 1_000_000_000_000
 	maxNrSeconds := 1_000_000
 	uh := &udpHandler{
@@ -47,6 +49,7 @@ func createUDPHandler(dstAddr string, dstDir string) *udpHandler {
 		lastPayload:    make([]byte, 7*tsPacketSize),
 		started:        false,
 		nrRepeatedPkts: 0,
+		gapThresholdMS: gapThresholdMS,
 	}
 	return uh
 }
@@ -110,7 +113,16 @@ func (u *udpHandler) AddPacket(dst string, udpPayload []byte, timestamp time.Tim
 			u.streams[dst] = false
 		}
 	}
+	if u.gapThresholdMS > 0 {
+		if timestamp.Sub(u.lastTime) > time.Duration(u.gapThresholdMS)*time.Millisecond && u.pktNr > 0 {
+			timeDiff := timestamp.Sub(u.lastTime)
+			if timeDiff > 2*time.Second {
+				log.Infof("gap detected: %.3fs before packet %d", timeDiff.Seconds(), u.pktNr)
+			}
+		}
+	}
 	u.pktNr++
+	u.lastTime = timestamp
 	if ok {
 		// Copy the full payload to lastPayload. First set the size to the same as udpPayload.
 		if len(udpPayload) > int(cap(u.lastPayload)) {
@@ -122,7 +134,6 @@ func (u *udpHandler) AddPacket(dst string, udpPayload []byte, timestamp time.Tim
 		if u.dstDir != "" {
 			_, _ = u.outfiles[dst].Write(udpPayload)
 		}
-		u.pktNr++
 		timeDiff := timestamp.Sub(u.startTimeStamp)
 		if u.pktNr%10000 == 0 {
 			log.Infof("Read and sent %d packets %.3fs (%d repeated)", u.pktNr, timeDiff.Seconds(), u.nrRepeatedPkts)

cmd/pcap-unpack/main.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/Eyevinn/pcap-tools/internal"
+)
+
+const (
+	appName = "pcap-unpack"
+)
+
+var usg = `Usage of %s:
+
+%s unpacks UDP streams from from a Wireshark/tcpdump capture.
+
+The output is saved as files with names input_destAddress_port.ts
+(assuming that the streams are MPEG-2 TS streams).
+`
+
+type options struct {
+	dst     string
+	version bool
+}
+
+func parseOptions(fs *flag.FlagSet, args []string) (*options, error) {
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, usg, appName, appName)
+		fmt.Fprintf(os.Stderr, "\n%s [options] pcapfile [pcapfile ....]\n\noptions:\n", appName)
+		fs.PrintDefaults()
+	}
+
+	opts := options{}
+	fs.StringVar(&opts.dst, "dst", "", "Destination directory for output files")
+	fs.BoolVar(&opts.version, "version", false, "Get mp4ff version")
+
+	err := fs.Parse(args[1:])
+	return &opts, err
+}
+
+func main() {
+	if err := run(os.Args); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run(args []string) error {
+	fs := flag.NewFlagSet(appName, flag.ContinueOnError)
+	opts, err := parseOptions(fs, args)
+
+	if err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return nil
+		}
+		return err
+	}
+
+	if opts.version {
+		fmt.Printf("%s %s\n", appName, internal.GetVersion())
+		return nil
+	}
+
+	pcapFiles := fs.Args()
+	if len(pcapFiles) == 0 {
+		return fmt.Errorf("no pcap files specified")
+	}
+
+	for _, pcapFile := range pcapFiles {
+		err := processPCAP(pcapFile, opts.dst)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/pcap-unpack/pcap.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+)
+
+func processPCAP(pcapFile string, dst string) error {
+	if pcapFile == "" {
+		return fmt.Errorf("pcapFile is required")
+	}
+	handle, err := pcap.OpenOffline(pcapFile)
+	if err != nil {
+		return err
+	}
+	defer handle.Close()
+
+	return processPcapHandle(handle, pcapFile, dst)
+}
+
+func processPcapHandle(handle *pcap.Handle, fileName, dstDir string) error {
+	// Loop through packets from source
+	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
+	packetChannel := packetSource.Packets()
+	udpDsts := make(map[string]io.WriteCloser)
+	for packet := range packetChannel {
+		if packet == nil {
+			return nil
+		}
+		udpLayer := packet.Layer(layers.LayerTypeUDP)
+		if udpLayer == nil {
+			continue
+		}
+		ipLayer := packet.Layer(layers.LayerTypeIPv4)
+		if ipLayer == nil {
+			continue
+		}
+		ip := ipLayer.(*layers.IPv4)
+		udp, _ := udpLayer.(*layers.UDP)
+		if udp == nil {
+			continue
+		}
+		dstPort := int(udp.DstPort)
+		dst := fmt.Sprintf("%s_%d.ts", ip.DstIP, dstPort)
+		if _, ok := udpDsts[dst]; !ok {
+			var dstPath string
+			switch {
+			case dstDir == "":
+				dstPath = fmt.Sprintf("%s_%s", fileName, dst)
+			default:
+				base := filepath.Base(fileName)
+				dstPath = filepath.Join(dstDir, fmt.Sprintf("%s_%s", base, dst))
+			}
+			fh, err := os.Create(dstPath)
+			if err != nil {
+				return err
+			}
+			defer fh.Close()
+			fmt.Println("Created", dstPath)
+			udpDsts[dst] = fh
+		}
+		fh := udpDsts[dst]
+		_, err := fh.Write(udp.Payload)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}