feat: improve output handling from commanands

.gitignore

@@ -4,6 +4,7 @@ __pycache__
 .venv
 *.pyc
 *.Dockerfile
+*.tar
 .poetry-version
 *.bak
 *.old

example/complex.yaml

@@ -15,7 +15,6 @@ labels:
 
     Let try templates here.
     Image test-case-2d on Alpine Linux {{ .alpine }}
-  org.opencontainers.image.test-case-2.name: alpine:{{ .alpine }}
 
 images:
   base:

pkg/cmd/cmd.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"os"
 	"os/exec"
@@ -15,6 +16,7 @@ type Cmd struct {
 	verbose  bool
 	preText  string
 	postText string
+	output   string
 }
 
 func New(c string) Cmd {
@@ -50,31 +52,81 @@ func (c Cmd) PostInfo(msg string) Cmd {
 	return c
 }
 
-func (c Cmd) Run(ctx context.Context) error {
+func (c Cmd) Run(ctx context.Context) (string, error) {
 	if c.preText != "" {
 		log.Info().Msg(c.preText)
 	}
 
 	cmd := exec.CommandContext(ctx, c.cmd, c.args...)
 
 	// pipe the commands output to the applications
+	var b bytes.Buffer
 	if c.verbose {
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
+	} else {
+		cmd.Stdout = &b
+		cmd.Stderr = &b
 	}
 
 	log.Debug().Str("cmd", c.cmd).Interface("args", c.args).Msg("Running")
-	if err := cmd.Run(); err != nil {
-		log.Error().Err(err).Msg("Could not run command")
-		return err
+	err := cmd.Run()
+
+	// Check for context cancellation or timeout
+	if ctx.Err() != nil {
+		// If the context was canceled, suppress output and return context error
+		if ctx.Err() == context.Canceled {
+			log.Warn().Str("cmd", c.cmd).Msg("Command was cancelled")
+		} else if ctx.Err() == context.DeadlineExceeded {
+			log.Warn().Str("cmd", c.cmd).Msg("Command timed out")
+		}
+		return "", ctx.Err()
+	}
+
+	// Handle other errors
+	if err != nil {
+		log.Error().Err(err).Str("cmd", c.cmd).Interface("args", c.args).Msg("Could not run command")
+		// c.setOutput(&b)
+		c.output = b.String()
+		log.Error().Msg(c.output)
+		return c.output, err
 	}
+	c.output = b.String()
 
 	if c.postText != "" {
 		log.Info().Msg(c.postText)
 	}
-	return nil
+	return c.output, nil
 }
 
 func (c Cmd) String() string {
 	return c.cmd + " " + strings.Join(c.args, " ")
 }
+
+func (c Cmd) Output() (string, error) {
+	cmd := exec.Command(c.cmd, c.args...)
+
+	// pipe the commands output to the applications
+	var b bytes.Buffer
+	if c.verbose {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	} else {
+		cmd.Stdout = &b
+		cmd.Stderr = &b
+	}
+
+	log.Debug().Str("cmd", c.cmd).Interface("args", c.args).Msg("Running")
+	err := cmd.Run()
+
+	// Handle other errors
+	if err != nil {
+		log.Error().Err(err).Str("cmd", c.cmd).Interface("args", c.args).Msg("Could not run command")
+		c.output = b.String()
+		log.Error().Msg(c.output)
+		return c.output, err
+	}
+	c.output = b.String()
+
+	return c.output, nil
+}

pkg/parser/inspect.go

@@ -0,0 +1,38 @@
+package parser
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/rs/zerolog/log"
+	"github.com/tgagor/template-dockerfiles/pkg/cmd"
+	"github.com/tgagor/template-dockerfiles/pkg/util"
+)
+
+type DockerInspect []struct {
+	Config struct {
+		Env        []string            `json:"Env"`
+		Cmd        []string            `json:"Cmd"`
+		Volumes    map[string]struct{} `json:"Volumes"`
+		WorkingDir string              `json:"WorkingDir"`
+		Entrypoint []string            `json:"Entrypoint"`
+		Labels     map[string]string   `json:"Labels"`
+	} `json:"Config"`
+}
+
+func inspectImg(image string) (DockerInspect, error) {
+	out, err := cmd.New("docker").Arg("inspect").Arg("--format").Arg("json").Arg(image).Output()
+	util.FailOnError(err)
+
+	// Create a variable to hold the unmarshaled data
+	var inspect DockerInspect
+
+	// Unmarshal the JSON data into the DockerInspect structure
+
+	if err := json.Unmarshal([]byte(out), &inspect); err != nil {
+		log.Error().Err(err).Msg("Error parsing JSON.")
+		os.Exit(1)
+	}
+
+	return inspect, nil
+}

pkg/parser/parser.go

@@ -35,11 +35,9 @@ func Run(workdir string, cfg *config.Config, flag config.Flags) error {
 		}
 
 		var tempFiles []string
+		var toSquash []string
 
 		buildTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
-		squashTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
-		exportTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
-		importTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
 		// labelling have to happen in order, so no parallelism
 		taggingTasks := runner.New().DryRun(!flag.Build)
 		pushTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
@@ -96,31 +94,7 @@ func Run(workdir string, cfg *config.Config, flag config.Flags) error {
 
 			// squash if demanded
 			if flag.Squash {
-				// first run it: docker run --name tgagor-${{ matrix.tag }} tgagor/centos:${{ matrix.tag }} true
-				runItFirst := cmd.New("docker").
-					Arg("run").
-					Arg("--name", sanitizeForFileName(currentImage)).
-					Arg(currentImage).
-					Arg("true"). // just a dummy command
-					SetVerbose(flag.Verbose)
-				squashTasks = squashTasks.AddTask(runItFirst)
-
-				// second, collect it's configuration
-				// imgMetadata := inspectImg(currentImage)
-
-				// third, export it
-				tmpTarFile := sanitizeForFileName(currentImage) + ".tar"
-				exportIt := cmd.New("docker").
-					Arg("export").
-					Arg(sanitizeForFileName(currentImage)).
-					Arg("-o", tmpTarFile)
-				exportTasks = exportTasks.AddTask(exportIt)
-				importIt := cmd.New("docker").
-					Arg("import").
-					// Arg("--change", "some change").
-					Arg(tmpTarFile).
-					Arg(sanitizeForFileName(currentImage))
-				importTasks = importTasks.AddTask(importIt)
+				toSquash = append(toSquash, currentImage)
 			}
 
 			// collect tagging commands to keep order
@@ -154,25 +128,29 @@ func Run(workdir string, cfg *config.Config, flag config.Flags) error {
 
 		if flag.Build {
 			err := buildTasks.Run()
-			util.FailOnError(err)
+			util.FailOnError(err, "Building failed with error, check error above. Exiting.")
 		}
+
+		// let squash it
 		if flag.Build && flag.Squash {
-			err := squashTasks.Run()
-			util.FailOnError(err)
-			err = exportTasks.Run()
-			util.FailOnError(err)
-			err = importTasks.Run()
-			util.FailOnError(err)
+			// first run it: docker run --name tgagor-${{ matrix.tag }} tgagor/centos:${{ matrix.tag }} true
+			// just a dummy command
+			// second, collect it's configuration
+			// third, export it
+			// now import and amend based on source metadata
+			squashImages(flag, toSquash)
 		}
+
+		// continue classical build
 		if flag.Build {
 			err := taggingTasks.Run()
-			util.FailOnError(err)
+			util.FailOnError(err, "Tagging failed with error, check error above. Exiting.")
 			err = cleanupTasks.Run()
-			util.FailOnError(err)
+			util.FailOnError(err, "Dropping temporary images failed. Exiting.")
 		}
 		if flag.Push {
 			err := pushTasks.Run()
-			util.FailOnError(err)
+			util.FailOnError(err, "Pushing images failed, check error above. Exiting.")
 		}
 
 		// Cleanup temporary files
@@ -189,11 +167,69 @@ func Run(workdir string, cfg *config.Config, flag config.Flags) error {
 	return nil
 }
 
-// func inspectImg(image string) error {
-// 	inspect := cmd.New("docker").Arg("inspect").Arg("--format").Arg("json").Arg(image)
+func squashImages(flag config.Flags, toSquash []string) {
+	runTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
+	exportTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
+	dropTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
+	importTasks := runner.New().Threads(flag.Threads).DryRun(!flag.Build)
+
+	for _, img := range toSquash {
+
+		runItFirst := cmd.New("docker").
+			Arg("run").
+			Arg("--name", sanitizeForFileName(img)).
+			Arg(img).
+			Arg("true").
+			SetVerbose(flag.Verbose)
+		runTasks = runTasks.AddTask(runItFirst)
+
+		imgMetadata, err := inspectImg(img)
+		util.FailOnError(err, "Couldn't inspect Docker image.")
+		log.Debug().Interface("data", imgMetadata).Msg("Docker inspect result")
+
+		tmpTarFile := sanitizeForFileName(img) + ".tar"
+		exportIt := cmd.New("docker").
+			Arg("export").
+			Arg(sanitizeForFileName(img)).
+			Arg("-o", tmpTarFile)
+		exportTasks = exportTasks.AddTask(exportIt)
+		dropIt := cmd.New("docker").Arg("rm").Arg("-f").Arg(sanitizeForFileName(img))
+		dropTasks = dropTasks.AddTask(dropIt)
+
+		importIt := cmd.New("docker").Arg("import")
+		for _, item := range imgMetadata {
+			for _, env := range item.Config.Env {
+				importIt = importIt.Arg("--change", "ENV "+env)
+			}
+			for _, command := range item.Config.Cmd {
+				importIt = importIt.Arg("--change", "CMD "+command)
+			}
+			for vol := range item.Config.Volumes {
+				importIt = importIt.Arg("--change", "VOLUME "+vol)
+			}
+			for key, value := range item.Config.Labels {
+				importIt = importIt.Arg("--change", fmt.Sprintf("LABEL '%s'='%s'", key, value))
+			}
+			if len(item.Config.Entrypoint) > 0 {
+				importIt = importIt.Arg("--change", "ENTRYPOINT [\""+strings.Join(item.Config.Entrypoint, "\", \"")+"\"]")
+			}
+			if item.Config.WorkingDir != "" {
+				importIt = importIt.Arg("--change", "WORKDIR "+item.Config.WorkingDir)
+			}
+		}
+		importIt = importIt.Arg(tmpTarFile).Arg(sanitizeForFileName(img))
+		importTasks = importTasks.AddTask(importIt)
+	}
 
-// 	return nil
-// }
+	err := runTasks.Run()
+	util.FailOnError(err)
+	err = exportTasks.Run()
+	util.FailOnError(err)
+	err = dropTasks.Run()
+	util.FailOnError(err)
+	err = importTasks.Run()
+	util.FailOnError(err)
+}
 
 func collectLabels(configSet map[string]interface{}) map[string]string {
 	labels, err := templateLabels(configSet["labels"].(map[string]string), configSet)

pkg/runner/runner.go

@@ -107,7 +107,7 @@ func (r Runner) Run() error {
 				if r.dryRun {
 					log.Debug().Str("cmd", c.String()).Msg("DRY-RUN: Run")
 				} else {
-					if err := c.Run(ctx); err != nil {
+					if _, err := c.Run(ctx); err != nil {
 						// Send the error to the results channel
 						results <- err
 						cancel() // Signal cancellation to all workers
@@ -126,7 +126,8 @@ func (r Runner) Run() error {
 
 	for err := range results {
 		if err != nil {
-			log.Error().Err(err).Msg("Worker encountered error")
+			// cmd prints it anyway
+			// log.Error().Err(err).Msg("Worker encountered error")
 			return err
 		}
 	}

tests/centos-squash/Dockerfile.tpl

@@ -0,0 +1,9 @@
+FROM quay.io/centos/centos:{{ .centos }}
+
+# upgrade packages
+RUN dnf upgrade --setopt=install_weak_deps=False -y && \
+    dnf clean all && \
+    rm -rf /tmp/* && \
+    rm -rf /var/cache/yum && \
+    rm -rf /var/cache/dnf && \
+    find /var/log -type f -name '*.log' -delete

tests/centos-squash/build.yaml

@@ -0,0 +1,33 @@
+---
+registry: ghcr.io
+prefix: tgagor
+maintainer: Tomasz Gągor <https://gagor.pro>
+
+labels:
+  org.opencontainers.image.licenses: GPL-2.0-only
+  org.opencontainers.image.url: https://hub.docker.com/repository/docker/tgagor/centos/general
+  org.opencontainers.image.documentation: https://github.com/tgagor/docker-centos/blob/master/README.md
+  org.opencontainers.image.title: Weekly updated CentOS Docker images
+  org.opencontainers.image.description: |
+    Those images are just standard CentOS base images, but:
+
+    1. With all the package updates installed weekly.
+    2. Squashed to single layer for smaller size.
+
+images:
+  centos:
+    dockerfile: Dockerfile.tpl
+    variables:
+      centos:
+        - stream9
+        - stream10
+    tags:
+      - centos:{{ .centos }}
+      - centos:{{ .centos | trimPrefix "stream" }}
+      - centos:{{ .tag | splitList "-" | first }}-{{ .centos }}
+      - centos:{{ .tag | splitList "-" | first }}
+      - centos:{{ .centos }}-{{ .tag | splitList "-" | rest | first }}
+      - centos:stream
+      - centos:latest
+    labels:
+      org.opencontainers.image.base.name: quay.io/centos/centos:{{ .centos }}