feat: better version tags (#56)

* feat: better version tags

- use `build-gh.sh` script
- better cache behaviour for npm

* fix(api): build image setting passed via env variable

---------

Co-authored-by: raphaelcoeffic <[email protected]>

Dockerfile

@@ -9,10 +9,12 @@ RUN make build
 FROM node:slim as ui-builder
 
 RUN mkdir /build
-ADD . /build/
 WORKDIR /build
 
+ADD ui/package*.json /build/ui/
 RUN npm ci --prefix ui/
+
+ADD . /build/
 RUN npm run build --prefix ui/
 
 FROM debian:bookworm-slim

artifactory/artifactory.go

@@ -93,7 +93,7 @@ func (artifactory *Artifactory) GetBuild(request *BuildRequest) (*BuildJobDto, e
 	if err != nil {
 		return nil, err
 	}
-	if err == nil && buildJob == nil {
+	if buildJob == nil {
 		return nil, ErrBuildNotFound
 	}
 
@@ -241,7 +241,7 @@ func (artifactory *Artifactory) Build(
 		return onBuildFailure(err, build)
 	}
 
-	firmwareBin, err := builder.Build(ctx, build.ContainerImage, flags)
+	firmwareBin, err := builder.Build(ctx, build.ContainerImage, build.Target, build.CommitRef, flags)
 	if err != nil {
 		return onBuildFailure(err, build)
 	}

artifactory/mocks_test.go

@@ -28,6 +28,8 @@ func (downloader *MockFirmwareBuilder) PullImage(ctx context.Context, buildConta
 func (downloader *MockFirmwareBuilder) Build(
 	ctx context.Context,
 	buildContainer string,
+	target string,
+	versionTag string,
 	flags []firmware.BuildFlag,
 ) ([]byte, error) {
 	args := downloader.Called(ctx, buildContainer, flags)

cli/cli.go

@@ -43,6 +43,8 @@ func ParseBuildFlags(buildFlagsJSON []byte) ([]firmware.BuildFlag, error) {
 }
 
 type BuildCommandArgs struct {
+	Target           string
+	VersionTag       string
 	CommitHash       string
 	BuildFlagsFile   string
 	BuildFlagsInline string
@@ -55,13 +57,15 @@ type BuildCommandArgs struct {
 func ParseArgs() (*BuildCommandArgs, error) {
 	config := &BuildCommandArgs{}
 
+	flag.StringVar(&config.Target, "target", "", "firmware target")
+	flag.StringVar(&config.VersionTag, "version", "", "version tag")
 	flag.StringVar(&config.CommitHash, "commit", "", "specify commit hash")
 	flag.StringVar(&config.BuildFlagsFile, "build-flags-file", "", "specify build flags json file location")
 	flag.StringVar(&config.BuildFlagsInline, "build-flags", "", "specify build flags inline")
 	flag.StringVar(
 		&config.BuildImage,
 		"build-image",
-		"ghcr.io/edgetx/edgetx-builder:2.5.1",
+		"ghcr.io/edgetx/edgetx-builder",
 		"specify podman image for building",
 	)
 
@@ -114,6 +118,8 @@ func ValidateBuildArgs(config *BuildCommandArgs) error {
 
 func Build(
 	ctx context.Context,
+	target string,
+	versionTag string,
 	buildImage string,
 	sourceRepository string,
 	commitHash string,
@@ -133,7 +139,7 @@ func Build(
 		return nil, err
 	}
 
-	firmwareBin, err := firmwareBuilder.Build(ctx, buildImage, buildFlags)
+	firmwareBin, err := firmwareBuilder.Build(ctx, buildImage, target, versionTag, buildFlags)
 	if err != nil {
 		return nil, err
 	}

cmd/ebuild/run/cmd.go

@@ -142,6 +142,7 @@ func NewRunCommand(ctx context.Context, o *config.CloudbuildOpts) *cobra.Command
 	o.BindCliOpts(cmd)
 	o.BindDBOpts(cmd)
 	o.BindStorageOpts(cmd)
+	o.BindBuildOpts(cmd)
 
 	s := newServerRunner(ctx, o)
 	cmd.AddCommand(NewAPICommand(s))

cmd/edgetx-build/main.go

@@ -48,7 +48,15 @@ func main() {
 			}
 		}
 
-		firmwareBin, err := cli.Build(ctx, config.BuildImage, config.SourceRepository, config.CommitHash, buildFlags)
+		firmwareBin, err := cli.Build(
+			ctx,
+			config.Target,
+			config.VersionTag,
+			config.BuildImage,
+			config.SourceRepository,
+			config.CommitHash,
+			buildFlags,
+		)
 		if err != nil {
 			log.Fatalf("failed to build firmware: %s", err)
 		}

config/options.go

@@ -149,6 +149,12 @@ func (o *CloudbuildOpts) BindStorageOpts(c *cobra.Command) {
 	)
 }
 
+func (o *CloudbuildOpts) BindBuildOpts(c *cobra.Command) {
+	c.PersistentFlags().StringVar(
+		&o.BuildImage, "build-img", o.BuildImage, "Build docker image",
+	)
+}
+
 func (o *CloudbuildOpts) BindAPIOpts(c *cobra.Command) {
 	c.Flags().Uint16VarP(&o.HTTPBindPort, "port", "p", o.HTTPBindPort, "HTTP listen port")
 	c.Flags().IPVarP(&o.HTTPBindAddress, "listen-ip", "l", net.IPv4zero, "HTTP listen IP")
@@ -157,7 +163,6 @@ func (o *CloudbuildOpts) BindAPIOpts(c *cobra.Command) {
 }
 
 func (o *CloudbuildOpts) BindWorkerOpts(c *cobra.Command) {
-	c.Flags().StringVar(&o.BuildImage, "build-img", o.BuildImage, "Build docker image")
 	c.Flags().StringVar(
 		&o.SourceRepository, "src-repo", o.SourceRepository, "Source repository")
 }

firmware/builder.go

@@ -6,5 +6,5 @@ import (
 
 type Builder interface {
 	PullImage(ctx context.Context, buildContainer string) error
-	Build(ctx context.Context, buildContainer string, flags []BuildFlag) ([]byte, error)
+	Build(ctx context.Context, buildContainer string, target string, versionTag string, flags []BuildFlag) ([]byte, error)
 }

firmware/podman_builder.go

@@ -5,20 +5,19 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path"
-	"strings"
+	"path/filepath"
 
 	"github.com/edgetx/cloudbuild/buildlogs"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
-type PodmanExecutor func(ctx context.Context, name string, args ...string) (string, error)
+type PodmanExecutor func(ctx context.Context, args ...string) (string, error)
 
 func DefaultPodmanExecutor(workingDir string) PodmanExecutor {
-	return func(ctx context.Context, name string, args ...string) (string, error) {
-		log.Debugf("podman cmd: %s %s", name, args)
-		cmd := exec.CommandContext(ctx, name, args...)
+	return func(ctx context.Context, args ...string) (string, error) {
+		log.Debugf("podman cmd: %s", args)
+		cmd := exec.CommandContext(ctx, "podman", args...)
 		cmd.Dir = workingDir
 		output, err := cmd.CombinedOutput()
 		return string(output), err
@@ -44,7 +43,7 @@ func NewPodmanBuilder(workingDir string, recorder *buildlogs.Recorder, cpuLimit
 }
 
 func (builder *PodmanBuilder) PullImage(ctx context.Context, buildContainer string) error {
-	output, err := builder.PodmanExecutor(ctx, "podman", "pull", "--quiet", buildContainer)
+	output, err := builder.PodmanExecutor(ctx, "pull", "--quiet", buildContainer)
 	builder.recorder.AddStdOut(output)
 	if err != nil {
 		return errors.Errorf("failed to pull container image: %s", err)
@@ -53,49 +52,68 @@ func (builder *PodmanBuilder) PullImage(ctx context.Context, buildContainer stri
 	return nil
 }
 
-func (builder *PodmanBuilder) Build(ctx context.Context, buildContainer string, flags []BuildFlag) ([]byte, error) {
-	err := builder.PullImage(ctx, buildContainer)
-	if err != nil {
-		return nil, err
-	}
-
-	commands := []string{
-		"rm -rf ./build",
-		"mkdir ./build",
-		"cd ./build",
-		fmt.Sprintf("cmake -Wno-dev %s ../", CmakeFlags(flags)),
-		"cd ../",
-		fmt.Sprintf("make --directory ./build -j%d firmware-size", builder.CPULimit*2),
-	}
-	output, err := builder.PodmanExecutor(
-		ctx,
-		"podman",
+func (builder *PodmanBuilder) buildCmdArgs(
+	buildContainer string,
+	target string,
+	versionTag string,
+	flags []BuildFlag,
+) []string {
+	args := []string{
 		"run",
 		"--tty",
-		"--userns=keep-id",
 		"--rm",
+		"--userns=keep-id",
 		fmt.Sprintf("--cpus=%d", builder.CPULimit),
-		"--workdir",
-		"/home/rootless/src",
 		"--volume",
 		fmt.Sprintf("%s:/home/rootless/src:Z", builder.workingDir),
-		buildContainer,
-		"bash",
-		"-c",
-		strings.Join(commands, " && "),
-	)
+	}
+
+	env := []string{
+		fmt.Sprintf("FLAVOR=%s", target),
+		fmt.Sprintf("EXTRA_OPTIONS=%s", CmakeFlags(flags)),
+	}
+	if versionTag == "nightly" {
+		env = append(env, "EDGETX_VERSION_SUFFIX=nightly")
+	} else {
+		env = append(
+			env,
+			"EDGETX_VERSION_SUFFIX=cloudbuild",
+			fmt.Sprintf("EDGETX_VERSION_TAG=%s", versionTag),
+		)
+	}
+	for _, varDef := range env {
+		args = append(args, "--env", varDef)
+	}
+
+	return append(args, buildContainer, "./tools/build-gh.sh")
+}
+
+func (builder *PodmanBuilder) Build(
+	ctx context.Context,
+	buildContainer string,
+	target string,
+	versionTag string,
+	flags []BuildFlag,
+) ([]byte, error) {
+	if err := builder.PullImage(ctx, buildContainer); err != nil {
+		return nil, err
+	}
+
+	args := builder.buildCmdArgs(buildContainer, target, versionTag, flags)
+	output, err := builder.PodmanExecutor(ctx, args...)
+
 	builder.recorder.AddStdOut(output)
 	log.Debugf("container build output: %s", output)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build: %w", err)
 	}
 
-	firmwarePath := path.Join(builder.workingDir, "build", "arm-none-eabi", "firmware.bin")
-	if _, err := os.Stat(firmwarePath); os.IsNotExist(err) {
-		return nil, fmt.Errorf("firmware.bin does not exist: %w", err)
+	firmwarePaths, err := filepath.Glob(filepath.Join(builder.workingDir, "*.bin"))
+	if err != nil || len(firmwarePaths) == 0 {
+		return nil, fmt.Errorf("cannot find build artifact: %w", err)
 	}
 
-	firmwareData, err := os.ReadFile(firmwarePath)
+	firmwareData, err := os.ReadFile(firmwarePaths[0])
 	if err != nil {
 		return nil, fmt.Errorf("failed to read firmware binary data: %w", err)
 	}

firmware/podman_builder_test.go

@@ -35,13 +35,17 @@ func TestFirmwareFirmwareBuild(t *testing.T) {
 		firmware.NewFlag("TRACE_SIMPGMSPACE", "NO"),
 		firmware.NewFlag("VERBOSE_CMAKELISTS", "YES"),
 		firmware.NewFlag("CMAKE_RULE_MESSAGES", "OFF"),
-		firmware.NewFlag("PCB", "X10"),
-		firmware.NewFlag("PCBREV", "T16"),
 		firmware.NewFlag("INTERNAL_MODULE_MULTI", "ON"),
 	}
 	ctx, cancel = context.WithTimeout(context.Background(), time.Minute*20)
 	defer cancel()
-	firmwareBin, err := firmwareBuilder.Build(ctx, "ghcr.io/edgetx/edgetx-commit-tests:latest", flags)
+	firmwareBin, err := firmwareBuilder.Build(
+		ctx,
+		"ghcr.io/edgetx/edgetx-builder",
+		"t16",
+		"test",
+		flags,
+	)
 	assert.Nil(t, err, "failed to build firmware")
 	assert.True(t, len(firmwareBin) > 0, "firmware bin is empty")
 }

processor/worker.go

@@ -39,7 +39,7 @@ func (worker *Worker) build(
 
 	recorder := buildlogs.NewRecorder()
 	gitDownloader := source.NewGitDownloader(sourceDir, recorder)
-	firmwareBuilder := firmware.NewPodmanBuilder(sourceDir, recorder, 2, 1024*1024*1024)
+	firmwareBuilder := firmware.NewPodmanBuilder(sourceDir, recorder, 4, 2*1024*1024*1024)
 
 	return worker.artifactory.Build(
 		ctx, job, recorder, gitDownloader, firmwareBuilder,
@@ -73,7 +73,7 @@ func (worker *Worker) PullImage(ctx context.Context, buildImage string) error {
 		We do this so actual build process is faster because of the cached build image
 	*/
 	recorder := buildlogs.NewRecorder()
-	firmwareBuilder := firmware.NewPodmanBuilder("/tmp", recorder, 2, 1024*1024*1024)
+	firmwareBuilder := firmware.NewPodmanBuilder("/tmp", recorder, 4, 1024*1024*1024)
 	ctx, cancel := context.WithTimeout(ctx, artifactory.MaxBuildDuration)
 	defer cancel()