From fa156d8271f04984b2e4663aafe1bb20e7f52c34 Mon Sep 17 00:00:00 2001 From: Nic Jackson Date: Tue, 15 Aug 2023 11:58:28 +0100 Subject: [PATCH] Add capability to copy files from build resources --- examples/build/build.hcl | 10 ++ examples/build/src/main.go | 2 +- pkg/clients/container/docker_tasks.go | 132 +++++++++++++++++++++++-- pkg/clients/tar/tar.go | 16 ++- pkg/config/resources/build/provider.go | 43 ++++++++ pkg/config/resources/build/resource.go | 8 ++ 6 files changed, 196 insertions(+), 15 deletions(-) diff --git a/examples/build/build.hcl b/examples/build/build.hcl index 465e4706..45071c0c 100644 --- a/examples/build/build.hcl +++ b/examples/build/build.hcl @@ -11,6 +11,16 @@ resource "build" "app" { dockerfile = "Dockerfile" context = "./src" } + + output { + source = "/bin/app" + destination = "${data("output_file")}/app" + } + + output { + source = "/bin" + destination = "${data("output_dir")}/bin" + } } resource "network" "onprem" { diff --git a/examples/build/src/main.go b/examples/build/src/main.go index 62cdf437..9f572440 100644 --- a/examples/build/src/main.go +++ b/examples/build/src/main.go @@ -10,5 +10,5 @@ func main() { fmt.Fprint(rw, "Hello world") }) - http.ListenAndServe(":9090", nil) + fmt.Println(http.ListenAndServe(":9090", nil)) } diff --git a/pkg/clients/container/docker_tasks.go b/pkg/clients/container/docker_tasks.go index e8381054..cf86de73 100644 --- a/pkg/clients/container/docker_tasks.go +++ b/pkg/clients/container/docker_tasks.go @@ -663,22 +663,50 @@ func (d *DockerTasks) CopyFromContainer(id, src, dst string) error { } defer reader.Close() - readBytes, err := ioutil.ReadAll(reader) + // untar the file to a temporary folder + dir, err := os.MkdirTemp(os.TempDir(), "") if err != nil { - return fmt.Errorf("unable to read file '%s' from container '%s': %w", src, id, err) + return fmt.Errorf("unable to create temporary directory, %s", err) } - // write to file, skipping the first 512 bytes which contain file metadata - // and trimming any NULL characters - trimBytes := bytes.Trim(readBytes[512:], "\x00") + // clean up the temp dir + defer os.RemoveAll(dir) - file, err := os.Create(dst) + // make sure the destination does not exist + if _, err := os.Stat(dst); err == nil { + err = os.RemoveAll(dst) + if err != nil { + return fmt.Errorf("unable to remove destination file or directory: %s", err) + } + } + + err = d.tg.Uncompress(reader, false, dir) if err != nil { - return fmt.Errorf("unable to create destination file '%s' when copying file '%s' from container '%s': %w", dst, src, id, err) + return fmt.Errorf("unable to extract tar file: %s", err) + } + + // copy the source temp to the destination + + // the source file or folder name is the last part of the src + filename := path.Base(src) + tmpPath := path.Join(dir, filename) + + // if tmpPath is a directory copy the directory + i, _ := os.Stat(tmpPath) + if i.IsDir() { + err = copyDir(tmpPath, dst) + if err != nil { + return fmt.Errorf("unable to copy temporary files to destination %s", err) + } + + return nil } - defer file.Close() - file.Write(trimBytes) + // else just copy the file + _, err = copy(tmpPath, dst) + if err != nil { + return fmt.Errorf("unable to copy temporary files to destination %s", err) + } return nil } @@ -1382,3 +1410,89 @@ func (d *DockerTasks) saveImageToTempFile(image, filename string) (string, error return tmpFileName, nil } + +func copyDir(src string, dest string) error { + + if dest[:len(src)] == src { + return fmt.Errorf("Cannot copy a folder into the folder itself!") + } + + f, err := os.Open(src) + if err != nil { + return err + } + + file, err := f.Stat() + if err != nil { + return err + } + if !file.IsDir() { + return fmt.Errorf("Source " + file.Name() + " is not a directory!") + } + + err = os.Mkdir(dest, 0755) + if err != nil { + return err + } + + files, err := ioutil.ReadDir(src) + if err != nil { + return err + } + + for _, f := range files { + + if f.IsDir() { + + err = copyDir(src+"/"+f.Name(), dest+"/"+f.Name()) + if err != nil { + return err + } + + } + + if !f.IsDir() { + + content, err := ioutil.ReadFile(src + "/" + f.Name()) + if err != nil { + return err + + } + + err = ioutil.WriteFile(dest+"/"+f.Name(), content, 0755) + if err != nil { + return err + + } + + } + + } + + return nil +} + +func copy(src, dst string) (int64, error) { + sourceFileStat, err := os.Stat(src) + if err != nil { + return 0, err + } + + if !sourceFileStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", src) + } + + source, err := os.Open(src) + if err != nil { + return 0, err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return 0, err + } + defer destination.Close() + nBytes, err := io.Copy(destination, source) + return nBytes, err +} diff --git a/pkg/clients/tar/tar.go b/pkg/clients/tar/tar.go index 0365960e..176ff94f 100644 --- a/pkg/clients/tar/tar.go +++ b/pkg/clients/tar/tar.go @@ -93,12 +93,18 @@ func (tg *TarGz) Compress(buf io.Writer, options *TarGzOptions, src ...string) e return nil } -func (tg *TarGz) Uncompress(src io.Reader, dst string) error { - // ungzip - zr, err := gzip.NewReader(src) - if err != nil { - return err +func (tg *TarGz) Uncompress(src io.Reader, gziped bool, dst string) error { + var zr io.Reader = src + var err error + + if gziped { + // ungzip + zr, err = gzip.NewReader(src) + if err != nil { + return err + } } + // untar tr := tar.NewReader(zr) diff --git a/pkg/config/resources/build/provider.go b/pkg/config/resources/build/provider.go index d0f7e4a2..87e943d8 100644 --- a/pkg/config/resources/build/provider.go +++ b/pkg/config/resources/build/provider.go @@ -76,6 +76,12 @@ func (b *Provider) Create() error { b.config.Image = name b.config.BuildChecksum = hash + // do we need to copy any files? + err = b.copyOutputs() + if err != nil { + return xerrors.Errorf("unable to copy files from build container: %w", err) + } + // clean up the previous builds only leaving the last 3 ids, err := b.client.FindImagesInLocalRegistry(fmt.Sprintf("jumppad.dev/localcache/%s", b.config.Name)) if err != nil { @@ -149,3 +155,40 @@ func (b *Provider) hasChanged() (bool, error) { return false, nil } + +func (b *Provider) copyOutputs() error { + if len(b.config.Outputs) < 1 { + return nil + } + + // start an instance of the container + c := types.Container{ + Image: &types.Image{ + Name: b.config.Image, + }, + Entrypoint: []string{}, + Command: []string{"tail", "-f", "/dev/null"}, + } + + b.log.Debug("Creating container to copy files", "ref", b.config.ID, "name", b.config.Image) + id, err := b.client.CreateContainer(&c) + if err != nil { + return err + } + + // always remove the temp container + defer func() { + b.log.Debug("Remove copy container", "ref", b.config.ID, "name", b.config.Image) + b.client.RemoveContainer(id, true) + }() + + for _, copy := range b.config.Outputs { + b.log.Debug("Copy file from container", "ref", b.config.ID, "source", copy.Source, "destination", copy.Destination) + err := b.client.CopyFromContainer(id, copy.Source, copy.Destination) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/config/resources/build/resource.go b/pkg/config/resources/build/resource.go index c5cde186..10803a85 100644 --- a/pkg/config/resources/build/resource.go +++ b/pkg/config/resources/build/resource.go @@ -15,6 +15,9 @@ type Build struct { Container BuildContainer `hcl:"container,block" json:"container"` + // Outputs allow files or directories to be copied from the container + Outputs []Output `hcl:"output,block" json:"outputs"` + // outputs // Image is the full local reference of the built image @@ -30,6 +33,11 @@ type BuildContainer struct { Args map[string]string `hcl:"args,optional" json:"args,omitempty"` // Build args to pass to the container } +type Output struct { + Source string `hcl:"source" json"source"` // Source file or directory in container + Destination string `hcl:"destination" json"destination"` // Destination for copied file or directory +} + func (b *Build) Process() error { b.Container.Context = utils.EnsureAbsolute(b.Container.Context, b.File)