Add subcommand to remove existing SoftPack built environments

README.md

@@ -120,7 +120,7 @@ spack:
   finalImage: "ubuntu:22.04"
   processorTarget: "x86_64_v3"
 
-coreURL: "http://x.y.z:9837/upload"
+coreURL: "http://x.y.z:9837/softpack"
 listenURL: "0.0.0.0:2456"
 ```
 

build/builder.go

@@ -52,6 +52,8 @@
 	builderOut             = "builder.out"
 	moduleForCoreBasename  = "module"
 	usageBasename          = "README.md"
+
+	uploadEndpoint = "/upload"
 )
 
 //go:embed singularity.tmpl
@@ -541,15 +543,15 @@
 
 	defer pw.Close()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, b.config.CoreURL+"?"+url.QueryEscape(envPath), pr)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimSuffix(b.config.CoreURL, "/")+uploadEndpoint+"?"+url.QueryEscape(envPath), pr)
 	if err != nil {
 		return err
 	}
 
 	req.Header.Add("Content-Type", writer.FormDataContentType())
 
 	resp, err := http.DefaultClient.Do(req)
-	slog.Debug("addArtifactsToRepo", "url", b.config.CoreURL+"?"+url.QueryEscape(envPath), "err", err)
+	slog.Debug("addArtifactsToRepo", "url", b.config.CoreURL+uploadEndpoint+"?"+url.QueryEscape(envPath), "err", err)
 
 	if err != nil {
 		return err

build/builder_test.go

@@ -369,7 +369,7 @@ Stage: final
 			modulePath := filepath.Join(conf.Module.ModuleInstallDir,
 				def.EnvironmentPath, def.EnvironmentName, def.EnvironmentVersion)
 			scriptsPath := filepath.Join(conf.Module.ScriptsInstallDir,
-				def.EnvironmentPath, def.EnvironmentName, def.EnvironmentVersion+scriptsDirSuffix)
+				def.EnvironmentPath, def.EnvironmentName, def.EnvironmentVersion+ScriptsDirSuffix)
 			imagePath := filepath.Join(scriptsPath, imageBasename)
 			expectedExes := []string{"python", "R", "Rscript", "xxhsum", "xxh32sum", "xxh64sum", "xxh128sum"}
 
@@ -535,7 +535,7 @@ packages:
 			})
 			So(ok, ShouldBeTrue)
 
-			expectedLog := "Post \\\"http://0.0.0.0:1234?groups%2Fhgi%2Fxxhash-0.8.1\\\"" +
+			expectedLog := "Post \\\"http://0.0.0.0:1234" + uploadEndpoint + "?groups%2Fhgi%2Fxxhash-0.8.1\\\"" +
 				": dial tcp 0.0.0.0:1234: connect: connection refused"
 			So(logWriter.String(), ShouldContainSubstring, expectedLog)
 

build/install.go

@@ -30,10 +30,11 @@ import (
 )
 
 const (
-	imageBasename    = "singularity.sif"
-	scriptsDirSuffix = "-scripts"
-	perms            = 0755
-	flags            = os.O_EXCL | os.O_CREATE | os.O_WRONLY
+	ScriptsDirSuffix = "-scripts"
+
+	imageBasename = "singularity.sif"
+	perms         = 0755
+	flags         = os.O_EXCL | os.O_CREATE | os.O_WRONLY
 )
 
 func installModule(scriptInstallBase, moduleInstallBase string, def *Definition, module,
@@ -67,7 +68,7 @@ func installModule(scriptInstallBase, moduleInstallBase string, def *Definition,
 
 func makeModuleDirs(scriptInstallBase, moduleInstallBase string, def *Definition) (string, string, error) {
 	scriptsDir := filepath.Join(scriptInstallBase, def.EnvironmentPath,
-		def.EnvironmentName, def.EnvironmentVersion+scriptsDirSuffix)
+		def.EnvironmentName, def.EnvironmentVersion+ScriptsDirSuffix)
 	moduleDir := filepath.Join(moduleInstallBase, def.EnvironmentPath, def.EnvironmentName)
 
 	if err := makeDirectory(scriptsDir, scriptInstallBase); err != nil {

build/install_test.go

@@ -54,7 +54,7 @@ func TestInstall(t *testing.T) {
 		createdModuleFile := readFile(t, filepath.Join(tmpModulesDir, def.EnvironmentPath,
 			def.EnvironmentName, def.EnvironmentVersion))
 		scriptsDir := filepath.Join(tmpScriptsDir, def.EnvironmentPath, def.EnvironmentName,
-			def.EnvironmentVersion+scriptsDirSuffix)
+			def.EnvironmentVersion+ScriptsDirSuffix)
 		createdImageFile := readFile(t, filepath.Join(scriptsDir, imageBasename))
 
 		So(createdModuleFile, ShouldEqual, moduleFile)

cmd/remove.go

@@ -0,0 +1,70 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"github.com/wtsi-hgi/go-softpack-builder/remove"
+	"github.com/wtsi-hgi/go-softpack-builder/s3"
+)
+
+var removeCmd = &cobra.Command{
+	Use:   "remove",
+	Short: "Remove an environment",
+	Long: `Remove an environment.
+
+Remove an existing environment; it's entry in the Core artefacts repo, the
+module files and the singularity image and symlinks.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) < 1 {
+			die("require a environment path")
+		}
+
+		const maxArgs = 2
+
+		if len(args) < maxArgs {
+			die("require a environment version")
+		}
+
+		if len(args) != maxArgs {
+			die("unexpected arguments")
+		}
+
+		conf := getConfig()
+
+		s, err := s3.New(conf.S3.BuildBase)
+		if err != nil {
+			die(err.Error())
+		}
+
+		envPath := filepath.Clean(string(filepath.Separator) + args[0])[1:]
+
+		if envPath != args[0] {
+			die("invalid environment path")
+		}
+
+		fmt.Printf(
+			"Will now remove environment %s-%s from artefacts repo and modules.\n"+
+				"Are you sure you sure you wish to proceed? [yN]: ",
+			envPath,
+			args[1],
+		)
+
+		var resp string
+
+		fmt.Scan(&resp)
+
+		if resp != "y" {
+			return
+		}
+
+		if err := remove.Remove(conf, s, envPath, args[1]); err != nil {
+			die(err.Error())
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(removeCmd)
+}

config/config_test.go

@@ -50,7 +50,7 @@ func TestConfig(t *testing.T) {
 		So(config.Spack.BuildImage, ShouldEqual, "spack/ubuntu-jammy:latest")
 		So(config.Spack.FinalImage, ShouldEqual, "ubuntu:22.04")
 		So(config.Spack.ProcessorTarget, ShouldEqual, "x86_64_v4")
-		So(config.CoreURL, ShouldEqual, "http://x.y.z:9837/graphql")
+		So(config.CoreURL, ShouldEqual, "http://x.y.z:9837/softpack")
 		So(config.ListenURL, ShouldEqual, "localhost:2456")
 	})
 }

internal/testdata/config.yml

@@ -18,5 +18,5 @@ spack:
   finalImage: "ubuntu:22.04"
   processorTarget: "x86_64_v4"
 
-coreURL: "http://x.y.z:9837/graphql"
+coreURL: "http://x.y.z:9837/softpack"
 listenURL: "localhost:2456"

remove/remove.go

@@ -0,0 +1,203 @@
+package remove
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/wtsi-hgi/go-softpack-builder/build"
+	"github.com/wtsi-hgi/go-softpack-builder/config"
+	"golang.org/x/sys/unix"
+)
+
+type Error string
+
+func (e Error) Error() string {
+	return string(e)
+}
+
+type coreResponse struct {
+	Data struct {
+		DeleteEnvironment struct {
+			Message string `json:"message,omitempty"`
+			Path    string `json:"path,omitempty"`
+			Name    string `json:"name,omitempty"`
+		} `json:"deleteEnvironment"`
+	} `json:"data"`
+}
+
+type S3Remover interface {
+	RemoveFile(string) error
+}
+
+const graphQLDeleteEnvironment = `mutation ($name: String!, $envPath: String!) {
+	deleteEnvironment(
+			name: $name
+			path: $envPath
+	) {
+		... on DeleteEnvironmentSuccess {
+				message
+		}
+		... on EnvironmentNotFoundError {
+				message
+				path
+				name
+		}
+	}
+}`
+
+const graphQLEndpoint = "/graphql"
+
+type graphQLDeleteEnvironmentMutation struct {
+	Query     string `json:"query"` //graphQLDeleteEnvironment
+	Variables struct {
+		Name    string `json:"name"`
+		EnvPath string `json:"envPath"`
+	} `json:"variables"`
+}
+
+func Remove(conf *config.Config, s S3Remover, envPath, version string) error {
+	envPath = filepath.Clean(string(filepath.Separator) + envPath)[1:]
+
+	modulePath := filepath.Join(conf.Module.ModuleInstallDir, envPath)
+	scriptPath := filepath.Join(conf.Module.ScriptsInstallDir, envPath, version+build.ScriptsDirSuffix)
+
+	if err := checkWriteAccess(modulePath, scriptPath); err != nil {
+		return err
+	}
+
+	if err := removeEnvFromCore(conf, envPath+"-"+version); err != nil {
+		return err
+	}
+
+	if err := removeLocalFiles(modulePath, scriptPath); err != nil {
+		return err
+	}
+
+	if err := removeFromS3(s, modulePath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func checkWriteAccess(modulePath, scriptPath string) error {
+	for _, p := range [...]string{
+		filepath.Dir(modulePath),
+		modulePath,
+		filepath.Dir(filepath.Dir(scriptPath)),
+		filepath.Dir(scriptPath),
+		scriptPath,
+	} {
+		if err := unix.Access(p, unix.W_OK); err != nil {
+			return Error(fmt.Sprintf("no write access to dir (%s): %s", p, err))
+		}
+	}
+
+	return nil
+}
+
+func removeEnvFromCore(conf *config.Config, envPath string) error {
+	log.Printf("Removing env %s from core\n", envPath)
+
+	req, err := http.NewRequestWithContext(
+		context.Background(),
+		http.MethodPost,
+		strings.TrimSuffix(conf.CoreURL, "/")+graphQLEndpoint,
+		createGraphQLPacket(envPath),
+	)
+	if err != nil {
+		return err
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	return handleCoreResponse(resp)
+}
+
+func createGraphQLPacket(envPath string) io.Reader {
+	mutation := graphQLDeleteEnvironmentMutation{
+		Query: graphQLDeleteEnvironment,
+	}
+	mutation.Variables.Name = filepath.Base(envPath)
+	mutation.Variables.EnvPath = filepath.Dir(envPath)
+
+	var buf bytes.Buffer
+
+	json.NewEncoder(&buf).Encode(mutation) //nolint:errcheck
+
+	return &buf
+}
+
+func handleCoreResponse(resp *http.Response) error {
+	var cr coreResponse
+
+	err := json.NewDecoder(resp.Body).Decode(&cr)
+	if err != nil {
+		return err
+	}
+
+	if cr.Data.DeleteEnvironment.Name != "" {
+		return Error(cr.Data.DeleteEnvironment.Message)
+	}
+
+	return nil
+}
+
+func removeLocalFiles(modulePath, scriptPath string) error {
+	if err := removeAllNoDescend(modulePath); err != nil {
+		return err
+	}
+
+	return removeAllNoDescend(scriptPath)
+}
+
+func removeAllNoDescend(path string) error {
+	files, err := os.ReadDir(path)
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		toRemove := filepath.Join(path, file.Name())
+
+		log.Printf("Removing file: %s\n", toRemove)
+
+		if err := os.Remove(toRemove); err != nil {
+			return err
+		}
+	}
+
+	log.Printf("Removing directory: %s\n", path)
+
+	return os.Remove(path)
+}
+
+var files = [...]string{"build.out", "singularity.def", "singularity.sif", "executables"} //nolint:gochecknoglobals
+
+func removeFromS3(s S3Remover, path string) error {
+	for _, file := range files {
+		toRemove := filepath.Join(path, file)
+
+		log.Printf("Removing file from S3: %s\n", toRemove)
+
+		if err := s.RemoveFile(toRemove); err != nil && !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+	}
+
+	return nil
+}