Add subcommand to remove existing SoftPack built environments

README.md

@@ -120,7 +120,7 @@ spack:
   finalImage: "ubuntu:22.04"
   processorTarget: "x86_64_v3"
 
-coreURL: "http://x.y.z:9837/upload"
+coreURL: "http://x.y.z:9837/softpack"
 listenURL: "0.0.0.0:2456"
 ```
 

build/builder.go

@@ -45,13 +45,15 @@ import (
 )
 
 const (
-	singularityDefBasename = "singularity.def"
-	exesBasename           = "executables"
-	softpackYaml           = "softpack.yml"
-	spackLock              = "spack.lock"
-	builderOut             = "builder.out"
+	SingularityDefBasename = "singularity.def"
+	ExesBasename           = "executables"
+	SoftpackYaml           = "softpack.yml"
+	SpackLockFile          = "spack.lock"
+	BuilderOut             = "builder.out"
 	moduleForCoreBasename  = "module"
-	usageBasename          = "README.md"
+	UsageBasename          = "README.md"
+
+	uploadEndpoint = "/upload"
 )
 
 //go:embed singularity.tmpl
@@ -285,7 +287,7 @@ func (b *Builder) generateAndUploadSingularityDef(def *Definition, s3Path string
 		return "", err
 	}
 
-	singDefUploadPath := filepath.Join(s3Path, singularityDefBasename)
+	singDefUploadPath := filepath.Join(s3Path, SingularityDefBasename)
 
 	err = b.s3.UploadData(strings.NewReader(singDef), singDefUploadPath)
 
@@ -348,22 +350,22 @@ func (b *Builder) asyncBuild(def *Definition, wrInput, s3Path, singDef string) e
 }
 
 func (b *Builder) addLogToRepo(s3Path, environmentPath string) {
-	log, err := b.s3.OpenFile(filepath.Join(s3Path, builderOut))
+	log, err := b.s3.OpenFile(filepath.Join(s3Path, BuilderOut))
 	if err != nil {
 		slog.Error("error getting build log file", "err", err)
 
 		return
 	}
 
 	if err := b.addArtifactsToRepo(map[string]io.Reader{
-		builderOut: log,
+		BuilderOut: log,
 	}, environmentPath); err != nil {
 		slog.Error("error sending build log file to core", "err", err)
 	}
 }
 
 func (b *Builder) getExes(s3Path string) ([]string, error) {
-	exeData, err := b.s3.OpenFile(filepath.Join(s3Path, exesBasename))
+	exeData, err := b.s3.OpenFile(filepath.Join(s3Path, ExesBasename))
 	if err != nil {
 		return nil, err
 	}
@@ -378,7 +380,7 @@ func (b *Builder) getExes(s3Path string) ([]string, error) {
 
 func (b *Builder) prepareAndInstallArtifacts(def *Definition, s3Path,
 	moduleFileData string, exes []string) error {
-	imageData, err := b.s3.OpenFile(filepath.Join(s3Path, imageBasename))
+	imageData, err := b.s3.OpenFile(filepath.Join(s3Path, ImageBasename))
 	if err != nil {
 		return err
 	}
@@ -408,24 +410,24 @@ func (b *Builder) prepareArtifactsFromS3AndSendToCoreAndS3(def *Definition, s3Pa
 
 	return b.addArtifactsToRepo(
 		map[string]io.Reader{
-			spackLock:              bytes.NewReader(lockData),
-			softpackYaml:           strings.NewReader(concreteSpackYAMLFile),
-			singularityDefBasename: strings.NewReader(singDef),
-			builderOut:             logData,
+			SpackLockFile:          bytes.NewReader(lockData),
+			SoftpackYaml:           strings.NewReader(concreteSpackYAMLFile),
+			SingularityDefBasename: strings.NewReader(singDef),
+			BuilderOut:             logData,
 			moduleForCoreBasename:  strings.NewReader(moduleFileData),
-			usageBasename:          strings.NewReader(readme),
+			UsageBasename:          strings.NewReader(readme),
 		},
 		def.FullEnvironmentPath(),
 	)
 }
 
 func (b *Builder) getArtifactDataFromS3(s3Path string) (io.Reader, []byte, error) {
-	logData, err := b.s3.OpenFile(filepath.Join(s3Path, builderOut))
+	logData, err := b.s3.OpenFile(filepath.Join(s3Path, BuilderOut))
 	if err != nil {
 		return nil, nil, err
 	}
 
-	lockFile, err := b.s3.OpenFile(filepath.Join(s3Path, spackLock))
+	lockFile, err := b.s3.OpenFile(filepath.Join(s3Path, SpackLockFile))
 	if err != nil {
 		return nil, nil, err
 	}
@@ -446,7 +448,7 @@ func (b *Builder) generateAndUploadSpackYAML(lockData []byte, description string
 	}
 
 	if err = b.s3.UploadData(strings.NewReader(concreteSpackYAMLFile),
-		filepath.Join(s3Path, softpackYaml)); err != nil {
+		filepath.Join(s3Path, SoftpackYaml)); err != nil {
 		return "", err
 	}
 
@@ -520,7 +522,7 @@ func SpackLockToSoftPackYML(spackLockData []byte, desc string, exes []string) (s
 func (b *Builder) generateAndUploadUsageFile(def *Definition, s3Path string) (string, error) {
 	readme := def.ModuleUsage(b.config.Module.LoadPath)
 
-	if err := b.s3.UploadData(strings.NewReader(readme), filepath.Join(s3Path, usageBasename)); err != nil {
+	if err := b.s3.UploadData(strings.NewReader(readme), filepath.Join(s3Path, UsageBasename)); err != nil {
 		return "", err
 	}
 
@@ -541,15 +543,17 @@ func (b *Builder) addArtifactsToRepo(artifacts map[string]io.Reader, envPath str
 
 	defer pw.Close()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, b.config.CoreURL+"?"+url.QueryEscape(envPath), pr)
+	uploadURL := strings.TrimSuffix(b.config.CoreURL, "/") + uploadEndpoint + "?" + url.QueryEscape(envPath)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadURL, pr)
 	if err != nil {
 		return err
 	}
 
 	req.Header.Add("Content-Type", writer.FormDataContentType())
 
 	resp, err := http.DefaultClient.Do(req)
-	slog.Debug("addArtifactsToRepo", "url", b.config.CoreURL+"?"+url.QueryEscape(envPath), "err", err)
+	slog.Debug("addArtifactsToRepo", "url", b.config.CoreURL+uploadEndpoint+"?"+url.QueryEscape(envPath), "err", err)
 
 	if err != nil {
 		return err

build/builder_test.go

@@ -79,19 +79,19 @@ func (m *mockS3) UploadData(data io.Reader, dest string) error {
 }
 
 func (m *mockS3) OpenFile(source string) (io.ReadCloser, error) {
-	if filepath.Base(source) == exesBasename {
+	if filepath.Base(source) == ExesBasename {
 		return io.NopCloser(strings.NewReader(m.exes)), nil
 	}
 
-	if filepath.Base(source) == builderOut {
+	if filepath.Base(source) == BuilderOut {
 		return io.NopCloser(strings.NewReader("output")), nil
 	}
 
-	if filepath.Base(source) == spackLock {
+	if filepath.Base(source) == SpackLockFile {
 		return io.NopCloser(strings.NewReader(`{"_meta":{"file-type":"spack-lockfile","lockfile-version":5,"specfile-version":4},"spack":{"version":"0.21.0.dev0","type":"git","commit":"dac3b453879439fd733b03d0106cc6fe070f71f6"},"roots":[{"hash":"oibd5a4hphfkgshqiav4fdkvw4hsq4ek","spec":"xxhash arch=None-None-x86_64_v3"}, {"hash":"1ibd5a4hphfkgshqiav4fdkvw4hsq4e1","spec":"py-anndata arch=None-None-x86_64_v3"}, {"hash":"2ibd5a4hphfkgshqiav4fdkvw4hsq4e2","spec":"r-seurat arch=None-None-x86_64_v3"}],"concrete_specs":{"oibd5a4hphfkgshqiav4fdkvw4hsq4ek":{"name":"xxhash","version":"0.8.1","arch":{"platform":"linux","platform_os":"ubuntu22.04","target":"x86_64_v3"},"compiler":{"name":"gcc","version":"11.4.0"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wuj5b2kjnmrzhtjszqovcvgc3q46m6hoehmiccimi5fs7nmsw22a====","hash":"oibd5a4hphfkgshqiav4fdkvw4hsq4ek"},"2ibd5a4hphfkgshqiav4fdkvw4hsq4e2":{"name":"r-seurat","version":"4","arch":{"platform":"linux","platform_os":"ubuntu22.04","target":"x86_64_v3"},"compiler":{"name":"gcc","version":"11.4.0"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"2uj5b2kjnmrzhtjszqovcvgc3q46m6hoehmiccimi5fs7nmsw222====","hash":"2ibd5a4hphfkgshqiav4fdkvw4hsq4e2"}, "1ibd5a4hphfkgshqiav4fdkvw4hsq4e1":{"name":"py-anndata","version":"3.14","arch":{"platform":"linux","platform_os":"ubuntu22.04","target":"x86_64_v3"},"compiler":{"name":"gcc","version":"11.4.0"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"2uj5b2kjnmrzhtjszqovcvgc3q46m6hoehmiccimi5fs7nmsw222====","hash":"1ibd5a4hphfkgshqiav4fdkvw4hsq4e1"}}}`)), nil //nolint:lll
 	}
 
-	if filepath.Base(source) == imageBasename {
+	if filepath.Base(source) == ImageBasename {
 		return io.NopCloser(strings.NewReader("image")), nil
 	}
 
@@ -369,8 +369,8 @@ Stage: final
 			modulePath := filepath.Join(conf.Module.ModuleInstallDir,
 				def.EnvironmentPath, def.EnvironmentName, def.EnvironmentVersion)
 			scriptsPath := filepath.Join(conf.Module.ScriptsInstallDir,
-				def.EnvironmentPath, def.EnvironmentName, def.EnvironmentVersion+scriptsDirSuffix)
-			imagePath := filepath.Join(scriptsPath, imageBasename)
+				def.EnvironmentPath, def.EnvironmentName, def.EnvironmentVersion+ScriptsDirSuffix)
+			imagePath := filepath.Join(scriptsPath, ImageBasename)
 			expectedExes := []string{"python", "R", "Rscript", "xxhsum", "xxh32sum", "xxh64sum", "xxh128sum"}
 
 			expectedFiles := []string{modulePath, scriptsPath, imagePath}
@@ -451,19 +451,19 @@ packages:
 			expectedReadmeContent := "module load " + moduleLoadPrefix + "/groups/hgi/xxhash/0.8.1"
 
 			for file, expectedData := range map[string]string{
-				softpackYaml:           expectedSoftpackYaml,
+				SoftpackYaml:           expectedSoftpackYaml,
 				moduleForCoreBasename:  "module-whatis",
-				singularityDefBasename: "specs:\n  - [email protected] arch=None-None-x86_64_v4",
-				spackLock:              `"concrete_specs":`,
-				builderOut:             "output",
-				usageBasename:          expectedReadmeContent,
+				SingularityDefBasename: "specs:\n  - [email protected] arch=None-None-x86_64_v4",
+				SpackLockFile:          `"concrete_specs":`,
+				BuilderOut:             "output",
+				UsageBasename:          expectedReadmeContent,
 			} {
 				data, okg := mc.getFile("groups/hgi/xxhash-0.8.1/" + file)
 				So(okg, ShouldBeTrue)
 				So(data, ShouldContainSubstring, expectedData)
 			}
 
-			_, ok = mc.getFile("groups/hgi/xxhash-0.8.1/" + imageBasename)
+			_, ok = mc.getFile("groups/hgi/xxhash-0.8.1/" + ImageBasename)
 			So(ok, ShouldBeFalse)
 
 			So(ms3.softpackYML, ShouldEqual, expectedSoftpackYaml)
@@ -491,7 +491,7 @@ packages:
 			So(logWriter.String(), ShouldContainSubstring,
 				"msg=\"Async part of build failed\" err=\"Mock error\" s3Path=some_path/groups/hgi/xxhash/0.8.1")
 
-			data, ok := mc.getFile("groups/hgi/xxhash-0.8.1/" + builderOut)
+			data, ok := mc.getFile("groups/hgi/xxhash-0.8.1/" + BuilderOut)
 			So(ok, ShouldBeTrue)
 			So(data, ShouldContainSubstring, "output")
 		})
@@ -535,7 +535,7 @@ packages:
 			})
 			So(ok, ShouldBeTrue)
 
-			expectedLog := "Post \\\"http://0.0.0.0:1234?groups%2Fhgi%2Fxxhash-0.8.1\\\"" +
+			expectedLog := "Post \\\"http://0.0.0.0:1234" + uploadEndpoint + "?groups%2Fhgi%2Fxxhash-0.8.1\\\"" +
 				": dial tcp 0.0.0.0:1234: connect: connection refused"
 			So(logWriter.String(), ShouldContainSubstring, expectedLog)
 

build/install.go

@@ -30,10 +30,11 @@ import (
 )
 
 const (
-	imageBasename    = "singularity.sif"
-	scriptsDirSuffix = "-scripts"
-	perms            = 0755
-	flags            = os.O_EXCL | os.O_CREATE | os.O_WRONLY
+	ScriptsDirSuffix = "-scripts"
+
+	ImageBasename = "singularity.sif"
+	perms         = 0755
+	flags         = os.O_EXCL | os.O_CREATE | os.O_WRONLY
 )
 
 func installModule(scriptInstallBase, moduleInstallBase string, def *Definition, module,
@@ -58,17 +59,17 @@ func installModule(scriptInstallBase, moduleInstallBase string, def *Definition,
 		return err
 	}
 
-	if err = installFile(image, filepath.Join(scriptsDir, imageBasename)); err != nil {
+	if err = installFile(image, filepath.Join(scriptsDir, ImageBasename)); err != nil {
 		return err
 	}
 
 	return createExeSymlinks(wrapperScript, scriptsDir, exes)
 }
 
 func makeModuleDirs(scriptInstallBase, moduleInstallBase string, def *Definition) (string, string, error) {
-	scriptsDir := filepath.Join(scriptInstallBase, def.EnvironmentPath,
-		def.EnvironmentName, def.EnvironmentVersion+scriptsDirSuffix)
-	moduleDir := filepath.Join(moduleInstallBase, def.EnvironmentPath, def.EnvironmentName)
+	scriptsDir := ScriptsDirFromNameAndVersion(scriptInstallBase, def.EnvironmentPath,
+		def.EnvironmentName, def.EnvironmentVersion)
+	moduleDir := ModuleDirFromName(moduleInstallBase, def.EnvironmentPath, def.EnvironmentName)
 
 	if err := makeDirectory(scriptsDir, scriptInstallBase); err != nil {
 		return "", "", err
@@ -81,6 +82,18 @@ func makeModuleDirs(scriptInstallBase, moduleInstallBase string, def *Definition
 	return scriptsDir, moduleDir, nil
 }
 
+// ScriptsDirFromNameAndVersion returns the canonical scripts path for an
+// environment.
+func ScriptsDirFromNameAndVersion(scriptInstallBase, path, name, version string) string {
+	return filepath.Join(scriptInstallBase, path,
+		name, version+ScriptsDirSuffix)
+}
+
+// ModuleDirFromName returns the canonical module path for an environment.
+func ModuleDirFromName(moduleInstallBase, path, name string) string {
+	return filepath.Join(moduleInstallBase, path, name)
+}
+
 // makeDirectory does a MkdirAll for leafDir, and then makes sure it and it's
 // parents up to baseDir are world accesible.
 func makeDirectory(leafDir, baseDir string) error {

build/install_test.go

@@ -54,8 +54,8 @@ func TestInstall(t *testing.T) {
 		createdModuleFile := readFile(t, filepath.Join(tmpModulesDir, def.EnvironmentPath,
 			def.EnvironmentName, def.EnvironmentVersion))
 		scriptsDir := filepath.Join(tmpScriptsDir, def.EnvironmentPath, def.EnvironmentName,
-			def.EnvironmentVersion+scriptsDirSuffix)
-		createdImageFile := readFile(t, filepath.Join(scriptsDir, imageBasename))
+			def.EnvironmentVersion+ScriptsDirSuffix)
+		createdImageFile := readFile(t, filepath.Join(scriptsDir, ImageBasename))
 
 		So(createdModuleFile, ShouldEqual, moduleFile)
 		So(createdImageFile, ShouldEqual, imageFile)

cmd/build.go

@@ -156,7 +156,7 @@ func printIfTTY(msg string) {
 		return
 	}
 
-	fmt.Println(msg)
+	cliPrint(msg)
 }
 
 func openOrDie(path string) *os.File {

cmd/remove.go

@@ -0,0 +1,79 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"github.com/wtsi-hgi/go-softpack-builder/remove"
+	"github.com/wtsi-hgi/go-softpack-builder/s3"
+)
+
+const numArgs = 2
+
+var removeCmd = &cobra.Command{
+	Use:   "remove",
+	Short: "Remove an environment",
+	Long: `Remove an environment.
+
+Remove an existing environment; it's entry in the Core artefacts repo, the
+module files and the singularity image and symlinks.
+
+Usage: gsb remove softpack/env/path version
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) < 1 {
+			die("environment path required")
+		}
+
+		if len(args) < numArgs {
+			die("environment version required")
+		}
+
+		if len(args) != numArgs {
+			die("unexpected arguments")
+		}
+
+		conf := getConfig()
+
+		s, err := s3.New(conf.S3.BuildBase)
+		if err != nil {
+			die(err.Error())
+		}
+
+		envPath := cleanEnvPath(args[0])
+
+		if envPath != args[0] {
+			die("invalid environment path")
+		}
+
+		cliPrint(
+			"Will now remove environment %s-%s from artefacts repo and modules.\n"+
+				"Are you sure you sure you wish to proceed? [yN]: ",
+			envPath,
+			args[1],
+		)
+
+		var resp string
+
+		fmt.Scan(&resp)
+
+		if resp != "y" {
+			return
+		}
+
+		if err := remove.Remove(conf, s, envPath, args[1]); err != nil {
+			die(err.Error())
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(removeCmd)
+}
+
+// cleanEnvPath strips out any attempts to manipulate the envPath in order to
+// get GSB to remove incorrect files.
+func cleanEnvPath(envPath string) string {
+	return filepath.Clean(string(filepath.Separator) + envPath)[1:]
+}

cmd/root.go

@@ -59,3 +59,8 @@ func die(msg string, a ...interface{}) {
 	slog.Error(fmt.Sprintf(msg, a...))
 	os.Exit(1)
 }
+
+// cliPrint outputs the message to STDOUT.
+func cliPrint(msg string, a ...interface{}) {
+	fmt.Fprintf(os.Stdout, msg, a...)
+}