Implement experimental virtual filesystem mode

go.mod

@@ -7,58 +7,31 @@ require (
 	github.com/alessio/shellescape v1.4.1
 	github.com/fatih/color v1.14.1
 	github.com/google/go-github/v39 v39.2.0
-	github.com/hashicorp/go-getter v1.6.2
 	github.com/muhammadmuzzammil1998/jsonc v1.0.0
 	github.com/nightlyone/lockfile v1.0.0
 	github.com/otiai10/copy v1.7.0
 	github.com/paul-mannino/go-fuzzywuzzy v0.0.0-20200127021948-54652b135d0e
 	github.com/spf13/cobra v1.6.1
+	github.com/stirante/dokan-go v0.0.0-20231031193153-dcfa431d507a
 	github.com/stirante/go-simple-eval v0.0.0-20230131075324-9ed520afbec1
 	go.uber.org/zap v1.23.0
 	golang.org/x/mod v0.6.0
-	golang.org/x/sys v0.4.0
+	golang.org/x/sys v0.13.0
 )
 
 replace github.com/hashicorp/go-getter => github.com/arikkfir/go-getter v1.6.3-0.20220803164326-281b7670b734
 
 require (
-	cloud.google.com/go v0.100.2 // indirect
-	cloud.google.com/go/compute v1.5.0 // indirect
-	cloud.google.com/go/iam v0.3.0 // indirect
-	cloud.google.com/go/storage v1.21.0 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect
-	github.com/aws/aws-sdk-go v1.43.25 // indirect
-	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/gammazero/deque v0.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.2.0 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-safetemp v1.0.0 // indirect
-	github.com/hashicorp/go-version v1.4.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/ulikunitz/xz v0.5.10 // indirect
-	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/crypto v0.4.0 // indirect
 	golang.org/x/exp v0.0.0-20230131013936-aae9b4e6329d // indirect
-	golang.org/x/net v0.1.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
 	golang.org/x/text v0.6.0 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.73.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect
-	google.golang.org/grpc v1.45.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
 )

main.go

@@ -268,6 +268,10 @@ func main() {
 		Short: "Runs Regolith using specified profile",
 		Long:  regolithRunDesc,
 		Run: func(cmd *cobra.Command, args []string) {
+			if regolith.IsExperimentEnabled(regolith.VFS) {
+				fmt.Println("Running experimental virtual filesystem mode")
+				fmt.Println("It requires Dokany v1.2, that you can download here: https://github.com/dokan-dev/dokany/releases/tag/v1.2.2.1000")
+			}
 			var profile string
 			if len(args) != 0 {
 				profile = args[0]

regolith/experiments.go

@@ -5,10 +5,12 @@ type Experiment int
 const (
 	// SizeTimeCheck is an experiment that checks the size and modification time when exporting
 	SizeTimeCheck Experiment = iota
+	VFS
 )
 
 var experimentNames = map[Experiment]string{
 	SizeTimeCheck: "size_time_check",
+	VFS:           "vfs",
 }
 
 var EnabledExperiments []string

regolith/export.go

@@ -98,9 +98,7 @@ func GetExportPaths(
 
 // ExportProject copies files from the tmp paths (tmp/BP and tmp/RP) into
 // the project's export target. The paths are generated with GetExportPaths.
-func ExportProject(
-	profile Profile, name, dataPath, dotRegolithPath string,
-) error {
+func ExportProject(profile Profile, name, dataPath, dotRegolithPath string, fs FileSystem) error {
 	MeasureStart("Export - GetExportPaths")
 	// Get the export target paths
 	exportTarget := profile.ExportTarget
@@ -176,6 +174,7 @@ func ExportProject(
 	// The root of the data path cannot be deleted because the
 	// "regolith watch" function would stop watching the file changes
 	// (due to Windows API limitation).
+	//TODO: Extend fs to handle this case
 	_, err = os.ReadDir(dataPath)
 	if err != nil {
 		var err1 error = nil
@@ -244,26 +243,26 @@ func ExportProject(
 	if IsExperimentEnabled(SizeTimeCheck) {
 		// Export BP
 		Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
-		err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
+		err = fs.SyncToPath("BP", bpPath, exportTarget.ReadOnly, true)
 		if err != nil {
 			return burrito.WrapError(err, "Failed to export behavior pack.")
 		}
 		// Export RP
 		Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
-		err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
+		err = fs.SyncToPath("RP", rpPath, exportTarget.ReadOnly, true)
 		if err != nil {
 			return burrito.WrapError(err, "Failed to export resource pack.")
 		}
 	} else {
 		// Export BP
 		Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
-		err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
+		err = fs.SaveToPath("BP", bpPath, exportTarget.ReadOnly, true)
 		if err != nil {
 			return burrito.WrapError(err, "Failed to export behavior pack.")
 		}
 		// Export RP
 		Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
-		err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
+		err = fs.SaveToPath("RP", rpPath, exportTarget.ReadOnly, true)
 		if err != nil {
 			return burrito.WrapError(err, "Failed to export resource pack.")
 		}

regolith/main_functions.go

@@ -371,7 +371,9 @@ func ApplyFilter(filterName string, filterArgs []string, debug bool) error {
 		return burrito.WrapErrorf(err, filterRunnerCheckError, filterName)
 	}
 	// Setup tmp directory
-	err = SetupTmpFiles(*config, dotRegolithPath)
+	fs, err := SetupTmpFiles(*config, dotRegolithPath)
+	// in case of VFS, cleanup must be called even in case of an error
+	defer fs.Close()
 	if err != nil {
 		return burrito.WrapErrorf(err, setupTmpFilesError, dotRegolithPath)
 	}
@@ -383,6 +385,7 @@ func ApplyFilter(filterName string, filterArgs []string, debug bool) error {
 	}
 	// Export files to the source files
 	Logger.Info("Overwriting the source files.")
+	//TODO: Use fs variable here
 	err = InplaceExportProject(config, dotRegolithPath)
 	if err != nil {
 		return burrito.WrapError(

regolith/profile.go

@@ -12,79 +12,145 @@ import (
 	"github.com/otiai10/copy"
 )
 
+type FileSystem interface {
+	Close()
+	SaveToPath(from, to string, makeReadOnly, copyParentAcl bool) error
+	SyncToPath(from, to string, makeReadOnly, copyParentAcl bool) error
+}
+
+type NoopFileSystem struct{}
+
+func (f *NoopFileSystem) Close() {
+
+}
+
+func (f *NoopFileSystem) SaveToPath(from, to string, makeReadOnly, copyParentAcl bool) error {
+	return nil
+}
+
+func (f *NoopFileSystem) SyncToPath(from, to string, makeReadOnly, copyParentAcl bool) error {
+	return nil
+}
+
+type TmpFileSystem struct {
+	dotRegolithPath string
+}
+
+func (f *TmpFileSystem) Close() {
+
+}
+
+func (f *TmpFileSystem) SaveToPath(from, to string, makeReadOnly, copyParentAcl bool) error {
+	return MoveOrCopy(filepath.Join(f.dotRegolithPath, "tmp", from), to, makeReadOnly, copyParentAcl)
+}
+
+func (f *TmpFileSystem) SyncToPath(from, to string, makeReadOnly, copyParentAcl bool) error {
+	return SyncDirectories(filepath.Join(f.dotRegolithPath, "tmp", from), to, makeReadOnly, copyParentAcl)
+}
+
 // SetupTmpFiles set up the workspace for the filters.
-func SetupTmpFiles(config Config, dotRegolithPath string) error {
-	start := time.Now()
+func SetupTmpFiles(config Config, dotRegolithPath string) (FileSystem, error) {
+	// Prepare cleanup function
+	var fs FileSystem = &NoopFileSystem{}
 	// Setup Directories
+	MeasureStart("SetupTmpFiles - Clean")
 	tmpPath := filepath.Join(dotRegolithPath, "tmp")
 	Logger.Debugf("Cleaning \"%s\"", tmpPath)
 	err := os.RemoveAll(tmpPath)
 	if err != nil {
-		return burrito.WrapErrorf(err, osRemoveError, tmpPath)
+		return fs, burrito.WrapErrorf(err, osRemoveError, tmpPath)
 	}
 
 	err = os.MkdirAll(tmpPath, 0755)
 	if err != nil {
-		return burrito.WrapErrorf(err, osMkdirError, tmpPath)
+		return fs, burrito.WrapErrorf(err, osMkdirError, tmpPath)
 	}
+	MeasureEnd()
 
-	// Copy the contents of the 'regolith' folder to '[dotRegolithPath]/tmp'
-	Logger.Debugf("Copying project files to \"%s\"", tmpPath)
-	// Avoid repetitive code of preparing ResourceFolder, BehaviorFolder
-	// and DataPath with a closure
-	setupTmpDirectory := func(
-		path, shortName, descriptiveName string,
-	) error {
-		p := filepath.Join(tmpPath, shortName)
-		if path != "" {
-			stats, err := os.Stat(path)
-			if err != nil {
-				if os.IsNotExist(err) {
-					Logger.Warnf(
-						"%s %q does not exist", descriptiveName, path)
-					err = os.MkdirAll(p, 0755)
+	if IsExperimentEnabled(VFS) {
+		Logger.Debugf("Setting up VFS at \"%s\"", tmpPath)
+		MeasureStart("SetupTmpFiles - Set up VFS")
+		fs, err = NewRegolithFS([]Root{
+			{
+				Source:    config.ResourceFolder,
+				MountName: "RP",
+			},
+			{
+				Source:    config.BehaviorFolder,
+				MountName: "BP",
+			},
+			{
+				Source:    config.DataPath,
+				MountName: "data",
+			},
+		}, tmpPath)
+		if err != nil {
+			return fs, burrito.WrapErrorf(err, "Failed to create VFS at %s.", tmpPath)
+		}
+		MeasureEnd()
+	} else {
+		fs = &TmpFileSystem{dotRegolithPath: dotRegolithPath}
+		// Copy the contents of the 'regolith' folder to '[dotRegolithPath]/tmp'
+		Logger.Debugf("Copying project files to \"%s\"", tmpPath)
+		// Avoid repetitive code of preparing ResourceFolder, BehaviorFolder
+		// and DataPath with a closure
+		setupTmpDirectory := func(
+			path, shortName, descriptiveName string,
+		) error {
+			p := filepath.Join(tmpPath, shortName)
+			if path != "" {
+				stats, err := os.Stat(path)
+				if err != nil {
+					if os.IsNotExist(err) {
+						Logger.Warnf(
+							"%s %q does not exist", descriptiveName, path)
+						err = os.MkdirAll(p, 0755)
+						if err != nil {
+							return burrito.WrapErrorf(err, osMkdirError, p)
+						}
+					}
+				} else if stats.IsDir() {
+					err = copy.Copy(
+						path,
+						p,
+						copy.Options{PreserveTimes: IsExperimentEnabled(SizeTimeCheck), Sync: false})
 					if err != nil {
-						return burrito.WrapErrorf(err, osMkdirError, p)
+						return burrito.WrapErrorf(err, osCopyError, path, p)
 					}
+				} else { // The folder paths leads to a file
+					return burrito.WrappedErrorf(isDirNotADirError, path)
 				}
-			} else if stats.IsDir() {
-				err = copy.Copy(
-					path,
-					p,
-					copy.Options{PreserveTimes: IsExperimentEnabled(SizeTimeCheck), Sync: false})
+			} else {
+				err = os.MkdirAll(p, 0755)
 				if err != nil {
-					return burrito.WrapErrorf(err, osCopyError, path, p)
+					return burrito.WrapErrorf(err, osMkdirError, p)
 				}
-			} else { // The folder paths leads to a file
-				return burrito.WrappedErrorf(isDirNotADirError, path)
-			}
-		} else {
-			err = os.MkdirAll(p, 0755)
-			if err != nil {
-				return burrito.WrapErrorf(err, osMkdirError, p)
 			}
+			return nil
 		}
-		return nil
-	}
 
-	err = setupTmpDirectory(config.ResourceFolder, "RP", "resource folder")
-	if err != nil {
-		return burrito.WrapErrorf(
-			err, "Failed to setup RP folder in the temporary directory.")
-	}
-	err = setupTmpDirectory(config.BehaviorFolder, "BP", "behavior folder")
-	if err != nil {
-		return burrito.WrapErrorf(
-			err, "Failed to setup BP folder in the temporary directory.")
-	}
-	err = setupTmpDirectory(config.DataPath, "data", "data folder")
-	if err != nil {
-		return burrito.WrapErrorf(
-			err, "Failed to setup data folder in the temporary directory.")
-	}
+		MeasureStart("SetupTmpFiles - RP")
+		err = setupTmpDirectory(config.ResourceFolder, "RP", "resource folder")
+		if err != nil {
+			return fs, burrito.WrapErrorf(
+				err, "Failed to setup RP folder in the temporary directory.")
+		}
+		MeasureStart("SetupTmpFiles - BP")
+		err = setupTmpDirectory(config.BehaviorFolder, "BP", "behavior folder")
+		if err != nil {
+			return fs, burrito.WrapErrorf(
+				err, "Failed to setup BP folder in the temporary directory.")
+		}
+		MeasureStart("SetupTmpFiles - Data")
+		err = setupTmpDirectory(config.DataPath, "data", "data folder")
+		if err != nil {
+			return fs, burrito.WrapErrorf(
+				err, "Failed to setup data folder in the temporary directory.")
+		}
 
-	Logger.Debug("Setup done in ", time.Since(start))
-	return nil
+		MeasureEnd()
+	}
+	return fs, nil
 }
 
 func CheckProfileImpl(
@@ -117,11 +183,15 @@ start:
 	if err != nil {
 		return burrito.WrapErrorf(err, runContextGetProfileError)
 	}
-	err = SetupTmpFiles(*context.Config, context.DotRegolithPath)
+	fs, err := SetupTmpFiles(*context.Config, context.DotRegolithPath)
+	// in case of VFS, cleanup must be called even in case of an error
+	defer fs.Close()
 	if err != nil {
 		return burrito.WrapErrorf(err, setupTmpFilesError, context.DotRegolithPath)
 	}
 	if context.IsInterrupted() {
+		// Dirty gotos break my deferred cleanup
+		fs.Close()
 		goto start
 	}
 	// Run the profile
@@ -130,17 +200,21 @@ start:
 		return burrito.PassError(err)
 	}
 	if interrupted {
+		// Dirty gotos break my deferred cleanup
+		fs.Close()
 		goto start
 	}
 	// Export files
 	Logger.Info("Moving files to target directory.")
 	start := time.Now()
 	err = ExportProject(
-		profile, context.Config.Name, context.Config.DataPath, context.DotRegolithPath)
+		profile, context.Config.Name, context.Config.DataPath, context.DotRegolithPath, fs)
 	if err != nil {
 		return burrito.WrapError(err, exportProjectError)
 	}
 	if context.IsInterrupted("data") {
+		// Dirty gotos break my deferred cleanup
+		fs.Close()
 		goto start
 	}
 	Logger.Debug("Done in ", time.Since(start))