Implement experiments flag and size_time_check experiment, that copie…

…s and deletes files based on size and modification time of both source and target directories

main.go

@@ -358,10 +358,11 @@ func main() {
 	}
 	subcommands = append(subcommands, cmdUpdateResolvers)
 
-	// add --debug and --timings flag to every command
+	// add --debug, --timings and --experiment flag to every command
 	for _, cmd := range subcommands {
 		cmd.Flags().BoolVarP(&burrito.PrintStackTrace, "debug", "", false, "Enables debugging")
 		cmd.Flags().BoolVarP(&regolith.EnableTimings, "timings", "", false, "Enables timing information")
+		cmd.Flags().StringSliceVar(&regolith.EnabledExperiments, "experiments", nil, "Enables experimental features")
 	}
 	// Build and run CLI
 	rootCmd.AddCommand(subcommands...)

regolith/experiments.go

@@ -0,0 +1,26 @@
+package regolith
+
+type Experiment int
+
+const (
+	// SizeTimeCheck is an experiment that checks the size and modification time when exporting
+	SizeTimeCheck Experiment = iota
+)
+
+var experimentNames = map[Experiment]string{
+	SizeTimeCheck: "size_time_check",
+}
+
+var EnabledExperiments []string
+
+func IsExperimentEnabled(exp Experiment) bool {
+	if EnabledExperiments == nil {
+		return false
+	}
+	for _, e := range EnabledExperiments {
+		if e == experimentNames[exp] {
+			return true
+		}
+	}
+	return false
+}

regolith/export.go

@@ -101,6 +101,7 @@ func GetExportPaths(
 func ExportProject(
 	profile Profile, name, dataPath, dotRegolithPath string,
 ) error {
+	MeasureStart("Export - GetExportPaths")
 	// Get the export target paths
 	exportTarget := profile.ExportTarget
 	bpPath, rpPath, err := GetExportPaths(exportTarget, name)
@@ -109,6 +110,7 @@ func ExportProject(
 			err, "Failed to get generate export paths.")
 	}
 
+	MeasureStart("Export - LoadEditedFiles")
 	// Loading edited_files.json or creating empty object
 	editedFiles := LoadEditedFiles(dotRegolithPath)
 	err = editedFiles.CheckDeletionSafety(rpPath, bpPath)
@@ -124,22 +126,28 @@ func ExportProject(
 			rpPath, bpPath)
 	}
 
-	// Clearing output locations
-	// Spooky, I hope file protection works, and it won't do any damage
-	err = os.RemoveAll(bpPath)
-	if err != nil {
-		return burrito.WrapErrorf(
-			err, "Failed to clear behavior pack from build path %q.\n"+
-				"Are user permissions correct?", bpPath)
-	}
-	err = os.RemoveAll(rpPath)
-	if err != nil {
-		return burrito.WrapErrorf(
-			err, "Failed to clear resource pack from build path %q.\n"+
-				"Are user permissions correct?", rpPath)
+	MeasureStart("Export - Clean")
+	// When comparing the size and modification time of the files, we need to
+	// keep the files in target paths.
+	if !IsExperimentEnabled(SizeTimeCheck) {
+		// Clearing output locations
+		// Spooky, I hope file protection works, and it won't do any damage
+		err = os.RemoveAll(bpPath)
+		if err != nil {
+			return burrito.WrapErrorf(
+				err, "Failed to clear behavior pack from build path %q.\n"+
+					"Are user permissions correct?", bpPath)
+		}
+		err = os.RemoveAll(rpPath)
+		if err != nil {
+			return burrito.WrapErrorf(
+				err, "Failed to clear resource pack from build path %q.\n"+
+					"Are user permissions correct?", rpPath)
+		}
 	}
+	MeasureEnd()
 	// List the names of the filters that opt-in to the data export process
-	exportedFilterNames := []string{}
+	var exportedFilterNames []string
 	for filter := range profile.Filters {
 		filter := profile.Filters[filter]
 		usingDataPath, err := filter.IsUsingDataExport(dotRegolithPath)
@@ -180,13 +188,15 @@ func ExportProject(
 				dataPath)
 		}
 	}
+	MeasureStart("Export - RevertibleOps")
 	// Create revertible operations object
 	backupPath := filepath.Join(dotRegolithPath, ".dataBackup")
 	revertibleOps, err := NewRevertibleFsOperations(backupPath)
 	if err != nil {
 		return burrito.WrapErrorf(err, newRevertibleFsOperationsError, backupPath)
 	}
 	// Export data
+	MeasureStart("Export - ExportData")
 	for _, exportedFilterName := range exportedFilterNames {
 		// Clear export target
 		targetPath := filepath.Join(dataPath, exportedFilterName)
@@ -230,18 +240,35 @@ func ExportProject(
 			return mainError
 		}
 	}
-	// Export BP
-	Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
-	err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
-	if err != nil {
-		return burrito.WrapError(err, "Failed to export behavior pack.")
-	}
-	// Export RP
-	Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
-	err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
-	if err != nil {
-		return burrito.WrapError(err, "Failed to export resource pack.")
+	MeasureStart("Export - MoveOrCopy")
+	if IsExperimentEnabled(SizeTimeCheck) {
+		// Export BP
+		Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
+		err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
+		if err != nil {
+			return burrito.WrapError(err, "Failed to export behavior pack.")
+		}
+		// Export RP
+		Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
+		err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
+		if err != nil {
+			return burrito.WrapError(err, "Failed to export resource pack.")
+		}
+	} else {
+		// Export BP
+		Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
+		err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
+		if err != nil {
+			return burrito.WrapError(err, "Failed to export behavior pack.")
+		}
+		// Export RP
+		Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
+		err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
+		if err != nil {
+			return burrito.WrapError(err, "Failed to export resource pack.")
+		}
 	}
+	MeasureStart("Export - UpdateFromPaths")
 	// Update or create edited_files.json
 	err = editedFiles.UpdateFromPaths(rpPath, bpPath)
 	if err != nil {
@@ -258,6 +285,7 @@ func ExportProject(
 	if err := revertibleOps.Close(); err != nil {
 		return burrito.PassError(err)
 	}
+	MeasureEnd()
 	return nil
 }
 

regolith/file_system.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/Bedrock-OSS/go-burrito/burrito"
 
@@ -869,3 +870,110 @@ func MoveOrCopy(
 	}
 	return nil
 }
+
+// SyncDirectories copies the source to destination while checking size and modification time.
+func SyncDirectories(
+	source string, destination string, makeReadOnly bool, copyParentAcl bool,
+) error {
+	// Make destination parent if not exists
+	destinationParent := filepath.Dir(destination)
+	if _, err := os.Stat(destinationParent); os.IsNotExist(err) {
+		err = os.MkdirAll(destinationParent, 0755)
+		if err != nil {
+			return burrito.WrapErrorf(
+				err, osMkdirError, destinationParent)
+		}
+	}
+	err := filepath.Walk(source, func(srcPath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(source, srcPath)
+		if err != nil {
+			return burrito.WrapErrorf(err, filepathRelError, source, srcPath)
+		}
+		destPath := filepath.Join(destination, relPath)
+
+		destInfo, err := os.Stat(destPath)
+		if err != nil && !os.IsNotExist(err) {
+			return burrito.WrapErrorf(err, osStatErrorAny, destPath)
+		}
+		if (err != nil && os.IsNotExist(err)) || info.ModTime() != destInfo.ModTime() || info.Size() != destInfo.Size() {
+			if info.IsDir() {
+				return os.MkdirAll(destPath, info.Mode())
+			}
+			Logger.Debugf("SYNC: Copying file %s to %s", srcPath, destPath)
+			return copyFile(srcPath, destPath, info)
+		} else {
+			Logger.Debugf("SYNC: Skipping file %s", srcPath)
+		}
+		return nil
+	})
+	if err != nil {
+		return burrito.WrapErrorf(err, osCopyError, source, destination)
+	}
+
+	// Remove files/folders in destination that are not in source
+	err = filepath.Walk(destination, func(destPath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(destination, destPath)
+		if err != nil {
+			return burrito.WrapErrorf(err, filepathRelError, destination, destPath)
+		}
+		srcPath := filepath.Join(source, relPath)
+		if _, err := os.Stat(srcPath); os.IsNotExist(err) {
+			Logger.Debugf("SYNC: Removing file %s", destPath)
+			return os.RemoveAll(destPath)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return burrito.WrapErrorf(err, osRemoveError, source, destination)
+	}
+
+	//TODO: copy ACL. To be honest, I have no clue why it was there in the first place
+
+	// Make files read only if this option is selected
+	if makeReadOnly {
+		Logger.Infof("Changing the access for output path to "+
+			"read-only.\n\tPath: %s", destination)
+		err := filepath.WalkDir(destination,
+			func(s string, d fs.DirEntry, e error) error {
+
+				if e != nil {
+					// Error message isn't important as it's not passed further
+					// in the code
+					return e
+				}
+				if !d.IsDir() {
+					os.Chmod(s, 0444)
+				}
+				return nil
+			})
+		if err != nil {
+			Logger.Warnf(
+				"Failed to change access of the output path to read-only.\n"+
+					"\tPath: %s",
+				destination)
+		}
+	}
+	return nil
+}
+
+func copyFile(src, dest string, info os.FileInfo) error {
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return burrito.WrapErrorf(err, fileReadError, src)
+	}
+	if err = os.WriteFile(dest, data, info.Mode()); err != nil {
+		return burrito.WrapErrorf(err, fileWriteError, dest)
+	}
+	err = os.Chtimes(dest, time.Now(), info.ModTime())
+	if err != nil {
+		return burrito.WrapErrorf(err, osChtimesError, dest)
+	}
+	return nil
+}

regolith/profile.go

@@ -51,7 +51,7 @@ func SetupTmpFiles(config Config, dotRegolithPath string) error {
 				err = copy.Copy(
 					path,
 					p,
-					copy.Options{PreserveTimes: false, Sync: false})
+					copy.Options{PreserveTimes: IsExperimentEnabled(SizeTimeCheck), Sync: false})
 				if err != nil {
 					return burrito.WrapErrorf(err, osCopyError, path, p)
 				}