Merge pull request #272 from Bedrock-OSS/exp-size_time_check

Implement experiments flag and size_time_check experiment

docs/docs/.vitepress/config.ts

@@ -126,6 +126,10 @@ export default defineConfig({
               text: 'Profiles',
               link: '/guide/profiles'
             },
+            {
+              text: 'Experiments',
+              link: '/guide/experiments'
+            },
             {
               text: 'Safety',
               link: '/guide/safety'

docs/docs/guide/experiments.md

@@ -0,0 +1,23 @@
+---
+title: Experiments
+---
+
+# Experiments
+
+Experiments are new experimental features of Regolith to be released in the future versions, once proven to be stable and useful. The experiments can be enabled with the `--experiments` flag.
+
+## Currently Available Experiments
+
+### `size_time_check`
+
+The `size_time_check` is an experiment that aims to speed up `regolith run` and `regolith watch` commands. It achieves this by checking the size and modification time of the files before moving them between working and output directories. If the source file is the same size and has the same modification time as the destination file, the target file will remain untouched (Regolith assumes that the files are the same).
+
+The `size_time_check` should greatly speed up the exports of large projects.
+
+The downside of this approach is that on the first run, the export will be slower, but on subsequent runs, the export will be much faster. This means that the `size_time_check` is not recommended for CI where Regolith is run only once.
+
+Usage:
+```
+regolith run --experiments size_time_check
+regolith watch --experiments size_time_check
+```

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/Bedrock-OSS/go-burrito/burrito"
 	"github.com/stirante/go-simple-eval/eval"
@@ -358,11 +359,22 @@ func main() {
 	}
 	subcommands = append(subcommands, cmdUpdateResolvers)
 
-	// add --debug and --timings flag to every command
+	// Generate the description for the experiments
+	experimentDescs := make([]string, len(regolith.AvailableExperiments))
+	for i, experiment := range regolith.AvailableExperiments {
+		experimentDescs[i] = "- " + experiment.Name + " - " + strings.Trim(experiment.Description, "\n")
+	}
+
+	// add --debug, --timings and --experiment flag to every command
 	for _, cmd := range subcommands {
 		cmd.Flags().BoolVarP(&burrito.PrintStackTrace, "debug", "", false, "Enables debugging")
 		cmd.Flags().BoolVarP(&regolith.EnableTimings, "timings", "", false, "Enables timing information")
+		cmd.Flags().StringSliceVar(
+			&regolith.EnabledExperiments, "experiments", nil,
+			"Enables experimental features. Currently supported experiments:\n"+
+				strings.Join(experimentDescs, "\n"))
 	}
+
 	// Build and run CLI
 	rootCmd.AddCommand(subcommands...)
 	rootCmd.Execute()

regolith/experiments.go

@@ -0,0 +1,40 @@
+package regolith
+
+type Experiment int
+
+const (
+	// SizeTimeCheck is an experiment that checks the size and modification time when exporting
+	SizeTimeCheck Experiment = iota
+)
+
+// The descriptions shouldn't be too wide, the text with their description is
+// indented a lot.
+const sizeTimeCheckDesc = `
+Activates optimization for file exporting by checking the size and
+modification time of files before exporting, and only exporting if
+the file has changed. This experiment applies to 'run' and 'watch'
+commands.
+`
+
+type ExperimentInfo struct {
+	Name        string
+	Description string
+}
+
+var AvailableExperiments = map[Experiment]ExperimentInfo{
+	SizeTimeCheck: {"size_time_check", sizeTimeCheckDesc},
+}
+
+var EnabledExperiments []string
+
+func IsExperimentEnabled(exp Experiment) bool {
+	if EnabledExperiments == nil {
+		return false
+	}
+	for _, e := range EnabledExperiments {
+		if e == AvailableExperiments[exp].Name {
+			return true
+		}
+	}
+	return false
+}

regolith/export.go

@@ -141,6 +141,7 @@ func GetExportNames(exportTarget ExportTarget, ctx RunContext) (bpName string, r
 // ExportProject copies files from the tmp paths (tmp/BP and tmp/RP) into
 // the project's export target. The paths are generated with GetExportPaths.
 func ExportProject(ctx RunContext) error {
+	MeasureStart("Export - GetExportPaths")
 	profile, err := ctx.GetProfile()
 	if err != nil {
 		return burrito.WrapError(err, runContextGetProfileError)
@@ -159,6 +160,7 @@ func ExportProject(ctx RunContext) error {
 			err, "Failed to get generate export paths.")
 	}
 
+	MeasureStart("Export - LoadEditedFiles")
 	// Loading edited_files.json or creating empty object
 	editedFiles := LoadEditedFiles(dotRegolithPath)
 	err = editedFiles.CheckDeletionSafety(rpPath, bpPath)
@@ -174,22 +176,28 @@ func ExportProject(ctx RunContext) error {
 			rpPath, bpPath)
 	}
 
-	// Clearing output locations
-	// Spooky, I hope file protection works, and it won't do any damage
-	err = os.RemoveAll(bpPath)
-	if err != nil {
-		return burrito.WrapErrorf(
-			err, "Failed to clear behavior pack from build path %q.\n"+
-				"Are user permissions correct?", bpPath)
-	}
-	err = os.RemoveAll(rpPath)
-	if err != nil {
-		return burrito.WrapErrorf(
-			err, "Failed to clear resource pack from build path %q.\n"+
-				"Are user permissions correct?", rpPath)
+	MeasureStart("Export - Clean")
+	// When comparing the size and modification time of the files, we need to
+	// keep the files in target paths.
+	if !IsExperimentEnabled(SizeTimeCheck) {
+		// Clearing output locations
+		// Spooky, I hope file protection works, and it won't do any damage
+		err = os.RemoveAll(bpPath)
+		if err != nil {
+			return burrito.WrapErrorf(
+				err, "Failed to clear behavior pack from build path %q.\n"+
+					"Are user permissions correct?", bpPath)
+		}
+		err = os.RemoveAll(rpPath)
+		if err != nil {
+			return burrito.WrapErrorf(
+				err, "Failed to clear resource pack from build path %q.\n"+
+					"Are user permissions correct?", rpPath)
+		}
 	}
+	MeasureEnd()
 	// List the names of the filters that opt-in to the data export process
-	exportedFilterNames := []string{}
+	var exportedFilterNames []string
 	for filter := range profile.Filters {
 		filter := profile.Filters[filter]
 		usingDataPath, err := filter.IsUsingDataExport(dotRegolithPath)
@@ -228,13 +236,15 @@ func ExportProject(ctx RunContext) error {
 			return burrito.WrapErrorf(err, osReadDirError, dataPath)
 		}
 	}
+	MeasureStart("Export - RevertibleOps")
 	// Create revertible operations object
 	backupPath := filepath.Join(dotRegolithPath, ".dataBackup")
 	revertibleOps, err := NewRevertibleFsOperations(backupPath)
 	if err != nil {
 		return burrito.WrapErrorf(err, newRevertibleFsOperationsError, backupPath)
 	}
 	// Export data
+	MeasureStart("Export - ExportData")
 	for _, exportedFilterName := range exportedFilterNames {
 		// Clear export target
 		targetPath := filepath.Join(dataPath, exportedFilterName)
@@ -278,18 +288,35 @@ func ExportProject(ctx RunContext) error {
 			return mainError
 		}
 	}
-	// Export BP
-	Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
-	err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
-	if err != nil {
-		return burrito.WrapError(err, "Failed to export behavior pack.")
-	}
-	// Export RP
-	Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
-	err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
-	if err != nil {
-		return burrito.WrapError(err, "Failed to export resource pack.")
+	MeasureStart("Export - MoveOrCopy")
+	if IsExperimentEnabled(SizeTimeCheck) {
+		// Export BP
+		Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
+		err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly)
+		if err != nil {
+			return burrito.WrapError(err, "Failed to export behavior pack.")
+		}
+		// Export RP
+		Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
+		err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly)
+		if err != nil {
+			return burrito.WrapError(err, "Failed to export resource pack.")
+		}
+	} else {
+		// Export BP
+		Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
+		err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
+		if err != nil {
+			return burrito.WrapError(err, "Failed to export behavior pack.")
+		}
+		// Export RP
+		Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
+		err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
+		if err != nil {
+			return burrito.WrapError(err, "Failed to export resource pack.")
+		}
 	}
+	MeasureStart("Export - UpdateFromPaths")
 	// Update or create edited_files.json
 	err = editedFiles.UpdateFromPaths(rpPath, bpPath)
 	if err != nil {
@@ -306,6 +333,7 @@ func ExportProject(ctx RunContext) error {
 	if err := revertibleOps.Close(); err != nil {
 		return burrito.PassError(err)
 	}
+	MeasureEnd()
 	return nil
 }
 

regolith/file_system.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/Bedrock-OSS/go-burrito/burrito"
 
@@ -872,3 +873,130 @@ func MoveOrCopy(
 	}
 	return nil
 }
+
+// SyncDirectories copies the source to destination while checking size and modification time.
+// If the file in the destination is different than the one in the source, it's overwritten,
+// otherwise it's skipped (the destination file is not modified).
+func SyncDirectories(
+	source string, destination string, makeReadOnly bool,
+) error {
+	// Make destination parent if not exists
+	destinationParent := filepath.Dir(destination)
+	if err := os.MkdirAll(destinationParent, 0755); err != nil {
+		return burrito.WrapErrorf(err, osMkdirError, destinationParent)
+	}
+	err := filepath.Walk(source, func(srcPath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(source, srcPath)
+		if err != nil {
+			return burrito.WrapErrorf(err, filepathRelError, source, srcPath)
+		}
+		destPath := filepath.Join(destination, relPath)
+
+		destInfo, err := os.Stat(destPath)
+		if err != nil && !os.IsNotExist(err) {
+			return burrito.WrapErrorf(err, osStatErrorAny, destPath)
+		}
+		if (err != nil && os.IsNotExist(err)) || info.ModTime() != destInfo.ModTime() || info.Size() != destInfo.Size() {
+			if info.IsDir() {
+				return os.MkdirAll(destPath, info.Mode())
+			}
+			Logger.Debugf("SYNC: Copying file %s to %s", srcPath, destPath)
+			// If file exists, we need to remove it first to avoid permission issues when it's
+			// read-only
+			if destInfo != nil {
+				err = os.Remove(destPath)
+				if err != nil {
+					return burrito.WrapErrorf(err, osRemoveError, destPath)
+				}
+			}
+			return copyFile(srcPath, destPath, info)
+		} else {
+			Logger.Debugf("SYNC: Skipping file %s", srcPath)
+		}
+		return nil
+	})
+	if err != nil {
+		return burrito.WrapErrorf(err, osCopyError, source, destination)
+	}
+
+	// Remove files/folders in destination that are not in source
+	toRemoveList := make([]string, 0)
+	err = filepath.Walk(destination, func(destPath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(destination, destPath)
+		if err != nil {
+			return burrito.WrapErrorf(err, filepathRelError, destination, destPath)
+		}
+		srcPath := filepath.Join(source, relPath)
+		if _, err := os.Stat(srcPath); os.IsNotExist(err) {
+			// TODO: Not sure if this is the best way to do this
+			// The toRemoveList might get pretty big
+			if !SliceAny[string](toRemoveList, func(s string) bool {
+				return strings.HasPrefix(destPath, s)
+			}) {
+				Logger.Debugf("SYNC: Removing file %s", destPath)
+				// Add to list of files to remove, because otherwise walk function might fail
+				// when trying to walk a directory that doesn't exist anymore
+				toRemoveList = append(toRemoveList, destPath)
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return burrito.PassError(err)
+	}
+
+	for _, path := range toRemoveList {
+		err = os.RemoveAll(path)
+		if err != nil {
+			return burrito.WrapErrorf(err, osRemoveError, path)
+		}
+	}
+
+	// Make files read only if this option is selected
+	if makeReadOnly {
+		Logger.Infof("Changing the access for output path to "+
+			"read-only.\n\tPath: %s", destination)
+		err := filepath.WalkDir(destination,
+			func(s string, d fs.DirEntry, e error) error {
+
+				if e != nil {
+					// Error message isn't important as it's not passed further
+					// in the code
+					return e
+				}
+				if !d.IsDir() {
+					os.Chmod(s, 0444)
+				}
+				return nil
+			})
+		if err != nil {
+			Logger.Warnf(
+				"Failed to change access of the output path to read-only.\n"+
+					"\tPath: %s",
+				destination)
+		}
+	}
+	return nil
+}
+
+func copyFile(src, dest string, info os.FileInfo) error {
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return burrito.WrapErrorf(err, fileReadError, src)
+	}
+	if err = os.WriteFile(dest, data, info.Mode()); err != nil {
+		return burrito.WrapErrorf(err, fileWriteError, dest)
+	}
+	err = os.Chtimes(dest, time.Now(), info.ModTime())
+	if err != nil {
+		return burrito.WrapErrorf(err, osChtimesError, dest)
+	}
+	return nil
+}