From 0c753d2226708d77d8e4b565294b0947c9a61ac6 Mon Sep 17 00:00:00 2001 From: stirante Date: Mon, 30 Oct 2023 22:51:55 +0100 Subject: [PATCH] Implement experiments flag and size_time_check experiment, that copies and deletes files based on size and modification time of both source and target directories --- main.go | 3 +- regolith/experiments.go | 26 ++++++++++ regolith/export.go | 78 +++++++++++++++++++---------- regolith/file_system.go | 108 ++++++++++++++++++++++++++++++++++++++++ regolith/profile.go | 2 +- 5 files changed, 190 insertions(+), 27 deletions(-) create mode 100644 regolith/experiments.go diff --git a/main.go b/main.go index 21590b0b..eed6a0df 100644 --- a/main.go +++ b/main.go @@ -358,10 +358,11 @@ func main() { } subcommands = append(subcommands, cmdUpdateResolvers) - // add --debug and --timings flag to every command + // add --debug, --timings and --experiment flag to every command for _, cmd := range subcommands { cmd.Flags().BoolVarP(&burrito.PrintStackTrace, "debug", "", false, "Enables debugging") cmd.Flags().BoolVarP(®olith.EnableTimings, "timings", "", false, "Enables timing information") + cmd.Flags().StringSliceVar(®olith.EnabledExperiments, "experiments", nil, "Enables experimental features") } // Build and run CLI rootCmd.AddCommand(subcommands...) diff --git a/regolith/experiments.go b/regolith/experiments.go new file mode 100644 index 00000000..d151759d --- /dev/null +++ b/regolith/experiments.go @@ -0,0 +1,26 @@ +package regolith + +type Experiment int + +const ( + // SizeTimeCheck is an experiment that checks the size and modification time when exporting + SizeTimeCheck Experiment = iota +) + +var experimentNames = map[Experiment]string{ + SizeTimeCheck: "size_time_check", +} + +var EnabledExperiments []string + +func IsExperimentEnabled(exp Experiment) bool { + if EnabledExperiments == nil { + return false + } + for _, e := range EnabledExperiments { + if e == experimentNames[exp] { + return true + } + } + return false +} diff --git a/regolith/export.go b/regolith/export.go index c19082cc..761fbccc 100644 --- a/regolith/export.go +++ b/regolith/export.go @@ -101,6 +101,7 @@ func GetExportPaths( func ExportProject( profile Profile, name, dataPath, dotRegolithPath string, ) error { + MeasureStart("Export - GetExportPaths") // Get the export target paths exportTarget := profile.ExportTarget bpPath, rpPath, err := GetExportPaths(exportTarget, name) @@ -109,6 +110,7 @@ func ExportProject( err, "Failed to get generate export paths.") } + MeasureStart("Export - LoadEditedFiles") // Loading edited_files.json or creating empty object editedFiles := LoadEditedFiles(dotRegolithPath) err = editedFiles.CheckDeletionSafety(rpPath, bpPath) @@ -124,22 +126,28 @@ func ExportProject( rpPath, bpPath) } - // Clearing output locations - // Spooky, I hope file protection works, and it won't do any damage - err = os.RemoveAll(bpPath) - if err != nil { - return burrito.WrapErrorf( - err, "Failed to clear behavior pack from build path %q.\n"+ - "Are user permissions correct?", bpPath) - } - err = os.RemoveAll(rpPath) - if err != nil { - return burrito.WrapErrorf( - err, "Failed to clear resource pack from build path %q.\n"+ - "Are user permissions correct?", rpPath) + MeasureStart("Export - Clean") + // When comparing the size and modification time of the files, we need to + // keep the files in target paths. + if !IsExperimentEnabled(SizeTimeCheck) { + // Clearing output locations + // Spooky, I hope file protection works, and it won't do any damage + err = os.RemoveAll(bpPath) + if err != nil { + return burrito.WrapErrorf( + err, "Failed to clear behavior pack from build path %q.\n"+ + "Are user permissions correct?", bpPath) + } + err = os.RemoveAll(rpPath) + if err != nil { + return burrito.WrapErrorf( + err, "Failed to clear resource pack from build path %q.\n"+ + "Are user permissions correct?", rpPath) + } } + MeasureEnd() // List the names of the filters that opt-in to the data export process - exportedFilterNames := []string{} + var exportedFilterNames []string for filter := range profile.Filters { filter := profile.Filters[filter] usingDataPath, err := filter.IsUsingDataExport(dotRegolithPath) @@ -180,6 +188,7 @@ func ExportProject( dataPath) } } + MeasureStart("Export - RevertibleOps") // Create revertible operations object backupPath := filepath.Join(dotRegolithPath, ".dataBackup") revertibleOps, err := NewRevertibleFsOperations(backupPath) @@ -187,6 +196,7 @@ func ExportProject( return burrito.WrapErrorf(err, newRevertibleFsOperationsError, backupPath) } // Export data + MeasureStart("Export - ExportData") for _, exportedFilterName := range exportedFilterNames { // Clear export target targetPath := filepath.Join(dataPath, exportedFilterName) @@ -230,18 +240,35 @@ func ExportProject( return mainError } } - // Export BP - Logger.Infof("Exporting behavior pack to \"%s\".", bpPath) - err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true) - if err != nil { - return burrito.WrapError(err, "Failed to export behavior pack.") - } - // Export RP - Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath)) - err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true) - if err != nil { - return burrito.WrapError(err, "Failed to export resource pack.") + MeasureStart("Export - MoveOrCopy") + if IsExperimentEnabled(SizeTimeCheck) { + // Export BP + Logger.Infof("Exporting behavior pack to \"%s\".", bpPath) + err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true) + if err != nil { + return burrito.WrapError(err, "Failed to export behavior pack.") + } + // Export RP + Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath)) + err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true) + if err != nil { + return burrito.WrapError(err, "Failed to export resource pack.") + } + } else { + // Export BP + Logger.Infof("Exporting behavior pack to \"%s\".", bpPath) + err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true) + if err != nil { + return burrito.WrapError(err, "Failed to export behavior pack.") + } + // Export RP + Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath)) + err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true) + if err != nil { + return burrito.WrapError(err, "Failed to export resource pack.") + } } + MeasureStart("Export - UpdateFromPaths") // Update or create edited_files.json err = editedFiles.UpdateFromPaths(rpPath, bpPath) if err != nil { @@ -258,6 +285,7 @@ func ExportProject( if err := revertibleOps.Close(); err != nil { return burrito.PassError(err) } + MeasureEnd() return nil } diff --git a/regolith/file_system.go b/regolith/file_system.go index 1f4e61cc..67fe35eb 100644 --- a/regolith/file_system.go +++ b/regolith/file_system.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/Bedrock-OSS/go-burrito/burrito" @@ -869,3 +870,110 @@ func MoveOrCopy( } return nil } + +// SyncDirectories copies the source to destination while checking size and modification time. +func SyncDirectories( + source string, destination string, makeReadOnly bool, copyParentAcl bool, +) error { + // Make destination parent if not exists + destinationParent := filepath.Dir(destination) + if _, err := os.Stat(destinationParent); os.IsNotExist(err) { + err = os.MkdirAll(destinationParent, 0755) + if err != nil { + return burrito.WrapErrorf( + err, osMkdirError, destinationParent) + } + } + err := filepath.Walk(source, func(srcPath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(source, srcPath) + if err != nil { + return burrito.WrapErrorf(err, filepathRelError, source, srcPath) + } + destPath := filepath.Join(destination, relPath) + + destInfo, err := os.Stat(destPath) + if err != nil && !os.IsNotExist(err) { + return burrito.WrapErrorf(err, osStatErrorAny, destPath) + } + if (err != nil && os.IsNotExist(err)) || info.ModTime() != destInfo.ModTime() || info.Size() != destInfo.Size() { + if info.IsDir() { + return os.MkdirAll(destPath, info.Mode()) + } + Logger.Debugf("SYNC: Copying file %s to %s", srcPath, destPath) + return copyFile(srcPath, destPath, info) + } else { + Logger.Debugf("SYNC: Skipping file %s", srcPath) + } + return nil + }) + if err != nil { + return burrito.WrapErrorf(err, osCopyError, source, destination) + } + + // Remove files/folders in destination that are not in source + err = filepath.Walk(destination, func(destPath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(destination, destPath) + if err != nil { + return burrito.WrapErrorf(err, filepathRelError, destination, destPath) + } + srcPath := filepath.Join(source, relPath) + if _, err := os.Stat(srcPath); os.IsNotExist(err) { + Logger.Debugf("SYNC: Removing file %s", destPath) + return os.RemoveAll(destPath) + } + return nil + }) + + if err != nil { + return burrito.WrapErrorf(err, osRemoveError, source, destination) + } + + //TODO: copy ACL. To be honest, I have no clue why it was there in the first place + + // Make files read only if this option is selected + if makeReadOnly { + Logger.Infof("Changing the access for output path to "+ + "read-only.\n\tPath: %s", destination) + err := filepath.WalkDir(destination, + func(s string, d fs.DirEntry, e error) error { + + if e != nil { + // Error message isn't important as it's not passed further + // in the code + return e + } + if !d.IsDir() { + os.Chmod(s, 0444) + } + return nil + }) + if err != nil { + Logger.Warnf( + "Failed to change access of the output path to read-only.\n"+ + "\tPath: %s", + destination) + } + } + return nil +} + +func copyFile(src, dest string, info os.FileInfo) error { + data, err := os.ReadFile(src) + if err != nil { + return burrito.WrapErrorf(err, fileReadError, src) + } + if err = os.WriteFile(dest, data, info.Mode()); err != nil { + return burrito.WrapErrorf(err, fileWriteError, dest) + } + err = os.Chtimes(dest, time.Now(), info.ModTime()) + if err != nil { + return burrito.WrapErrorf(err, osChtimesError, dest) + } + return nil +} diff --git a/regolith/profile.go b/regolith/profile.go index 0d2ca749..74c69960 100644 --- a/regolith/profile.go +++ b/regolith/profile.go @@ -51,7 +51,7 @@ func SetupTmpFiles(config Config, dotRegolithPath string) error { err = copy.Copy( path, p, - copy.Options{PreserveTimes: false, Sync: false}) + copy.Options{PreserveTimes: IsExperimentEnabled(SizeTimeCheck), Sync: false}) if err != nil { return burrito.WrapErrorf(err, osCopyError, path, p) }