Skip to content

Commit

Permalink
Implement experiments flag and size_time_check experiment, that copie…
Browse files Browse the repository at this point in the history
…s and deletes files based on size and modification time of both source and target directories
  • Loading branch information
stirante committed Oct 30, 2023
1 parent d651e5b commit 0c753d2
Show file tree
Hide file tree
Showing 5 changed files with 190 additions and 27 deletions.
3 changes: 2 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -358,10 +358,11 @@ func main() {
}
subcommands = append(subcommands, cmdUpdateResolvers)

// add --debug and --timings flag to every command
// add --debug, --timings and --experiment flag to every command
for _, cmd := range subcommands {
cmd.Flags().BoolVarP(&burrito.PrintStackTrace, "debug", "", false, "Enables debugging")
cmd.Flags().BoolVarP(&regolith.EnableTimings, "timings", "", false, "Enables timing information")
cmd.Flags().StringSliceVar(&regolith.EnabledExperiments, "experiments", nil, "Enables experimental features")
}
// Build and run CLI
rootCmd.AddCommand(subcommands...)
Expand Down
26 changes: 26 additions & 0 deletions regolith/experiments.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package regolith

type Experiment int

const (
// SizeTimeCheck is an experiment that checks the size and modification time when exporting
SizeTimeCheck Experiment = iota
)

var experimentNames = map[Experiment]string{
SizeTimeCheck: "size_time_check",
}

var EnabledExperiments []string

func IsExperimentEnabled(exp Experiment) bool {
if EnabledExperiments == nil {
return false
}
for _, e := range EnabledExperiments {
if e == experimentNames[exp] {
return true
}
}
return false
}
78 changes: 53 additions & 25 deletions regolith/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ func GetExportPaths(
func ExportProject(
profile Profile, name, dataPath, dotRegolithPath string,
) error {
MeasureStart("Export - GetExportPaths")
// Get the export target paths
exportTarget := profile.ExportTarget
bpPath, rpPath, err := GetExportPaths(exportTarget, name)
Expand All @@ -109,6 +110,7 @@ func ExportProject(
err, "Failed to get generate export paths.")
}

MeasureStart("Export - LoadEditedFiles")
// Loading edited_files.json or creating empty object
editedFiles := LoadEditedFiles(dotRegolithPath)
err = editedFiles.CheckDeletionSafety(rpPath, bpPath)
Expand All @@ -124,22 +126,28 @@ func ExportProject(
rpPath, bpPath)
}

// Clearing output locations
// Spooky, I hope file protection works, and it won't do any damage
err = os.RemoveAll(bpPath)
if err != nil {
return burrito.WrapErrorf(
err, "Failed to clear behavior pack from build path %q.\n"+
"Are user permissions correct?", bpPath)
}
err = os.RemoveAll(rpPath)
if err != nil {
return burrito.WrapErrorf(
err, "Failed to clear resource pack from build path %q.\n"+
"Are user permissions correct?", rpPath)
MeasureStart("Export - Clean")
// When comparing the size and modification time of the files, we need to
// keep the files in target paths.
if !IsExperimentEnabled(SizeTimeCheck) {
// Clearing output locations
// Spooky, I hope file protection works, and it won't do any damage
err = os.RemoveAll(bpPath)
if err != nil {
return burrito.WrapErrorf(
err, "Failed to clear behavior pack from build path %q.\n"+
"Are user permissions correct?", bpPath)
}
err = os.RemoveAll(rpPath)
if err != nil {
return burrito.WrapErrorf(
err, "Failed to clear resource pack from build path %q.\n"+
"Are user permissions correct?", rpPath)
}
}
MeasureEnd()
// List the names of the filters that opt-in to the data export process
exportedFilterNames := []string{}
var exportedFilterNames []string
for filter := range profile.Filters {
filter := profile.Filters[filter]
usingDataPath, err := filter.IsUsingDataExport(dotRegolithPath)
Expand Down Expand Up @@ -180,13 +188,15 @@ func ExportProject(
dataPath)
}
}
MeasureStart("Export - RevertibleOps")
// Create revertible operations object
backupPath := filepath.Join(dotRegolithPath, ".dataBackup")
revertibleOps, err := NewRevertibleFsOperations(backupPath)
if err != nil {
return burrito.WrapErrorf(err, newRevertibleFsOperationsError, backupPath)
}
// Export data
MeasureStart("Export - ExportData")
for _, exportedFilterName := range exportedFilterNames {
// Clear export target
targetPath := filepath.Join(dataPath, exportedFilterName)
Expand Down Expand Up @@ -230,18 +240,35 @@ func ExportProject(
return mainError
}
}
// Export BP
Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export behavior pack.")
}
// Export RP
Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export resource pack.")
MeasureStart("Export - MoveOrCopy")
if IsExperimentEnabled(SizeTimeCheck) {
// Export BP
Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export behavior pack.")
}
// Export RP
Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export resource pack.")
}
} else {
// Export BP
Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export behavior pack.")
}
// Export RP
Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export resource pack.")
}
}
MeasureStart("Export - UpdateFromPaths")
// Update or create edited_files.json
err = editedFiles.UpdateFromPaths(rpPath, bpPath)
if err != nil {
Expand All @@ -258,6 +285,7 @@ func ExportProject(
if err := revertibleOps.Close(); err != nil {
return burrito.PassError(err)
}
MeasureEnd()
return nil
}

Expand Down
108 changes: 108 additions & 0 deletions regolith/file_system.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"sort"
"strconv"
"strings"
"time"

"github.com/Bedrock-OSS/go-burrito/burrito"

Expand Down Expand Up @@ -869,3 +870,110 @@ func MoveOrCopy(
}
return nil
}

// SyncDirectories copies the source to destination while checking size and modification time.
func SyncDirectories(
source string, destination string, makeReadOnly bool, copyParentAcl bool,
) error {
// Make destination parent if not exists
destinationParent := filepath.Dir(destination)
if _, err := os.Stat(destinationParent); os.IsNotExist(err) {
err = os.MkdirAll(destinationParent, 0755)
if err != nil {
return burrito.WrapErrorf(
err, osMkdirError, destinationParent)
}
}
err := filepath.Walk(source, func(srcPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
relPath, err := filepath.Rel(source, srcPath)
if err != nil {
return burrito.WrapErrorf(err, filepathRelError, source, srcPath)
}
destPath := filepath.Join(destination, relPath)

destInfo, err := os.Stat(destPath)
if err != nil && !os.IsNotExist(err) {
return burrito.WrapErrorf(err, osStatErrorAny, destPath)
}
if (err != nil && os.IsNotExist(err)) || info.ModTime() != destInfo.ModTime() || info.Size() != destInfo.Size() {
if info.IsDir() {
return os.MkdirAll(destPath, info.Mode())
}
Logger.Debugf("SYNC: Copying file %s to %s", srcPath, destPath)
return copyFile(srcPath, destPath, info)
} else {
Logger.Debugf("SYNC: Skipping file %s", srcPath)
}
return nil
})
if err != nil {
return burrito.WrapErrorf(err, osCopyError, source, destination)
}

// Remove files/folders in destination that are not in source
err = filepath.Walk(destination, func(destPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
relPath, err := filepath.Rel(destination, destPath)
if err != nil {
return burrito.WrapErrorf(err, filepathRelError, destination, destPath)
}
srcPath := filepath.Join(source, relPath)
if _, err := os.Stat(srcPath); os.IsNotExist(err) {
Logger.Debugf("SYNC: Removing file %s", destPath)
return os.RemoveAll(destPath)
}
return nil
})

if err != nil {
return burrito.WrapErrorf(err, osRemoveError, source, destination)
}

//TODO: copy ACL. To be honest, I have no clue why it was there in the first place

// Make files read only if this option is selected
if makeReadOnly {
Logger.Infof("Changing the access for output path to "+
"read-only.\n\tPath: %s", destination)
err := filepath.WalkDir(destination,
func(s string, d fs.DirEntry, e error) error {

if e != nil {
// Error message isn't important as it's not passed further
// in the code
return e
}
if !d.IsDir() {
os.Chmod(s, 0444)
}
return nil
})
if err != nil {
Logger.Warnf(
"Failed to change access of the output path to read-only.\n"+
"\tPath: %s",
destination)
}
}
return nil
}

func copyFile(src, dest string, info os.FileInfo) error {
data, err := os.ReadFile(src)
if err != nil {
return burrito.WrapErrorf(err, fileReadError, src)
}
if err = os.WriteFile(dest, data, info.Mode()); err != nil {
return burrito.WrapErrorf(err, fileWriteError, dest)
}
err = os.Chtimes(dest, time.Now(), info.ModTime())
if err != nil {
return burrito.WrapErrorf(err, osChtimesError, dest)
}
return nil
}
2 changes: 1 addition & 1 deletion regolith/profile.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func SetupTmpFiles(config Config, dotRegolithPath string) error {
err = copy.Copy(
path,
p,
copy.Options{PreserveTimes: false, Sync: false})
copy.Options{PreserveTimes: IsExperimentEnabled(SizeTimeCheck), Sync: false})
if err != nil {
return burrito.WrapErrorf(err, osCopyError, path, p)
}
Expand Down

0 comments on commit 0c753d2

Please sign in to comment.