Skip to content

Commit

Permalink
Merge pull request #272 from Bedrock-OSS/exp-size_time_check
Browse files Browse the repository at this point in the history
Implement experiments flag and size_time_check experiment
  • Loading branch information
Nusiq authored Mar 3, 2024
2 parents 7619d3e + 01209f4 commit ddf62eb
Show file tree
Hide file tree
Showing 56 changed files with 574 additions and 37 deletions.
4 changes: 4 additions & 0 deletions docs/docs/.vitepress/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,10 @@ export default defineConfig({
text: 'Profiles',
link: '/guide/profiles'
},
{
text: 'Experiments',
link: '/guide/experiments'
},
{
text: 'Safety',
link: '/guide/safety'
Expand Down
23 changes: 23 additions & 0 deletions docs/docs/guide/experiments.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
---
title: Experiments
---

# Experiments

Experiments are new experimental features of Regolith to be released in the future versions, once proven to be stable and useful. The experiments can be enabled with the `--experiments` flag.

## Currently Available Experiments

### `size_time_check`

The `size_time_check` is an experiment that aims to speed up `regolith run` and `regolith watch` commands. It achieves this by checking the size and modification time of the files before moving them between working and output directories. If the source file is the same size and has the same modification time as the destination file, the target file will remain untouched (Regolith assumes that the files are the same).

The `size_time_check` should greatly speed up the exports of large projects.

The downside of this approach is that on the first run, the export will be slower, but on subsequent runs, the export will be much faster. This means that the `size_time_check` is not recommended for CI where Regolith is run only once.

Usage:
```
regolith run --experiments size_time_check
regolith watch --experiments size_time_check
```
14 changes: 13 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"fmt"
"os"
"strings"

"github.com/Bedrock-OSS/go-burrito/burrito"
"github.com/stirante/go-simple-eval/eval"
Expand Down Expand Up @@ -358,11 +359,22 @@ func main() {
}
subcommands = append(subcommands, cmdUpdateResolvers)

// add --debug and --timings flag to every command
// Generate the description for the experiments
experimentDescs := make([]string, len(regolith.AvailableExperiments))
for i, experiment := range regolith.AvailableExperiments {
experimentDescs[i] = "- " + experiment.Name + " - " + strings.Trim(experiment.Description, "\n")
}

// add --debug, --timings and --experiment flag to every command
for _, cmd := range subcommands {
cmd.Flags().BoolVarP(&burrito.PrintStackTrace, "debug", "", false, "Enables debugging")
cmd.Flags().BoolVarP(&regolith.EnableTimings, "timings", "", false, "Enables timing information")
cmd.Flags().StringSliceVar(
&regolith.EnabledExperiments, "experiments", nil,
"Enables experimental features. Currently supported experiments:\n"+
strings.Join(experimentDescs, "\n"))
}

// Build and run CLI
rootCmd.AddCommand(subcommands...)
rootCmd.Execute()
Expand Down
40 changes: 40 additions & 0 deletions regolith/experiments.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package regolith

type Experiment int

const (
// SizeTimeCheck is an experiment that checks the size and modification time when exporting
SizeTimeCheck Experiment = iota
)

// The descriptions shouldn't be too wide, the text with their description is
// indented a lot.
const sizeTimeCheckDesc = `
Activates optimization for file exporting by checking the size and
modification time of files before exporting, and only exporting if
the file has changed. This experiment applies to 'run' and 'watch'
commands.
`

type ExperimentInfo struct {
Name string
Description string
}

var AvailableExperiments = map[Experiment]ExperimentInfo{
SizeTimeCheck: {"size_time_check", sizeTimeCheckDesc},
}

var EnabledExperiments []string

func IsExperimentEnabled(exp Experiment) bool {
if EnabledExperiments == nil {
return false
}
for _, e := range EnabledExperiments {
if e == AvailableExperiments[exp].Name {
return true
}
}
return false
}
78 changes: 53 additions & 25 deletions regolith/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ func GetExportNames(exportTarget ExportTarget, ctx RunContext) (bpName string, r
// ExportProject copies files from the tmp paths (tmp/BP and tmp/RP) into
// the project's export target. The paths are generated with GetExportPaths.
func ExportProject(ctx RunContext) error {
MeasureStart("Export - GetExportPaths")
profile, err := ctx.GetProfile()
if err != nil {
return burrito.WrapError(err, runContextGetProfileError)
Expand All @@ -159,6 +160,7 @@ func ExportProject(ctx RunContext) error {
err, "Failed to get generate export paths.")
}

MeasureStart("Export - LoadEditedFiles")
// Loading edited_files.json or creating empty object
editedFiles := LoadEditedFiles(dotRegolithPath)
err = editedFiles.CheckDeletionSafety(rpPath, bpPath)
Expand All @@ -174,22 +176,28 @@ func ExportProject(ctx RunContext) error {
rpPath, bpPath)
}

// Clearing output locations
// Spooky, I hope file protection works, and it won't do any damage
err = os.RemoveAll(bpPath)
if err != nil {
return burrito.WrapErrorf(
err, "Failed to clear behavior pack from build path %q.\n"+
"Are user permissions correct?", bpPath)
}
err = os.RemoveAll(rpPath)
if err != nil {
return burrito.WrapErrorf(
err, "Failed to clear resource pack from build path %q.\n"+
"Are user permissions correct?", rpPath)
MeasureStart("Export - Clean")
// When comparing the size and modification time of the files, we need to
// keep the files in target paths.
if !IsExperimentEnabled(SizeTimeCheck) {
// Clearing output locations
// Spooky, I hope file protection works, and it won't do any damage
err = os.RemoveAll(bpPath)
if err != nil {
return burrito.WrapErrorf(
err, "Failed to clear behavior pack from build path %q.\n"+
"Are user permissions correct?", bpPath)
}
err = os.RemoveAll(rpPath)
if err != nil {
return burrito.WrapErrorf(
err, "Failed to clear resource pack from build path %q.\n"+
"Are user permissions correct?", rpPath)
}
}
MeasureEnd()
// List the names of the filters that opt-in to the data export process
exportedFilterNames := []string{}
var exportedFilterNames []string
for filter := range profile.Filters {
filter := profile.Filters[filter]
usingDataPath, err := filter.IsUsingDataExport(dotRegolithPath)
Expand Down Expand Up @@ -228,13 +236,15 @@ func ExportProject(ctx RunContext) error {
return burrito.WrapErrorf(err, osReadDirError, dataPath)
}
}
MeasureStart("Export - RevertibleOps")
// Create revertible operations object
backupPath := filepath.Join(dotRegolithPath, ".dataBackup")
revertibleOps, err := NewRevertibleFsOperations(backupPath)
if err != nil {
return burrito.WrapErrorf(err, newRevertibleFsOperationsError, backupPath)
}
// Export data
MeasureStart("Export - ExportData")
for _, exportedFilterName := range exportedFilterNames {
// Clear export target
targetPath := filepath.Join(dataPath, exportedFilterName)
Expand Down Expand Up @@ -278,18 +288,35 @@ func ExportProject(ctx RunContext) error {
return mainError
}
}
// Export BP
Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export behavior pack.")
}
// Export RP
Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export resource pack.")
MeasureStart("Export - MoveOrCopy")
if IsExperimentEnabled(SizeTimeCheck) {
// Export BP
Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly)
if err != nil {
return burrito.WrapError(err, "Failed to export behavior pack.")
}
// Export RP
Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly)
if err != nil {
return burrito.WrapError(err, "Failed to export resource pack.")
}
} else {
// Export BP
Logger.Infof("Exporting behavior pack to \"%s\".", bpPath)
err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export behavior pack.")
}
// Export RP
Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath))
err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true)
if err != nil {
return burrito.WrapError(err, "Failed to export resource pack.")
}
}
MeasureStart("Export - UpdateFromPaths")
// Update or create edited_files.json
err = editedFiles.UpdateFromPaths(rpPath, bpPath)
if err != nil {
Expand All @@ -306,6 +333,7 @@ func ExportProject(ctx RunContext) error {
if err := revertibleOps.Close(); err != nil {
return burrito.PassError(err)
}
MeasureEnd()
return nil
}

Expand Down
128 changes: 128 additions & 0 deletions regolith/file_system.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"sort"
"strconv"
"strings"
"time"

"github.com/Bedrock-OSS/go-burrito/burrito"

Expand Down Expand Up @@ -872,3 +873,130 @@ func MoveOrCopy(
}
return nil
}

// SyncDirectories copies the source to destination while checking size and modification time.
// If the file in the destination is different than the one in the source, it's overwritten,
// otherwise it's skipped (the destination file is not modified).
func SyncDirectories(
source string, destination string, makeReadOnly bool,
) error {
// Make destination parent if not exists
destinationParent := filepath.Dir(destination)
if err := os.MkdirAll(destinationParent, 0755); err != nil {
return burrito.WrapErrorf(err, osMkdirError, destinationParent)
}
err := filepath.Walk(source, func(srcPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
relPath, err := filepath.Rel(source, srcPath)
if err != nil {
return burrito.WrapErrorf(err, filepathRelError, source, srcPath)
}
destPath := filepath.Join(destination, relPath)

destInfo, err := os.Stat(destPath)
if err != nil && !os.IsNotExist(err) {
return burrito.WrapErrorf(err, osStatErrorAny, destPath)
}
if (err != nil && os.IsNotExist(err)) || info.ModTime() != destInfo.ModTime() || info.Size() != destInfo.Size() {
if info.IsDir() {
return os.MkdirAll(destPath, info.Mode())
}
Logger.Debugf("SYNC: Copying file %s to %s", srcPath, destPath)
// If file exists, we need to remove it first to avoid permission issues when it's
// read-only
if destInfo != nil {
err = os.Remove(destPath)
if err != nil {
return burrito.WrapErrorf(err, osRemoveError, destPath)
}
}
return copyFile(srcPath, destPath, info)
} else {
Logger.Debugf("SYNC: Skipping file %s", srcPath)
}
return nil
})
if err != nil {
return burrito.WrapErrorf(err, osCopyError, source, destination)
}

// Remove files/folders in destination that are not in source
toRemoveList := make([]string, 0)
err = filepath.Walk(destination, func(destPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
relPath, err := filepath.Rel(destination, destPath)
if err != nil {
return burrito.WrapErrorf(err, filepathRelError, destination, destPath)
}
srcPath := filepath.Join(source, relPath)
if _, err := os.Stat(srcPath); os.IsNotExist(err) {
// TODO: Not sure if this is the best way to do this
// The toRemoveList might get pretty big
if !SliceAny[string](toRemoveList, func(s string) bool {
return strings.HasPrefix(destPath, s)
}) {
Logger.Debugf("SYNC: Removing file %s", destPath)
// Add to list of files to remove, because otherwise walk function might fail
// when trying to walk a directory that doesn't exist anymore
toRemoveList = append(toRemoveList, destPath)
}
}
return nil
})

if err != nil {
return burrito.PassError(err)
}

for _, path := range toRemoveList {
err = os.RemoveAll(path)
if err != nil {
return burrito.WrapErrorf(err, osRemoveError, path)
}
}

// Make files read only if this option is selected
if makeReadOnly {
Logger.Infof("Changing the access for output path to "+
"read-only.\n\tPath: %s", destination)
err := filepath.WalkDir(destination,
func(s string, d fs.DirEntry, e error) error {

if e != nil {
// Error message isn't important as it's not passed further
// in the code
return e
}
if !d.IsDir() {
os.Chmod(s, 0444)
}
return nil
})
if err != nil {
Logger.Warnf(
"Failed to change access of the output path to read-only.\n"+
"\tPath: %s",
destination)
}
}
return nil
}

func copyFile(src, dest string, info os.FileInfo) error {
data, err := os.ReadFile(src)
if err != nil {
return burrito.WrapErrorf(err, fileReadError, src)
}
if err = os.WriteFile(dest, data, info.Mode()); err != nil {
return burrito.WrapErrorf(err, fileWriteError, dest)
}
err = os.Chtimes(dest, time.Now(), info.ModTime())
if err != nil {
return burrito.WrapErrorf(err, osChtimesError, dest)
}
return nil
}
Loading

0 comments on commit ddf62eb

Please sign in to comment.