From b9d3d55113bbe962fbcdf01d674bde1e251f6403 Mon Sep 17 00:00:00 2001 From: plastikfan Date: Sat, 23 Dec 2023 13:23:50 +0000 Subject: [PATCH] ref(proxy): implement transparent with profile (#80) --- .vscode/settings.json | 1 + src/app/proxy/controller-sampler_test.go | 10 +- src/app/proxy/controller.go | 13 +- src/app/proxy/enter-shrink.go | 10 ++ src/app/proxy/execution-step.go | 42 ++++- src/app/proxy/file-manager.go | 52 ++++-- src/app/proxy/path-finder.go | 213 ++++++++++++++++++++--- src/app/proxy/proxy-defs.go | 4 + 8 files changed, 282 insertions(+), 63 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a4eeef9..e3b8e85 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,6 +4,7 @@ "--fast" ], "cSpell.words": [ + "arity", "beezledub", "bodyclose", "chardata", diff --git a/src/app/proxy/controller-sampler_test.go b/src/app/proxy/controller-sampler_test.go index ff52947..6c54716 100644 --- a/src/app/proxy/controller-sampler_test.go +++ b/src/app/proxy/controller-sampler_test.go @@ -281,8 +281,8 @@ var _ = Describe("SamplerController", Ordered, func() { Entry(nil, &samplerTE{ controllerTE: controllerTE{ - given: "profile", - should: "sample(first) with glob filter using the defined profile", + given: "run transparent with profile", + should: "sample(first) with glob filter, result file takes place of input", relative: backyardWorldsPlanet9Scan01, args: []string{ "--sample", @@ -296,7 +296,7 @@ var _ = Describe("SamplerController", Ordered, func() { }, }), - Entry(nil, &samplerTE{ + XEntry(nil, &samplerTE{ controllerTE: controllerTE{ given: "profile", should: "sample(last) with glob filter using the defined profile", @@ -312,7 +312,7 @@ var _ = Describe("SamplerController", Ordered, func() { }, }), - Entry(nil, &samplerTE{ + XEntry(nil, &samplerTE{ controllerTE: controllerTE{ given: "profile without no-files in args", should: "sample(first) with glob filter, using no-files from config", @@ -344,7 +344,7 @@ var _ = Describe("SamplerController", Ordered, func() { // === - Entry(nil, &samplerTE{ + XEntry(nil, &samplerTE{ controllerTE: controllerTE{ given: "scheme", should: "sample all profiles in the scheme", diff --git a/src/app/proxy/controller.go b/src/app/proxy/controller.go index abcaf1a..351ffb2 100644 --- a/src/app/proxy/controller.go +++ b/src/app/proxy/controller.go @@ -15,6 +15,7 @@ import ( type controller struct { shared *SharedControllerInfo + local localControllerInfo } func (c *controller) profileSequence( @@ -94,8 +95,11 @@ func (c *controller) Run(item *nav.TraverseItem, sequence Sequence) error { ) iterator := collections.ForwardRunIt[Step, error](sequence, zero) - each := func(s Step) error { - return s.Run(c.shared) + each := func(step Step) error { + return step.Run(&RunStepInfo{ + Item: item, + Source: c.local.destination, + }) } while := func(_ Step, err error) bool { if resultErr == nil { @@ -115,8 +119,8 @@ func (c *controller) Run(item *nav.TraverseItem, sequence Sequence) error { // Perhaps we have an error policy including one that implements // a retry. // - if err := c.shared.fileManager.Setup(item); err != nil { - return err + if c.local.destination, resultErr = c.shared.fileManager.Setup(item); resultErr != nil { + return resultErr } iterator.RunAll(each, while) @@ -125,4 +129,5 @@ func (c *controller) Run(item *nav.TraverseItem, sequence Sequence) error { } func (c *controller) Reset() { + c.local.destination = "" } diff --git a/src/app/proxy/enter-shrink.go b/src/app/proxy/enter-shrink.go index 7b94319..56d76b8 100644 --- a/src/app/proxy/enter-shrink.go +++ b/src/app/proxy/enter-shrink.go @@ -108,13 +108,23 @@ func (e *ShrinkEntry) createFinder() *PathFinder { output: &inlineOutputStrategy{}, deletion: &inlineDeletionStrategy{}, }, + arity: 1, + } + + if finder.Scheme != "" { + schemeCFG, _ := e.SamplerCFG.Scheme(finder.Scheme) + finder.arity = len(schemeCFG.Profiles) } if e.Inputs.ParamSet.Native.OutputPath != "" { + finder.Output = e.Inputs.ParamSet.Native.OutputPath finder.behaviours.output = &ejectOutputStrategy{} + } else { + finder.transparentInput = true } if e.Inputs.ParamSet.Native.TrashPath != "" { + finder.Trash = e.Inputs.ParamSet.Native.TrashPath finder.behaviours.deletion = &ejectOutputStrategy{} } diff --git a/src/app/proxy/execution-step.go b/src/app/proxy/execution-step.go index c90b313..65d05ad 100644 --- a/src/app/proxy/execution-step.go +++ b/src/app/proxy/execution-step.go @@ -1,16 +1,26 @@ package proxy import ( + "path/filepath" + "github.com/snivilised/cobrass/src/clif" + "github.com/snivilised/extendio/xfs/nav" ) // Step -type Step interface { - Run(*SharedControllerInfo) error -} +type ( + RunStepInfo struct { + Item *nav.TraverseItem + Source string + } -// Sequence -type Sequence []Step + Step interface { + Run(rsi *RunStepInfo) error + } + + // Sequence + Sequence []Step +) // magickStep knows how to combine parameters together so that the program // can be invoked correctly; but it does not know how to compose the input @@ -27,8 +37,24 @@ type magickStep struct { } // Run -func (s *magickStep) Run(*SharedControllerInfo) error { - positional := []string{s.sourcePath} +func (s *magickStep) Run(rsi *RunStepInfo) error { + folder, file := s.shared.finder.Result(&resultInfo{ + pathInfo: pathInfo{ + item: rsi.Item, + origin: rsi.Item.Extension.Parent, + }, + scheme: s.scheme, + profile: s.profile, + }) + result := filepath.Join(folder, file) + input := []string{rsi.Source} + + // if transparent, then we need to ask the fm to move the + // existing file out of the way. But shouldn't that already have happened + // during setup? See, which mean setup in not working properly in + // this scenario. - return s.shared.program.Execute(clif.Expand(positional, s.thirdPartyCL, s.outputPath)...) + return s.shared.program.Execute( + clif.Expand(input, s.thirdPartyCL, result)..., + ) } diff --git a/src/app/proxy/file-manager.go b/src/app/proxy/file-manager.go index 521e078..5f28d94 100644 --- a/src/app/proxy/file-manager.go +++ b/src/app/proxy/file-manager.go @@ -11,7 +11,8 @@ import ( ) const ( - beezledub = os.FileMode(0o666) + beezledub = os.FileMode(0o666) + errorDestination = "" ) // FileManager knows how to translate requests into invocations on the file @@ -23,40 +24,57 @@ type FileManager struct { // Setup prepares for operation by moving existing file out of the way, // if applicable. -func (fm *FileManager) Setup(item *nav.TraverseItem) error { +func (fm *FileManager) Setup(item *nav.TraverseItem) (destination string, err error) { + if !fm.finder.transparentInput { + // Any result file must not clash with the input file, so the input + // file must stay in place + return item.Path, nil + } + // https://pkg.go.dev/os#Rename LinkError may result // // this might not be right. it may be that we want to leave the // original alone and create other outputs; in this scenario // we don't want to rename/move the source... // - from := &destinationInfo{ - item: item, - origin: item.Parent.Path, - transparent: true, // might come from a flag + from := &pathInfo{ + item: item, + origin: item.Parent.Path, } if folder, file := fm.finder.Destination(from); folder != "" { - if err := fm.vfs.MkdirAll(folder, beezledub); err != nil { - return errors.Wrapf(err, "could not create parent setup for '%v'", item.Path) + if err = fm.vfs.MkdirAll(folder, beezledub); err != nil { + return errorDestination, errors.Wrapf( + err, "could not create parent setup for '%v'", item.Path, + ) } - destination := filepath.Join(folder, file) + // THIS DESTINATION IS NOT REPORTED BACK + // TO BE USED AS THE INPUT + destination = filepath.Join(folder, file) if !fm.vfs.FileExists(item.Path) { - return fmt.Errorf("source file: '%v' does not exist", item.Path) + return errorDestination, fmt.Errorf( + "source file: '%v' does not exist", item.Path, + ) } - if fm.vfs.FileExists(destination) { - return fmt.Errorf("destination file: '%v' already exists", destination) - } - - if err := fm.vfs.Rename(item.Path, destination); err != nil { - return errors.Wrapf(err, "could not complete setup for '%v'", item.Path) + if item.Path != destination { + if fm.vfs.FileExists(destination) { + return errorDestination, fmt.Errorf( + "destination file: '%v' already exists", destination, + ) + } + + if err := fm.vfs.Rename(item.Path, destination); err != nil { + return errorDestination, errors.Wrapf( + err, "could not complete setup for '%v'", item.Path, + ) + } } } - return nil + return destination, nil } func (fm *FileManager) move(from, to string) error { diff --git a/src/app/proxy/path-finder.go b/src/app/proxy/path-finder.go index d653038..74c67d3 100644 --- a/src/app/proxy/path-finder.go +++ b/src/app/proxy/path-finder.go @@ -12,8 +12,11 @@ type pfPath uint const ( pfPathUndefined pfPath = iota - pfPathSetupInlineDestFolder - pfPathSetupInlineDestFileOriginalExt + pfPathInputDestinationFolder + pfPathTxInputDestinationFolder + pfPathInputDestinationFileOriginalExt + pfPathResultFolder + pfPathResultFile ) const ( @@ -30,18 +33,41 @@ var ( pfTemplates pfTemplatesCollection ) +/* +📚 FIELD DICTIONARY: +- ADHOC: (static): tag that indicates no profile or scheme is active +- INPUT-DESTINATION: the path where the input file is moved to +- ITEM-FULL-NAME: the original item.Name, which includes the original extension +- OUTPUT-ROOT: --output flag +- ITEM-SUB-PATH: item.Extension.SubPath +- RESULT-NAME: the path of the result file +- SUPPLEMENT: ${{ADHOC}} | / --> created dynamically +- TRASH-LABEL: (static) input file tag marked for deletion +*/ + func init() { pfTemplates = pfTemplatesCollection{ - // we probably have to come up with better key names... - // - pfPathSetupInlineDestFolder: templateSegments{ - "${{OUTPUT-ROOT}}", + pfPathInputDestinationFolder: templateSegments{ + "${{INPUT-DESTINATION}}", "${{ITEM-SUB-PATH}}", + "${{SUPPLEMENT}}", "${{TRASH-LABEL}}", }, + pfPathTxInputDestinationFolder: templateSegments{ + "${{OUTPUT-ROOT}}", + }, - pfPathSetupInlineDestFileOriginalExt: templateSegments{ - "${{ITEM-NAME-ORIG-EXT}}", + pfPathInputDestinationFileOriginalExt: templateSegments{ + "${{ITEM-FULL-NAME}}", + }, + + pfPathResultFolder: templateSegments{ + "${{OUTPUT-ROOT}}", + "${{ITEM-SUB-PATH}}", + "${{SUPPLEMENT}}", + }, + pfPathResultFile: templateSegments{ + "${{RESULT-NAME}}", }, } } @@ -71,6 +97,44 @@ func (tc pfTemplatesCollection) evaluate( return filepath.Clean(result) } +// eval returns a string representing a file system path from a +// template string containing place-holders and field values. +// +// Make sure that the keys of the values passed in match the segments. +// If they differ, then the result will contain unresolved segments (ie, +// 1 or more segments that are not evaluated and still contain the +// template placeholder.) +func (tc pfTemplatesCollection) eval( + values pfFieldValues, + segments ...string, +) string { + // There is a very subtle but important point to note about the eval + // method, in particular the parameters being passed in. It might seem + // to the reader that the segments being passed in is redundant, because + // they could be derived from the keys of the values map. However, map + // entries do not have a guaranteed iteration order. Only arrays are + // guaranteed to remain in the same order in which they were created. This + // is the purpose of the segments parameter; it dictates the order in which + // the segments of a path are evaluated. We can't even use the OrderedKeys + // map, because entries are sorted lexically, which is not what we want. + // + const ( + quantity = 1 + ) + + // expand + sourceTemplate := filepath.Join(segments...) + + // evaluate + result := lo.Reduce(segments, func(acc, field string, _ int) string { + return strings.Replace(acc, field, values[field], quantity) + }, + sourceTemplate, + ) + + return filepath.Clean(result) +} + // INLINE-MODE: EJECT | INLINE (should we call this a strategy? // they do the same thing but create a different output structure => OutputStrategy) // @@ -158,6 +222,9 @@ type PathFinder struct { // I think this depends on the mode (tidy/preserve) Trash string + arity int + transparentInput bool + behaviours strategies } @@ -166,11 +233,11 @@ type staticInfo struct { legacyLabel string } -type destinationInfo struct { +type pathInfo struct { item *nav.TraverseItem origin string // in:item.Parent.Path, ej:eject-path(output???) // statics *staticInfo - transparent bool + // // transparent=true should be the default scenario. This means // that any changes that occur leave the file system in a state @@ -210,16 +277,71 @@ type destinationInfo struct { // is extracted from the source path and attached to the output // folder. // -// should return empty string if no move is required -func (f *PathFinder) Destination(info *destinationInfo) (destinationFolder, destinationFile string) { +// Destination creates a path for the input; should return empty +// string for the folder, if no move is required (ie transparent) +// The PathFinder will only call this function when the input +// is not transparent +func (f *PathFinder) Destination(info *pathInfo) (folder, file string) { // TODO: we still need to get the rest of the mirror sub-path - // .///TRASH///<.item.Name>..ext // legacyLabel := "LEGACY" trashLabel := "TRASH" // this does not take into account transparent, without modification; // ie what happens if we don;t want any supplemented paths? + to := lo.TernaryF(f.Trash != "", + func() string { + return f.Trash // eject + }, + func() string { + return info.origin // inline + }, + ) + + folder = func() string { + segments := pfTemplates[pfPathInputDestinationFolder] + + // return lo.TernaryF(f.transparentInput && f.arity == 1, + // func() string { + // return to + // }, + // func() string { + // return pfTemplates.eval(pfFieldValues{ + // "${{INPUT-DESTINATION}}": to, + // "${{ITEM-SUB-PATH}}": info.item.Extension.SubPath, + // "${{TRASH-LABEL}}": trashLabel, + // }, segments...) + // }, + // ) + + return pfTemplates.eval(pfFieldValues{ + "${{INPUT-DESTINATION}}": to, + "${{ITEM-SUB-PATH}}": info.item.Extension.SubPath, + "${{SUPPLEMENT}}": f.supplement(), + "${{TRASH-LABEL}}": trashLabel, + }, segments...) + }() + + file = func() string { + segments := pfTemplates[pfPathInputDestinationFileOriginalExt] + + return pfTemplates.eval(pfFieldValues{ + "${{ITEM-FULL-NAME}}": info.item.Extension.Name, + }, segments...) + }() + + return folder, file +} + +type resultInfo struct { + pathInfo + scheme string + profile string +} + +// Result creates a path for each result so should be called by the +// execution step +func (f *PathFinder) Result(info *resultInfo) (folder, file string) { to := lo.TernaryF(f.Output != "", func() string { return f.Output // eject @@ -229,27 +351,60 @@ func (f *PathFinder) Destination(info *destinationInfo) (destinationFolder, dest }, ) - destinationFolder = func() string { - segments := pfTemplates[pfPathSetupInlineDestFolder] - path := pfTemplates.expand(filepath.Join(segments...)) - - return pfTemplates.evaluate(path, segments, pfFieldValues{ - "${{OUTPUT-ROOT}}": to, - "${{ITEM-SUB-PATH}}": info.item.Extension.SubPath, - "${{TRASH-LABEL}}": trashLabel, - }) + folder = func() string { + segments := pfTemplates[pfPathInputDestinationFolder] + + return lo.TernaryF(f.transparentInput && f.arity == 1, + func() string { + // The result file has to be in the same folder + // as the input + // + segments = pfTemplates[pfPathTxInputDestinationFolder] + + return pfTemplates.eval(pfFieldValues{ + "${{OUTPUT-ROOT}}": info.origin, + }, segments...) + }, + func() string { + // If there is no scheme of profile, then the user is + // only relying flags on the command line, ie running adhoc + // so the result path should include an adhoc label. Otherwise, + // the result should reflect the supplementary path. + // + + return pfTemplates.eval(pfFieldValues{ + "${{OUTPUT-ROOT}}": to, + "${{SUPPLEMENT}}": f.supplement(), + "${{ITEM-SUB-PATH}}": info.item.Extension.SubPath, + }, segments...) + }, + ) }() - destinationFile = func() string { - segments := pfTemplates[pfPathSetupInlineDestFileOriginalExt] - path := pfTemplates.expand(filepath.Join(segments...)) + file = func() string { + // The file name just matches the input file name. The folder name + // provides the context. + // + segments := pfTemplates[pfPathResultFile] - return pfTemplates.evaluate(path, segments, pfFieldValues{ - "${{ITEM-NAME-ORIG-EXT}}": info.item.Extension.Name, - }) + return pfTemplates.eval(pfFieldValues{ + "${{RESULT-NAME}}": info.item.Extension.Name, + }, segments...) }() - return destinationFolder, destinationFile + return folder, file +} + +func (f *PathFinder) supplement() string { + return lo.TernaryF(f.Scheme == "" && f.Profile == "", + func() string { + adhocLabel := "ADHOC" + return adhocLabel + }, + func() string { + return filepath.Join(f.Scheme, f.Profile) + }, + ) } /* diff --git a/src/app/proxy/proxy-defs.go b/src/app/proxy/proxy-defs.go index 5d60abd..d76f4c7 100644 --- a/src/app/proxy/proxy-defs.go +++ b/src/app/proxy/proxy-defs.go @@ -23,6 +23,10 @@ type SharedControllerInfo struct { fileManager *FileManager } +type localControllerInfo struct { + destination string +} + // ItemController type ItemController interface { OnNewShrinkItem(item *nav.TraverseItem,