From f8fbd69e3c1a5a9e96d86b77be1a5ac4e42891f1 Mon Sep 17 00:00:00 2001 From: William Perreault Date: Mon, 9 Sep 2024 20:37:19 -0400 Subject: [PATCH] Preprocessing stage (#25) * Implement Preprocessing Stage * Make specterutils DependencyResolutionProcessor and LintingProcessors implement the Preprocessors as well --- pkg/specter/pipeline.go | 50 ++++--- pkg/specter/pipelinedefault.go | 175 +++++++++++++++++----- pkg/specter/pipelinedefault_test.go | 125 +++++++++++++++- pkg/specter/pipelinedefaultexport_test.go | 1 + pkg/specter/unitloading.go | 16 +- pkg/specter/unitloading_test.go | 12 +- pkg/specter/unitpreproc.go | 41 +++++ pkg/specter/unitpreproc_test.go | 87 +++++++++++ pkg/specterutils/depresolve.go | 40 +++-- pkg/specterutils/linting.go | 38 ++++- 10 files changed, 497 insertions(+), 88 deletions(-) create mode 100644 pkg/specter/unitpreproc.go create mode 100644 pkg/specter/unitpreproc_test.go diff --git a/pkg/specter/pipeline.go b/pkg/specter/pipeline.go index b359057..28370d2 100644 --- a/pkg/specter/pipeline.go +++ b/pkg/specter/pipeline.go @@ -26,10 +26,12 @@ const ( StopAfterSourceLoadingStage RunMode = "stop-after-source-loading-stage" StopAfterUnitLoadingStage RunMode = "stop-after-unit-loading-stage" StopAfterUnitProcessingStage RunMode = "stop-after-unit-processing-stage" + StopAfterPreprocessingStage RunMode = "stop-after-preprocessing-stage" ) const SourceLoadingFailedErrorCode = "specter.source_loading_failed" const UnitLoadingFailedErrorCode = "specter.unit_loading_failed" +const UnitPreprocessingFailedErrorCode = "specter.unit_preprocessing_failed" const UnitProcessingFailedErrorCode = "specter.unit_processing_failed" const ArtifactProcessingFailedErrorCode = "specter.artifact_processing_failed" @@ -57,7 +59,7 @@ type PipelineContextData struct { } type Pipeline interface { - Run(ctx context.Context, sourceLocations []string, runMode RunMode) (result PipelineResult, err error) + Run(ctx context.Context, runMode RunMode, sourceLocations []string) (PipelineResult, error) } type SourceLoadingStage interface { @@ -65,45 +67,57 @@ type SourceLoadingStage interface { } type UnitLoadingStage interface { - Run(ctx PipelineContext, sources []Source) ([]Unit, error) + Run(PipelineContext, []Source) ([]Unit, error) +} + +type UnitPreprocessingStage interface { + Run(PipelineContext, []Unit) ([]Unit, error) } type UnitProcessingStage interface { - Run(ctx PipelineContext, units []Unit) ([]Artifact, error) + Run(PipelineContext, []Unit) ([]Artifact, error) } type ArtifactProcessingStage interface { - Run(ctx PipelineContext, artifacts []Artifact) error + Run(PipelineContext, []Artifact) error } type SourceLoadingStageHooks interface { - Before(ctx PipelineContext) error - After(ctx PipelineContext) error + Before(PipelineContext) error + After(PipelineContext) error BeforeSourceLocation(ctx PipelineContext, sourceLocation string) error AfterSourceLocation(ctx PipelineContext, sourceLocation string) error - OnError(ctx PipelineContext, err error) error + OnError(PipelineContext, error) error } type UnitLoadingStageHooks interface { - Before(ctx PipelineContext) error - After(ctx PipelineContext) error - BeforeSource(ctx PipelineContext, source Source) error - AfterSource(ctx PipelineContext, source Source) error - OnError(ctx PipelineContext, err error) error + Before(PipelineContext) error + After(PipelineContext) error + BeforeSource(PipelineContext, Source) error + AfterSource(PipelineContext, Source) error + OnError(PipelineContext, error) error +} + +type UnitPreprocessingStageHooks interface { + Before(PipelineContext) error + After(PipelineContext) error + BeforePreprocessor(ctx PipelineContext, preprocessorName string) error + AfterPreprocessor(ctx PipelineContext, preprocessorName string) error + OnError(PipelineContext, error) error } type UnitProcessingStageHooks interface { - Before(ctx PipelineContext) error - After(ctx PipelineContext) error + Before(PipelineContext) error + After(PipelineContext) error BeforeProcessor(ctx PipelineContext, processorName string) error AfterProcessor(ctx PipelineContext, processorName string) error - OnError(ctx PipelineContext, err error) error + OnError(PipelineContext, error) error } type ArtifactProcessingStageHooks interface { - Before(ctx PipelineContext) error - After(ctx PipelineContext) error + Before(PipelineContext) error + After(PipelineContext) error BeforeProcessor(ctx PipelineContext, processorName string) error AfterProcessor(ctx PipelineContext, processorName string) error - OnError(ctx PipelineContext, err error) error + OnError(PipelineContext, error) error } diff --git a/pkg/specter/pipelinedefault.go b/pkg/specter/pipelinedefault.go index ff716f1..5740140 100644 --- a/pkg/specter/pipelinedefault.go +++ b/pkg/specter/pipelinedefault.go @@ -26,12 +26,13 @@ type DefaultPipeline struct { SourceLoadingStage SourceLoadingStage UnitLoadingStage UnitLoadingStage + UnitPreprocessingStage UnitPreprocessingStage UnitProcessingStage UnitProcessingStage ArtifactProcessingStage ArtifactProcessingStage } // Run the DefaultPipeline from start to finish. -func (p DefaultPipeline) Run(ctx context.Context, sourceLocations []string, runMode RunMode) (PipelineResult, error) { +func (p DefaultPipeline) Run(ctx context.Context, runMode RunMode, sourceLocations []string) (PipelineResult, error) { if runMode == "" { runMode = RunThrough } @@ -70,6 +71,13 @@ func (p DefaultPipeline) run(pctx *PipelineContext, sourceLocations []string, ru return nil } + if err := p.runUnitPreprocessingStage(pctx); err != nil { + return err + } + if runMode == StopAfterPreprocessingStage { + return nil + } + if err := p.runUnitProcessingStage(pctx); err != nil { return err } @@ -83,14 +91,14 @@ func (p DefaultPipeline) run(pctx *PipelineContext, sourceLocations []string, ru return nil } -func (p DefaultPipeline) runSourceLoadingStage(pctx *PipelineContext, sourceLocations []string) error { - if err := pctx.Err(); err != nil { +func (p DefaultPipeline) runSourceLoadingStage(ctx *PipelineContext, sourceLocations []string) error { + if err := ctx.Err(); err != nil { return err } var err error if p.SourceLoadingStage != nil { - pctx.Sources, err = p.SourceLoadingStage.Run(*pctx, sourceLocations) + ctx.Sources, err = p.SourceLoadingStage.Run(*ctx, sourceLocations) if err != nil { return errors.WrapWithMessage(err, SourceLoadingFailedErrorCode, "failed loading sources") } @@ -99,14 +107,14 @@ func (p DefaultPipeline) runSourceLoadingStage(pctx *PipelineContext, sourceLoca return nil } -func (p DefaultPipeline) runUnitLoadingStage(pctx *PipelineContext) error { - if err := pctx.Err(); err != nil { +func (p DefaultPipeline) runUnitLoadingStage(ctx *PipelineContext) error { + if err := ctx.Err(); err != nil { return err } var err error if p.UnitLoadingStage != nil { - pctx.Units, err = p.UnitLoadingStage.Run(*pctx, pctx.Sources) + ctx.Units, err = p.UnitLoadingStage.Run(*ctx, ctx.Sources) if err != nil { return errors.WrapWithMessage(err, UnitLoadingFailedErrorCode, "failed loading units") } @@ -115,14 +123,14 @@ func (p DefaultPipeline) runUnitLoadingStage(pctx *PipelineContext) error { return nil } -func (p DefaultPipeline) runUnitProcessingStage(pctx *PipelineContext) error { - if err := pctx.Err(); err != nil { +func (p DefaultPipeline) runUnitProcessingStage(ctx *PipelineContext) error { + if err := ctx.Err(); err != nil { return err } - var err error if p.UnitProcessingStage != nil { - pctx.Artifacts, err = p.UnitProcessingStage.Run(*pctx, pctx.Units) + var err error + ctx.Artifacts, err = p.UnitProcessingStage.Run(*ctx, ctx.Units) if err != nil { return errors.WrapWithMessage(err, UnitProcessingFailedErrorCode, "failed processing units") } @@ -130,32 +138,33 @@ func (p DefaultPipeline) runUnitProcessingStage(pctx *PipelineContext) error { return nil } -func (p DefaultPipeline) runArtifactProcessingStage(pctx *PipelineContext) error { - if err := pctx.Err(); err != nil { +func (p DefaultPipeline) runArtifactProcessingStage(ctx *PipelineContext) error { + if err := ctx.Err(); err != nil { return err } if p.ArtifactProcessingStage != nil { - if err := p.ArtifactProcessingStage.Run(*pctx, pctx.Artifacts); err != nil { + if err := p.ArtifactProcessingStage.Run(*ctx, ctx.Artifacts); err != nil { return errors.WrapWithMessage(err, ArtifactProcessingFailedErrorCode, "failed processing artifacts") } } return nil } -type SourceLoadingStageHooksAdapter struct{} +func (p DefaultPipeline) runUnitPreprocessingStage(ctx *PipelineContext) error { + if err := ctx.Err(); err != nil { + return err + } -func (_ SourceLoadingStageHooksAdapter) Before(_ PipelineContext) error { return nil } -func (_ SourceLoadingStageHooksAdapter) After(_ PipelineContext) error { return nil } -func (_ SourceLoadingStageHooksAdapter) BeforeSourceLocation(_ PipelineContext, _ string) error { - return nil -} -func (_ SourceLoadingStageHooksAdapter) AfterSourceLocation(_ PipelineContext, _ string) error { + if p.UnitPreprocessingStage != nil { + var err error + ctx.Units, err = p.UnitPreprocessingStage.Run(*ctx, ctx.Units) + if err != nil { + return errors.WrapWithMessage(err, UnitPreprocessingFailedErrorCode, "failed preprocessing units") + } + } return nil } -func (_ SourceLoadingStageHooksAdapter) OnError(_ PipelineContext, err error) error { - return err -} type sourceLoadingStage struct { SourceLoaders []SourceLoader @@ -212,7 +221,7 @@ func (s sourceLoadingStage) run(ctx PipelineContext, sourceLocations []string) ( return ctx.Sources, errors.GroupOrNil(errs) } -func (s sourceLoadingStage) processSourceLocation(ctx PipelineContext, sl string) ([]Source, error) { +func (s sourceLoadingStage) processSourceLocation(_ PipelineContext, sl string) ([]Source, error) { var sources []Source for _, l := range s.SourceLoaders { if !l.Supports(sl) { @@ -227,26 +236,19 @@ func (s sourceLoadingStage) processSourceLocation(ctx PipelineContext, sl string return sources, nil } -type UnitLoadingStageHooksAdapter struct{} - -func (_ UnitLoadingStageHooksAdapter) Before(_ PipelineContext) error { return nil } -func (_ UnitLoadingStageHooksAdapter) After(_ PipelineContext) error { return nil } -func (_ UnitLoadingStageHooksAdapter) BeforeSource(_ PipelineContext, _ Source) error { return nil } -func (_ UnitLoadingStageHooksAdapter) AfterSource(_ PipelineContext, _ Source) error { return nil } -func (_ UnitLoadingStageHooksAdapter) OnError(_ PipelineContext, err error) error { return err } - -type UnitProcessingStageHooksAdapter struct { -} +type SourceLoadingStageHooksAdapter struct{} -func (_ UnitProcessingStageHooksAdapter) Before(_ PipelineContext) error { return nil } -func (_ UnitProcessingStageHooksAdapter) After(_ PipelineContext) error { return nil } -func (_ UnitProcessingStageHooksAdapter) BeforeProcessor(_ PipelineContext, _ string) error { +func (_ SourceLoadingStageHooksAdapter) Before(_ PipelineContext) error { return nil } +func (_ SourceLoadingStageHooksAdapter) After(_ PipelineContext) error { return nil } +func (_ SourceLoadingStageHooksAdapter) BeforeSourceLocation(_ PipelineContext, _ string) error { return nil } -func (_ UnitProcessingStageHooksAdapter) AfterProcessor(_ PipelineContext, _ string) error { +func (_ SourceLoadingStageHooksAdapter) AfterSourceLocation(_ PipelineContext, _ string) error { return nil } -func (_ UnitProcessingStageHooksAdapter) OnError(_ PipelineContext, err error) error { return err } +func (_ SourceLoadingStageHooksAdapter) OnError(_ PipelineContext, err error) error { + return err +} type unitLoadingStage struct { Loaders []UnitLoader @@ -314,6 +316,84 @@ func (s unitLoadingStage) runLoader(src Source) ([]Unit, error) { return units, nil } +type UnitLoadingStageHooksAdapter struct{} + +func (_ UnitLoadingStageHooksAdapter) Before(_ PipelineContext) error { return nil } +func (_ UnitLoadingStageHooksAdapter) After(_ PipelineContext) error { return nil } +func (_ UnitLoadingStageHooksAdapter) BeforeSource(_ PipelineContext, _ Source) error { return nil } +func (_ UnitLoadingStageHooksAdapter) AfterSource(_ PipelineContext, _ Source) error { return nil } +func (_ UnitLoadingStageHooksAdapter) OnError(_ PipelineContext, err error) error { return err } + +type unitPreprocessingStage struct { + Preprocessors []UnitPreprocessor + Hooks UnitPreprocessingStageHooks +} + +func (s unitPreprocessingStage) Run(ctx PipelineContext, units []Unit) ([]Unit, error) { + if s.Hooks == nil { + s.Hooks = UnitPreprocessingStageHooksAdapter{} + } + + if err := s.Hooks.Before(ctx); err != nil { + return nil, newFailedToRunHookErr(err, "Before") + } + + units, err := s.run(ctx, units) + if err != nil { + return nil, s.Hooks.OnError(ctx, errors.WrapWithMessage(err, UnitProcessingFailedErrorCode, "failed preprocessing units")) + } + + if err := s.Hooks.After(ctx); err != nil { + return nil, newFailedToRunHookErr(err, "After") + } + + return units, nil +} + +func (s unitPreprocessingStage) run(ctx PipelineContext, units []Unit) ([]Unit, error) { + var err error + for _, p := range s.Preprocessors { + if err := ctx.Err(); err != nil { + return nil, err + } + + if err := s.Hooks.BeforePreprocessor(ctx, p.Name()); err != nil { + return nil, newFailedToRunHookErr(err, "BeforePreprocessor") + } + + units, err = p.Preprocess(ctx, units) + if err != nil { + return nil, fmt.Errorf("preprocessor %q returned an error: %w", p.Name(), err) + } + + units = append(ctx.Units, units...) + ctx.Units = units + + if err := s.Hooks.AfterPreprocessor(ctx, p.Name()); err != nil { + return nil, newFailedToRunHookErr(err, "AfterPreprocessor") + } + } + + return units, nil +} + +type UnitPreprocessingStageHooksAdapter struct { +} + +func (u UnitPreprocessingStageHooksAdapter) Before(PipelineContext) error { return nil } + +func (u UnitPreprocessingStageHooksAdapter) After(PipelineContext) error { return nil } + +func (u UnitPreprocessingStageHooksAdapter) BeforePreprocessor(PipelineContext, string) error { + return nil +} + +func (u UnitPreprocessingStageHooksAdapter) AfterPreprocessor(PipelineContext, string) error { + return nil +} + +func (u UnitPreprocessingStageHooksAdapter) OnError(_ PipelineContext, err error) error { return err } + type unitProcessingStage struct { Processors []UnitProcessor Hooks UnitProcessingStageHooks @@ -356,7 +436,7 @@ func (s unitProcessingStage) run(ctx PipelineContext, units []Unit) ([]Artifact, Artifacts: ctx.Artifacts, }) if err != nil { - return nil, fmt.Errorf("processor %q returned an error :%w", processor.Name(), err) + return nil, fmt.Errorf("processor %q returned an error: %w", processor.Name(), err) } ctx.Artifacts = append(ctx.Artifacts, artifacts...) @@ -369,6 +449,19 @@ func (s unitProcessingStage) run(ctx PipelineContext, units []Unit) ([]Artifact, return ctx.Artifacts, nil } +type UnitProcessingStageHooksAdapter struct { +} + +func (_ UnitProcessingStageHooksAdapter) Before(_ PipelineContext) error { return nil } +func (_ UnitProcessingStageHooksAdapter) After(_ PipelineContext) error { return nil } +func (_ UnitProcessingStageHooksAdapter) BeforeProcessor(_ PipelineContext, _ string) error { + return nil +} +func (_ UnitProcessingStageHooksAdapter) AfterProcessor(_ PipelineContext, _ string) error { + return nil +} +func (_ UnitProcessingStageHooksAdapter) OnError(_ PipelineContext, err error) error { return err } + type ArtifactProcessingStageHooksAdapter struct { } diff --git a/pkg/specter/pipelinedefault_test.go b/pkg/specter/pipelinedefault_test.go index 5f29814..c52f79c 100644 --- a/pkg/specter/pipelinedefault_test.go +++ b/pkg/specter/pipelinedefault_test.go @@ -377,7 +377,7 @@ func TestDefaultPipeline_Run(t *testing.T) { UnitProcessingStage: tt.given.UnitProcessingStage, ArtifactProcessingStage: tt.given.ArtifactProcessingStage, } - got, err := p.Run(tt.when.ctx, tt.when.sourceLocations, tt.when.runMode) + got, err := p.Run(tt.when.ctx, tt.when.runMode, tt.when.sourceLocations) tt.then.expectedError(t, err) assert.Equal(t, tt.then.expectedResult, got) }) @@ -484,7 +484,7 @@ func Test_unitLoadingStage_Run(t *testing.T) { stage := specter.DefaultUnitLoadingStage{ Loaders: []specter.UnitLoader{ - specter.FunctionalUnitLoader{ + specter.UnitLoaderAdapter{ LoadFunc: func(s specter.Source) ([]specter.Unit, error) { return nil, nil }, @@ -515,7 +515,7 @@ func Test_unitLoadingStage_Run(t *testing.T) { stage := specter.DefaultUnitLoadingStage{ Loaders: []specter.UnitLoader{ - specter.FunctionalUnitLoader{ + specter.UnitLoaderAdapter{ LoadFunc: func(s specter.Source) ([]specter.Unit, error) { return nil, assert.AnError }, @@ -549,7 +549,7 @@ func Test_unitLoadingStage_Run(t *testing.T) { } stage := specter.DefaultUnitLoadingStage{ Loaders: []specter.UnitLoader{ - specter.FunctionalUnitLoader{ + specter.UnitLoaderAdapter{ LoadFunc: func(s specter.Source) ([]specter.Unit, error) { return expectedUnits, nil }, @@ -571,6 +571,83 @@ func Test_unitLoadingStage_Run(t *testing.T) { }) } +func Test_unitPreprocessingStage_Run(t *testing.T) { + t.Run("should call all hooks under normal processing", func(t *testing.T) { + recorder := unitPreprocessingStageHooksCallRecorder{} + + expectedUnits := []specter.Unit{ + testutils.NewUnitStub("id", "kind", specter.Source{}), + } + stage := specter.DefaultUnitPreprocessingStage{ + Preprocessors: []specter.UnitPreprocessor{ + specter.UnitPreprocessorFunc("unit-tester", func(pipelineContext specter.PipelineContext, units []specter.Unit) ([]specter.Unit, error) { + return units, nil + }), + }, + Hooks: &recorder, + } + + units, err := stage.Run(specter.PipelineContext{Context: context.Background()}, expectedUnits) + require.NoError(t, err) + require.Equal(t, expectedUnits, units) + + assert.True(t, recorder.beforeCalled) + assert.True(t, recorder.beforePreprocessorCalled) + assert.Equal(t, "unit-tester", recorder.beforePreprocessorName) + assert.True(t, recorder.afterPreprocessorCalled) + assert.Equal(t, "unit-tester", recorder.afterPreprocessorName) + assert.True(t, recorder.afterCalled) + }) + + t.Run("should call hooks until error", func(t *testing.T) { + recorder := unitPreprocessingStageHooksCallRecorder{} + + stage := specter.DefaultUnitPreprocessingStage{ + Preprocessors: []specter.UnitPreprocessor{ + specter.UnitPreprocessorFunc("unit-tester", func(pipelineContext specter.PipelineContext, units []specter.Unit) ([]specter.Unit, error) { + return units, assert.AnError + }), + }, + Hooks: &recorder, + } + + units, err := stage.Run(specter.PipelineContext{Context: context.Background()}, []specter.Unit{ + testutils.NewUnitStub("id", "kind", specter.Source{}), + }) + require.Error(t, err) + require.Nil(t, units) + + assert.True(t, recorder.beforeCalled) + assert.True(t, recorder.beforePreprocessorCalled) + assert.Equal(t, "unit-tester", recorder.beforePreprocessorName) + assert.False(t, recorder.afterPreprocessorCalled) + assert.Equal(t, "", recorder.afterPreprocessorName) + assert.False(t, recorder.afterCalled) + assert.True(t, recorder.onErrorCalled) + }) + + t.Run("should return the loaded units", func(t *testing.T) { + + recorder := unitPreprocessingStageHooksCallRecorder{} + + expectedUnits := []specter.Unit{ + testutils.NewUnitStub("id", "kind", specter.Source{}), + } + stage := specter.DefaultUnitPreprocessingStage{ + Preprocessors: []specter.UnitPreprocessor{ + specter.UnitPreprocessorFunc("unit-tester", func(pipelineContext specter.PipelineContext, units []specter.Unit) ([]specter.Unit, error) { + return units, nil + }), + }, + Hooks: &recorder, + } + + units, err := stage.Run(specter.PipelineContext{Context: context.Background()}, expectedUnits) + require.NoError(t, err) + require.Equal(t, expectedUnits, units) + }) +} + func Test_unitProcessingStage_Run(t *testing.T) { t.Run("should call all hooks under normal processing", func(t *testing.T) { recorder := unitProcessingStageHooksCallRecorder{} @@ -801,6 +878,46 @@ func (s *sourceLoadingStageHooksCallRecorder) OnError(_ specter.PipelineContext, return err } +type unitPreprocessingStageHooksCallRecorder struct { + beforeCalled bool + afterCalled bool + + beforePreprocessorCalled bool + beforePreprocessorName string + + afterPreprocessorCalled bool + afterPreprocessorName string + + onErrorCalled bool +} + +func (u *unitPreprocessingStageHooksCallRecorder) Before(specter.PipelineContext) error { + u.beforeCalled = true + return nil +} + +func (u *unitPreprocessingStageHooksCallRecorder) After(specter.PipelineContext) error { + u.afterCalled = true + return nil +} + +func (u *unitPreprocessingStageHooksCallRecorder) BeforePreprocessor(_ specter.PipelineContext, preprocessorName string) error { + u.beforePreprocessorCalled = true + u.beforePreprocessorName = preprocessorName + return nil +} + +func (u *unitPreprocessingStageHooksCallRecorder) AfterPreprocessor(_ specter.PipelineContext, preprocessorName string) error { + u.afterPreprocessorCalled = true + u.afterPreprocessorName = preprocessorName + return nil +} + +func (u *unitPreprocessingStageHooksCallRecorder) OnError(_ specter.PipelineContext, err error) error { + u.onErrorCalled = true + return err +} + type unitProcessingStageHooksCallRecorder struct { beforeCalled bool afterCalled bool diff --git a/pkg/specter/pipelinedefaultexport_test.go b/pkg/specter/pipelinedefaultexport_test.go index 3cd4e44..c7114e1 100644 --- a/pkg/specter/pipelinedefaultexport_test.go +++ b/pkg/specter/pipelinedefaultexport_test.go @@ -16,5 +16,6 @@ package specter type DefaultSourceLoadingStage = sourceLoadingStage type DefaultUnitLoadingStage = unitLoadingStage +type DefaultUnitPreprocessingStage = unitPreprocessingStage type DefaultUnitProcessingStage = unitProcessingStage type DefaultArtifactProcessingStage = artifactProcessingStage diff --git a/pkg/specter/unitloading.go b/pkg/specter/unitloading.go index 4c53fe7..36347a7 100644 --- a/pkg/specter/unitloading.go +++ b/pkg/specter/unitloading.go @@ -185,15 +185,21 @@ func MapUnitGroup[T any](g UnitGroup, p func(u Unit) T) []T { return mapped } -type FunctionalUnitLoader struct { +type UnitLoaderAdapter struct { LoadFunc func(s Source) ([]Unit, error) SupportsSourceFunc func(s Source) bool } -func (u FunctionalUnitLoader) Load(s Source) ([]Unit, error) { - return u.LoadFunc(s) +func (u UnitLoaderAdapter) Load(s Source) ([]Unit, error) { + if u.LoadFunc != nil { + return u.LoadFunc(s) + } + return nil, nil } -func (u FunctionalUnitLoader) SupportsSource(s Source) bool { - return u.SupportsSourceFunc(s) +func (u UnitLoaderAdapter) SupportsSource(s Source) bool { + if u.SupportsSourceFunc != nil { + return u.SupportsSourceFunc(s) + } + return false } diff --git a/pkg/specter/unitloading_test.go b/pkg/specter/unitloading_test.go index 8b63f1e..8c45fe5 100644 --- a/pkg/specter/unitloading_test.go +++ b/pkg/specter/unitloading_test.go @@ -376,7 +376,7 @@ func TestUnitWithIDsMatcher(t *testing.T) { } } -func TestFunctionalUnitLoader(t *testing.T) { +func TestUnitLoaderAdapter(t *testing.T) { t.Run("functions should be called", func(t *testing.T) { expectedSource := specter.Source{ Location: "/path/to/file", @@ -385,7 +385,7 @@ func TestFunctionalUnitLoader(t *testing.T) { specter.UnitOf(0, "", "", expectedSource), } - f := specter.FunctionalUnitLoader{ + f := specter.UnitLoaderAdapter{ LoadFunc: func(s specter.Source) ([]specter.Unit, error) { assert.Equal(t, expectedSource, s) return expectedUnits, nil @@ -401,4 +401,12 @@ func TestFunctionalUnitLoader(t *testing.T) { require.NoError(t, err) assert.Equal(t, expectedUnits, units) }) + t.Run("functions should not fail if no parameters provided", func(t *testing.T) { + f := specter.UnitLoaderAdapter{} + assert.False(t, f.SupportsSource(specter.Source{})) + + units, err := f.Load(specter.Source{}) + require.NoError(t, err) + assert.Nil(t, units) + }) } diff --git a/pkg/specter/unitpreproc.go b/pkg/specter/unitpreproc.go new file mode 100644 index 0000000..a4b844b --- /dev/null +++ b/pkg/specter/unitpreproc.go @@ -0,0 +1,41 @@ +// Copyright 2024 Morébec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package specter + +type UnitPreprocessor interface { + Preprocess(PipelineContext, []Unit) ([]Unit, error) + Name() string +} + +type UnitPreprocessorAdapter struct { + PreprocessFunc func(PipelineContext, []Unit) ([]Unit, error) + name string +} + +func (u UnitPreprocessorAdapter) Preprocess(ctx PipelineContext, units []Unit) ([]Unit, error) { + if u.PreprocessFunc == nil { + return units, nil + } + + return u.PreprocessFunc(ctx, units) +} + +func (u UnitPreprocessorAdapter) Name() string { + return u.name +} + +func UnitPreprocessorFunc(name string, processFunc func(PipelineContext, []Unit) ([]Unit, error)) *UnitPreprocessorAdapter { + return &UnitPreprocessorAdapter{PreprocessFunc: processFunc, name: name} +} diff --git a/pkg/specter/unitpreproc_test.go b/pkg/specter/unitpreproc_test.go new file mode 100644 index 0000000..09c258f --- /dev/null +++ b/pkg/specter/unitpreproc_test.go @@ -0,0 +1,87 @@ +// Copyright 2024 Morébec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package specter_test + +import ( + "github.com/morebec/specter/pkg/specter" + "github.com/morebec/specter/pkg/testutils" + "github.com/stretchr/testify/require" + "testing" +) + +func TestUnitPreprocessorAdapter_Process(t *testing.T) { + type given struct { + PreprocessFunc func(specter.PipelineContext, []specter.Unit) ([]specter.Unit, error) + } + type when struct { + ctx specter.PipelineContext + units []specter.Unit + } + type then struct { + units []specter.Unit + err require.ErrorAssertionFunc + } + tests := []struct { + name string + given given + when when + then then + }{ + { + name: "GIVEN no PreprocessFunc WHEN units THEN return same units and no error", + given: given{ + PreprocessFunc: nil, + }, + when: when{ + units: []specter.Unit{ + testutils.NewUnitStub("id", "kind", specter.Source{}), + }, + }, + then: then{ + units: []specter.Unit{ + testutils.NewUnitStub("id", "kind", specter.Source{}), + }, + err: require.NoError, + }, + }, + { + name: "GIVEN PreprocessFunc returns specific_units WHEN units THEN return specific_units and no error", + given: given{ + PreprocessFunc: func(specter.PipelineContext, []specter.Unit) ([]specter.Unit, error) { + return []specter.Unit{ + testutils.NewUnitStub("id", "kind", specter.Source{}), + }, nil + }, + }, + when: when{ + units: nil, + }, + then: then{ + units: []specter.Unit{ + testutils.NewUnitStub("id", "kind", specter.Source{}), + }, + err: require.NoError, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := specter.UnitPreprocessorFunc("preprocessor", tt.given.PreprocessFunc) + got, err := p.Preprocess(tt.when.ctx, tt.when.units) + tt.then.err(t, err) + require.Equal(t, tt.then.units, got) + }) + } +} diff --git a/pkg/specterutils/depresolve.go b/pkg/specterutils/depresolve.go index f8546c4..c236857 100644 --- a/pkg/specterutils/depresolve.go +++ b/pkg/specterutils/depresolve.go @@ -25,6 +25,10 @@ const DependencyResolutionFailed = "specter.dependency_resolution_failed" const ResolvedDependenciesArtifactID = "_resolved_dependencies" +func GetResolvedDependenciesFromContext(ctx specter.UnitProcessingContext) ResolvedDependencies { + return specter.GetContextArtifact[ResolvedDependencies](ctx, ResolvedDependenciesArtifactID) +} + // ResolvedDependencies represents an ordered list of Unit that should be // processed in that specific order based on their dependencies. type ResolvedDependencies specter.UnitGroup @@ -40,6 +44,11 @@ type DependencyProvider interface { var _ specter.UnitProcessor = DependencyResolutionProcessor{} +// DependencyResolutionProcessor is both a specter.UnitPreprocessor and a specter.UnitProcessor that resolves the dependencies +// of units based on specific DependencyProvider. +// When used as a specter.UnitPreprocessor units will be topologically sorted as a result in the pipeline. +// When used as a specter.UnitProcessor the ResolvedDependencies will be available as a specter.Artifact under the key ResolvedDependenciesArtifactID. +// A helper function GetResolvedDependenciesFromContext can be used in other processors to get access to the artifacts. type DependencyResolutionProcessor struct { providers []DependencyProvider } @@ -52,15 +61,31 @@ func (p DependencyResolutionProcessor) Name() string { return "dependency_resolution_processor" } +func (p DependencyResolutionProcessor) Preprocess(context specter.PipelineContext, units []specter.Unit) ([]specter.Unit, error) { + deps, err := p.resolveDependencies(units) + if err != nil { + return nil, err + } + return deps, nil +} + func (p DependencyResolutionProcessor) Process(ctx specter.UnitProcessingContext) ([]specter.Artifact, error) { + deps, err := p.resolveDependencies(ctx.Units) + if err != nil { + return nil, err + } + return []specter.Artifact{deps}, nil +} + +func (p DependencyResolutionProcessor) resolveDependencies(units []specter.Unit) (ResolvedDependencies, error) { var nodes []dependencyNode - for _, s := range ctx.Units { - node := dependencyNode{Unit: s, Dependencies: nil} + for _, u := range units { + node := dependencyNode{Unit: u, Dependencies: nil} for _, provider := range p.providers { - if !provider.Supports(s) { + if !provider.Supports(u) { continue } - deps := provider.Provide(s) + deps := provider.Provide(u) node.Dependencies = newDependencySet(deps...) break } @@ -71,12 +96,7 @@ func (p DependencyResolutionProcessor) Process(ctx specter.UnitProcessingContext if err != nil { return nil, errors.WrapWithMessage(err, DependencyResolutionFailed, "failed resolving dependencies") } - - return []specter.Artifact{deps}, nil -} - -func GetResolvedDependenciesFromContext(ctx specter.UnitProcessingContext) ResolvedDependencies { - return specter.GetContextArtifact[ResolvedDependencies](ctx, ResolvedDependenciesArtifactID) + return deps, nil } type dependencySet map[specter.UnitID]struct{} diff --git a/pkg/specterutils/linting.go b/pkg/specterutils/linting.go index a15a4d0..1f68d63 100644 --- a/pkg/specterutils/linting.go +++ b/pkg/specterutils/linting.go @@ -36,6 +36,7 @@ const ( WarningSeverity LinterResultSeverity = "warning" ) +// LintingProcessor is both a specter.UnitProcessor and specter.UnitPreprocessor that allows to lint a the loaded specter.Unit. type LintingProcessor struct { linters []UnitLinter Logger specter.Logger @@ -49,17 +50,37 @@ func (l LintingProcessor) Name() string { return "linting_processor" } -func (l LintingProcessor) Process(ctx specter.UnitProcessingContext) (artifacts []specter.Artifact, err error) { +func (l LintingProcessor) Preprocess(ctx specter.PipelineContext, units []specter.Unit) ([]specter.Unit, error) { if l.Logger == nil { l.Logger = specter.NewDefaultLogger(specter.DefaultLoggerConfig{Writer: io.Discard}) } - linter := CompositeUnitLinter(l.linters...) + lr := l.lintUnits(ctx.Units) + if err := l.handleLinterResultSet(lr); err != nil { + return nil, err + } + + return units, nil +} + +func (l LintingProcessor) Process(ctx specter.UnitProcessingContext) (artifacts []specter.Artifact, err error) { + if l.Logger == nil { + l.Logger = specter.NewDefaultLogger(specter.DefaultLoggerConfig{Writer: io.Discard}) + } - lr := linter.Lint(ctx.Units) + lr := l.lintUnits(ctx.Units) artifacts = append(artifacts, lr) + if err = l.handleLinterResultSet(lr); err != nil { + return artifacts, err + } + + return artifacts, err +} + +func (l LintingProcessor) handleLinterResultSet(lr LinterResultSet) error { + var err error if lr.HasWarnings() { for _, w := range lr.Warnings() { l.Logger.Warning(fmt.Sprintf("Warning: %s\n", w.Message)) @@ -72,13 +93,14 @@ func (l LintingProcessor) Process(ctx specter.UnitProcessingContext) (artifacts } err = lr.Errors() } + return err +} - if !lr.HasWarnings() && !lr.HasErrors() { - l.Logger.Success("Units linted successfully.") - } - - return artifacts, err +func (l LintingProcessor) lintUnits(units []specter.Unit) LinterResultSet { + linter := CompositeUnitLinter(l.linters...) + lr := linter.Lint(units) + return lr } func GetLintingResultsFromContext(ctx specter.UnitProcessingContext) LinterResultSet {