Skip to content

Commit

Permalink
Merge pull request #1315 from merico-dev/feat-apps-cicd-pass-vars
Browse files Browse the repository at this point in the history
feat: pass variable from ci to cd
  • Loading branch information
daniel-hutao authored Dec 7, 2022
2 parents 2349a04 + e093747 commit 593ccc2
Show file tree
Hide file tree
Showing 26 changed files with 690 additions and 290 deletions.
29 changes: 29 additions & 0 deletions docs/plugins/argocdapp.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,32 @@ In the example above:
- We used `repo-scaffolding.golang-github`'s output as input for the `githubactions-golang` plugin.

Pay attention to the `${{ xxx }}` part in the example. `${{ TOOL_NAME.PLUGIN.outputs.var}}` is the syntax for using an output.

## Automatically Create Helm Configuration

This plugin can push helm configuration automatically when your source.path helm config not exist, so you can use this plugin with helm configured alreay. For example:

```yaml
---
tools:
- name: go-webapp-argocd-deploy
plugin: argocdapp
dependsOn: ["repo-scaffolding.golang-github"]
options:
app:
name: hello
namespace: argocd
destination:
server: https://kubernetes.default.svc
namespace: default
source:
valuefile: values.yaml
path: charts/go-hello-http
repoURL: ${{repo-scaffolding.golang-github.outputs.repoURL}}
imageRepo:
url: http://test.barbor.com/library
user: test_owner
tag: "1.0.0"
```

This config will push default helm config to repo `${{repo-scaffolding.golang-github.outputs.repoURL}}`, and the generated config will use image `http://test.barbor.com/library/test_owner/hello:1.0.0` as inital image for helm.
92 changes: 59 additions & 33 deletions internal/pkg/configmanager/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/devstream-io/devstream/pkg/util/log"
"github.com/devstream-io/devstream/pkg/util/scm"
"github.com/devstream-io/devstream/pkg/util/scm/git"
)

const (
Expand All @@ -23,41 +24,49 @@ type app struct {
RepoTemplate *repoTemplate `yaml:"repoTemplate" mapstructure:"repoTemplate"`
CIRawConfigs []pipelineRaw `yaml:"ci" mapstructure:"ci"`
CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"`

// these two variables is used internal for convince
// repoInfo is generated from Repo field with setDefault method
repoInfo *git.RepoInfo `yaml:"-" mapstructure:"-"`
// repoTemplateInfo is generated from RepoTemplate field with setDefault method
repoTemplateInfo *git.RepoInfo `yaml:"-" mapstructure:"-"`
}

func (a *app) getTools(vars map[string]any, templateMap map[string]string) (Tools, error) {
// generate app repo and template repo from scmInfo
a.setDefault()
repoScaffoldingTool, err := a.getRepoTemplateTool()
if err != nil {
return nil, fmt.Errorf("app[%s] can't get valid repo config: %w", a.Name, err)
// 1. set app default field repoInfo and repoTemplateInfo
if err := a.setDefault(); err != nil {
return nil, err
}

// get ci/cd pipelineTemplates
// 2. get ci/cd pipelineTemplates
appVars := a.Spec.merge(vars)
tools, err := a.generateCICDTools(templateMap, appVars)
if err != nil {
return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", a.Name, err)
}

// 3. generate app repo and template repo from scmInfo
repoScaffoldingTool := a.generateRepoTemplateTool()
if repoScaffoldingTool != nil {
tools = append(tools, repoScaffoldingTool)
}

log.Debugf("Have got %d tools from app %s.", len(tools), a.Name)

return tools, nil
}

// getAppPipelineTool generate ci/cd tools from app config
// generateAppPipelineTool generate ci/cd tools from app config
func (a *app) generateCICDTools(templateMap map[string]string, appVars map[string]any) (Tools, error) {
allPipelineRaw := append(a.CIRawConfigs, a.CDRawConfigs...)
var tools Tools
// pipelineGlobalVars is used to pass variable from ci/cd pipelines
pipelineGlobalVars := a.newPipelineGlobalOptionFromApp()
for _, p := range allPipelineRaw {
t, err := p.getPipelineTemplate(templateMap, appVars)
if err != nil {
return nil, err
}
pipelineTool, err := t.generatePipelineTool(a)
t.updatePipelineVars(pipelineGlobalVars)
pipelineTool, err := t.generatePipelineTool(pipelineGlobalVars)
if err != nil {
return nil, err
}
Expand All @@ -67,41 +76,48 @@ func (a *app) generateCICDTools(templateMap map[string]string, appVars map[strin
return tools, nil
}

// getRepoTemplateTool will use repo-scaffolding plugin for app
func (a *app) getRepoTemplateTool() (*Tool, error) {
if a.Repo == nil {
return nil, fmt.Errorf("app.repo field can't be empty")
}
appRepo, err := a.Repo.BuildRepoInfo()
if err != nil {
return nil, fmt.Errorf("configmanager[app] parse repo failed: %w", err)
}
if a.RepoTemplate != nil {
// generateRepoTemplateTool will use repo-scaffolding plugin for app
func (a *app) generateRepoTemplateTool() *Tool {
if a.repoTemplateInfo != nil {
templateVars := make(RawOptions)
// templateRepo doesn't need auth info
templateRepo, err := a.RepoTemplate.BuildRepoInfo()
templateRepo.NeedAuth = false
if err != nil {
return nil, fmt.Errorf("configmanager[app] parse repoTemplate failed: %w", err)
}
if a.RepoTemplate.Vars == nil {
a.RepoTemplate.Vars = make(RawOptions)
templateVars = make(RawOptions)
}
return newTool(
repoScaffoldingPluginName, a.Name, RawOptions{
"destinationRepo": RawOptions(appRepo.Encode()),
"sourceRepo": RawOptions(templateRepo.Encode()),
"vars": a.RepoTemplate.Vars,
"destinationRepo": RawOptions(a.repoInfo.Encode()),
"sourceRepo": RawOptions(a.repoTemplateInfo.Encode()),
"vars": templateVars,
},
), nil
)
}
return nil, nil
return nil
}

// setDefault will set repoName to appName if repo.name field is empty
func (a *app) setDefault() {
if a.Repo != nil && a.Repo.Name == "" {
func (a *app) setDefault() error {
if a.Repo == nil {
return fmt.Errorf("configmanager[app] is invalid, repo field must be configured")
}
if a.Repo.Name == "" {
a.Repo.Name = a.Name
}
appRepo, err := a.Repo.BuildRepoInfo()
if err != nil {
return fmt.Errorf("configmanager[app] parse repo failed: %w", err)
}
a.repoInfo = appRepo
if a.RepoTemplate != nil {
// templateRepo doesn't need auth info
templateRepo, err := a.RepoTemplate.BuildRepoInfo()
if err != nil {
return fmt.Errorf("configmanager[app] parse repoTemplate failed: %w", err)
}
templateRepo.NeedAuth = false
a.repoTemplateInfo = templateRepo
}
return nil
}

// since all plugin depends on code is deployed, get dependsOn for repoTemplate
Expand All @@ -113,3 +129,13 @@ func (a *app) getRepoTemplateDependants() []string {
}
return dependsOn
}

// newPipelineGlobalOptionFromApp generate pipeline options used for pipeline option configuration
func (a *app) newPipelineGlobalOptionFromApp() *pipelineGlobalOption {
return &pipelineGlobalOption{
RepoInfo: a.repoInfo,
AppSpec: a.Spec,
Scm: a.Repo,
AppName: a.Name,
}
}
55 changes: 52 additions & 3 deletions internal/pkg/configmanager/app_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package configmanager

import (
"fmt"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

Expand All @@ -29,7 +27,7 @@ var _ = Describe("app struct", func() {
It("should return error", func() {
_, err := a.getTools(vars, templateMap)
Expect(err).Should(HaveOccurred())
Expect(err.Error()).Should(ContainSubstring(fmt.Sprintf("app[%s] can't get valid repo config", appName)))
Expect(err.Error()).Should(ContainSubstring("configmanager[app] is invalid, repo field must be configured"))
})
})
When("ci/cd template is not valid", func() {
Expand All @@ -52,6 +50,57 @@ var _ = Describe("app struct", func() {
Expect(err.Error()).Should(ContainSubstring("not found in pipelineTemplates"))
})
})
When("app repo template is empty", func() {
BeforeEach(func() {
a = &app{
Name: appName,
Repo: &scm.SCMInfo{
CloneURL: "http://test.com/test/test_app",
},
}
})
It("should return empty tools", func() {
tools, err := a.getTools(vars, templateMap)
Expect(err).ShouldNot(HaveOccurred())
Expect(len(tools)).Should(Equal(0))
})
})
When("repo url is not valid", func() {
BeforeEach(func() {
a = &app{
Name: appName,
Repo: &scm.SCMInfo{
CloneURL: "not_valid_url",
},
}
})
It("should return empty tools", func() {
_, err := a.getTools(vars, templateMap)
Expect(err).Should(HaveOccurred())
Expect(err.Error()).Should(ContainSubstring("configmanager[app] parse repo failed"))
})
})
When("template repo url is not valid", func() {
BeforeEach(func() {
a = &app{
Name: appName,
RepoTemplate: &repoTemplate{
SCMInfo: &scm.SCMInfo{
CloneURL: "not_valid_url",
},
},
Repo: &scm.SCMInfo{
CloneURL: "http://test.com/test/test_app",
},
}
})
It("should return empty tools", func() {
_, err := a.getTools(vars, templateMap)
Expect(err).Should(HaveOccurred())
Expect(err.Error()).Should(ContainSubstring("configmanager[app] parse repoTemplate failed"))
})
})

})

Context("generateCICDTools method", func() {
Expand Down
5 changes: 1 addition & 4 deletions internal/pkg/configmanager/appspec.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,7 @@ func (s *appSpec) merge(vars map[string]any) map[string]any {
log.Warnf("appspec %+v decode failed: %+v", s, err)
return map[string]any{}
}
if err := mergo.Merge(&specMap, vars); err != nil {
log.Warnf("appSpec %+v merge map failed: %+v", s, err)
return vars
}
_ = mergo.Merge(&specMap, vars)
return specMap
}

Expand Down
12 changes: 1 addition & 11 deletions internal/pkg/configmanager/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package configmanager

import (
"fmt"

"gopkg.in/yaml.v3"
)

// Config is a general config in DevStream.
Expand All @@ -24,19 +22,11 @@ func (c *Config) renderInstanceIDtoOptions() {

func (c *Config) validate() error {
if c.Config.State == nil {
return fmt.Errorf("state is not defined")
return fmt.Errorf("config.state is not defined")
}

if err := c.Tools.validateAll(); err != nil {
return err
}
return nil
}

func (c *Config) String() string {
bs, err := yaml.Marshal(c)
if err != nil {
return err.Error()
}
return string(bs)
}
46 changes: 46 additions & 0 deletions internal/pkg/configmanager/config_test.go
Original file line number Diff line number Diff line change
@@ -1 +1,47 @@
package configmanager

import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("Config struct", func() {
var (
c *Config
toolName, instanceID string
)
BeforeEach(func() {
c = &Config{}
toolName = "test_tool"
instanceID = "test_instance"
})
Context("renderInstanceIDtoOptions method", func() {
When("tool option is null", func() {
BeforeEach(func() {
c.Tools = Tools{
{Name: toolName, InstanceID: instanceID},
}
})
It("should set nil to RawOptions", func() {
c.renderInstanceIDtoOptions()
Expect(len(c.Tools)).Should(Equal(1))
tool := c.Tools[0]
Expect(tool.Options).Should(Equal(RawOptions{
"instanceID": instanceID,
}))
})
})
})
Context("validate method", func() {
When("config state is null", func() {
BeforeEach(func() {
c.Config.State = nil
})
It("should return err", func() {
err := c.validate()
Expect(err).Should(HaveOccurred())
Expect(err.Error()).Should(Equal("config.state is not defined"))
})
})
})
})
Loading

0 comments on commit 593ccc2

Please sign in to comment.